Refactor

cmd/khepri/cmd_backup.go

@@ -1,117 +1,13 @@
 package main
 
 import (
-	"crypto/sha256"
 	"errors"
 	"fmt"
-	"io"
 	"log"
-	"os"
-	"path/filepath"
 
 	"github.com/fd0/khepri"
 )
 
-func hash(filename string) (khepri.ID, error) {
-	h := sha256.New()
-	f, err := os.Open(filename)
-	if err != nil {
-		return nil, err
-	}
-
-	io.Copy(h, f)
-	return h.Sum([]byte{}), nil
-}
-
-func store_file(repo *khepri.Repository, path string) (khepri.ID, error) {
-	obj, idch, err := repo.Create(khepri.TYPE_BLOB)
-	if err != nil {
-		return nil, err
-	}
-
-	file, err := os.Open(path)
-	defer func() {
-		file.Close()
-	}()
-
-	_, err = io.Copy(obj, file)
-	if err != nil {
-		return nil, err
-	}
-
-	err = obj.Close()
-	if err != nil {
-		return nil, err
-	}
-
-	return <-idch, nil
-}
-
-func archive_dir(repo *khepri.Repository, path string) (khepri.ID, error) {
-	log.Printf("archiving dir %q", path)
-
-	dir, err := os.Open(path)
-	if err != nil {
-		log.Printf("open(%q): %v\n", path, err)
-		return nil, err
-	}
-
-	entries, err := dir.Readdir(-1)
-	if err != nil {
-		log.Printf("readdir(%q): %v\n", path, err)
-		return nil, err
-	}
-
-	// use nil ID for empty directories
-	if len(entries) == 0 {
-		return nil, nil
-	}
-
-	t := khepri.NewTree()
-	for _, e := range entries {
-		node := khepri.NodeFromFileInfo(e)
-
-		var id khepri.ID
-		var err error
-
-		if e.IsDir() {
-			id, err = archive_dir(repo, filepath.Join(path, e.Name()))
-		} else {
-			id, err = store_file(repo, filepath.Join(path, e.Name()))
-		}
-
-		node.Content = id
-
-		t.Nodes = append(t.Nodes, node)
-
-		if err != nil {
-			log.Printf("  error storing %q: %v\n", e.Name(), err)
-			continue
-		}
-	}
-
-	log.Printf("  dir %q: %v entries", path, len(t.Nodes))
-
-	obj, idch, err := repo.Create(khepri.TYPE_BLOB)
-
-	if err != nil {
-		log.Printf("error creating object for tree: %v", err)
-		return nil, err
-	}
-
-	err = t.Save(obj)
-	if err != nil {
-		log.Printf("error saving tree to repo: %v", err)
-	}
-
-	obj.Close()
-
-	id := <-idch
-	log.Printf("tree for %q saved at %s", path, id)
-
-	return id, nil
-}
-
 func commandBackup(repo *khepri.Repository, args []string) error {
 	if len(args) != 1 {
 		return errors.New("usage: backup dir")
@@ -119,13 +15,18 @@ func commandBackup(repo *khepri.Repository, args []string) error {
 
 	target := args[0]
 
-	id, err := archive_dir(repo, target)
+	tree, err := khepri.NewTreeFromPath(repo, target)
+	if err != nil {
+		return err
+	}
+
+	id, err := tree.Save(repo)
 	if err != nil {
 		return err
 	}
 
 	sn := khepri.NewSnapshot(target)
-	sn.TreeID = id
+	sn.Content = id
 	snid, err := sn.Save(repo)
 
 	if err != nil {

cmd/khepri/cmd_dump.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"os"
+
+	"github.com/fd0/khepri"
+)
+
+func dump_tree(repo *khepri.Repository, id khepri.ID) error {
+	tree, err := khepri.NewTreeFromRepo(repo, id)
+	if err != nil {
+		return err
+	}
+
+	buf, err := json.MarshalIndent(tree, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("tree %s\n%s\n", id, buf)
+
+	for _, node := range tree.Nodes {
+		if node.Type == "dir" {
+			err = dump_tree(repo, node.Subtree)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "error: %v\n", err)
+			}
+		}
+	}
+
+	return nil
+}
+
+func dump_snapshot(repo *khepri.Repository, id khepri.ID) error {
+	sn, err := khepri.LoadSnapshot(repo, id)
+	if err != nil {
+		log.Fatalf("error loading snapshot %s", id)
+	}
+
+	buf, err := json.MarshalIndent(sn, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("%s\n%s\n", sn, buf)
+
+	return dump_tree(repo, sn.Content)
+}
+
+func dump_file(repo *khepri.Repository, id khepri.ID) error {
+	rd, err := repo.Get(khepri.TYPE_BLOB, id)
+	if err != nil {
+		return err
+	}
+
+	io.Copy(os.Stdout, rd)
+
+	return nil
+}
+
+func commandDump(repo *khepri.Repository, args []string) error {
+	if len(args) != 2 {
+		return errors.New("usage: dump [snapshot|tree|file] ID")
+	}
+
+	tpe := args[0]
+
+	id, err := khepri.ParseID(args[1])
+	if err != nil {
+		errx(1, "invalid id %q: %v", args[0], err)
+	}
+
+	switch tpe {
+	case "snapshot":
+		return dump_snapshot(repo, id)
+	case "tree":
+		return dump_tree(repo, id)
+	case "file":
+		return dump_file(repo, id)
+	default:
+		return fmt.Errorf("invalid type %q", tpe)
+	}
+
+	return nil
+}

cmd/khepri/cmd_fsck.go

@@ -41,7 +41,7 @@ func fsck_snapshot(repo *khepri.Repository, id khepri.ID) (bool, error) {
 		return false, err
 	}
 
-	return fsck_tree(repo, sn.TreeID)
+	return fsck_tree(repo, sn.Content)
 }
 
 func commandFsck(repo *khepri.Repository, args []string) error {

cmd/khepri/cmd_restore.go

@@ -2,44 +2,95 @@ package main
 
 import (
 	"errors"
+	"fmt"
 	"io"
 	"log"
 	"os"
-	"path"
+	"path/filepath"
+	"syscall"
 
 	"github.com/fd0/khepri"
 )
 
-func restore_file(repo *khepri.Repository, node khepri.Node, target string) error {
-	log.Printf("  restore file %q\n", target)
+func restore_file(repo *khepri.Repository, node *khepri.Node, path string) (err error) {
+	switch node.Type {
+	case "file":
+		// TODO: handle hard links
+		rd, err := repo.Get(khepri.TYPE_BLOB, node.Content)
+		if err != nil {
+			return err
+		}
 
-	rd, err := repo.Get(khepri.TYPE_BLOB, node.Content)
+		f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0600)
+		defer f.Close()
+		if err != nil {
+			return err
+		}
+
+		_, err = io.Copy(f, rd)
+		if err != nil {
+			return err
+		}
+
+	case "symlink":
+		err = os.Symlink(node.LinkTarget, path)
+		if err != nil {
+			return err
+		}
+
+		err = os.Lchown(path, int(node.UID), int(node.GID))
+		if err != nil {
+			return err
+		}
+
+		f, err := os.OpenFile(path, khepri.O_PATH|syscall.O_NOFOLLOW, 0600)
+		defer f.Close()
+		if err != nil {
+			return err
+		}
+
+		var utimes = []syscall.Timeval{
+			syscall.NsecToTimeval(node.AccessTime.UnixNano()),
+			syscall.NsecToTimeval(node.ModTime.UnixNano()),
+		}
+		err = syscall.Futimes(int(f.Fd()), utimes)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	case "dev":
+		err = syscall.Mknod(path, syscall.S_IFBLK|0600, int(node.Device))
+		if err != nil {
+			return err
+		}
+	case "chardev":
+		err = syscall.Mknod(path, syscall.S_IFCHR|0600, int(node.Device))
+		if err != nil {
+			return err
+		}
+	case "fifo":
+		err = syscall.Mkfifo(path, 0600)
+		if err != nil {
+			return err
+		}
+	case "socket":
+		// nothing to do, we do not restore sockets
+	default:
+		return fmt.Errorf("filetype %q not implemented!\n", node.Type)
+	}
+
+	err = os.Chmod(path, node.Mode)
 	if err != nil {
 		return err
 	}
 
-	f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY, 0600)
-	defer f.Close()
+	err = os.Chown(path, int(node.UID), int(node.GID))
 	if err != nil {
 		return err
 	}
 
-	_, err = io.Copy(f, rd)
-	if err != nil {
-		return err
-	}
-
-	err = f.Chmod(node.Mode)
-	if err != nil {
-		return err
-	}
-
-	err = f.Chown(int(node.User), int(node.Group))
-	if err != nil {
-		return err
-	}
-
-	err = os.Chtimes(target, node.AccessTime, node.ModTime)
+	err = os.Chtimes(path, node.AccessTime, node.ModTime)
 	if err != nil {
 		return err
 	}
@@ -47,61 +98,48 @@ func restore_file(repo *khepri.Repository, node khepri.Node, target string) erro
 	return nil
 }
 
-func restore_dir(repo *khepri.Repository, id khepri.ID, target string) error {
-	log.Printf("  restore dir %q\n", target)
-	rd, err := repo.Get(khepri.TYPE_BLOB, id)
-	if err != nil {
-		return err
-	}
+func restore_subtree(repo *khepri.Repository, tree *khepri.Tree, path string) {
+	fmt.Printf("restore_subtree(%s)\n", path)
 
-	t := khepri.NewTree()
-	err = t.Restore(rd)
-	if err != nil {
-		return err
-	}
+	for _, node := range tree.Nodes {
+		nodepath := filepath.Join(path, node.Name)
+		// fmt.Printf("%s:%s\n", node.Type, nodepath)
 
-	for _, node := range t.Nodes {
-		name := path.Base(node.Name)
-		if name == "." || name == ".." {
-			return errors.New("invalid path")
-		}
-
-		nodepath := path.Join(target, name)
-		if node.Mode.IsDir() {
-			err = os.Mkdir(nodepath, 0700)
+		if node.Type == "dir" {
+			err := os.Mkdir(nodepath, 0700)
 			if err != nil {
-				return err
+				fmt.Fprintf(os.Stderr, "%s\n", err)
+				continue
 			}
 
 			err = os.Chmod(nodepath, node.Mode)
 			if err != nil {
-				return err
+				fmt.Fprintf(os.Stderr, "%s\n", err)
+				continue
 			}
 
-			err = os.Chown(nodepath, int(node.User), int(node.Group))
+			err = os.Chown(nodepath, int(node.UID), int(node.GID))
 			if err != nil {
-				return err
+				fmt.Fprintf(os.Stderr, "%s\n", err)
+				continue
 			}
 
-			err = restore_dir(repo, node.Content, nodepath)
-			if err != nil {
-				return err
-			}
+			restore_subtree(repo, node.Tree, filepath.Join(path, node.Name))
 
 			err = os.Chtimes(nodepath, node.AccessTime, node.ModTime)
 			if err != nil {
-				return err
+				fmt.Fprintf(os.Stderr, "%s\n", err)
+				continue
 			}
 
 		} else {
-			err = restore_file(repo, node, nodepath)
+			err := restore_file(repo, node, nodepath)
 			if err != nil {
-				return err
+				fmt.Fprintf(os.Stderr, "%s\n", err)
+				continue
 			}
 		}
 	}
-
-	return nil
 }
 
 func commandRestore(repo *khepri.Repository, args []string) error {
@@ -126,11 +164,13 @@ func commandRestore(repo *khepri.Repository, args []string) error {
 		log.Fatalf("error loading snapshot %s", id)
 	}
 
-	err = restore_dir(repo, sn.TreeID, target)
+	tree, err := khepri.NewTreeFromRepo(repo, sn.Content)
 	if err != nil {
-		return err
+		log.Fatalf("error loading tree %s", sn.Content)
 	}
 
+	restore_subtree(repo, tree, target)
+
 	log.Printf("%q restored to %q\n", id, target)
 
 	return nil

cmd/khepri/main.go

@@ -34,6 +34,7 @@ func init() {
 	commands["list"] = commandList
 	commands["snapshots"] = commandSnapshots
 	commands["fsck"] = commandFsck
+	commands["dump"] = commandDump
 }
 
 func main() {

cmd/stat/stat.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/fd0/khepri"
+)
+
+func main() {
+	if len(os.Args) == 1 {
+		fmt.Printf("usage: %s [file] [file] [...]\n", os.Args[0])
+		os.Exit(1)
+	}
+
+	for _, path := range os.Args[1:] {
+		fmt.Printf("lstat %s\n", path)
+
+		fi, err := os.Lstat(path)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%v", err)
+			continue
+		}
+
+		node, err := khepri.NodeFromFileInfo(path, fi)
+		if err != nil {
+			fmt.Printf("err: %v\n", err)
+		}
+
+		buf, err := json.MarshalIndent(node, "", "  ")
+		if err != nil {
+			panic(err)
+		}
+		fmt.Printf("%s\n", string(buf))
+	}
+}

cmd/tree_serialise/main.go

@@ -0,0 +1,222 @@
+package main
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func check(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+// References content within a repository.
+type ID []byte
+
+func (id ID) String() string {
+	return hex.EncodeToString(id)
+}
+
+func (id ID) MarshalJSON() ([]byte, error) {
+	return json.Marshal(id.String())
+}
+
+func (id *ID) UnmarshalJSON(b []byte) error {
+	var s string
+	err := json.Unmarshal(b, &s)
+	if err != nil {
+		return err
+	}
+
+	*id = make([]byte, len(s)/2)
+	_, err = hex.Decode(*id, []byte(s))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ParseID converts the given string to an ID.
+func ParseID(s string) ID {
+	b, err := hex.DecodeString(s)
+
+	if err != nil {
+		panic(err)
+	}
+
+	return ID(b)
+}
+
+type Repository interface {
+	Store([]byte) ID
+	Get(ID) []byte
+}
+
+type Repo map[string][]byte
+
+func (r Repo) Store(buf []byte) ID {
+	hash := sha256.New()
+	_, err := hash.Write(buf)
+	check(err)
+
+	id := ID(hash.Sum([]byte{}))
+	r[id.String()] = buf
+
+	return id
+}
+
+func (r Repo) Get(id ID) []byte {
+	buf, ok := r[id.String()]
+	if !ok {
+		panic("no such id")
+	}
+
+	return buf
+}
+
+func (r Repo) Dump(wr io.Writer) {
+	for k, v := range r {
+		_, err := wr.Write([]byte(k))
+		check(err)
+		_, err = wr.Write([]byte(":"))
+		check(err)
+		_, err = wr.Write(v)
+		check(err)
+		_, err = wr.Write([]byte("\n"))
+		check(err)
+	}
+}
+
+type Tree struct {
+	Nodes []*Node `json:"nodes,omitempty"`
+}
+
+type Node struct {
+	Name    string `json:"name"`
+	Tree    *Tree  `json:"tree,omitempty"`
+	Subtree ID     `json:"subtree,omitempty"`
+	Content ID     `json:"content,omitempty"`
+}
+
+func (tree Tree) Save(repo Repository) ID {
+	// fmt.Printf("nodes: %#v\n", tree.Nodes)
+	for _, node := range tree.Nodes {
+		if node.Tree != nil {
+			node.Subtree = node.Tree.Save(repo)
+			node.Tree = nil
+		}
+	}
+
+	buf, err := json.Marshal(tree)
+	check(err)
+
+	return repo.Store(buf)
+}
+
+func (tree Tree) PP(wr io.Writer) {
+	tree.pp(0, wr)
+}
+
+func (tree Tree) pp(indent int, wr io.Writer) {
+	for _, node := range tree.Nodes {
+		if node.Tree != nil {
+			fmt.Printf("%s%s/\n", strings.Repeat("    ", indent), node.Name)
+			node.Tree.pp(indent+1, wr)
+		} else {
+			fmt.Printf("%s%s [%s]\n", strings.Repeat("    ", indent), node.Name, node.Content)
+		}
+
+	}
+}
+
+func create_tree(path string) *Tree {
+	dir, err := os.Open(path)
+	check(err)
+
+	entries, err := dir.Readdir(-1)
+	check(err)
+
+	tree := &Tree{
+		Nodes: make([]*Node, 0, len(entries)),
+	}
+
+	for _, entry := range entries {
+		node := &Node{}
+		node.Name = entry.Name()
+
+		if !entry.Mode().IsDir() && entry.Mode()&os.ModeType != 0 {
+			fmt.Fprintf(os.Stderr, "skipping %q\n", filepath.Join(path, entry.Name()))
+			continue
+		}
+
+		tree.Nodes = append(tree.Nodes, node)
+
+		if entry.IsDir() {
+			node.Tree = create_tree(filepath.Join(path, entry.Name()))
+			continue
+		}
+
+		file, err := os.Open(filepath.Join(path, entry.Name()))
+		defer file.Close()
+		check(err)
+
+		hash := sha256.New()
+		io.Copy(hash, file)
+
+		node.Content = hash.Sum([]byte{})
+	}
+
+	return tree
+}
+
+func load_tree(repo Repository, id ID) *Tree {
+	tree := &Tree{}
+
+	buf := repo.Get(id)
+	json.Unmarshal(buf, tree)
+
+	for _, node := range tree.Nodes {
+		if node.Subtree != nil {
+			node.Tree = load_tree(repo, node.Subtree)
+			node.Subtree = nil
+		}
+	}
+
+	return tree
+}
+
+func main() {
+	repo := make(Repo)
+
+	tree := create_tree(os.Args[1])
+	// encoder := json.NewEncoder(os.Stdout)
+	// fmt.Println("---------------------------")
+	// encoder.Encode(tree)
+	// fmt.Println("---------------------------")
+
+	id := tree.Save(repo)
+
+	// for k, v := range repo {
+	// 	fmt.Printf("%s: %s\n", k, v)
+	// }
+
+	// fmt.Println("---------------------------")
+
+	tree2 := load_tree(repo, id)
+	tree2.PP(os.Stdout)
+	// encoder.Encode(tree2)
+
+	// dumpfile, err := os.Create("dump")
+	// defer dumpfile.Close()
+	// check(err)
+
+	// repo.Dump(dumpfile)
+}

cmd/tree_test/khepri-repo/blobs/44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a

@@ -0,0 +1 @@
+{}

cmd/tree_test/khepri-repo/blobs/655ee6b9ff7fb7324f1e8566eca49c67d251459787765f9c5c6af513e3db0c72

@@ -0,0 +1 @@
+{"nodes":[{"name":"blobs","type":"dir","mode":2147484096,"mtime":"2014-08-11T19:51:11.894770622+02:00","atime":"2014-08-11T19:51:11.894770622+02:00","ctime":"2014-08-11T19:51:11.894770622+02:00","uid":1000,"gid":100,"user":"fd0","inode":428349318,"subtree":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"},{"name":"refs","type":"dir","mode":2147484096,"mtime":"2014-08-11T19:51:11.894770622+02:00","atime":"2014-08-11T19:51:11.894770622+02:00","ctime":"2014-08-11T19:51:11.894770622+02:00","uid":1000,"gid":100,"user":"fd0","inode":4883656,"subtree":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"},{"name":"tmp","type":"dir","mode":2147484096,"mtime":"2014-08-11T19:51:11.894770622+02:00","atime":"2014-08-11T19:51:11.894770622+02:00","ctime":"2014-08-11T19:51:11.894770622+02:00","uid":1000,"gid":100,"user":"fd0","inode":169971890,"subtree":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"}]}

cmd/tree_test/khepri-repo/blobs/9e3528071fbfda7702010f2b711174c17df3156b77c14015ce633c9722c0e146

@@ -0,0 +1 @@
+{"nodes":[{"name":"main.go","type":"file","mode":420,"mtime":"2014-08-10T23:18:15.815999516+02:00","atime":"2014-08-10T23:18:15.815999516+02:00","ctime":"2014-08-10T23:18:15.819332884+02:00","uid":1000,"gid":100,"user":"fd0","inode":193352413,"size":1181,"links":1,"content":"c45d3975908245296eb5730cf4bbbe9d8c39b0736658b5df9beada0739d40569"},{"name":"khepri-repo","type":"dir","mode":2147484096,"mtime":"2014-08-11T19:51:11.894770622+02:00","atime":"2014-08-11T19:51:11.894770622+02:00","ctime":"2014-08-11T19:51:11.894770622+02:00","uid":1000,"gid":100,"user":"fd0","inode":275443328,"subtree":"655ee6b9ff7fb7324f1e8566eca49c67d251459787765f9c5c6af513e3db0c72"}]}

cmd/tree_test/main.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/fd0/khepri"
+)
+
+func check(err error) {
+	if err == nil {
+		return
+	}
+
+	fmt.Fprintf(os.Stderr, "error: %v\n", err)
+	os.Exit(1)
+}
+
+func save(repo *khepri.Repository, path string) {
+	tree, err := khepri.NewTreeFromPath(repo, path)
+
+	check(err)
+
+	id, err := tree.Save(repo)
+
+	fmt.Printf("saved tree as %s\n", id)
+}
+
+func restore(repo *khepri.Repository, idstr string) {
+	id, err := khepri.ParseID(idstr)
+	check(err)
+
+	tree, err := khepri.NewTreeFromRepo(repo, id)
+	check(err)
+
+	walk(0, tree)
+}
+
+func walk(indent int, tree *khepri.Tree) {
+	for _, node := range tree.Nodes {
+		if node.Type == "dir" {
+			fmt.Printf("%s%s:%s/\n", strings.Repeat(" ", indent), node.Type, node.Name)
+			walk(indent+1, node.Tree)
+		} else {
+			fmt.Printf("%s%s:%s\n", strings.Repeat(" ", indent), node.Type, node.Name)
+		}
+	}
+}
+
+func main() {
+	if len(os.Args) != 3 {
+		fmt.Fprintf(os.Stderr, "usage: %s [save|restore] DIR\n", os.Args[0])
+		os.Exit(1)
+	}
+
+	command := os.Args[1]
+	arg := os.Args[2]
+
+	repo, err := khepri.NewRepository("khepri-repo")
+	check(err)
+
+	switch command {
+	case "save":
+		save(repo, arg)
+	case "restore":
+		restore(repo, arg)
+	}
+}