Add decrypt, refactor

backend/doc.go

@@ -0,0 +1,2 @@
+// Package backend provides local and remote storage for khepri backups.
+package backend

backend/generic.go

@@ -0,0 +1,84 @@
+package backend
+
+import (
+	"bytes"
+	"compress/zlib"
+	"crypto/sha256"
+	"io/ioutil"
+)
+
+// Each lists all entries of type t in the backend and calls function f() with
+// the id and data.
+func Each(be Server, t Type, f func(id ID, data []byte, err error)) error {
+	ids, err := be.List(t)
+	if err != nil {
+		return err
+	}
+
+	for _, id := range ids {
+		data, err := be.Get(t, id)
+		if err != nil {
+			f(id, nil, err)
+			continue
+		}
+
+		f(id, data, nil)
+	}
+
+	return nil
+}
+
+// Each lists all entries of type t in the backend and calls function f() with
+// the id.
+func EachID(be Server, t Type, f func(ID)) error {
+	ids, err := be.List(t)
+	if err != nil {
+		return err
+	}
+
+	for _, id := range ids {
+		f(id)
+	}
+
+	return nil
+}
+
+// Compress applies zlib compression to data.
+func Compress(data []byte) []byte {
+	// apply zlib compression
+	var b bytes.Buffer
+	w := zlib.NewWriter(&b)
+	_, err := w.Write(data)
+	if err != nil {
+		panic(err)
+	}
+	w.Close()
+
+	return b.Bytes()
+}
+
+// Uncompress reverses zlib compression on data.
+func Uncompress(data []byte) []byte {
+	b := bytes.NewBuffer(data)
+	r, err := zlib.NewReader(b)
+	if err != nil {
+		panic(err)
+	}
+
+	buf, err := ioutil.ReadAll(r)
+	if err != nil {
+		panic(err)
+	}
+
+	r.Close()
+
+	return buf
+}
+
+// Hash returns the ID for data.
+func Hash(data []byte) ID {
+	h := sha256.Sum256(data)
+	id := make(ID, 32)
+	copy(id, h[:])
+	return id
+}

backend/generic_test.go

@@ -0,0 +1,36 @@
+package backend_test
+
+import (
+	"fmt"
+	"path/filepath"
+	"reflect"
+	"runtime"
+	"testing"
+)
+
+// assert fails the test if the condition is false.
+func assert(tb testing.TB, condition bool, msg string, v ...interface{}) {
+	if !condition {
+		_, file, line, _ := runtime.Caller(1)
+		fmt.Printf("\033[31m%s:%d: "+msg+"\033[39m\n\n", append([]interface{}{filepath.Base(file), line}, v...)...)
+		tb.FailNow()
+	}
+}
+
+// ok fails the test if an err is not nil.
+func ok(tb testing.TB, err error) {
+	if err != nil {
+		_, file, line, _ := runtime.Caller(1)
+		fmt.Printf("\033[31m%s:%d: unexpected error: %s\033[39m\n\n", filepath.Base(file), line, err.Error())
+		tb.FailNow()
+	}
+}
+
+// equals fails the test if exp is not equal to act.
+func equals(tb testing.TB, exp, act interface{}) {
+	if !reflect.DeepEqual(exp, act) {
+		_, file, line, _ := runtime.Caller(1)
+		fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act)
+		tb.FailNow()
+	}
+}

backend/id.go

@@ -0,0 +1,105 @@
+package backend
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+)
+
+const sha256_length = 32 // in bytes
+
+// References content within a repository.
+type ID []byte
+
+// ParseID converts the given string to an ID.
+func ParseID(s string) (ID, error) {
+	b, err := hex.DecodeString(s)
+
+	if err != nil {
+		return nil, err
+	}
+
+	if len(b) != sha256_length {
+		return nil, errors.New("invalid length for sha256 hash")
+	}
+
+	return ID(b), nil
+}
+
+func (id ID) String() string {
+	return hex.EncodeToString(id)
+}
+
+// Equal compares an ID to another other.
+func (id ID) Equal(other ID) bool {
+	return bytes.Equal(id, other)
+}
+
+// EqualString compares this ID to another one, given as a string.
+func (id ID) EqualString(other string) (bool, error) {
+	s, err := hex.DecodeString(other)
+	if err != nil {
+		return false, err
+	}
+
+	return id.Equal(ID(s)), nil
+}
+
+func (id ID) MarshalJSON() ([]byte, error) {
+	return json.Marshal(id.String())
+}
+
+func (id *ID) UnmarshalJSON(b []byte) error {
+	var s string
+	err := json.Unmarshal(b, &s)
+	if err != nil {
+		return err
+	}
+
+	*id = make([]byte, len(s)/2)
+	_, err = hex.Decode(*id, []byte(s))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func IDFromData(d []byte) ID {
+	hash := sha256.Sum256(d)
+	id := make([]byte, sha256_length)
+	copy(id, hash[:])
+	return id
+}
+
+type IDs []ID
+
+func (ids IDs) Len() int {
+	return len(ids)
+}
+
+func (ids IDs) Less(i, j int) bool {
+	if len(ids[i]) < len(ids[j]) {
+		return true
+	}
+
+	for k, b := range ids[i] {
+		if b == ids[j][k] {
+			continue
+		}
+
+		if b < ids[j][k] {
+			return true
+		} else {
+			return false
+		}
+	}
+
+	return false
+}
+
+func (ids IDs) Swap(i, j int) {
+	ids[i], ids[j] = ids[j], ids[i]
+}

backend/interface.go

@@ -0,0 +1,21 @@
+package backend
+
+type Type string
+
+const (
+	Blob     Type = "blob"
+	Key           = "key"
+	Lock          = "lock"
+	Snapshot      = "snapshot"
+	Tree          = "tree"
+)
+
+type Server interface {
+	Create(Type, []byte) (ID, error)
+	Get(Type, ID) ([]byte, error)
+	List(Type) (IDs, error)
+	Test(Type, ID) (bool, error)
+	Remove(Type, ID) error
+
+	Location() string
+}

backend/local.go

@@ -0,0 +1,213 @@
+package backend
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+const (
+	dirMode      = 0700
+	blobPath     = "blobs"
+	snapshotPath = "snapshots"
+	treePath     = "trees"
+	lockPath     = "locks"
+	keyPath      = "keys"
+	tempPath     = "tmp"
+)
+
+type Local struct {
+	p string
+}
+
+// OpenLocal opens the local backend at dir.
+func OpenLocal(dir string) (*Local, error) {
+	items := []string{
+		dir,
+		filepath.Join(dir, blobPath),
+		filepath.Join(dir, snapshotPath),
+		filepath.Join(dir, treePath),
+		filepath.Join(dir, lockPath),
+		filepath.Join(dir, keyPath),
+		filepath.Join(dir, tempPath),
+	}
+
+	// test if all necessary dirs and files are there
+	for _, d := range items {
+		if _, err := os.Stat(d); err != nil {
+			return nil, fmt.Errorf("%s does not exist", d)
+		}
+	}
+
+	return &Local{p: dir}, nil
+}
+
+// CreateLocal creates all the necessary files and directories for a new local
+// backend at dir.
+func CreateLocal(dir string) (*Local, error) {
+	dirs := []string{
+		dir,
+		filepath.Join(dir, blobPath),
+		filepath.Join(dir, snapshotPath),
+		filepath.Join(dir, treePath),
+		filepath.Join(dir, lockPath),
+		filepath.Join(dir, keyPath),
+		filepath.Join(dir, tempPath),
+	}
+
+	// test if directories already exist
+	for _, d := range dirs[1:] {
+		if _, err := os.Stat(d); err == nil {
+			return nil, fmt.Errorf("dir %s already exists", d)
+		}
+	}
+
+	// create paths for blobs, refs and temp
+	for _, d := range dirs {
+		err := os.MkdirAll(d, dirMode)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// open repository
+	return OpenLocal(dir)
+}
+
+// Location returns this backend's location (the directory name).
+func (b *Local) Location() string {
+	return b.p
+}
+
+// Return temp directory in correct directory for this backend.
+func (b *Local) tempFile() (*os.File, error) {
+	return ioutil.TempFile(filepath.Join(b.p, tempPath), "temp-")
+}
+
+// Rename temp file to final name according to type and ID.
+func (b *Local) renameFile(file *os.File, t Type, id ID) error {
+	filename := filepath.Join(b.dir(t), id.String())
+	return os.Rename(file.Name(), filename)
+}
+
+// Construct directory for given Type.
+func (b *Local) dir(t Type) string {
+	var n string
+	switch t {
+	case Blob:
+		n = blobPath
+	case Snapshot:
+		n = snapshotPath
+	case Tree:
+		n = treePath
+	case Lock:
+		n = lockPath
+	case Key:
+		n = keyPath
+	}
+	return filepath.Join(b.p, n)
+}
+
+// Create stores new content of type t and data and returns the ID.
+func (b *Local) Create(t Type, data []byte) (ID, error) {
+	// TODO: make sure that tempfile is removed upon error
+
+	// create tempfile in repository
+	var err error
+	file, err := b.tempFile()
+	if err != nil {
+		return nil, err
+	}
+
+	// write data to tempfile
+	_, err = file.Write(data)
+	if err != nil {
+		return nil, err
+	}
+
+	// close tempfile, return id
+	id := IDFromData(data)
+	err = b.renameFile(file, t, id)
+	if err != nil {
+		return nil, err
+	}
+
+	return id, nil
+}
+
+// Construct path for given Type and ID.
+func (b *Local) filename(t Type, id ID) string {
+	return filepath.Join(b.dir(t), id.String())
+}
+
+// Get returns the content stored under the given ID.
+func (b *Local) Get(t Type, id ID) ([]byte, error) {
+	// try to open file
+	file, err := os.Open(b.filename(t, id))
+	defer file.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	// read all
+	buf, err := ioutil.ReadAll(file)
+	if err != nil {
+		return nil, err
+	}
+
+	return buf, nil
+}
+
+// Test returns true if a blob of the given type and ID exists in the backend.
+func (b *Local) Test(t Type, id ID) (bool, error) {
+	// try to open file
+	file, err := os.Open(b.filename(t, id))
+	defer func() {
+		file.Close()
+	}()
+
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
+	}
+
+	return true, nil
+}
+
+// Remove removes the content stored at ID.
+func (b *Local) Remove(t Type, id ID) error {
+	return os.Remove(b.filename(t, id))
+}
+
+// List lists all objects of a given type.
+func (b *Local) List(t Type) (IDs, error) {
+	// TODO: use os.Open() and d.Readdirnames() instead of Glob()
+	pattern := filepath.Join(b.dir(t), "*")
+
+	matches, err := filepath.Glob(pattern)
+	if err != nil {
+		return nil, err
+	}
+
+	ids := make(IDs, 0, len(matches))
+
+	for _, m := range matches {
+		base := filepath.Base(m)
+
+		if base == "" {
+			continue
+		}
+		id, err := ParseID(base)
+
+		if err != nil {
+			continue
+		}
+
+		ids = append(ids, id)
+	}
+
+	return ids, nil
+}

backend/local_test.go

@@ -0,0 +1,171 @@
+package backend_test
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"sort"
+	"testing"
+
+	"github.com/fd0/khepri/backend"
+)
+
+var testCleanup = flag.Bool("test.cleanup", true, "clean up after running tests (remove local backend directory with all content)")
+
+var TestStrings = []struct {
+	id   string
+	data string
+}{
+	{"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", "foobar"},
+	{"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"},
+	{"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", "foo/bar"},
+	{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"},
+}
+
+func setupBackend(t *testing.T) *backend.Local {
+	tempdir, err := ioutil.TempDir("", "khepri-test-")
+	ok(t, err)
+
+	b, err := backend.CreateLocal(tempdir)
+	ok(t, err)
+
+	t.Logf("created local backend at %s", tempdir)
+
+	return b
+}
+
+func teardownBackend(t *testing.T, b *backend.Local) {
+	if !*testCleanup {
+		t.Logf("leaving local backend at %s\n", b.Location())
+		return
+	}
+
+	ok(t, os.RemoveAll(b.Location()))
+}
+
+func testBackend(b backend.Server, t *testing.T) {
+	for _, tpe := range []backend.Type{backend.Blob, backend.Key, backend.Lock, backend.Snapshot, backend.Tree} {
+		// detect non-existing files
+		for _, test := range TestStrings {
+			id, err := backend.ParseID(test.id)
+			ok(t, err)
+
+			// test if blob is already in repository
+			ret, err := b.Test(tpe, id)
+			ok(t, err)
+			assert(t, !ret, "blob was found to exist before creating")
+
+			// try to open not existing blob
+			d, err := b.Get(tpe, id)
+			assert(t, err != nil && d == nil, "blob data could be extracted befor creation")
+
+			// try to get string out, should fail
+			ret, err = b.Test(tpe, id)
+			ok(t, err)
+			assert(t, !ret, fmt.Sprintf("id %q was found (but should not have)", test.id))
+		}
+
+		// add files
+		for _, test := range TestStrings {
+			// store string in backend
+			id, err := b.Create(tpe, []byte(test.data))
+			ok(t, err)
+
+			equals(t, test.id, id.String())
+
+			// try to get it out again
+			buf, err := b.Get(tpe, id)
+			ok(t, err)
+			assert(t, buf != nil, "Get() returned nil")
+
+			// compare content
+			equals(t, test.data, string(buf))
+		}
+
+		// list items
+		IDs := backend.IDs{}
+
+		for _, test := range TestStrings {
+			id, err := backend.ParseID(test.id)
+			ok(t, err)
+			IDs = append(IDs, id)
+		}
+
+		ids, err := b.List(tpe)
+		ok(t, err)
+
+		sort.Sort(ids)
+		sort.Sort(IDs)
+		equals(t, IDs, ids)
+
+		// remove content if requested
+		if *testCleanup {
+			for _, test := range TestStrings {
+				id, err := backend.ParseID(test.id)
+				ok(t, err)
+
+				found, err := b.Test(tpe, id)
+				ok(t, err)
+				assert(t, found, fmt.Sprintf("id %q was not found before removal"))
+
+				ok(t, b.Remove(tpe, id))
+
+				found, err = b.Test(tpe, id)
+				ok(t, err)
+				assert(t, !found, fmt.Sprintf("id %q was not found before removal"))
+			}
+		}
+
+	}
+}
+
+func TestBackend(t *testing.T) {
+	// test for non-existing backend
+	b, err := backend.OpenLocal("/invalid-khepri-test")
+	assert(t, err != nil, "opening invalid repository at /invalid-khepri-test should have failed, but err is nil")
+	assert(t, b == nil, fmt.Sprintf("opening invalid repository at /invalid-khepri-test should have failed, but b is not nil: %v", b))
+
+	b = setupBackend(t)
+	defer teardownBackend(t, b)
+
+	testBackend(b, t)
+}
+
+func TestLocalBackendCreationFailures(t *testing.T) {
+	b := setupBackend(t)
+	defer teardownBackend(t, b)
+
+	// test failure to create a new repository at the same location
+	b2, err := backend.CreateLocal(b.Location())
+	assert(t, err != nil && b2 == nil, fmt.Sprintf("creating a repository at %s for the second time should have failed", b.Location()))
+
+	// test failure to create a new repository at the same location without a config file
+	b2, err = backend.CreateLocal(b.Location())
+	assert(t, err != nil && b2 == nil, fmt.Sprintf("creating a repository at %s for the second time should have failed", b.Location()))
+}
+
+func TestID(t *testing.T) {
+	for _, test := range TestStrings {
+		id, err := backend.ParseID(test.id)
+		ok(t, err)
+
+		id2, err := backend.ParseID(test.id)
+		ok(t, err)
+		assert(t, id.Equal(id2), "ID.Equal() does not work as expected")
+
+		ret, err := id.EqualString(test.id)
+		ok(t, err)
+		assert(t, ret, "ID.EqualString() returned wrong value")
+
+		// test json marshalling
+		buf, err := id.MarshalJSON()
+		ok(t, err)
+		equals(t, "\""+test.id+"\"", string(buf))
+
+		var id3 backend.ID
+		err = id3.UnmarshalJSON(buf)
+		ok(t, err)
+		equals(t, id, id3)
+	}
+}