Refactor repository structure

Merge Map data type into Tree.

backend/id.go

@@ -36,6 +36,12 @@ func (id ID) String() string {
 	return hex.EncodeToString(id)
 }
 
+const shortStr = 4
+
+func (id ID) Str() string {
+	return hex.EncodeToString(id[:shortStr])
+}
+
 // Equal compares an ID to another other.
 func (id ID) Equal(other ID) bool {
 	return bytes.Equal(id, other)

backend/id_set.go

@@ -0,0 +1,68 @@
+package backend
+
+import (
+	"errors"
+	"sort"
+	"sync"
+)
+
+type IDSet struct {
+	list IDs
+	m    sync.Mutex
+}
+
+func NewIDSet() *IDSet {
+	return &IDSet{
+		list: make(IDs, 0),
+	}
+}
+
+func (s *IDSet) find(id ID) (int, error) {
+	pos := sort.Search(len(s.list), func(i int) bool {
+		return id.Compare(s.list[i]) >= 0
+	})
+
+	if pos < len(s.list) {
+		candID := s.list[pos]
+		if id.Compare(candID) == 0 {
+			return pos, nil
+		}
+	}
+
+	return pos, errors.New("ID not found")
+}
+
+func (s *IDSet) insert(id ID) {
+	pos, err := s.find(id)
+	if err == nil {
+		// already present
+		return
+	}
+
+	// insert blob
+	// https://code.google.com/p/go-wiki/wiki/SliceTricks
+	s.list = append(s.list, ID{})
+	copy(s.list[pos+1:], s.list[pos:])
+	s.list[pos] = id
+
+	return
+}
+
+func (s *IDSet) Insert(id ID) {
+	s.m.Lock()
+	defer s.m.Unlock()
+
+	s.insert(id)
+}
+
+func (s *IDSet) Find(id ID) error {
+	s.m.Lock()
+	defer s.m.Unlock()
+
+	_, err := s.find(id)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

backend/id_set_test.go

@@ -0,0 +1,40 @@
+package backend_test
+
+import (
+	"crypto/rand"
+	"io"
+	"testing"
+
+	"github.com/restic/restic/backend"
+)
+
+func randomID() []byte {
+	buf := make([]byte, backend.IDSize)
+	_, err := io.ReadFull(rand.Reader, buf)
+	if err != nil {
+		panic(err)
+	}
+	return buf
+}
+
+func TestSet(t *testing.T) {
+	s := backend.NewIDSet()
+
+	testID := randomID()
+	err := s.Find(testID)
+	assert(t, err != nil, "found test ID in IDSet before insertion")
+
+	for i := 0; i < 238; i++ {
+		s.Insert(randomID())
+	}
+
+	s.Insert(testID)
+	ok(t, s.Find(testID))
+
+	for i := 0; i < 80; i++ {
+		s.Insert(randomID())
+	}
+
+	s.Insert(testID)
+	ok(t, s.Find(testID))
+}

backend/interface.go

@@ -10,7 +10,6 @@ const (
 	Lock          = "lock"
 	Snapshot      = "snapshot"
 	Tree          = "tree"
-	Map           = "map"
 )
 
 const (

backend/local.go

@@ -17,7 +17,6 @@ const (
 	dataPath        = "data"
 	snapshotPath    = "snapshots"
 	treePath        = "trees"
-	mapPath         = "maps"
 	lockPath        = "locks"
 	keyPath         = "keys"
 	tempPath        = "tmp"
@@ -38,7 +37,6 @@ func OpenLocal(dir string) (*Local, error) {
 		filepath.Join(dir, dataPath),
 		filepath.Join(dir, snapshotPath),
 		filepath.Join(dir, treePath),
-		filepath.Join(dir, mapPath),
 		filepath.Join(dir, lockPath),
 		filepath.Join(dir, keyPath),
 		filepath.Join(dir, tempPath),
@@ -90,7 +88,6 @@ func CreateLocal(dir string) (*Local, error) {
 		filepath.Join(dir, dataPath),
 		filepath.Join(dir, snapshotPath),
 		filepath.Join(dir, treePath),
-		filepath.Join(dir, mapPath),
 		filepath.Join(dir, lockPath),
 		filepath.Join(dir, keyPath),
 		filepath.Join(dir, tempPath),
@@ -176,8 +173,6 @@ func (b *Local) dirname(t Type, id ID) string {
 		if id != nil {
 			n = filepath.Join(treePath, fmt.Sprintf("%02x", id[0]))
 		}
-	case Map:
-		n = mapPath
 	case Lock:
 		n = lockPath
 	case Key:

backend/local_test.go

@@ -44,7 +44,7 @@ func teardownBackend(t *testing.T, b *backend.Local) {
 }
 
 func testBackend(b *backend.Local, t *testing.T) {
-	for _, tpe := range []backend.Type{backend.Data, backend.Key, backend.Lock, backend.Snapshot, backend.Tree, backend.Map} {
+	for _, tpe := range []backend.Type{backend.Data, backend.Key, backend.Lock, backend.Snapshot, backend.Tree} {
 		// detect non-existing files
 		for _, test := range TestStrings {
 			id, err := backend.ParseID(test.id)
@@ -106,13 +106,13 @@ func testBackend(b *backend.Local, t *testing.T) {
 
 				found, err := b.Test(tpe, id)
 				ok(t, err)
-				assert(t, found, fmt.Sprintf("id %q was not found before removal"))
+				assert(t, found, fmt.Sprintf("id %q was not found before removal", id))
 
 				ok(t, b.Remove(tpe, id))
 
 				found, err = b.Test(tpe, id)
 				ok(t, err)
-				assert(t, !found, fmt.Sprintf("id %q was not found before removal"))
+				assert(t, !found, fmt.Sprintf("id %q not found after removal", id))
 			}
 		}
 

backend/sftp.go

@@ -77,7 +77,6 @@ func OpenSFTP(dir string, program string, args ...string) (*SFTP, error) {
 		filepath.Join(dir, dataPath),
 		filepath.Join(dir, snapshotPath),
 		filepath.Join(dir, treePath),
-		filepath.Join(dir, mapPath),
 		filepath.Join(dir, lockPath),
 		filepath.Join(dir, keyPath),
 		filepath.Join(dir, tempPath),
@@ -134,7 +133,6 @@ func CreateSFTP(dir string, program string, args ...string) (*SFTP, error) {
 		filepath.Join(dir, dataPath),
 		filepath.Join(dir, snapshotPath),
 		filepath.Join(dir, treePath),
-		filepath.Join(dir, mapPath),
 		filepath.Join(dir, lockPath),
 		filepath.Join(dir, keyPath),
 		filepath.Join(dir, tempPath),
@@ -242,7 +240,7 @@ func (r *SFTP) mkdirAll(dir string, mode os.FileMode) error {
 	fi, err = r.c.Lstat(dir)
 	if err != nil {
 		// return previous errors
-		return fmt.Errorf("mkdirAll(%s): unable to create directories: %v, %v", errMkdirAll, errMkdir)
+		return fmt.Errorf("mkdirAll(%s): unable to create directories: %v, %v", dir, errMkdirAll, errMkdir)
 	}
 
 	if !fi.IsDir() {
@@ -284,8 +282,6 @@ func (r *SFTP) dirname(t Type, id ID) string {
 		if id != nil {
 			n = filepath.Join(treePath, fmt.Sprintf("%02x", id[0]))
 		}
-	case Map:
-		n = mapPath
 	case Lock:
 		n = lockPath
 	case Key: