repository: optimize MasterIndex.Each

Sending data through a channel at very high frequency is extremely
inefficient. Thus use simple callbacks instead of channels.

> name                old time/op  new time/op  delta
> MasterIndexEach-16   6.68s ±24%   0.96s ± 2%  -85.64%  (p=0.008 n=5+5)

internal/repository/index.go

@@ -217,34 +217,22 @@ func (idx *Index) AddToSupersedes(ids ...restic.ID) error {
 	return nil
 }
 
-// Each returns a channel that yields all blobs known to the index. When the
-// context is cancelled, the background goroutine terminates. This blocks any
+// Each passes all blobs known to the index to the callback fn. This blocks any
 // modification of the index.
-func (idx *Index) Each(ctx context.Context) <-chan restic.PackedBlob {
+func (idx *Index) Each(ctx context.Context, fn func(restic.PackedBlob)) {
 	idx.m.Lock()
+	defer idx.m.Unlock()
 
-	ch := make(chan restic.PackedBlob)
-
-	go func() {
-		defer idx.m.Unlock()
-		defer func() {
-			close(ch)
-		}()
-
-		for typ := range idx.byType {
-			m := &idx.byType[typ]
-			m.foreach(func(e *indexEntry) bool {
-				select {
-				case <-ctx.Done():
-					return false
-				case ch <- idx.toPackedBlob(e, restic.BlobType(typ)):
-					return true
-				}
-			})
-		}
-	}()
-
-	return ch
+	for typ := range idx.byType {
+		m := &idx.byType[typ]
+		m.foreach(func(e *indexEntry) bool {
+			if ctx.Err() != nil {
+				return false
+			}
+			fn(idx.toPackedBlob(e, restic.BlobType(typ)))
+			return true
+		})
+	}
 }
 
 type EachByPackResult struct {

internal/repository/index_test.go

@@ -355,11 +355,11 @@ func TestIndexUnserialize(t *testing.T) {
 }
 
 func listPack(idx *repository.Index, id restic.ID) (pbs []restic.PackedBlob) {
-	for pb := range idx.Each(context.TODO()) {
+	idx.Each(context.TODO(), func(pb restic.PackedBlob) {
 		if pb.PackID.Equal(id) {
 			pbs = append(pbs, pb)
 		}
-	}
+	})
 	return pbs
 }
 

internal/repository/master_index.go

@@ -234,30 +234,15 @@ func (mi *MasterIndex) finalizeFullIndexes() []*Index {
 	return list
 }
 
-// Each returns a channel that yields all blobs known to the index. When the
-// context is cancelled, the background goroutine terminates. This blocks any
-// modification of the index.
-func (mi *MasterIndex) Each(ctx context.Context) <-chan restic.PackedBlob {
+// Each runs fn on all blobs known to the index. When the context is cancelled,
+// the index iteration return immediately. This blocks any modification of the index.
+func (mi *MasterIndex) Each(ctx context.Context, fn func(restic.PackedBlob)) {
 	mi.idxMutex.RLock()
+	defer mi.idxMutex.RUnlock()
 
-	ch := make(chan restic.PackedBlob)
-
-	go func() {
-		defer mi.idxMutex.RUnlock()
-		defer close(ch)
-
-		for _, idx := range mi.idx {
-			for pb := range idx.Each(ctx) {
-				select {
-				case <-ctx.Done():
-					return
-				case ch <- pb:
-				}
-			}
-		}
-	}()
-
-	return ch
+	for _, idx := range mi.idx {
+		idx.Each(ctx, fn)
+	}
 }
 
 // MergeFinalIndexes merges all final indexes together.
@@ -450,11 +435,11 @@ func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan
 			if len(packBlob) == 0 {
 				continue
 			}
-			for pb := range mi.Each(ctx) {
+			mi.Each(ctx, func(pb restic.PackedBlob) {
 				if packs.Has(pb.PackID) && pb.PackID[0]&0xf == i {
 					packBlob[pb.PackID] = append(packBlob[pb.PackID], pb.Blob)
 				}
-			}
+			})
 
 			// pass on packs
 			for packID, pbs := range packBlob {

internal/repository/master_index_test.go

@@ -163,9 +163,9 @@ func TestMasterMergeFinalIndexes(t *testing.T) {
 	rtest.Equals(t, 1, idxCount)
 
 	blobCount := 0
-	for range mIdx.Each(context.TODO()) {
+	mIdx.Each(context.TODO(), func(pb restic.PackedBlob) {
 		blobCount++
-	}
+	})
 	rtest.Equals(t, 2, blobCount)
 
 	blobs := mIdx.Lookup(bhInIdx1)
@@ -195,9 +195,9 @@ func TestMasterMergeFinalIndexes(t *testing.T) {
 	rtest.Equals(t, []restic.PackedBlob{blob2}, blobs)
 
 	blobCount = 0
-	for range mIdx.Each(context.TODO()) {
+	mIdx.Each(context.TODO(), func(pb restic.PackedBlob) {
 		blobCount++
-	}
+	})
 	rtest.Equals(t, 2, blobCount)
 }
 
@@ -316,9 +316,9 @@ func BenchmarkMasterIndexEach(b *testing.B) {
 
 	for i := 0; i < b.N; i++ {
 		entries := 0
-		for _ = range mIdx.Each(context.TODO()) {
+		mIdx.Each(context.TODO(), func(pb restic.PackedBlob) {
 			entries++
-		}
+		})
 	}
 }
 

internal/repository/repository.go

@@ -595,10 +595,15 @@ func (r *Repository) LoadIndex(ctx context.Context) error {
 		// sanity check
 		ctx, cancel := context.WithCancel(ctx)
 		defer cancel()
-		for blob := range r.idx.Each(ctx) {
+
+		invalidIndex := false
+		r.idx.Each(ctx, func(blob restic.PackedBlob) {
 			if blob.IsCompressed() {
-				return errors.Fatal("index uses feature not supported by repository version 1")
+				invalidIndex = true
 			}
+		})
+		if invalidIndex {
+			return errors.Fatal("index uses feature not supported by repository version 1")
 		}
 	}
 

internal/repository/repository_test.go

@@ -362,13 +362,13 @@ func testRepositoryIncrementalIndex(t *testing.T, version uint) {
 		idx, err := loadIndex(context.TODO(), repo, id)
 		rtest.OK(t, err)
 
-		for pb := range idx.Each(context.TODO()) {
+		idx.Each(context.TODO(), func(pb restic.PackedBlob) {
 			if _, ok := packEntries[pb.PackID]; !ok {
 				packEntries[pb.PackID] = make(map[restic.ID]struct{})
 			}
 
 			packEntries[pb.PackID][id] = struct{}{}
-		}
+		})
 		return nil
 	})
 	if err != nil {