Fix non-intuitive repository behavior

- The SaveBlob method now checks for duplicates.
- Moves handling of pending blobs to MasterIndex.
  -> also cleans up pending index entries when they are saved in the index
  -> when using SaveBlob no need to care about index any longer
- Always check for full index and save it when storing packs.
  -> removes the need of an index uploader
  -> also removes the verbose "uploaded intermediate index" messages
- The Flush method now also saves the index
- Fix race condition when checking and saving full/non-finalized indexes

internal/archiver/blob_saver.go

@@ -2,7 +2,6 @@ package archiver
 
 import (
 	"context"
-	"sync"
 
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/restic"
@@ -11,18 +10,14 @@ import (
 
 // Saver allows saving a blob.
 type Saver interface {
-	SaveBlob(ctx context.Context, t restic.BlobType, data []byte, id restic.ID) (restic.ID, error)
+	SaveBlob(ctx context.Context, t restic.BlobType, data []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, error)
 	Index() restic.Index
 }
 
 // BlobSaver concurrently saves incoming blobs to the repo.
 type BlobSaver struct {
 	repo Saver
-
-	m          sync.Mutex
-	knownBlobs restic.BlobSet
-
-	ch chan<- saveBlobJob
+	ch   chan<- saveBlobJob
 }
 
 // NewBlobSaver returns a new blob. A worker pool is started, it is stopped
@@ -30,9 +25,8 @@ type BlobSaver struct {
 func NewBlobSaver(ctx context.Context, t *tomb.Tomb, repo Saver, workers uint) *BlobSaver {
 	ch := make(chan saveBlobJob)
 	s := &BlobSaver{
-		repo:       repo,
-		knownBlobs: restic.NewBlobSet(),
-		ch:         ch,
+		repo: repo,
+		ch:   ch,
 	}
 
 	for i := uint(0); i < workers; i++ {
@@ -106,45 +100,15 @@ type saveBlobResponse struct {
 }
 
 func (s *BlobSaver) saveBlob(ctx context.Context, t restic.BlobType, buf []byte) (saveBlobResponse, error) {
-	id := restic.Hash(buf)
-	h := restic.BlobHandle{ID: id, Type: t}
+	id, known, err := s.repo.SaveBlob(ctx, t, buf, restic.ID{}, false)
 
-	// check if another goroutine has already saved this blob
-	known := false
-	s.m.Lock()
-	if s.knownBlobs.Has(h) {
-		known = true
-	} else {
-		s.knownBlobs.Insert(h)
-		known = false
-	}
-	s.m.Unlock()
-
-	// blob is already known, nothing to do
-	if known {
-		return saveBlobResponse{
-			id:    id,
-			known: true,
-		}, nil
-	}
-
-	// check if the repo knows this blob
-	if s.repo.Index().Has(id, t) {
-		return saveBlobResponse{
-			id:    id,
-			known: true,
-		}, nil
-	}
-
-	// otherwise we're responsible for saving it
-	_, err := s.repo.SaveBlob(ctx, t, buf, id)
 	if err != nil {
 		return saveBlobResponse{}, err
 	}
 
 	return saveBlobResponse{
 		id:    id,
-		known: false,
+		known: known,
 	}, nil
 }