Add plumbing to calculate backend specific file hash for upload

This enables the backends to request the calculation of a
backend-specific hash. For the currently supported backends this will
always be MD5. The hash calculation happens as early as possible, for
pack files this is during assembly of the pack file. That way the hash
would even capture corruptions of the temporary pack file on disk.

internal/repository/key.go

@@ -279,7 +279,7 @@ func AddKey(ctx context.Context, s *Repository, password, username, hostname str
 		Name: restic.Hash(buf).String(),
 	}
 
-	err = s.be.Save(ctx, h, restic.NewByteReader(buf))
+	err = s.be.Save(ctx, h, restic.NewByteReader(buf, s.be.Hasher()))
 	if err != nil {
 		return nil, err
 	}

internal/repository/packer_manager.go

@@ -2,6 +2,8 @@ package repository
 
 import (
 	"context"
+	"hash"
+	"io"
 	"os"
 	"sync"
 
@@ -20,12 +22,14 @@ import (
 // Saver implements saving data in a backend.
 type Saver interface {
 	Save(context.Context, restic.Handle, restic.RewindReader) error
+	Hasher() hash.Hash
 }
 
 // Packer holds a pack.Packer together with a hash writer.
 type Packer struct {
 	*pack.Packer
 	hw      *hashing.Writer
+	beHw    *hashing.Writer
 	tmpfile *os.File
 }
 
@@ -71,10 +75,19 @@ func (r *packerManager) findPacker() (packer *Packer, err error) {
 		return nil, errors.Wrap(err, "fs.TempFile")
 	}
 
-	hw := hashing.NewWriter(tmpfile, sha256.New())
+	w := io.Writer(tmpfile)
+	beHasher := r.be.Hasher()
+	var beHw *hashing.Writer
+	if beHasher != nil {
+		beHw = hashing.NewWriter(w, beHasher)
+		w = beHw
+	}
+
+	hw := hashing.NewWriter(w, sha256.New())
 	p := pack.NewPacker(r.key, hw)
 	packer = &Packer{
 		Packer:  p,
+		beHw:    beHw,
 		hw:      hw,
 		tmpfile: tmpfile,
 	}
@@ -101,8 +114,11 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packe
 
 	id := restic.IDFromHash(p.hw.Sum(nil))
 	h := restic.Handle{Type: restic.PackFile, Name: id.String()}
-
-	rd, err := restic.NewFileReader(p.tmpfile)
+	var beHash []byte
+	if p.beHw != nil {
+		beHash = p.beHw.Sum(nil)
+	}
+	rd, err := restic.NewFileReader(p.tmpfile, beHash)
 	if err != nil {
 		return err
 	}

internal/repository/packer_manager_test.go

@@ -33,11 +33,11 @@ func min(a, b int) int {
 	return b
 }
 
-func saveFile(t testing.TB, be Saver, length int, f *os.File, id restic.ID) {
+func saveFile(t testing.TB, be Saver, length int, f *os.File, id restic.ID, hash []byte) {
 	h := restic.Handle{Type: restic.PackFile, Name: id.String()}
 	t.Logf("save file %v", h)
 
-	rd, err := restic.NewFileReader(f)
+	rd, err := restic.NewFileReader(f, hash)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -90,7 +90,11 @@ func fillPacks(t testing.TB, rnd *rand.Rand, be Saver, pm *packerManager, buf []
 		}
 
 		packID := restic.IDFromHash(packer.hw.Sum(nil))
-		saveFile(t, be, int(packer.Size()), packer.tmpfile, packID)
+		var beHash []byte
+		if packer.beHw != nil {
+			beHash = packer.beHw.Sum(nil)
+		}
+		saveFile(t, be, int(packer.Size()), packer.tmpfile, packID, beHash)
 	}
 
 	return bytes
@@ -106,7 +110,11 @@ func flushRemainingPacks(t testing.TB, be Saver, pm *packerManager) (bytes int)
 			bytes += int(n)
 
 			packID := restic.IDFromHash(packer.hw.Sum(nil))
-			saveFile(t, be, int(packer.Size()), packer.tmpfile, packID)
+			var beHash []byte
+			if packer.beHw != nil {
+				beHash = packer.beHw.Sum(nil)
+			}
+			saveFile(t, be, int(packer.Size()), packer.tmpfile, packID, beHash)
 		}
 	}
 

internal/repository/repository.go

@@ -322,7 +322,7 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by
 	}
 	h := restic.Handle{Type: t, Name: id.String()}
 
-	err = r.be.Save(ctx, h, restic.NewByteReader(ciphertext))
+	err = r.be.Save(ctx, h, restic.NewByteReader(ciphertext, r.be.Hasher()))
 	if err != nil {
 		debug.Log("error saving blob %v: %v", h, err)
 		return restic.ID{}, err