Add []byte to repo.LoadAndDecrypt and utils.LoadAll

This commit changes the signatures for repository.LoadAndDecrypt and
utils.LoadAll to allow passing in a []byte as the buffer to use. This
buffer is enlarged as needed, and returned back to the caller for
further use.

In later commits, this allows reducing allocations by reusing a buffer
for multiple calls, e.g. in a worker function.

internal/repository/index.go

@@ -552,7 +552,7 @@ func DecodeOldIndex(buf []byte) (idx *Index, err error) {
 func LoadIndexWithDecoder(ctx context.Context, repo restic.Repository, id restic.ID, fn func([]byte) (*Index, error)) (idx *Index, err error) {
 	debug.Log("Loading index %v", id)
 
-	buf, err := repo.LoadAndDecrypt(ctx, restic.IndexFile, id)
+	buf, err := repo.LoadAndDecrypt(ctx, nil, restic.IndexFile, id)
 	if err != nil {
 		return nil, err
 	}

internal/repository/key.go

@@ -184,7 +184,7 @@ func SearchKey(ctx context.Context, s *Repository, password string, maxKeys int,
 // LoadKey loads a key from the backend.
 func LoadKey(ctx context.Context, s *Repository, name string) (k *Key, err error) {
 	h := restic.Handle{Type: restic.KeyFile, Name: name}
-	data, err := backend.LoadAll(ctx, s.be, h)
+	data, err := backend.LoadAll(ctx, nil, s.be, h)
 	if err != nil {
 		return nil, err
 	}

internal/repository/repository.go

@@ -9,7 +9,6 @@ import (
 	"io"
 	"os"
 
-	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/cache"
 	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/debug"
@@ -67,15 +66,29 @@ func (r *Repository) PrefixLength(t restic.FileType) (int, error) {
 	return restic.PrefixLength(r.be, t)
 }
 
-// LoadAndDecrypt loads and decrypts data identified by t and id from the
-// backend.
-func (r *Repository) LoadAndDecrypt(ctx context.Context, t restic.FileType, id restic.ID) (buf []byte, err error) {
+// LoadAndDecrypt loads and decrypts the file with the given type and ID, using
+// the supplied buffer (which must be empty). If the buffer is nil, a new
+// buffer will be allocated and returned.
+func (r *Repository) LoadAndDecrypt(ctx context.Context, buf []byte, t restic.FileType, id restic.ID) ([]byte, error) {
+	if len(buf) != 0 {
+		panic("buf is not empty")
+	}
+
 	debug.Log("load %v with id %v", t, id)
 
 	h := restic.Handle{Type: t, Name: id.String()}
-	buf, err = backend.LoadAll(ctx, r.be, h)
+	err := r.be.Load(ctx, h, 0, 0, func(rd io.Reader) error {
+		// make sure this call is idempotent, in case an error occurs
+		wr := bytes.NewBuffer(buf[:0])
+		_, cerr := io.Copy(wr, rd)
+		if cerr != nil {
+			return cerr
+		}
+		buf = wr.Bytes()
+		return nil
+	})
+
 	if err != nil {
-		debug.Log("error loading %v: %v", h, err)
 		return nil, err
 	}
 
@@ -188,7 +201,7 @@ func (r *Repository) loadBlob(ctx context.Context, id restic.ID, t restic.BlobTy
 // LoadJSONUnpacked decrypts the data and afterwards calls json.Unmarshal on
 // the item.
 func (r *Repository) LoadJSONUnpacked(ctx context.Context, t restic.FileType, id restic.ID, item interface{}) (err error) {
-	buf, err := r.LoadAndDecrypt(ctx, t, id)
+	buf, err := r.LoadAndDecrypt(ctx, nil, t, id)
 	if err != nil {
 		return err
 	}

internal/repository/repository_test.go

@@ -244,7 +244,7 @@ func BenchmarkLoadAndDecrypt(b *testing.B) {
 	b.SetBytes(int64(length))
 
 	for i := 0; i < b.N; i++ {
-		data, err := repo.LoadAndDecrypt(context.TODO(), restic.DataFile, storageID)
+		data, err := repo.LoadAndDecrypt(context.TODO(), nil, restic.DataFile, storageID)
 		rtest.OK(b, err)
 		if len(data) != length {
 			b.Errorf("wanted %d bytes, got %d", length, len(data))