mirror of
https://github.com/restic/restic.git
synced 2025-10-09 07:33:53 +00:00
repository: move backend.LoadAll to repository.LoadRaw
LoadRaw also includes improved context cancellation handling similar to the implementation in repository.LoadUnpacked. The removed cache backend test will be added again later on.
This commit is contained in:
Michael Eischer
@@ -36,6 +36,19 @@ func beTest(ctx context.Context, be backend.Backend, h backend.Handle) (bool, er
|
||||
return err == nil, err
|
||||
}
|
||||
|
||||
func LoadAll(ctx context.Context, be backend.Backend, h backend.Handle) ([]byte, error) {
|
||||
var buf []byte
|
||||
err := be.Load(ctx, h, 0, 0, func(rd io.Reader) error {
|
||||
var err error
|
||||
buf, err = io.ReadAll(rd)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
// TestStripPasswordCall tests that the StripPassword method of a factory can be called without crashing.
|
||||
// It does not verify whether passwords are removed correctly
|
||||
func (s *Suite[C]) TestStripPasswordCall(_ *testing.T) {
|
||||
@@ -94,7 +107,7 @@ func (s *Suite[C]) TestConfig(t *testing.T) {
|
||||
var testString = "Config"
|
||||
|
||||
// create config and read it back
|
||||
_, err := backend.LoadAll(context.TODO(), nil, b, backend.Handle{Type: backend.ConfigFile})
|
||||
_, err := LoadAll(context.TODO(), b, backend.Handle{Type: backend.ConfigFile})
|
||||
if err == nil {
|
||||
t.Fatalf("did not get expected error for non-existing config")
|
||||
}
|
||||
@@ -110,7 +123,7 @@ func (s *Suite[C]) TestConfig(t *testing.T) {
|
||||
// same config
|
||||
for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} {
|
||||
h := backend.Handle{Type: backend.ConfigFile, Name: name}
|
||||
buf, err := backend.LoadAll(context.TODO(), nil, b, h)
|
||||
buf, err := LoadAll(context.TODO(), b, h)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to read config with name %q: %+v", name, err)
|
||||
}
|
||||
@@ -519,7 +532,7 @@ func (s *Suite[C]) TestSave(t *testing.T) {
|
||||
err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher()))
|
||||
test.OK(t, err)
|
||||
|
||||
buf, err := backend.LoadAll(context.TODO(), nil, b, h)
|
||||
buf, err := LoadAll(context.TODO(), b, h)
|
||||
test.OK(t, err)
|
||||
if len(buf) != len(data) {
|
||||
t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf))
|
||||
@@ -821,7 +834,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
|
||||
|
||||
// test Load()
|
||||
h := backend.Handle{Type: tpe, Name: ts.id}
|
||||
buf, err := backend.LoadAll(context.TODO(), nil, b, h)
|
||||
buf, err := LoadAll(context.TODO(), b, h)
|
||||
test.OK(t, err)
|
||||
test.Equals(t, ts.data, string(buf))
|
||||
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
package backend
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/minio/sha256-simd"
|
||||
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
)
|
||||
|
||||
func verifyContentMatchesName(s string, data []byte) (bool, error) {
|
||||
if len(s) != hex.EncodedLen(sha256.Size) {
|
||||
return false, fmt.Errorf("invalid length for ID: %q", s)
|
||||
}
|
||||
|
||||
b, err := hex.DecodeString(s)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("invalid ID: %s", err)
|
||||
}
|
||||
var id [sha256.Size]byte
|
||||
copy(id[:], b)
|
||||
|
||||
hashed := sha256.Sum256(data)
|
||||
return id == hashed, nil
|
||||
}
|
||||
|
||||
// LoadAll reads all data stored in the backend for the handle into the given
|
||||
// buffer, which is truncated. If the buffer is not large enough or nil, a new
|
||||
// one is allocated.
|
||||
func LoadAll(ctx context.Context, buf []byte, be Backend, h Handle) ([]byte, error) {
|
||||
retriedInvalidData := false
|
||||
err := be.Load(ctx, h, 0, 0, func(rd io.Reader) error {
|
||||
// make sure this is idempotent, in case an error occurs this function may be called multiple times!
|
||||
wr := bytes.NewBuffer(buf[:0])
|
||||
_, cerr := io.Copy(wr, rd)
|
||||
if cerr != nil {
|
||||
return cerr
|
||||
}
|
||||
buf = wr.Bytes()
|
||||
|
||||
// retry loading damaged data only once. If a file fails to download correctly
|
||||
// the second time, then it is likely corrupted at the backend. Return the data
|
||||
// to the caller in that case to let it decide what to do with the data.
|
||||
if !retriedInvalidData && h.Type != ConfigFile {
|
||||
if matches, err := verifyContentMatchesName(h.Name, buf); err == nil && !matches {
|
||||
debug.Log("retry loading broken blob %v", h)
|
||||
retriedInvalidData = true
|
||||
return errors.Errorf("loadAll(%v): invalid data returned", h)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buf, nil
|
||||
}
|
||||
@@ -1,149 +0,0 @@
|
||||
package backend_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"math/rand"
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/backend"
|
||||
"github.com/restic/restic/internal/backend/mem"
|
||||
"github.com/restic/restic/internal/backend/mock"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
const KiB = 1 << 10
|
||||
const MiB = 1 << 20
|
||||
|
||||
func TestLoadAll(t *testing.T) {
|
||||
b := mem.New()
|
||||
var buf []byte
|
||||
|
||||
for i := 0; i < 20; i++ {
|
||||
data := rtest.Random(23+i, rand.Intn(MiB)+500*KiB)
|
||||
|
||||
id := restic.Hash(data)
|
||||
h := backend.Handle{Name: id.String(), Type: backend.PackFile}
|
||||
err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher()))
|
||||
rtest.OK(t, err)
|
||||
|
||||
buf, err := backend.LoadAll(context.TODO(), buf, b, backend.Handle{Type: backend.PackFile, Name: id.String()})
|
||||
rtest.OK(t, err)
|
||||
|
||||
if len(buf) != len(data) {
|
||||
t.Errorf("length of returned buffer does not match, want %d, got %d", len(data), len(buf))
|
||||
continue
|
||||
}
|
||||
|
||||
if !bytes.Equal(buf, data) {
|
||||
t.Errorf("wrong data returned")
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func save(t testing.TB, be backend.Backend, buf []byte) backend.Handle {
|
||||
id := restic.Hash(buf)
|
||||
h := backend.Handle{Name: id.String(), Type: backend.PackFile}
|
||||
err := be.Save(context.TODO(), h, backend.NewByteReader(buf, be.Hasher()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
type quickRetryBackend struct {
|
||||
backend.Backend
|
||||
}
|
||||
|
||||
func (be *quickRetryBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
|
||||
err := be.Backend.Load(ctx, h, length, offset, fn)
|
||||
if err != nil {
|
||||
// retry
|
||||
err = be.Backend.Load(ctx, h, length, offset, fn)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func TestLoadAllBroken(t *testing.T) {
|
||||
b := mock.NewBackend()
|
||||
|
||||
data := rtest.Random(23, rand.Intn(MiB)+500*KiB)
|
||||
id := restic.Hash(data)
|
||||
// damage buffer
|
||||
data[0] ^= 0xff
|
||||
|
||||
b.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
|
||||
return io.NopCloser(bytes.NewReader(data)), nil
|
||||
}
|
||||
|
||||
// must fail on first try
|
||||
_, err := backend.LoadAll(context.TODO(), nil, b, backend.Handle{Type: backend.PackFile, Name: id.String()})
|
||||
if err == nil {
|
||||
t.Fatalf("missing expected error")
|
||||
}
|
||||
|
||||
// must return the broken data after a retry
|
||||
be := &quickRetryBackend{Backend: b}
|
||||
buf, err := backend.LoadAll(context.TODO(), nil, be, backend.Handle{Type: backend.PackFile, Name: id.String()})
|
||||
rtest.OK(t, err)
|
||||
|
||||
if !bytes.Equal(buf, data) {
|
||||
t.Fatalf("wrong data returned")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAllAppend(t *testing.T) {
|
||||
b := mem.New()
|
||||
|
||||
h1 := save(t, b, []byte("foobar test string"))
|
||||
randomData := rtest.Random(23, rand.Intn(MiB)+500*KiB)
|
||||
h2 := save(t, b, randomData)
|
||||
|
||||
var tests = []struct {
|
||||
handle backend.Handle
|
||||
buf []byte
|
||||
want []byte
|
||||
}{
|
||||
{
|
||||
handle: h1,
|
||||
buf: nil,
|
||||
want: []byte("foobar test string"),
|
||||
},
|
||||
{
|
||||
handle: h1,
|
||||
buf: []byte("xxx"),
|
||||
want: []byte("foobar test string"),
|
||||
},
|
||||
{
|
||||
handle: h2,
|
||||
buf: nil,
|
||||
want: randomData,
|
||||
},
|
||||
{
|
||||
handle: h2,
|
||||
buf: make([]byte, 0, 200),
|
||||
want: randomData,
|
||||
},
|
||||
{
|
||||
handle: h2,
|
||||
buf: []byte("foobarbaz"),
|
||||
want: randomData,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run("", func(t *testing.T) {
|
||||
buf, err := backend.LoadAll(context.TODO(), test.buf, b, test.handle)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(buf, test.want) {
|
||||
t.Errorf("wrong data returned, want %q, got %q", test.want, buf)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user