TheArchive/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2025-12-04 00:01:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4499 from MichaelEischer/modular-backend-code

Browse Source 
Split backend code from restic package
This commit is contained in:
Michael Eischer
2023-10-27 20:19:20 +02:00
committed by GitHub
parent 50ef01131a 249605843b
commit 45962c2847
132 changed files with 1145 additions and 1062 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
internal/restic/backend.go
View File

@@ -1,112 +0,0 @@
package restic
import (
"context"
"hash"
"io"
)
// Backend is used to store and access data.
//
// Backend operations that return an error will be retried when a Backend is
// wrapped in a RetryBackend. To prevent that from happening, the operations
// should return a github.com/cenkalti/backoff/v4.PermanentError. Errors from
// the context package need not be wrapped, as context cancellation is checked
// separately by the retrying logic.
type Backend interface {
// Location returns a string that describes the type and location of the
// repository.
Location() string
// Connections returns the maxmimum number of concurrent backend operations.
Connections() uint
// Hasher may return a hash function for calculating a content hash for the backend
Hasher() hash.Hash
// HasAtomicReplace returns whether Save() can atomically replace files
HasAtomicReplace() bool
// Remove removes a File described by h.
Remove(ctx context.Context, h Handle) error
// Close the backend
Close() error
// Save stores the data from rd under the given handle.
Save(ctx context.Context, h Handle, rd RewindReader) error
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset. If length is larger than zero, only a portion of the file
// is read.
//
// The function fn may be called multiple times during the same Load invocation
// and therefore must be idempotent.
//
// Implementations are encouraged to use backend.DefaultLoad
Load(ctx context.Context, h Handle, length int, offset int64, fn func(rd io.Reader) error) error
// Stat returns information about the File identified by h.
Stat(ctx context.Context, h Handle) (FileInfo, error)
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
//
// The function fn is called exactly once for each file during successful
// execution and at most once in case of an error.
//
// The function fn is called in the same Goroutine that List() is called
// from.
List(ctx context.Context, t FileType, fn func(FileInfo) error) error
// IsNotExist returns true if the error was caused by a non-existing file
// in the backend.
//
// The argument may be a wrapped error. The implementation is responsible
// for unwrapping it.
IsNotExist(err error) bool
// Delete removes all data in the backend.
Delete(ctx context.Context) error
}
type BackendUnwrapper interface {
// Unwrap returns the underlying backend or nil if there is none.
Unwrap() Backend
}
func AsBackend[B Backend](b Backend) B {
for b != nil {
if be, ok := b.(B); ok {
return be
}
if be, ok := b.(BackendUnwrapper); ok {
b = be.Unwrap()
} else {
// not the backend we're looking for
break
}
}
var be B
return be
}
type FreezeBackend interface {
Backend
// Freeze blocks all backend operations except those on lock files
Freeze()
// Unfreeze allows all backend operations to continue
Unfreeze()
}
// FileInfo is contains information about a file in the backend.
type FileInfo struct {
Size int64
Name string
}
// ApplyEnvironmenter fills in a backend configuration from the environment
type ApplyEnvironmenter interface {
ApplyEnvironment(prefix string)
}

14
internal/restic/backend_find.go
View File

@@ -3,8 +3,6 @@ package restic
import (
"context"
"fmt"
"github.com/restic/restic/internal/debug"
)
// A MultipleIDMatchesError is returned by Find() when multiple IDs with a
@@ -32,15 +30,9 @@ func Find(ctx context.Context, be Lister, t FileType, prefix string) (ID, error)
ctx, cancel := context.WithCancel(ctx)
defer cancel()
err := be.List(ctx, t, func(fi FileInfo) error {
// ignore filename which are not an id
id, err := ParseID(fi.Name)
if err != nil {
debug.Log("unable to parse %v as an ID", fi.Name)
return nil
}
if len(fi.Name) >= len(prefix) && prefix == fi.Name[:len(prefix)] {
err := be.List(ctx, t, func(id ID, size int64) error {
name := id.String()
if len(name) >= len(prefix) && prefix == name[:len(prefix)] {
if match.IsNull() {
match = id
} else {

52
internal/restic/backend_find_test.go
View File

@@ -1,37 +1,31 @@
package restic
package restic_test
import (
"context"
"strings"
"testing"
"github.com/restic/restic/internal/restic"
)
type mockBackend struct {
list func(context.Context, FileType, func(FileInfo) error) error
}
func (m mockBackend) List(ctx context.Context, t FileType, fn func(FileInfo) error) error {
return m.list(ctx, t, fn)
}
var samples = IDs{
TestParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff"),
TestParseID("20bdc1402a6fc9b633ccd578c4a92d0f4ef1a457fa2e16c596bc73fb409d6cc0"),
TestParseID("20bdc1402a6fc9b633ffffffffffffffffffffffffffffffffffffffffffffff"),
TestParseID("20ff988befa5fc40350f00d531a767606efefe242c837aaccb80673f286be53d"),
TestParseID("326cb59dfe802304f96ee9b5b9af93bdee73a30f53981e5ec579aedb6f1d0f07"),
TestParseID("86b60b9594d1d429c4aa98fa9562082cabf53b98c7dc083abe5dae31074dd15a"),
TestParseID("96c8dbe225079e624b5ce509f5bd817d1453cd0a85d30d536d01b64a8669aeae"),
TestParseID("fa31d65b87affcd167b119e9d3d2a27b8236ca4836cb077ed3e96fcbe209b792"),
var samples = restic.IDs{
restic.TestParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff"),
restic.TestParseID("20bdc1402a6fc9b633ccd578c4a92d0f4ef1a457fa2e16c596bc73fb409d6cc0"),
restic.TestParseID("20bdc1402a6fc9b633ffffffffffffffffffffffffffffffffffffffffffffff"),
restic.TestParseID("20ff988befa5fc40350f00d531a767606efefe242c837aaccb80673f286be53d"),
restic.TestParseID("326cb59dfe802304f96ee9b5b9af93bdee73a30f53981e5ec579aedb6f1d0f07"),
restic.TestParseID("86b60b9594d1d429c4aa98fa9562082cabf53b98c7dc083abe5dae31074dd15a"),
restic.TestParseID("96c8dbe225079e624b5ce509f5bd817d1453cd0a85d30d536d01b64a8669aeae"),
restic.TestParseID("fa31d65b87affcd167b119e9d3d2a27b8236ca4836cb077ed3e96fcbe209b792"),
}
func TestFind(t *testing.T) {
list := samples
m := mockBackend{}
m.list = func(ctx context.Context, t FileType, fn func(FileInfo) error) error {
m := &ListHelper{}
m.ListFn = func(ctx context.Context, t restic.FileType, fn func(id restic.ID, size int64) error) error {
for _, id := range list {
err := fn(FileInfo{Name: id.String()})
err := fn(id, 0)
if err != nil {
return err
}
@@ -39,17 +33,17 @@ func TestFind(t *testing.T) {
return nil
}
f, err := Find(context.TODO(), m, SnapshotFile, "20bdc1402a6fc9b633aa")
f, err := restic.Find(context.TODO(), m, restic.SnapshotFile, "20bdc1402a6fc9b633aa")
if err != nil {
t.Error(err)
}
expectedMatch := TestParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff")
expectedMatch := restic.TestParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff")
if f != expectedMatch {
t.Errorf("Wrong match returned want %s, got %s", expectedMatch, f)
}
f, err = Find(context.TODO(), m, SnapshotFile, "NotAPrefix")
if _, ok := err.(*NoIDByPrefixError); !ok || !strings.Contains(err.Error(), "NotAPrefix") {
f, err = restic.Find(context.TODO(), m, restic.SnapshotFile, "NotAPrefix")
if _, ok := err.(*restic.NoIDByPrefixError); !ok || !strings.Contains(err.Error(), "NotAPrefix") {
t.Error("Expected no snapshots to be found.")
}
if !f.IsNull() {
@@ -58,8 +52,8 @@ func TestFind(t *testing.T) {
// Try to match with a prefix longer than any ID.
extraLengthID := samples[0].String() + "f"
f, err = Find(context.TODO(), m, SnapshotFile, extraLengthID)
if _, ok := err.(*NoIDByPrefixError); !ok || !strings.Contains(err.Error(), extraLengthID) {
f, err = restic.Find(context.TODO(), m, restic.SnapshotFile, extraLengthID)
if _, ok := err.(*restic.NoIDByPrefixError); !ok || !strings.Contains(err.Error(), extraLengthID) {
t.Errorf("Wrong error %v for no snapshots matched", err)
}
if !f.IsNull() {
@@ -67,8 +61,8 @@ func TestFind(t *testing.T) {
}
// Use a prefix that will match the prefix of multiple Ids in `samples`.
f, err = Find(context.TODO(), m, SnapshotFile, "20bdc140")
if _, ok := err.(*MultipleIDMatchesError); !ok {
f, err = restic.Find(context.TODO(), m, restic.SnapshotFile, "20bdc140")
if _, ok := err.(*restic.MultipleIDMatchesError); !ok {
t.Errorf("Wrong error %v for multiple snapshots", err)
}
if !f.IsNull() {

38
internal/restic/backend_test.go
View File

@@ -1,38 +0,0 @@
package restic_test
import (
"testing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
type testBackend struct {
restic.Backend
}
func (t *testBackend) Unwrap() restic.Backend {
return nil
}
type otherTestBackend struct {
restic.Backend
}
func (t *otherTestBackend) Unwrap() restic.Backend {
return t.Backend
}
func TestAsBackend(t *testing.T) {
other := otherTestBackend{}
test.Assert(t, restic.AsBackend[*testBackend](other) == nil, "otherTestBackend is not a testBackend backend")
testBe := &testBackend{}
test.Assert(t, restic.AsBackend[*testBackend](testBe) == testBe, "testBackend was not returned")
wrapper := &otherTestBackend{Backend: testBe}
test.Assert(t, restic.AsBackend[*testBackend](wrapper) == testBe, "failed to unwrap testBackend backend")
wrapper.Backend = other
test.Assert(t, restic.AsBackend[*testBackend](wrapper) == nil, "a wrapped otherTestBackend is not a testBackend")
}

9
internal/restic/blob.go
View File

@@ -75,6 +75,15 @@ func (t BlobType) String() string {
return fmt.Sprintf("<BlobType %d>", t)
}
func (t BlobType) IsMetadata() bool {
switch t {
case TreeBlob:
return true
default:
return false
}
}
// MarshalJSON encodes the BlobType into JSON.
func (t BlobType) MarshalJSON() ([]byte, error) {
switch t {

79
internal/restic/file.go
View File

@@ -1,79 +0,0 @@
package restic
import (
"fmt"
"github.com/restic/restic/internal/errors"
)
// FileType is the type of a file in the backend.
type FileType uint8
// These are the different data types a backend can store.
const (
PackFile FileType = 1 + iota
KeyFile
LockFile
SnapshotFile
IndexFile
ConfigFile
)
func (t FileType) String() string {
s := "invalid"
switch t {
case PackFile:
// Spelled "data" instead of "pack" for historical reasons.
s = "data"
case KeyFile:
s = "key"
case LockFile:
s = "lock"
case SnapshotFile:
s = "snapshot"
case IndexFile:
s = "index"
case ConfigFile:
s = "config"
}
return s
}
// Handle is used to store and access data in a backend.
type Handle struct {
Type FileType
ContainedBlobType BlobType
Name string
}
func (h Handle) String() string {
name := h.Name
if len(name) > 10 {
name = name[:10]
}
return fmt.Sprintf("<%s/%s>", h.Type, name)
}
// Valid returns an error if h is not valid.
func (h Handle) Valid() error {
switch h.Type {
case PackFile:
case KeyFile:
case LockFile:
case SnapshotFile:
case IndexFile:
case ConfigFile:
default:
return errors.Errorf("invalid Type %d", h.Type)
}
if h.Type == ConfigFile {
return nil
}
if h.Name == "" {
return errors.New("invalid Name")
}
return nil
}

36
internal/restic/file_test.go
View File

@@ -1,36 +0,0 @@
package restic
import (
"testing"
rtest "github.com/restic/restic/internal/test"
)
func TestHandleString(t *testing.T) {
rtest.Equals(t, "<data/foobar>", Handle{Type: PackFile, Name: "foobar"}.String())
rtest.Equals(t, "<lock/1>", Handle{Type: LockFile, Name: "1"}.String())
}
func TestHandleValid(t *testing.T) {
var handleTests = []struct {
h Handle
valid bool
}{
{Handle{Name: "foo"}, false},
{Handle{Type: 0}, false},
{Handle{Type: ConfigFile, Name: ""}, true},
{Handle{Type: PackFile, Name: ""}, false},
{Handle{Type: LockFile, Name: "010203040506"}, true},
}
for i, test := range handleTests {
err := test.h.Valid()
if err != nil && test.valid {
t.Errorf("test %v failed: error returned for valid handle: %v", i, err)
}
if !test.valid && err == nil {
t.Errorf("test %v failed: expected error for invalid handle not found", i)
}
}
}

52
internal/restic/lister.go Normal file
View File

@@ -0,0 +1,52 @@
package restic
import (
"context"
"fmt"
)
type fileInfo struct {
id ID
size int64
}
type memorizedLister struct {
fileInfos []fileInfo
tpe FileType
}
func (m *memorizedLister) List(ctx context.Context, t FileType, fn func(ID, int64) error) error {
if t != m.tpe {
return fmt.Errorf("filetype mismatch, expected %s got %s", m.tpe, t)
}
for _, fi := range m.fileInfos {
if ctx.Err() != nil {
break
}
err := fn(fi.id, fi.size)
if err != nil {
return err
}
}
return ctx.Err()
}
func MemorizeList(ctx context.Context, be Lister, t FileType) (Lister, error) {
if _, ok := be.(*memorizedLister); ok {
return be, nil
}
var fileInfos []fileInfo
err := be.List(ctx, t, func(id ID, size int64) error {
fileInfos = append(fileInfos, fileInfo{id, size})
return nil
})
if err != nil {
return nil, err
}
return &memorizedLister{
fileInfos: fileInfos,
tpe: t,
}, nil
}

68
internal/restic/lister_test.go Normal file
View File

@@ -0,0 +1,68 @@
package restic_test
import (
"context"
"fmt"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
type ListHelper struct {
ListFn func(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error
}
func (l *ListHelper) List(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error {
return l.ListFn(ctx, t, fn)
}
func TestMemoizeList(t *testing.T) {
// setup backend to serve as data source for memoized list
be := &ListHelper{}
type FileInfo struct {
ID restic.ID
Size int64
}
files := []FileInfo{
{ID: restic.NewRandomID(), Size: 42},
{ID: restic.NewRandomID(), Size: 45},
}
be.ListFn = func(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error {
for _, fi := range files {
if err := fn(fi.ID, fi.Size); err != nil {
return err
}
}
return nil
}
mem, err := restic.MemorizeList(context.TODO(), be, backend.SnapshotFile)
rtest.OK(t, err)
err = mem.List(context.TODO(), backend.IndexFile, func(id restic.ID, size int64) error {
t.Fatal("file type mismatch")
return nil // the memoized lister must return an error by itself
})
rtest.Assert(t, err != nil, "missing error on file typ mismatch")
var memFiles []FileInfo
err = mem.List(context.TODO(), backend.SnapshotFile, func(id restic.ID, size int64) error {
memFiles = append(memFiles, FileInfo{ID: id, Size: size})
return nil
})
rtest.OK(t, err)
rtest.Equals(t, files, memFiles)
}
func TestMemoizeListError(t *testing.T) {
// setup backend to serve as data source for memoized list
be := &ListHelper{}
be.ListFn = func(ctx context.Context, t backend.FileType, fn func(restic.ID, int64) error) error {
return fmt.Errorf("list error")
}
_, err := restic.MemorizeList(context.TODO(), be, backend.SnapshotFile)
rtest.Assert(t, err != nil, "missing error on list error")
}

21
internal/restic/lock.go
View File

@@ -12,6 +12,7 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/debug"
@@ -213,7 +214,7 @@ func (l *Lock) Unlock() error {
return nil
}
return l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: l.lockID.String()})
return l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: l.lockID.String()})
}
var StaleLockTimeout = 30 * time.Minute
@@ -273,7 +274,7 @@ func (l *Lock) Refresh(ctx context.Context) error {
oldLockID := l.lockID
l.lockID = &id
return l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: oldLockID.String()})
return l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: oldLockID.String()})
}
// RefreshStaleLock is an extended variant of Refresh that can also refresh stale lock files.
@@ -302,13 +303,13 @@ func (l *Lock) RefreshStaleLock(ctx context.Context) error {
exists, err = l.checkExistence(ctx)
if err != nil {
// cleanup replacement lock
_ = l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: id.String()})
_ = l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: id.String()})
return err
}
if !exists {
// cleanup replacement lock
_ = l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: id.String()})
_ = l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: id.String()})
return ErrRemovedLock
}
@@ -319,7 +320,7 @@ func (l *Lock) RefreshStaleLock(ctx context.Context) error {
oldLockID := l.lockID
l.lockID = &id
return l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: oldLockID.String()})
return l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: oldLockID.String()})
}
func (l *Lock) checkExistence(ctx context.Context) (bool, error) {
@@ -328,7 +329,7 @@ func (l *Lock) checkExistence(ctx context.Context) (bool, error) {
exists := false
err := l.repo.Backend().List(ctx, LockFile, func(fi FileInfo) error {
err := l.repo.Backend().List(ctx, LockFile, func(fi backend.FileInfo) error {
if fi.Name == l.lockID.String() {
exists = true
}
@@ -387,7 +388,7 @@ func RemoveStaleLocks(ctx context.Context, repo Repository) (uint, error) {
}
if lock.Stale() {
err = repo.Backend().Remove(ctx, Handle{Type: LockFile, Name: id.String()})
err = repo.Backend().Remove(ctx, backend.Handle{Type: LockFile, Name: id.String()})
if err == nil {
processed++
}
@@ -402,8 +403,8 @@ func RemoveStaleLocks(ctx context.Context, repo Repository) (uint, error) {
// RemoveAllLocks removes all locks forcefully.
func RemoveAllLocks(ctx context.Context, repo Repository) (uint, error) {
var processed uint32
err := ParallelList(ctx, repo.Backend(), LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error {
err := repo.Backend().Remove(ctx, Handle{Type: LockFile, Name: id.String()})
err := ParallelList(ctx, repo, LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error {
err := repo.Backend().Remove(ctx, backend.Handle{Type: LockFile, Name: id.String()})
if err == nil {
atomic.AddUint32(&processed, 1)
}
@@ -420,7 +421,7 @@ func ForAllLocks(ctx context.Context, repo Repository, excludeID *ID, fn func(ID
var m sync.Mutex
// For locks decoding is nearly for free, thus just assume were only limited by IO
return ParallelList(ctx, repo.Backend(), LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error {
return ParallelList(ctx, repo, LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error {
if excludeID != nil && id.Equal(*excludeID) {
return nil
}

11
internal/restic/lock_test.go
View File

@@ -8,6 +8,7 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mem"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
@@ -53,10 +54,10 @@ func TestMultipleLock(t *testing.T) {
}
type failLockLoadingBackend struct {
restic.Backend
backend.Backend
}
func (be *failLockLoadingBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *failLockLoadingBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
if h.Type == restic.LockFile {
return fmt.Errorf("error loading lock")
}
@@ -130,7 +131,7 @@ func createFakeLock(repo restic.Repository, t time.Time, pid int) (restic.ID, er
}
func removeLock(repo restic.Repository, id restic.ID) error {
h := restic.Handle{Type: restic.LockFile, Name: id.String()}
h := backend.Handle{Type: restic.LockFile, Name: id.String()}
return repo.Backend().Remove(context.TODO(), h)
}
@@ -191,7 +192,7 @@ func TestLockStale(t *testing.T) {
}
func lockExists(repo restic.Repository, t testing.TB, id restic.ID) bool {
h := restic.Handle{Type: restic.LockFile, Name: id.String()}
h := backend.Handle{Type: restic.LockFile, Name: id.String()}
_, err := repo.Backend().Stat(context.TODO(), h)
if err != nil && !repo.Backend().IsNotExist(err) {
t.Fatal(err)
@@ -317,7 +318,7 @@ func TestLockRefreshStaleMissing(t *testing.T) {
lockID := checkSingleLock(t, repo)
// refresh must fail if lock was removed
rtest.OK(t, repo.Backend().Remove(context.TODO(), restic.Handle{Type: restic.LockFile, Name: lockID.String()}))
rtest.OK(t, repo.Backend().Remove(context.TODO(), backend.Handle{Type: restic.LockFile, Name: lockID.String()}))
time.Sleep(time.Millisecond)
err = lock.RefreshStaleLock(context.TODO())
rtest.Assert(t, err == restic.ErrRemovedLock, "unexpected error, expected %v, got %v", restic.ErrRemovedLock, err)

11
internal/restic/parallel.go
View File

@@ -8,7 +8,6 @@ import (
)
func ParallelList(ctx context.Context, r Lister, t FileType, parallelism uint, fn func(context.Context, ID, int64) error) error {
type FileIDInfo struct {
ID
Size int64
@@ -22,17 +21,11 @@ func ParallelList(ctx context.Context, r Lister, t FileType, parallelism uint, f
// send list of index files through ch, which is closed afterwards
wg.Go(func() error {
defer close(ch)
return r.List(ctx, t, func(fi FileInfo) error {
id, err := ParseID(fi.Name)
if err != nil {
debug.Log("unable to parse %v as an ID", fi.Name)
return nil
}
return r.List(ctx, t, func(id ID, size int64) error {
select {
case <-ctx.Done():
return nil
case ch <- FileIDInfo{id, fi.Size}:
case ch <- FileIDInfo{id, size}:
}
return nil
})

23
internal/restic/repository.go
View File

@@ -3,6 +3,7 @@ package restic
import (
"context"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/ui/progress"
@@ -17,7 +18,7 @@ var ErrInvalidData = errors.New("invalid data returned")
type Repository interface {
// Backend returns the backend used by the repository
Backend() Backend
Backend() backend.Backend
// Connections returns the maximum number of concurrent backend operations
Connections() uint
@@ -56,10 +57,17 @@ type Repository interface {
SaveUnpacked(context.Context, FileType, []byte) (ID, error)
}
// Lister allows listing files in a backend.
type Lister interface {
List(context.Context, FileType, func(FileInfo) error) error
}
type FileType = backend.FileType
// These are the different data types a backend can store.
const (
PackFile FileType = backend.PackFile
KeyFile FileType = backend.KeyFile
LockFile FileType = backend.LockFile
SnapshotFile FileType = backend.SnapshotFile
IndexFile FileType = backend.IndexFile
ConfigFile FileType = backend.ConfigFile
)
// LoaderUnpacked allows loading a blob not stored in a pack file
type LoaderUnpacked interface {
@@ -92,3 +100,8 @@ type MasterIndex interface {
Save(ctx context.Context, repo SaverUnpacked, packBlacklist IDSet, extraObsolete IDs, p *progress.Counter) (obsolete IDSet, err error)
}
// Lister allows listing files in a backend.
type Lister interface {
List(ctx context.Context, t FileType, fn func(ID, int64) error) error
}

117
internal/restic/rewind_reader.go
View File

@@ -1,117 +0,0 @@
package restic
import (
"bytes"
"hash"
"io"
"github.com/restic/restic/internal/errors"
)
// RewindReader allows resetting the Reader to the beginning of the data.
type RewindReader interface {
io.Reader
// Rewind rewinds the reader so the same data can be read again from the
// start.
Rewind() error
// Length returns the number of bytes that can be read from the Reader
// after calling Rewind.
Length() int64
// Hash return a hash of the data if requested by the backed.
Hash() []byte
}
// ByteReader implements a RewindReader for a byte slice.
type ByteReader struct {
*bytes.Reader
Len int64
hash []byte
}
// Rewind restarts the reader from the beginning of the data.
func (b *ByteReader) Rewind() error {
_, err := b.Reader.Seek(0, io.SeekStart)
return err
}
// Length returns the number of bytes read from the reader after Rewind is
// called.
func (b *ByteReader) Length() int64 {
return b.Len
}
// Hash return a hash of the data if requested by the backed.
func (b *ByteReader) Hash() []byte {
return b.hash
}
// statically ensure that *ByteReader implements RewindReader.
var _ RewindReader = &ByteReader{}
// NewByteReader prepares a ByteReader that can then be used to read buf.
func NewByteReader(buf []byte, hasher hash.Hash) *ByteReader {
var hash []byte
if hasher != nil {
// must never fail according to interface
_, err := hasher.Write(buf)
if err != nil {
panic(err)
}
hash = hasher.Sum(nil)
}
return &ByteReader{
Reader: bytes.NewReader(buf),
Len: int64(len(buf)),
hash: hash,
}
}
// statically ensure that *FileReader implements RewindReader.
var _ RewindReader = &FileReader{}
// FileReader implements a RewindReader for an open file.
type FileReader struct {
io.ReadSeeker
Len int64
hash []byte
}
// Rewind seeks to the beginning of the file.
func (f *FileReader) Rewind() error {
_, err := f.ReadSeeker.Seek(0, io.SeekStart)
return errors.Wrap(err, "Seek")
}
// Length returns the length of the file.
func (f *FileReader) Length() int64 {
return f.Len
}
// Hash return a hash of the data if requested by the backed.
func (f *FileReader) Hash() []byte {
return f.hash
}
// NewFileReader wraps f in a *FileReader.
func NewFileReader(f io.ReadSeeker, hash []byte) (*FileReader, error) {
pos, err := f.Seek(0, io.SeekEnd)
if err != nil {
return nil, errors.Wrap(err, "Seek")
}
fr := &FileReader{
ReadSeeker: f,
Len: pos,
hash: hash,
}
err = fr.Rewind()
if err != nil {
return nil, err
}
return fr, nil
}

175
internal/restic/rewind_reader_test.go
View File

@@ -1,175 +0,0 @@
package restic
import (
"bytes"
"crypto/md5"
"hash"
"io"
"math/rand"
"os"
"path/filepath"
"testing"
"time"
"github.com/restic/restic/internal/test"
)
func TestByteReader(t *testing.T) {
buf := []byte("foobar")
for _, hasher := range []hash.Hash{nil, md5.New()} {
fn := func() RewindReader {
return NewByteReader(buf, hasher)
}
testRewindReader(t, fn, buf)
}
}
func TestFileReader(t *testing.T) {
buf := []byte("foobar")
d := test.TempDir(t)
filename := filepath.Join(d, "file-reader-test")
err := os.WriteFile(filename, buf, 0600)
if err != nil {
t.Fatal(err)
}
f, err := os.Open(filename)
if err != nil {
t.Fatal(err)
}
defer func() {
err := f.Close()
if err != nil {
t.Fatal(err)
}
}()
for _, hasher := range []hash.Hash{nil, md5.New()} {
fn := func() RewindReader {
var hash []byte
if hasher != nil {
// must never fail according to interface
_, err := hasher.Write(buf)
if err != nil {
panic(err)
}
hash = hasher.Sum(nil)
}
rd, err := NewFileReader(f, hash)
if err != nil {
t.Fatal(err)
}
return rd
}
testRewindReader(t, fn, buf)
}
}
func testRewindReader(t *testing.T, fn func() RewindReader, data []byte) {
seed := time.Now().UnixNano()
t.Logf("seed is %d", seed)
rnd := rand.New(rand.NewSource(seed))
type ReaderTestFunc func(t testing.TB, r RewindReader, data []byte)
var tests = []ReaderTestFunc{
func(t testing.TB, rd RewindReader, data []byte) {
if rd.Length() != int64(len(data)) {
t.Fatalf("wrong length returned, want %d, got %d", int64(len(data)), rd.Length())
}
buf := make([]byte, len(data))
_, err := io.ReadFull(rd, buf)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(buf, data) {
t.Fatalf("wrong data returned")
}
if rd.Length() != int64(len(data)) {
t.Fatalf("wrong length returned, want %d, got %d", int64(len(data)), rd.Length())
}
err = rd.Rewind()
if err != nil {
t.Fatal(err)
}
if rd.Length() != int64(len(data)) {
t.Fatalf("wrong length returned, want %d, got %d", int64(len(data)), rd.Length())
}
buf2 := make([]byte, int64(len(data)))
_, err = io.ReadFull(rd, buf2)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(buf2, data) {
t.Fatalf("wrong data returned")
}
if rd.Length() != int64(len(data)) {
t.Fatalf("wrong length returned, want %d, got %d", int64(len(data)), rd.Length())
}
if rd.Hash() != nil {
hasher := md5.New()
// must never fail according to interface
_, _ = hasher.Write(buf2)
if !bytes.Equal(rd.Hash(), hasher.Sum(nil)) {
t.Fatal("hash does not match data")
}
}
},
func(t testing.TB, rd RewindReader, data []byte) {
// read first bytes
buf := make([]byte, rnd.Intn(len(data)))
_, err := io.ReadFull(rd, buf)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(buf, data[:len(buf)]) {
t.Fatalf("wrong data returned")
}
err = rd.Rewind()
if err != nil {
t.Fatal(err)
}
buf2 := make([]byte, rnd.Intn(len(data)))
_, err = io.ReadFull(rd, buf2)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(buf2, data[:len(buf2)]) {
t.Fatalf("wrong data returned")
}
// read remainder
buf3 := make([]byte, len(data)-len(buf2))
_, err = io.ReadFull(rd, buf3)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(buf3, data[len(buf2):]) {
t.Fatalf("wrong data returned")
}
},
}
for _, test := range tests {
t.Run("", func(t *testing.T) {
rd := fn()
test(t, rd, data)
})
}
}

8
internal/restic/snapshot_find_test.go
View File

@@ -16,7 +16,7 @@ func TestFindLatestSnapshot(t *testing.T) {
latestSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1)
f := restic.SnapshotFilter{Hosts: []string{"foo"}}
sn, _, err := f.FindLatest(context.TODO(), repo.Backend(), repo, "latest")
sn, _, err := f.FindLatest(context.TODO(), repo, repo, "latest")
if err != nil {
t.Fatalf("FindLatest returned error: %v", err)
}
@@ -35,7 +35,7 @@ func TestFindLatestSnapshotWithMaxTimestamp(t *testing.T) {
sn, _, err := (&restic.SnapshotFilter{
Hosts: []string{"foo"},
TimestampLimit: parseTimeUTC("2018-08-08 08:08:08"),
}).FindLatest(context.TODO(), repo.Backend(), repo, "latest")
}).FindLatest(context.TODO(), repo, repo, "latest")
if err != nil {
t.Fatalf("FindLatest returned error: %v", err)
}
@@ -62,7 +62,7 @@ func TestFindLatestWithSubpath(t *testing.T) {
{desiredSnapshot.ID().String() + ":subfolder", "subfolder"},
} {
t.Run("", func(t *testing.T) {
sn, subfolder, err := (&restic.SnapshotFilter{}).FindLatest(context.TODO(), repo.Backend(), repo, exp.query)
sn, subfolder, err := (&restic.SnapshotFilter{}).FindLatest(context.TODO(), repo, repo, exp.query)
if err != nil {
t.Fatalf("FindLatest returned error: %v", err)
}
@@ -78,7 +78,7 @@ func TestFindAllSubpathError(t *testing.T) {
desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1)
count := 0
test.OK(t, (&restic.SnapshotFilter{}).FindAll(context.TODO(), repo.Backend(), repo,
test.OK(t, (&restic.SnapshotFilter{}).FindAll(context.TODO(), repo, repo,
[]string{"latest:subfolder", desiredSnapshot.ID().Str() + ":subfolder"},
func(id string, sn *restic.Snapshot, err error) error {
if err == restic.ErrInvalidSnapshotSyntax {

2
internal/restic/testing_test.go
View File

@@ -20,7 +20,7 @@ const (
// LoadAllSnapshots returns a list of all snapshots in the repo.
// If a snapshot ID is in excludeIDs, it will not be included in the result.
func loadAllSnapshots(ctx context.Context, repo restic.Repository, excludeIDs restic.IDSet) (snapshots restic.Snapshots, err error) {
err = restic.ForAllSnapshots(ctx, repo.Backend(), repo, excludeIDs, func(id restic.ID, sn *restic.Snapshot, err error) error {
err = restic.ForAllSnapshots(ctx, repo, repo, excludeIDs, func(id restic.ID, sn *restic.Snapshot, err error) error {
if err != nil {
return err
}