TheArchive/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2025-12-03 21:21:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4499 from MichaelEischer/modular-backend-code

Browse Source 
Split backend code from restic package
This commit is contained in:
Michael Eischer
2023-10-27 20:19:20 +02:00
committed by GitHub
parent 50ef01131a 249605843b
commit 45962c2847
132 changed files with 1145 additions and 1062 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
internal/repository/key.go
View File

@@ -116,7 +116,7 @@ func SearchKey(ctx context.Context, s *Repository, password string, maxKeys int,
checked := 0
if len(keyHint) > 0 {
id, err := restic.Find(ctx, s.Backend(), restic.KeyFile, keyHint)
id, err := restic.Find(ctx, s, restic.KeyFile, keyHint)
if err == nil {
key, err := OpenKey(ctx, s, id, password)
@@ -178,7 +178,7 @@ func SearchKey(ctx context.Context, s *Repository, password string, maxKeys int,
// LoadKey loads a key from the backend.
func LoadKey(ctx context.Context, s *Repository, id restic.ID) (k *Key, err error) {
h := restic.Handle{Type: restic.KeyFile, Name: id.String()}
h := backend.Handle{Type: restic.KeyFile, Name: id.String()}
data, err := backend.LoadAll(ctx, nil, s.be, h)
if err != nil {
return nil, err
@@ -270,12 +270,12 @@ func AddKey(ctx context.Context, s *Repository, password, username, hostname str
id := restic.Hash(buf)
// store in repository and return
h := restic.Handle{
h := backend.Handle{
Type: restic.KeyFile,
Name: id.String(),
}
err = s.be.Save(ctx, h, restic.NewByteReader(buf, s.be.Hasher()))
err = s.be.Save(ctx, h, backend.NewByteReader(buf, s.be.Hasher()))
if err != nil {
return nil, err
}

7
internal/repository/packer_manager.go
View File

@@ -8,6 +8,7 @@ import (
"runtime"
"sync"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/hashing"
"github.com/restic/restic/internal/restic"
@@ -145,7 +146,7 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packe
// calculate sha256 hash in a second pass
var rd io.Reader
rd, err = restic.NewFileReader(p.tmpfile, nil)
rd, err = backend.NewFileReader(p.tmpfile, nil)
if err != nil {
return err
}
@@ -163,12 +164,12 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packe
}
id := restic.IDFromHash(hr.Sum(nil))
h := restic.Handle{Type: restic.PackFile, Name: id.String(), ContainedBlobType: t}
h := backend.Handle{Type: backend.PackFile, Name: id.String(), IsMetadata: t.IsMetadata()}
var beHash []byte
if beHr != nil {
beHash = beHr.Sum(nil)
}
rrd, err := restic.NewFileReader(p.tmpfile, beHash)
rrd, err := backend.NewFileReader(p.tmpfile, beHash)
if err != nil {
return err
}

5
internal/repository/repack_test.go
View File

@@ -6,6 +6,7 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
@@ -157,7 +158,7 @@ func repack(t *testing.T, repo restic.Repository, packs restic.IDSet, blobs rest
}
for id := range repackedBlobs {
err = repo.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()})
err = repo.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()})
if err != nil {
t.Fatal(err)
}
@@ -191,7 +192,7 @@ func rebuildIndex(t *testing.T, repo restic.Repository) {
}
err = repo.List(context.TODO(), restic.IndexFile, func(id restic.ID, size int64) error {
h := restic.Handle{
h := backend.Handle{
Type: restic.IndexFile,
Name: id.String(),
}

38
internal/repository/repository.go
View File

@@ -36,7 +36,7 @@ const MaxPackSize = 128 * 1024 * 1024
// Repository is used to access a repository in a backend.
type Repository struct {
be restic.Backend
be backend.Backend
cfg restic.Config
key *crypto.Key
keyID restic.ID
@@ -109,7 +109,7 @@ func (c *CompressionMode) Type() string {
}
// New returns a new repository with backend be.
func New(be restic.Backend, opts Options) (*Repository, error) {
func New(be backend.Backend, opts Options) (*Repository, error) {
if opts.Compression == CompressionInvalid {
return nil, errors.New("invalid compression mode")
}
@@ -181,7 +181,7 @@ func (r *Repository) LoadUnpacked(ctx context.Context, t restic.FileType, id res
ctx, cancel := context.WithCancel(ctx)
h := restic.Handle{Type: t, Name: id.String()}
h := backend.Handle{Type: t, Name: id.String()}
retriedInvalidData := false
var dataErr error
wr := new(bytes.Buffer)
@@ -232,7 +232,7 @@ func (r *Repository) LoadUnpacked(ctx context.Context, t restic.FileType, id res
}
type haver interface {
Has(restic.Handle) bool
Has(backend.Handle) bool
}
// sortCachedPacksFirst moves all cached pack files to the front of blobs.
@@ -250,7 +250,7 @@ func sortCachedPacksFirst(cache haver, blobs []restic.PackedBlob) {
noncached := make([]restic.PackedBlob, 0, len(blobs)/2)
for _, blob := range blobs {
if cache.Has(restic.Handle{Type: restic.PackFile, Name: blob.PackID.String()}) {
if cache.Has(backend.Handle{Type: restic.PackFile, Name: blob.PackID.String()}) {
cached = append(cached, blob)
continue
}
@@ -284,7 +284,7 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic.
}
// load blob from pack
h := restic.Handle{Type: restic.PackFile, Name: blob.PackID.String(), ContainedBlobType: t}
h := backend.Handle{Type: restic.PackFile, Name: blob.PackID.String(), IsMetadata: t.IsMetadata()}
switch {
case cap(buf) < int(blob.Length):
@@ -494,9 +494,9 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by
} else {
id = restic.Hash(ciphertext)
}
h := restic.Handle{Type: t, Name: id.String()}
h := backend.Handle{Type: t, Name: id.String()}
err = r.be.Save(ctx, h, restic.NewByteReader(ciphertext, r.be.Hasher()))
err = r.be.Save(ctx, h, backend.NewByteReader(ciphertext, r.be.Hasher()))
if err != nil {
debug.Log("error saving blob %v: %v", h, err)
return restic.ID{}, err
@@ -561,7 +561,7 @@ func (r *Repository) flushPacks(ctx context.Context) error {
}
// Backend returns the backend for the repository.
func (r *Repository) Backend() restic.Backend {
func (r *Repository) Backend() backend.Backend {
return r.be
}
@@ -584,20 +584,14 @@ func (r *Repository) SetIndex(i restic.MasterIndex) error {
func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error {
debug.Log("Loading index")
indexList, err := backend.MemorizeList(ctx, r.Backend(), restic.IndexFile)
indexList, err := restic.MemorizeList(ctx, r, restic.IndexFile)
if err != nil {
return err
}
if p != nil {
var numIndexFiles uint64
err := indexList.List(ctx, restic.IndexFile, func(fi restic.FileInfo) error {
_, err := restic.ParseID(fi.Name)
if err != nil {
debug.Log("unable to parse %v as an ID", fi.Name)
return nil
}
err := indexList.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error {
numIndexFiles++
return nil
})
@@ -773,7 +767,7 @@ func (r *Repository) Init(ctx context.Context, version uint, password string, ch
return fmt.Errorf("repository version %v too low", version)
}
_, err := r.be.Stat(ctx, restic.Handle{Type: restic.ConfigFile})
_, err := r.be.Stat(ctx, backend.Handle{Type: restic.ConfigFile})
if err != nil && !r.be.IsNotExist(err) {
return err
}
@@ -818,7 +812,7 @@ func (r *Repository) KeyID() restic.ID {
// List runs fn for all files of type t in the repo.
func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error {
return r.be.List(ctx, t, func(fi restic.FileInfo) error {
return r.be.List(ctx, t, func(fi backend.FileInfo) error {
id, err := restic.ParseID(fi.Name)
if err != nil {
debug.Log("unable to parse %v as an ID", fi.Name)
@@ -831,7 +825,7 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic
// ListPack returns the list of blobs saved in the pack id and the length of
// the pack header.
func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) ([]restic.Blob, uint32, error) {
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
h := backend.Handle{Type: restic.PackFile, Name: id.String()}
return pack.List(r.Key(), backend.ReaderAt(ctx, r.Backend(), h), size)
}
@@ -881,7 +875,7 @@ func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte
return newID, known, size, err
}
type BackendLoadFn func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error
type BackendLoadFn func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error
// Skip sections with more than 4MB unused blobs
const maxUnusedRange = 4 * 1024 * 1024
@@ -922,7 +916,7 @@ func StreamPack(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, pack
}
func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
h := restic.Handle{Type: restic.PackFile, Name: packID.String(), ContainedBlobType: restic.DataBlob}
h := backend.Handle{Type: restic.PackFile, Name: packID.String(), IsMetadata: false}
dataStart := blobs[0].Offset
dataEnd := blobs[len(blobs)-1].Offset + blobs[len(blobs)-1].Length

13
internal/repository/repository_internal_test.go
View File

@@ -5,13 +5,14 @@ import (
"sort"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
type mapcache map[restic.Handle]bool
type mapcache map[backend.Handle]bool
func (c mapcache) Has(h restic.Handle) bool { return c[h] }
func (c mapcache) Has(h backend.Handle) bool { return c[h] }
func TestSortCachedPacksFirst(t *testing.T) {
var (
@@ -27,15 +28,15 @@ func TestSortCachedPacksFirst(t *testing.T) {
blobs[i] = restic.PackedBlob{PackID: id}
if i%3 == 0 {
h := restic.Handle{Name: id.String(), Type: restic.PackFile}
h := backend.Handle{Name: id.String(), Type: backend.PackFile}
cache[h] = true
}
}
copy(sorted[:], blobs[:])
sort.SliceStable(sorted[:], func(i, j int) bool {
hi := restic.Handle{Type: restic.PackFile, Name: sorted[i].PackID.String()}
hj := restic.Handle{Type: restic.PackFile, Name: sorted[j].PackID.String()}
hi := backend.Handle{Type: backend.PackFile, Name: sorted[i].PackID.String()}
hj := backend.Handle{Type: backend.PackFile, Name: sorted[j].PackID.String()}
return cache.Has(hi) && !cache.Has(hj)
})
@@ -58,7 +59,7 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) {
blobs[i] = restic.PackedBlob{PackID: id}
if i%3 == 0 {
h := restic.Handle{Name: id.String(), Type: restic.PackFile}
h := backend.Handle{Name: id.String(), Type: backend.PackFile}
cache[h] = true
}
}

13
internal/repository/repository_test.go
View File

@@ -16,6 +16,7 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/klauspost/compress/zstd"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/local"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/index"
@@ -278,13 +279,13 @@ func TestRepositoryLoadUnpackedBroken(t *testing.T) {
data := rtest.Random(23, 12345)
id := restic.Hash(data)
h := restic.Handle{Type: restic.IndexFile, Name: id.String()}
h := backend.Handle{Type: restic.IndexFile, Name: id.String()}
// damage buffer
data[0] ^= 0xff
repo := repository.TestOpenLocal(t, repodir)
// store broken file
err := repo.Backend().Save(context.TODO(), h, restic.NewByteReader(data, nil))
err := repo.Backend().Save(context.TODO(), h, backend.NewByteReader(data, nil))
rtest.OK(t, err)
// without a retry backend this will just return an error that the file is broken
@@ -296,10 +297,10 @@ func TestRepositoryLoadUnpackedBroken(t *testing.T) {
}
type damageOnceBackend struct {
restic.Backend
backend.Backend
}
func (be *damageOnceBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *damageOnceBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
// don't break the config file as we can't retry it
if h.Type == restic.ConfigFile {
return be.Backend.Load(ctx, h, length, offset, fn)
@@ -352,7 +353,7 @@ func benchmarkLoadIndex(b *testing.B, version uint) {
rtest.OK(b, err)
b.Logf("index saved as %v", id.Str())
fi, err := repo.Backend().Stat(context.TODO(), restic.Handle{Type: restic.IndexFile, Name: id.String()})
fi, err := repo.Backend().Stat(context.TODO(), backend.Handle{Type: restic.IndexFile, Name: id.String()})
rtest.OK(b, err)
b.Logf("filesize is %v", fi.Size)
@@ -528,7 +529,7 @@ func testStreamPack(t *testing.T, version uint) {
packfileBlobs, packfile := buildPackfileWithoutHeader(blobSizes, &key, compress)
loadCalls := 0
load := func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
load := func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
data := packfile
if offset > int64(len(data)) {

7
internal/repository/testing.go
View File

@@ -6,6 +6,7 @@ import (
"os"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/local"
"github.com/restic/restic/internal/backend/mem"
"github.com/restic/restic/internal/backend/retry"
@@ -34,7 +35,7 @@ func TestUseLowSecurityKDFParameters(t logger) {
}
// TestBackend returns a fully configured in-memory backend.
func TestBackend(_ testing.TB) restic.Backend {
func TestBackend(_ testing.TB) backend.Backend {
return mem.New()
}
@@ -43,7 +44,7 @@ const TestChunkerPol = chunker.Pol(0x3DA3358B4DC173)
// TestRepositoryWithBackend returns a repository initialized with a test
// password. If be is nil, an in-memory backend is used. A constant polynomial
// is used for the chunker and low-security test parameters.
func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint) restic.Repository {
func TestRepositoryWithBackend(t testing.TB, be backend.Backend, version uint) restic.Repository {
t.Helper()
TestUseLowSecurityKDFParameters(t)
restic.TestDisableCheckPolynomial(t)
@@ -98,7 +99,7 @@ func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository {
// TestOpenLocal opens a local repository.
func TestOpenLocal(t testing.TB, dir string) (r restic.Repository) {
var be restic.Backend
var be backend.Backend
be, err := local.Open(context.TODO(), local.Config{Path: dir, Connections: 2})
if err != nil {
t.Fatal(err)