TheArchive/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2025-12-03 22:21:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Return real size from SaveBlob

Browse Source
This commit is contained in:
Alexander Neumann
2022-05-01 14:26:57 +02:00
committed by Michael Eischer
parent fdc53a9d32
commit 99634c0936
13 changed files with 50 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/repository/fuzz_test.go
View File

@@ -23,7 +23,7 @@ func FuzzSaveLoadBlob(f *testing.F) {
id := restic.Hash(blob)
repo, _ := TestRepositoryWithBackend(t, mem.New(), 2)
_, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, blob, id, false)
_, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, blob, id, false)
if err != nil {
t.Fatal(err)
}

2
internal/repository/repack.go
View File

@@ -75,7 +75,7 @@ func Repack(ctx context.Context, repo restic.Repository, dstRepo restic.Reposito
}
// We do want to save already saved blobs!
_, _, err = dstRepo.SaveBlob(wgCtx, blob.Type, buf, blob.ID, true)
_, _, _, err = dstRepo.SaveBlob(wgCtx, blob.Type, buf, blob.ID, true)
if err != nil {
return err
}

4
internal/repository/repack_test.go
View File

@@ -32,7 +32,7 @@ func createRandomBlobs(t testing.TB, repo restic.Repository, blobs int, pData fl
buf := make([]byte, length)
rand.Read(buf)
id, exists, err := repo.SaveBlob(context.TODO(), tpe, buf, restic.ID{}, false)
id, exists, _, err := repo.SaveBlob(context.TODO(), tpe, buf, restic.ID{}, false)
if err != nil {
t.Fatalf("SaveFrom() error %v", err)
}
@@ -62,7 +62,7 @@ func createRandomWrongBlob(t testing.TB, repo restic.Repository) {
// invert first data byte
buf[0] ^= 0xff
_, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, id, false)
_, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, id, false)
if err != nil {
t.Fatalf("SaveFrom() error %v", err)
}

34
internal/repository/repository.go
View File

@@ -378,9 +378,10 @@ func (r *Repository) getZstdDecoder() *zstd.Decoder {
}
// saveAndEncrypt encrypts data and stores it to the backend as type t. If data
// is small enough, it will be packed together with other small blobs.
// The caller must ensure that the id matches the data.
func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data []byte, id restic.ID) error {
// is small enough, it will be packed together with other small blobs. The
// caller must ensure that the id matches the data. Returned is the size data
// occupies in the repo (compressed or not, including the encryption overhead).
func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data []byte, id restic.ID) (size int, err error) {
debug.Log("save id %v (%v, %d bytes)", id, t, len(data))
uncompressedLength := 0
@@ -417,24 +418,29 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data
packer, err := pm.findPacker()
if err != nil {
return err
return 0, err
}
// save ciphertext
_, err = packer.Add(t, id, ciphertext, uncompressedLength)
size, err = packer.Add(t, id, ciphertext, uncompressedLength)
if err != nil {
return err
return 0, err
}
// if the pack is not full enough, put back to the list
if packer.Size() < minPackSize {
debug.Log("pack is not full enough (%d bytes)", packer.Size())
pm.insertPacker(packer)
return nil
return size, nil
}
// else write the pack to the backend
return r.savePacker(ctx, t, packer)
err = r.savePacker(ctx, t, packer)
if err != nil {
return 0, err
}
return size, nil
}
// SaveJSONUnpacked serialises item as JSON and encrypts and saves it in the
@@ -815,8 +821,10 @@ func (r *Repository) Close() error {
// It takes care that no duplicates are saved; this can be overwritten
// by setting storeDuplicate to true.
// If id is the null id, it will be computed and returned.
// Also returns if the blob was already known before
func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (newID restic.ID, known bool, err error) {
// Also returns if the blob was already known before.
// If the blob was not known before, it returns the number of bytes the blob
// occupies in the repo (compressed or not, including encryption overhead).
func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (newID restic.ID, known bool, size int, err error) {
// compute plaintext hash if not already set
if id.IsNull() {
@@ -830,10 +838,10 @@ func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte
// only save when needed or explicitly told
if !known || storeDuplicate {
err = r.saveAndEncrypt(ctx, t, buf, newID)
size, err = r.saveAndEncrypt(ctx, t, buf, newID)
}
return newID, known, err
return newID, known, size, err
}
// LoadTree loads a tree from the repository.
@@ -867,7 +875,7 @@ func (r *Repository) SaveTree(ctx context.Context, t *restic.Tree) (restic.ID, e
// adds a newline after each object)
buf = append(buf, '\n')
id, _, err := r.SaveBlob(ctx, restic.TreeBlob, buf, restic.ID{}, false)
id, _, _, err := r.SaveBlob(ctx, restic.TreeBlob, buf, restic.ID{}, false)
return id, err
}

12
internal/repository/repository_test.go
View File

@@ -44,7 +44,7 @@ func testSave(t *testing.T, version uint) {
id := restic.Hash(data)
// save
sid, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, restic.ID{}, false)
sid, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, restic.ID{}, false)
rtest.OK(t, err)
rtest.Equals(t, id, sid)
@@ -83,7 +83,7 @@ func testSaveFrom(t *testing.T, version uint) {
id := restic.Hash(data)
// save
id2, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, id, false)
id2, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, id, false)
rtest.OK(t, err)
rtest.Equals(t, id, id2)
@@ -125,7 +125,7 @@ func benchmarkSaveAndEncrypt(t *testing.B, version uint) {
t.SetBytes(int64(size))
for i := 0; i < t.N; i++ {
_, _, err = repo.SaveBlob(context.TODO(), restic.DataBlob, data, id, true)
_, _, _, err = repo.SaveBlob(context.TODO(), restic.DataBlob, data, id, true)
rtest.OK(t, err)
}
}
@@ -187,7 +187,7 @@ func testLoadBlob(t *testing.T, version uint) {
_, err := io.ReadFull(rnd, buf)
rtest.OK(t, err)
id, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false)
id, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false)
rtest.OK(t, err)
rtest.OK(t, repo.Flush(context.Background()))
@@ -220,7 +220,7 @@ func benchmarkLoadBlob(b *testing.B, version uint) {
_, err := io.ReadFull(rnd, buf)
rtest.OK(b, err)
id, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false)
id, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false)
rtest.OK(b, err)
rtest.OK(b, repo.Flush(context.Background()))
@@ -396,7 +396,7 @@ func saveRandomDataBlobs(t testing.TB, repo restic.Repository, num int, sizeMax
_, err := io.ReadFull(rnd, buf)
rtest.OK(t, err)
_, _, err = repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false)
_, _, _, err = repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false)
rtest.OK(t, err)
}
}