TheArchive/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2025-10-10 11:32:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

archiver: Incrementally serialize tree nodes

Browse Source 
That way it is not necessary to keep both the Nodes forming a Tree and
the serialized JSON version in memory.
This commit is contained in:
Michael Eischer
2022-05-21 13:33:08 +02:00
parent c206a101a3
commit b817681a11
5 changed files with 101 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
internal/archiver/archiver.go
View File

@@ -2,7 +2,6 @@ package archiver
import (
"context"
"encoding/json"
"os"
"path"
"runtime"
@@ -175,17 +174,13 @@ func (arch *Archiver) error(item string, err error) error {
// saveTree stores a tree in the repo. It checks the index and the known blobs
// before saving anything.
func (arch *Archiver) saveTree(ctx context.Context, t *restic.Tree) (restic.ID, ItemStats, error) {
func (arch *Archiver) saveTree(ctx context.Context, t *restic.TreeJSONBuilder) (restic.ID, ItemStats, error) {
var s ItemStats
buf, err := json.Marshal(t)
buf, err := t.Finalize()
if err != nil {
return restic.ID{}, s, errors.Wrap(err, "MarshalJSON")
return restic.ID{}, s, err
}
// append a newline so that the data is always consistent (json.Encoder
// adds a newline after each object)
buf = append(buf, '\n')
b := &Buffer{Data: buf}
res := arch.blobSaver.Save(ctx, restic.TreeBlob, b)
@@ -620,7 +615,11 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree,
return nil, err
}
id, nodeStats, err := arch.saveTree(ctx, subtree)
tb, err := restic.TreeToBuilder(subtree)
if err != nil {
return nil, err
}
id, nodeStats, err := arch.saveTree(ctx, tb)
if err != nil {
return nil, err
}
@@ -834,7 +833,11 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps
return errors.New("snapshot is empty")
}
rootTreeID, stats, err = arch.saveTree(wgCtx, tree)
tb, err := restic.TreeToBuilder(tree)
if err != nil {
return err
}
rootTreeID, stats, err = arch.saveTree(wgCtx, tb)
arch.stopWorkers()
return err
})