TheArchive/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2025-12-12 07:41:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

data: split node and snapshot code from restic package

Browse Source
This commit is contained in:
Michael Eischer
2025-09-23 20:01:09 +02:00
parent c85b157e0e
commit 56ac8360c7
166 changed files with 1170 additions and 1107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
internal/walker/rewriter.go
View File

@@ -5,11 +5,12 @@ import (
"fmt"
"path"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
)
type NodeRewriteFunc func(node *restic.Node, path string) *restic.Node
type NodeRewriteFunc func(node *data.Node, path string) *data.Node
type FailedTreeRewriteFunc func(nodeID restic.ID, path string, err error) (restic.ID, error)
type QueryRewrittenSizeFunc func() SnapshotSize
@@ -45,7 +46,7 @@ func NewTreeRewriter(opts RewriteOpts) *TreeRewriter {
}
// setup default implementations
if rw.opts.RewriteNode == nil {
rw.opts.RewriteNode = func(node *restic.Node, _ string) *restic.Node {
rw.opts.RewriteNode = func(node *data.Node, _ string) *data.Node {
return node
}
}
@@ -63,9 +64,9 @@ func NewSnapshotSizeRewriter(rewriteNode NodeRewriteFunc) (*TreeRewriter, QueryR
var size uint64
t := NewTreeRewriter(RewriteOpts{
RewriteNode: func(node *restic.Node, path string) *restic.Node {
RewriteNode: func(node *data.Node, path string) *data.Node {
node = rewriteNode(node, path)
if node != nil && node.Type == restic.NodeTypeFile {
if node != nil && node.Type == data.NodeTypeFile {
count++
size += node.Size
}
@@ -94,7 +95,7 @@ func (t *TreeRewriter) RewriteTree(ctx context.Context, repo BlobLoadSaver, node
}
// a nil nodeID will lead to a load error
curTree, err := restic.LoadTree(ctx, repo, nodeID)
curTree, err := data.LoadTree(ctx, repo, nodeID)
if err != nil {
return t.opts.RewriteFailedTree(nodeID, nodepath, err)
}
@@ -103,7 +104,7 @@ func (t *TreeRewriter) RewriteTree(ctx context.Context, repo BlobLoadSaver, node
// check that we can properly encode this tree without losing information
// The alternative of using json/Decoder.DisallowUnknownFields() doesn't work as we use
// a custom UnmarshalJSON to decode trees, see also https://github.com/golang/go/issues/41144
testID, err := restic.SaveTree(ctx, repo, curTree)
testID, err := data.SaveTree(ctx, repo, curTree)
if err != nil {
return restic.ID{}, err
}
@@ -114,7 +115,7 @@ func (t *TreeRewriter) RewriteTree(ctx context.Context, repo BlobLoadSaver, node
debug.Log("filterTree: %s, nodeId: %s\n", nodepath, nodeID.Str())
tb := restic.NewTreeJSONBuilder()
tb := data.NewTreeJSONBuilder()
for _, node := range curTree.Nodes {
if ctx.Err() != nil {
return restic.ID{}, ctx.Err()
@@ -126,7 +127,7 @@ func (t *TreeRewriter) RewriteTree(ctx context.Context, repo BlobLoadSaver, node
continue
}
if node.Type != restic.NodeTypeDir {
if node.Type != data.NodeTypeDir {
err = tb.AddNode(node)
if err != nil {
return restic.ID{}, err

15
internal/walker/rewriter_test.go
View File

@@ -5,6 +5,7 @@ import (
"testing"
"github.com/pkg/errors"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
@@ -44,7 +45,7 @@ func checkRewriteItemOrder(want []string) checkRewriteFunc {
pos := 0
return func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB)) {
rewriter = NewTreeRewriter(RewriteOpts{
RewriteNode: func(node *restic.Node, path string) *restic.Node {
RewriteNode: func(node *data.Node, path string) *data.Node {
if pos >= len(want) {
t.Errorf("additional unexpected path found: %v", path)
return nil
@@ -74,7 +75,7 @@ func checkRewriteSkips(skipFor map[string]struct{}, want []string, disableCache
return func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB)) {
rewriter = NewTreeRewriter(RewriteOpts{
RewriteNode: func(node *restic.Node, path string) *restic.Node {
RewriteNode: func(node *data.Node, path string) *data.Node {
if pos >= len(want) {
t.Errorf("additional unexpected path found: %v", path)
return nil
@@ -108,8 +109,8 @@ func checkRewriteSkips(skipFor map[string]struct{}, want []string, disableCache
func checkIncreaseNodeSize(increase uint64) checkRewriteFunc {
return func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB)) {
rewriter = NewTreeRewriter(RewriteOpts{
RewriteNode: func(node *restic.Node, path string) *restic.Node {
if node.Type == restic.NodeTypeFile {
RewriteNode: func(node *data.Node, path string) *data.Node {
if node.Type == data.NodeTypeFile {
node.Size += increase
}
return node
@@ -324,11 +325,11 @@ func TestSnapshotSizeQuery(t *testing.T) {
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
rewriteNode := func(node *restic.Node, path string) *restic.Node {
rewriteNode := func(node *data.Node, path string) *data.Node {
if path == "/bar" {
return nil
}
if node.Type == restic.NodeTypeFile {
if node.Type == data.NodeTypeFile {
node.Size += 21
}
return node
@@ -366,7 +367,7 @@ func TestRewriterFailOnUnknownFields(t *testing.T) {
defer cancel()
rewriter := NewTreeRewriter(RewriteOpts{
RewriteNode: func(node *restic.Node, path string) *restic.Node {
RewriteNode: func(node *data.Node, path string) *data.Node {
// tree loading must not succeed
t.Fail()
return node

13
internal/walker/walker.go
View File

@@ -7,6 +7,7 @@ import (
"github.com/pkg/errors"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/restic"
)
@@ -21,7 +22,7 @@ var ErrSkipNode = errors.New("skip this node")
// When the special value ErrSkipNode is returned and node is a dir node, it is
// not walked. When the node is not a dir node, the remaining items in this
// tree are skipped.
type WalkFunc func(parentTreeID restic.ID, path string, node *restic.Node, nodeErr error) (err error)
type WalkFunc func(parentTreeID restic.ID, path string, node *data.Node, nodeErr error) (err error)
type WalkVisitor struct {
// If the node is a `dir`, it will be entered afterwards unless `ErrSkipNode`
@@ -35,7 +36,7 @@ type WalkVisitor struct {
// error, it is passed up the call stack. The trees in ignoreTrees are not
// walked. If walkFn ignores trees, these are added to the set.
func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, visitor WalkVisitor) error {
tree, err := restic.LoadTree(ctx, repo, root)
tree, err := data.LoadTree(ctx, repo, root)
err = visitor.ProcessNode(root, "/", nil, err)
if err != nil {
@@ -51,7 +52,7 @@ func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, visitor W
// walk recursively traverses the tree, ignoring subtrees when the ID of the
// subtree is in ignoreTrees. If err is nil and ignore is true, the subtree ID
// will be added to ignoreTrees by walk.
func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree *restic.Tree, visitor WalkVisitor) (err error) {
func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree *data.Tree, visitor WalkVisitor) (err error) {
sort.Slice(tree.Nodes, func(i, j int) bool {
return tree.Nodes[i].Name < tree.Nodes[j].Name
})
@@ -63,11 +64,11 @@ func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTree
p := path.Join(prefix, node.Name)
if node.Type == restic.NodeTypeInvalid {
if node.Type == data.NodeTypeInvalid {
return errors.Errorf("node type is empty for node %q", node.Name)
}
if node.Type != restic.NodeTypeDir {
if node.Type != data.NodeTypeDir {
err := visitor.ProcessNode(parentTreeID, p, node, nil)
if err != nil {
if err == ErrSkipNode {
@@ -85,7 +86,7 @@ func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTree
return errors.Errorf("subtree for node %v in tree %v is nil", node.Name, p)
}
subtree, err := restic.LoadTree(ctx, repo, *node.Subtree)
subtree, err := data.LoadTree(ctx, repo, *node.Subtree)
err = visitor.ProcessNode(parentTreeID, p, node, err)
if err != nil {
if err == ErrSkipNode {

19
internal/walker/walker_test.go
View File

@@ -7,6 +7,7 @@ import (
"testing"
"github.com/pkg/errors"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@@ -26,7 +27,7 @@ func BuildTreeMap(tree TestTree) (m TreeMap, root restic.ID) {
}
func buildTreeMap(tree TestTree, m TreeMap) restic.ID {
tb := restic.NewTreeJSONBuilder()
tb := data.NewTreeJSONBuilder()
var names []string
for name := range tree {
names = append(names, name)
@@ -37,9 +38,9 @@ func buildTreeMap(tree TestTree, m TreeMap) restic.ID {
item := tree[name]
switch elem := item.(type) {
case TestFile:
err := tb.AddNode(&restic.Node{
err := tb.AddNode(&data.Node{
Name: name,
Type: restic.NodeTypeFile,
Type: data.NodeTypeFile,
Size: elem.Size,
})
if err != nil {
@@ -47,10 +48,10 @@ func buildTreeMap(tree TestTree, m TreeMap) restic.ID {
}
case TestTree:
id := buildTreeMap(elem, m)
err := tb.AddNode(&restic.Node{
err := tb.AddNode(&data.Node{
Name: name,
Subtree: &id,
Type: restic.NodeTypeDir,
Type: data.NodeTypeDir,
})
if err != nil {
panic(err)
@@ -100,7 +101,7 @@ type checkFunc func(t testing.TB) (walker WalkFunc, leaveDir func(path string) e
func checkItemOrder(want []string) checkFunc {
pos := 0
return func(t testing.TB) (walker WalkFunc, leaveDir func(path string) error, final func(testing.TB, error)) {
walker = func(treeID restic.ID, path string, node *restic.Node, err error) error {
walker = func(treeID restic.ID, path string, node *data.Node, err error) error {
if err != nil {
t.Errorf("error walking %v: %v", path, err)
return err
@@ -137,7 +138,7 @@ func checkItemOrder(want []string) checkFunc {
func checkParentTreeOrder(want []string) checkFunc {
pos := 0
return func(t testing.TB) (walker WalkFunc, leaveDir func(path string) error, final func(testing.TB, error)) {
walker = func(treeID restic.ID, path string, node *restic.Node, err error) error {
walker = func(treeID restic.ID, path string, node *data.Node, err error) error {
if err != nil {
t.Errorf("error walking %v: %v", path, err)
return err
@@ -172,7 +173,7 @@ func checkSkipFor(skipFor map[string]struct{}, wantPaths []string) checkFunc {
var pos int
return func(t testing.TB) (walker WalkFunc, leaveDir func(path string) error, final func(testing.TB, error)) {
walker = func(treeID restic.ID, path string, node *restic.Node, err error) error {
walker = func(treeID restic.ID, path string, node *data.Node, err error) error {
if err != nil {
t.Errorf("error walking %v: %v", path, err)
return err
@@ -214,7 +215,7 @@ func checkErrorReturned(errForPath string) checkFunc {
expectedErr := fmt.Errorf("error for %v", errForPath)
return func(t testing.TB) (walker WalkFunc, leaveDir func(path string) error, final func(testing.TB, error)) {
walker = func(treeID restic.ID, path string, node *restic.Node, err error) error {
walker = func(treeID restic.ID, path string, node *data.Node, err error) error {
if path == errForPath {
return expectedErr
}