diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 5e884de87..6d6483b19 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -19,6 +19,7 @@ import ( "golang.org/x/sync/errgroup" "github.com/restic/restic/internal/archiver" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/filter" @@ -76,7 +77,7 @@ type BackupOptions struct { filter.ExcludePatternOptions Parent string - GroupBy restic.SnapshotGroupByOptions + GroupBy data.SnapshotGroupByOptions Force bool ExcludeOtherFS bool ExcludeIfPresent []string @@ -86,7 +87,7 @@ type BackupOptions struct { Stdin bool StdinFilename string StdinCommand bool - Tags restic.TagLists + Tags data.TagLists Host string FilesFrom []string FilesFromVerbatim []string @@ -104,7 +105,7 @@ type BackupOptions struct { func (opts *BackupOptions) AddFlags(f *pflag.FlagSet) { f.StringVar(&opts.Parent, "parent", "", "use this parent `snapshot` (default: latest snapshot in the group determined by --group-by and not newer than the timestamp determined by --time)") - opts.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true} + opts.GroupBy = data.SnapshotGroupByOptions{Host: true, Path: true} f.VarP(&opts.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')") f.BoolVarP(&opts.Force, "force", "f", false, `force re-reading the source files/directories (overrides the "parent" flag)`) @@ -446,7 +447,7 @@ func collectTargets(opts BackupOptions, args []string, warnf func(msg string, ar // parent returns the ID of the parent snapshot. If there is none, nil is // returned. -func findParentSnapshot(ctx context.Context, repo restic.ListerLoaderUnpacked, opts BackupOptions, targets []string, timeStampLimit time.Time) (*restic.Snapshot, error) { +func findParentSnapshot(ctx context.Context, repo restic.ListerLoaderUnpacked, opts BackupOptions, targets []string, timeStampLimit time.Time) (*data.Snapshot, error) { if opts.Force { return nil, nil } @@ -455,7 +456,7 @@ func findParentSnapshot(ctx context.Context, repo restic.ListerLoaderUnpacked, o if snName == "" { snName = "latest" } - f := restic.SnapshotFilter{TimestampLimit: timeStampLimit} + f := data.SnapshotFilter{TimestampLimit: timeStampLimit} if opts.GroupBy.Host { f.Hosts = []string{opts.Host} } @@ -463,12 +464,12 @@ func findParentSnapshot(ctx context.Context, repo restic.ListerLoaderUnpacked, o f.Paths = targets } if opts.GroupBy.Tag { - f.Tags = []restic.TagList{opts.Tags.Flatten()} + f.Tags = []data.TagList{opts.Tags.Flatten()} } sn, _, err := f.FindLatest(ctx, repo, repo, snName) // Snapshot not found is ok if no explicit parent was set - if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) { + if opts.Parent == "" && errors.Is(err, data.ErrNoSnapshotFound) { err = nil } return sn, err @@ -529,7 +530,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter return err } - var parentSnapshot *restic.Snapshot + var parentSnapshot *data.Snapshot if !opts.Stdin { parentSnapshot, err = findParentSnapshot(ctx, repo, opts, targets, timeStamp) if err != nil { diff --git a/cmd/restic/cmd_backup_integration_test.go b/cmd/restic/cmd_backup_integration_test.go index 004b09a1d..ff58420cd 100644 --- a/cmd/restic/cmd_backup_integration_test.go +++ b/cmd/restic/cmd_backup_integration_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -22,7 +23,7 @@ func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts defer cleanup() } - opts.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true} + opts.GroupBy = data.SnapshotGroupByOptions{Host: true, Path: true} return runBackup(ctx, opts, gopts, gopts.term, target) }) } @@ -473,7 +474,7 @@ func TestBackupTags(t *testing.T) { "expected no tags, got %v", newest.Tags) parent := newest - opts.Tags = restic.TagLists{[]string{"NL"}} + opts.Tags = data.TagLists{[]string{"NL"}} testRunBackup(t, "", []string{env.testdata}, opts, env.gopts) testRunCheck(t, env.gopts) newest, _ = testRunSnapshots(t, env.gopts) diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index 967687171..1b5c44830 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -7,6 +7,7 @@ import ( "github.com/spf13/cobra" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -105,7 +106,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string, term ui.Ter printer.S(string(buf)) return nil case "snapshot": - sn, _, err := restic.FindSnapshot(ctx, repo, repo, args[1]) + sn, _, err := data.FindSnapshot(ctx, repo, repo, args[1]) if err != nil { return errors.Fatalf("could not find snapshot: %v", err) } @@ -190,7 +191,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string, term ui.Ter return errors.Fatal("blob not found") case "tree": - sn, subfolder, err := restic.FindSnapshot(ctx, repo, repo, args[1]) + sn, subfolder, err := data.FindSnapshot(ctx, repo, repo, args[1]) if err != nil { return errors.Fatalf("could not find snapshot: %v", err) } @@ -200,7 +201,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string, term ui.Ter return err } - sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder) + sn.Tree, err = data.FindTreeDirectory(ctx, repo, sn.Tree, subfolder) if err != nil { return err } diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 9e772a6e9..d21e5e4f5 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" @@ -59,7 +60,7 @@ Exit status is 12 if the password is incorrect. // CopyOptions bundles all options for the copy command. type CopyOptions struct { secondaryRepoOptions - restic.SnapshotFilter + data.SnapshotFilter } func (opts *CopyOptions) AddFlags(f *pflag.FlagSet) { @@ -109,7 +110,7 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args [] return err } - dstSnapshotByOriginal := make(map[restic.ID][]*restic.Snapshot) + dstSnapshotByOriginal := make(map[restic.ID][]*data.Snapshot) for sn := range FindFilteredSnapshots(ctx, dstSnapshotLister, dstRepo, &opts.SnapshotFilter, nil, printer) { if sn.Original != nil && !sn.Original.IsNull() { dstSnapshotByOriginal[*sn.Original] = append(dstSnapshotByOriginal[*sn.Original], sn) @@ -158,7 +159,7 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args [] if sn.Original == nil { sn.Original = sn.ID() } - newID, err := restic.SaveSnapshot(ctx, dstRepo, sn) + newID, err := data.SaveSnapshot(ctx, dstRepo, sn) if err != nil { return err } @@ -167,7 +168,7 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args [] return ctx.Err() } -func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool { +func similarSnapshots(sna *data.Snapshot, snb *data.Snapshot) bool { // everything except Parent and Original must match if !sna.Time.Equal(snb.Time) || !sna.Tree.Equal(*snb.Tree) || sna.Hostname != snb.Hostname || sna.Username != snb.Username || sna.UID != snb.UID || sna.GID != snb.GID || @@ -191,7 +192,7 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep wg, wgCtx := errgroup.WithContext(ctx) - treeStream := restic.StreamTrees(wgCtx, wg, srcRepo, restic.IDs{rootTreeID}, func(treeID restic.ID) bool { + treeStream := data.StreamTrees(wgCtx, wg, srcRepo, restic.IDs{rootTreeID}, func(treeID restic.ID) bool { visited := visitedTrees.Has(treeID) visitedTrees.Insert(treeID) return visited diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 00e312b13..b5d4d646c 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -22,6 +22,7 @@ import ( "golang.org/x/sync/errgroup" "github.com/restic/restic/internal/crypto" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository/index" @@ -115,7 +116,7 @@ func prettyPrintJSON(wr io.Writer, item interface{}) error { } func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io.Writer) error { - return restic.ForAllSnapshots(ctx, repo, repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error { + return data.ForAllSnapshots(ctx, repo, repo, nil, func(id restic.ID, snapshot *data.Snapshot, err error) error { if err != nil { return err } diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index a25c79fd9..908d536e6 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -7,6 +7,7 @@ import ( "reflect" "sort" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -69,8 +70,8 @@ func (opts *DiffOptions) AddFlags(f *pflag.FlagSet) { f.BoolVar(&opts.ShowMetadata, "metadata", false, "print changes in metadata") } -func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.LoaderUnpacked, desc string) (*restic.Snapshot, string, error) { - sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc) +func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.LoaderUnpacked, desc string) (*data.Snapshot, string, error) { + sn, subfolder, err := data.FindSnapshot(ctx, be, repo, desc) if err != nil { return nil, "", errors.Fatalf("%s", err) } @@ -106,15 +107,15 @@ type DiffStat struct { } // Add adds stats information for node to s. -func (s *DiffStat) Add(node *restic.Node) { +func (s *DiffStat) Add(node *data.Node) { if node == nil { return } switch node.Type { - case restic.NodeTypeFile: + case data.NodeTypeFile: s.Files++ - case restic.NodeTypeDir: + case data.NodeTypeDir: s.Dirs++ default: s.Others++ @@ -122,13 +123,13 @@ func (s *DiffStat) Add(node *restic.Node) { } // addBlobs adds the blobs of node to s. -func addBlobs(bs restic.BlobSet, node *restic.Node) { +func addBlobs(bs restic.BlobSet, node *data.Node) { if node == nil { return } switch node.Type { - case restic.NodeTypeFile: + case data.NodeTypeFile: for _, blob := range node.Content { h := restic.BlobHandle{ ID: blob, @@ -136,7 +137,7 @@ func addBlobs(bs restic.BlobSet, node *restic.Node) { } bs.Insert(h) } - case restic.NodeTypeDir: + case data.NodeTypeDir: h := restic.BlobHandle{ ID: *node.Subtree, Type: restic.TreeBlob, @@ -177,7 +178,7 @@ func updateBlobs(repo restic.Loader, blobs restic.BlobSet, stats *DiffStat, prin func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, blobs restic.BlobSet, prefix string, id restic.ID) error { debug.Log("print %v tree %v", mode, id) - tree, err := restic.LoadTree(ctx, c.repo, id) + tree, err := data.LoadTree(ctx, c.repo, id) if err != nil { return err } @@ -188,14 +189,14 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b } name := path.Join(prefix, node.Name) - if node.Type == restic.NodeTypeDir { + if node.Type == data.NodeTypeDir { name += "/" } c.printChange(NewChange(name, mode)) stats.Add(node) addBlobs(blobs, node) - if node.Type == restic.NodeTypeDir { + if node.Type == data.NodeTypeDir { err := c.printDir(ctx, mode, stats, blobs, name, *node.Subtree) if err != nil && err != context.Canceled { c.printError("error: %v", err) @@ -208,7 +209,7 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b func (c *Comparer) collectDir(ctx context.Context, blobs restic.BlobSet, id restic.ID) error { debug.Log("print tree %v", id) - tree, err := restic.LoadTree(ctx, c.repo, id) + tree, err := data.LoadTree(ctx, c.repo, id) if err != nil { return err } @@ -220,7 +221,7 @@ func (c *Comparer) collectDir(ctx context.Context, blobs restic.BlobSet, id rest addBlobs(blobs, node) - if node.Type == restic.NodeTypeDir { + if node.Type == data.NodeTypeDir { err := c.collectDir(ctx, blobs, *node.Subtree) if err != nil && err != context.Canceled { c.printError("error: %v", err) @@ -231,15 +232,15 @@ func (c *Comparer) collectDir(ctx context.Context, blobs restic.BlobSet, id rest return ctx.Err() } -func uniqueNodeNames(tree1, tree2 *restic.Tree) (tree1Nodes, tree2Nodes map[string]*restic.Node, uniqueNames []string) { +func uniqueNodeNames(tree1, tree2 *data.Tree) (tree1Nodes, tree2Nodes map[string]*data.Node, uniqueNames []string) { names := make(map[string]struct{}) - tree1Nodes = make(map[string]*restic.Node) + tree1Nodes = make(map[string]*data.Node) for _, node := range tree1.Nodes { tree1Nodes[node.Name] = node names[node.Name] = struct{}{} } - tree2Nodes = make(map[string]*restic.Node) + tree2Nodes = make(map[string]*data.Node) for _, node := range tree2.Nodes { tree2Nodes[node.Name] = node names[node.Name] = struct{}{} @@ -256,12 +257,12 @@ func uniqueNodeNames(tree1, tree2 *restic.Tree) (tree1Nodes, tree2Nodes map[stri func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, prefix string, id1, id2 restic.ID) error { debug.Log("diffing %v to %v", id1, id2) - tree1, err := restic.LoadTree(ctx, c.repo, id1) + tree1, err := data.LoadTree(ctx, c.repo, id1) if err != nil { return err } - tree2, err := restic.LoadTree(ctx, c.repo, id2) + tree2, err := data.LoadTree(ctx, c.repo, id2) if err != nil { return err } @@ -288,12 +289,12 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref mod += "T" } - if node2.Type == restic.NodeTypeDir { + if node2.Type == data.NodeTypeDir { name += "/" } - if node1.Type == restic.NodeTypeFile && - node2.Type == restic.NodeTypeFile && + if node1.Type == data.NodeTypeFile && + node2.Type == data.NodeTypeFile && !reflect.DeepEqual(node1.Content, node2.Content) { mod += "M" stats.ChangedFiles++ @@ -315,7 +316,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref c.printChange(NewChange(name, mod)) } - if node1.Type == restic.NodeTypeDir && node2.Type == restic.NodeTypeDir { + if node1.Type == data.NodeTypeDir && node2.Type == data.NodeTypeDir { var err error if (*node1.Subtree).Equal(*node2.Subtree) { err = c.collectDir(ctx, stats.BlobsCommon, *node1.Subtree) @@ -328,13 +329,13 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref } case t1 && !t2: prefix := path.Join(prefix, name) - if node1.Type == restic.NodeTypeDir { + if node1.Type == data.NodeTypeDir { prefix += "/" } c.printChange(NewChange(prefix, "-")) stats.Removed.Add(node1) - if node1.Type == restic.NodeTypeDir { + if node1.Type == data.NodeTypeDir { err := c.printDir(ctx, "-", &stats.Removed, stats.BlobsBefore, prefix, *node1.Subtree) if err != nil && err != context.Canceled { c.printError("error: %v", err) @@ -342,13 +343,13 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref } case !t1 && t2: prefix := path.Join(prefix, name) - if node2.Type == restic.NodeTypeDir { + if node2.Type == data.NodeTypeDir { prefix += "/" } c.printChange(NewChange(prefix, "+")) stats.Added.Add(node2) - if node2.Type == restic.NodeTypeDir { + if node2.Type == data.NodeTypeDir { err := c.printDir(ctx, "+", &stats.Added, stats.BlobsAfter, prefix, *node2.Subtree) if err != nil && err != context.Canceled { c.printError("error: %v", err) @@ -403,12 +404,12 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args [] return errors.Errorf("snapshot %v has nil tree", sn2.ID().Str()) } - sn1.Tree, err = restic.FindTreeDirectory(ctx, repo, sn1.Tree, subfolder1) + sn1.Tree, err = data.FindTreeDirectory(ctx, repo, sn1.Tree, subfolder1) if err != nil { return err } - sn2.Tree, err = restic.FindTreeDirectory(ctx, repo, sn2.Tree, subfolder2) + sn2.Tree, err = data.FindTreeDirectory(ctx, repo, sn2.Tree, subfolder2) if err != nil { return err } diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index b36fb2236..a3406328f 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -7,6 +7,7 @@ import ( "path" "path/filepath" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/dump" "github.com/restic/restic/internal/errors" @@ -57,7 +58,7 @@ Exit status is 12 if the password is incorrect. // DumpOptions collects all options for the dump command. type DumpOptions struct { - restic.SnapshotFilter + data.SnapshotFilter Archive string Target string } @@ -77,7 +78,7 @@ func splitPath(p string) []string { return append(s, f) } -func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoader, prefix string, pathComponents []string, d *dump.Dumper, canWriteArchiveFunc func() error) error { +func printFromTree(ctx context.Context, tree *data.Tree, repo restic.BlobLoader, prefix string, pathComponents []string, d *dump.Dumper, canWriteArchiveFunc func() error) error { // If we print / we need to assume that there are multiple nodes at that // level in the tree. if pathComponents[0] == "" { @@ -98,26 +99,26 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoade // first item it finds and dump that according to the switch case below. if node.Name == pathComponents[0] { switch { - case l == 1 && node.Type == restic.NodeTypeFile: + case l == 1 && node.Type == data.NodeTypeFile: return d.WriteNode(ctx, node) - case l > 1 && node.Type == restic.NodeTypeDir: - subtree, err := restic.LoadTree(ctx, repo, *node.Subtree) + case l > 1 && node.Type == data.NodeTypeDir: + subtree, err := data.LoadTree(ctx, repo, *node.Subtree) if err != nil { return errors.Wrapf(err, "cannot load subtree for %q", item) } return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d, canWriteArchiveFunc) - case node.Type == restic.NodeTypeDir: + case node.Type == data.NodeTypeDir: if err := canWriteArchiveFunc(); err != nil { return err } - subtree, err := restic.LoadTree(ctx, repo, *node.Subtree) + subtree, err := data.LoadTree(ctx, repo, *node.Subtree) if err != nil { return err } return d.DumpTree(ctx, subtree, item) case l > 1: return fmt.Errorf("%q should be a dir, but is a %q", item, node.Type) - case node.Type != restic.NodeTypeFile: + case node.Type != data.NodeTypeFile: return fmt.Errorf("%q should be a file, but is a %q", item, node.Type) } } @@ -151,7 +152,7 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] } defer unlock() - sn, subfolder, err := (&restic.SnapshotFilter{ + sn, subfolder, err := (&data.SnapshotFilter{ Hosts: opts.Hosts, Paths: opts.Paths, Tags: opts.Tags, @@ -165,12 +166,12 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] return err } - sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder) + sn.Tree, err = data.FindTreeDirectory(ctx, repo, sn.Tree, subfolder) if err != nil { return err } - tree, err := restic.LoadTree(ctx, repo, *sn.Tree) + tree, err := data.LoadTree(ctx, repo, *sn.Tree) if err != nil { return errors.Fatalf("loading tree for snapshot %q failed: %v", snapshotIDString, err) } diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index bb3c0b55a..70043d7ca 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -12,6 +12,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/filter" @@ -70,7 +71,7 @@ type FindOptions struct { ListLong bool HumanReadable bool Reverse bool - restic.SnapshotFilter + data.SnapshotFilter } func (opts *FindOptions) AddFlags(f *pflag.FlagSet) { @@ -124,8 +125,8 @@ type statefulOutput struct { HumanReadable bool JSON bool inuse bool - newsn *restic.Snapshot - oldsn *restic.Snapshot + newsn *data.Snapshot + oldsn *data.Snapshot hits int printer interface { S(string, ...interface{}) @@ -135,8 +136,8 @@ type statefulOutput struct { stdout io.Writer } -func (s *statefulOutput) PrintPatternJSON(path string, node *restic.Node) { - type findNode restic.Node +func (s *statefulOutput) PrintPatternJSON(path string, node *data.Node) { + type findNode data.Node b, err := json.Marshal(struct { // Add these attributes Path string `json:"path,omitempty"` @@ -179,7 +180,7 @@ func (s *statefulOutput) PrintPatternJSON(path string, node *restic.Node) { s.hits++ } -func (s *statefulOutput) PrintPatternNormal(path string, node *restic.Node) { +func (s *statefulOutput) PrintPatternNormal(path string, node *data.Node) { if s.newsn != s.oldsn { if s.oldsn != nil { s.printer.P("") @@ -190,7 +191,7 @@ func (s *statefulOutput) PrintPatternNormal(path string, node *restic.Node) { s.printer.S(formatNode(path, node, s.ListLong, s.HumanReadable)) } -func (s *statefulOutput) PrintPattern(path string, node *restic.Node) { +func (s *statefulOutput) PrintPattern(path string, node *data.Node) { if s.JSON { s.PrintPatternJSON(path, node) } else { @@ -198,7 +199,7 @@ func (s *statefulOutput) PrintPattern(path string, node *restic.Node) { } } -func (s *statefulOutput) PrintObjectJSON(kind, id, nodepath, treeID string, sn *restic.Snapshot) { +func (s *statefulOutput) PrintObjectJSON(kind, id, nodepath, treeID string, sn *data.Snapshot) { b, err := json.Marshal(struct { // Add these attributes ObjectType string `json:"object_type"` @@ -230,7 +231,7 @@ func (s *statefulOutput) PrintObjectJSON(kind, id, nodepath, treeID string, sn * s.hits++ } -func (s *statefulOutput) PrintObjectNormal(kind, id, nodepath, treeID string, sn *restic.Snapshot) { +func (s *statefulOutput) PrintObjectNormal(kind, id, nodepath, treeID string, sn *data.Snapshot) { s.printer.S("Found %s %s", kind, id) if kind == "blob" { s.printer.S(" ... in file %s", nodepath) @@ -241,7 +242,7 @@ func (s *statefulOutput) PrintObjectNormal(kind, id, nodepath, treeID string, sn s.printer.S(" ... in snapshot %s (%s)", sn.ID().Str(), sn.Time.Local().Format(TimeFormat)) } -func (s *statefulOutput) PrintObject(kind, id, nodepath, treeID string, sn *restic.Snapshot) { +func (s *statefulOutput) PrintObject(kind, id, nodepath, treeID string, sn *data.Snapshot) { if s.JSON { s.PrintObjectJSON(kind, id, nodepath, treeID, sn) } else { @@ -279,7 +280,7 @@ type Finder struct { } } -func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error { +func (f *Finder) findInSnapshot(ctx context.Context, sn *data.Snapshot) error { debug.Log("searching in snapshot %s\n for entries within [%s %s]", sn.ID(), f.pat.oldest, f.pat.newest) if sn.Tree == nil { @@ -287,7 +288,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error } f.out.newsn = sn - return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error { + return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *data.Node, err error) error { if err != nil { debug.Log("Error loading tree %v: %v", parentTreeID, err) @@ -320,7 +321,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error } var errIfNoMatch error - if node.Type == restic.NodeTypeDir { + if node.Type == data.NodeTypeDir { var childMayMatch bool for _, pat := range f.pat.pattern { mayMatch, err := filter.ChildMatch(pat, normalizedNodepath) @@ -378,7 +379,7 @@ func (f *Finder) findTree(treeID restic.ID, nodepath string) error { return nil } -func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { +func (f *Finder) findIDs(ctx context.Context, sn *data.Snapshot) error { debug.Log("searching IDs in snapshot %s", sn.ID()) if sn.Tree == nil { @@ -386,7 +387,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { } f.out.newsn = sn - return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error { + return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *data.Node, err error) error { if err != nil { debug.Log("Error loading tree %v: %v", parentTreeID, err) @@ -411,7 +412,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { } } - if node.Type == restic.NodeTypeFile && f.blobIDs != nil { + if node.Type == data.NodeTypeFile && f.blobIDs != nil { for _, id := range node.Content { if ctx.Err() != nil { return ctx.Err() @@ -654,7 +655,7 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args [] } } - var filteredSnapshots []*restic.Snapshot + var filteredSnapshots []*data.Snapshot for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, opts.Snapshots, printer) { filteredSnapshots = append(filteredSnapshots, sn) } diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index 51613987c..a3d6246c2 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -7,6 +7,7 @@ import ( "io" "strconv" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui" @@ -102,21 +103,21 @@ type ForgetOptions struct { Weekly ForgetPolicyCount Monthly ForgetPolicyCount Yearly ForgetPolicyCount - Within restic.Duration - WithinHourly restic.Duration - WithinDaily restic.Duration - WithinWeekly restic.Duration - WithinMonthly restic.Duration - WithinYearly restic.Duration - KeepTags restic.TagLists + Within data.Duration + WithinHourly data.Duration + WithinDaily data.Duration + WithinWeekly data.Duration + WithinMonthly data.Duration + WithinYearly data.Duration + KeepTags data.TagLists UnsafeAllowRemoveAll bool - restic.SnapshotFilter + data.SnapshotFilter Compact bool // Grouping - GroupBy restic.SnapshotGroupByOptions + GroupBy data.SnapshotGroupByOptions DryRun bool Prune bool } @@ -147,7 +148,7 @@ func (opts *ForgetOptions) AddFlags(f *pflag.FlagSet) { initMultiSnapshotFilter(f, &opts.SnapshotFilter, false) f.BoolVarP(&opts.Compact, "compact", "c", false, "use compact output format") - opts.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true} + opts.GroupBy = data.SnapshotGroupByOptions{Host: true, Path: true} f.VarP(&opts.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')") f.BoolVarP(&opts.DryRun, "dry-run", "n", false, "do not delete anything, just print what would be done") f.BoolVar(&opts.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed") @@ -161,7 +162,7 @@ func verifyForgetOptions(opts *ForgetOptions) error { return errors.Fatal("negative values other than -1 are not allowed for --keep-*") } - for _, d := range []restic.Duration{opts.Within, opts.WithinHourly, opts.WithinDaily, + for _, d := range []data.Duration{opts.Within, opts.WithinHourly, opts.WithinDaily, opts.WithinMonthly, opts.WithinWeekly, opts.WithinYearly} { if d.Hours < 0 || d.Days < 0 || d.Months < 0 || d.Years < 0 { return errors.Fatal("durations containing negative values are not allowed for --keep-within*") @@ -193,7 +194,7 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption } defer unlock() - var snapshots restic.Snapshots + var snapshots data.Snapshots removeSnIDs := restic.NewIDSet() for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args, printer) { @@ -211,12 +212,12 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption removeSnIDs.Insert(*sn.ID()) } } else { - snapshotGroups, _, err := restic.GroupSnapshots(snapshots, opts.GroupBy) + snapshotGroups, _, err := data.GroupSnapshots(snapshots, opts.GroupBy) if err != nil { return err } - policy := restic.ExpirePolicy{ + policy := data.ExpirePolicy{ Last: int(opts.Last), Hourly: int(opts.Hourly), Daily: int(opts.Daily), @@ -257,7 +258,7 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption } } - var key restic.SnapshotGroupKey + var key data.SnapshotGroupKey if json.Unmarshal([]byte(k), &key) != nil { return err } @@ -267,7 +268,7 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption fg.Host = key.Hostname fg.Paths = key.Paths - keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy) + keep, remove, reasons := data.ApplyPolicy(snapshotGroup, policy) if !policy.Empty() && len(keep) == 0 { return fmt.Errorf("refusing to delete last snapshot of snapshot group \"%v\"", key.String()) @@ -361,7 +362,7 @@ type ForgetGroup struct { Reasons []KeepReason `json:"reasons"` } -func asJSONSnapshots(list restic.Snapshots) []Snapshot { +func asJSONSnapshots(list data.Snapshots) []Snapshot { var resultList []Snapshot for _, sn := range list { k := Snapshot{ @@ -380,7 +381,7 @@ type KeepReason struct { Matches []string `json:"matches"` } -func asJSONKeeps(list []restic.KeepReason) []KeepReason { +func asJSONKeeps(list []data.KeepReason) []KeepReason { var resultList []KeepReason for _, keep := range list { k := KeepReason{ diff --git a/cmd/restic/cmd_forget_integration_test.go b/cmd/restic/cmd_forget_integration_test.go index 3ce19182b..49f7ed96b 100644 --- a/cmd/restic/cmd_forget_integration_test.go +++ b/cmd/restic/cmd_forget_integration_test.go @@ -6,7 +6,7 @@ import ( "strings" "testing" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/data" rtest "github.com/restic/restic/internal/test" ) @@ -38,8 +38,8 @@ func TestRunForgetSafetyNet(t *testing.T) { // --keep-tags invalid err := testRunForgetMayFail(t, env.gopts, ForgetOptions{ - KeepTags: restic.TagLists{restic.TagList{"invalid"}}, - GroupBy: restic.SnapshotGroupByOptions{Host: true, Path: true}, + KeepTags: data.TagLists{data.TagList{"invalid"}}, + GroupBy: data.SnapshotGroupByOptions{Host: true, Path: true}, }) rtest.Assert(t, strings.Contains(err.Error(), `refusing to delete last snapshot of snapshot group "host example, path`), "wrong error message got %v", err) @@ -56,8 +56,8 @@ func TestRunForgetSafetyNet(t *testing.T) { // `forget --host example --unsafe-allow-remove-all` should work testRunForget(t, env.gopts, ForgetOptions{ UnsafeAllowRemoveAll: true, - GroupBy: restic.SnapshotGroupByOptions{Host: true, Path: true}, - SnapshotFilter: restic.SnapshotFilter{ + GroupBy: data.SnapshotGroupByOptions{Host: true, Path: true}, + SnapshotFilter: data.SnapshotFilter{ Hosts: []string{opts.Host}, }, }) diff --git a/cmd/restic/cmd_forget_test.go b/cmd/restic/cmd_forget_test.go index 0d60f7445..b4175042a 100644 --- a/cmd/restic/cmd_forget_test.go +++ b/cmd/restic/cmd_forget_test.go @@ -3,7 +3,7 @@ package main import ( "testing" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/data" rtest "github.com/restic/restic/internal/test" "github.com/spf13/pflag" ) @@ -69,18 +69,18 @@ func TestForgetOptionValues(t *testing.T) { {ForgetOptions{Weekly: -2}, negValErrorMsg}, {ForgetOptions{Monthly: -2}, negValErrorMsg}, {ForgetOptions{Yearly: -2}, negValErrorMsg}, - {ForgetOptions{Within: restic.ParseDurationOrPanic("1y2m3d3h")}, ""}, - {ForgetOptions{WithinHourly: restic.ParseDurationOrPanic("1y2m3d3h")}, ""}, - {ForgetOptions{WithinDaily: restic.ParseDurationOrPanic("1y2m3d3h")}, ""}, - {ForgetOptions{WithinWeekly: restic.ParseDurationOrPanic("1y2m3d3h")}, ""}, - {ForgetOptions{WithinMonthly: restic.ParseDurationOrPanic("2y4m6d8h")}, ""}, - {ForgetOptions{WithinYearly: restic.ParseDurationOrPanic("2y4m6d8h")}, ""}, - {ForgetOptions{Within: restic.ParseDurationOrPanic("-1y2m3d3h")}, negDurationValErrorMsg}, - {ForgetOptions{WithinHourly: restic.ParseDurationOrPanic("1y-2m3d3h")}, negDurationValErrorMsg}, - {ForgetOptions{WithinDaily: restic.ParseDurationOrPanic("1y2m-3d3h")}, negDurationValErrorMsg}, - {ForgetOptions{WithinWeekly: restic.ParseDurationOrPanic("1y2m3d-3h")}, negDurationValErrorMsg}, - {ForgetOptions{WithinMonthly: restic.ParseDurationOrPanic("-2y4m6d8h")}, negDurationValErrorMsg}, - {ForgetOptions{WithinYearly: restic.ParseDurationOrPanic("2y-4m6d8h")}, negDurationValErrorMsg}, + {ForgetOptions{Within: data.ParseDurationOrPanic("1y2m3d3h")}, ""}, + {ForgetOptions{WithinHourly: data.ParseDurationOrPanic("1y2m3d3h")}, ""}, + {ForgetOptions{WithinDaily: data.ParseDurationOrPanic("1y2m3d3h")}, ""}, + {ForgetOptions{WithinWeekly: data.ParseDurationOrPanic("1y2m3d3h")}, ""}, + {ForgetOptions{WithinMonthly: data.ParseDurationOrPanic("2y4m6d8h")}, ""}, + {ForgetOptions{WithinYearly: data.ParseDurationOrPanic("2y4m6d8h")}, ""}, + {ForgetOptions{Within: data.ParseDurationOrPanic("-1y2m3d3h")}, negDurationValErrorMsg}, + {ForgetOptions{WithinHourly: data.ParseDurationOrPanic("1y-2m3d3h")}, negDurationValErrorMsg}, + {ForgetOptions{WithinDaily: data.ParseDurationOrPanic("1y2m-3d3h")}, negDurationValErrorMsg}, + {ForgetOptions{WithinWeekly: data.ParseDurationOrPanic("1y2m3d-3h")}, negDurationValErrorMsg}, + {ForgetOptions{WithinMonthly: data.ParseDurationOrPanic("-2y4m6d8h")}, negDurationValErrorMsg}, + {ForgetOptions{WithinYearly: data.ParseDurationOrPanic("2y-4m6d8h")}, negDurationValErrorMsg}, } for _, testCase := range testCases { diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index de015f180..a65867994 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -15,6 +15,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" @@ -70,7 +71,7 @@ Exit status is 12 if the password is incorrect. // LsOptions collects all options for the ls command. type LsOptions struct { ListLong bool - restic.SnapshotFilter + data.SnapshotFilter Recursive bool HumanReadable bool Ncdu bool @@ -89,8 +90,8 @@ func (opts *LsOptions) AddFlags(f *pflag.FlagSet) { } type lsPrinter interface { - Snapshot(sn *restic.Snapshot) error - Node(path string, node *restic.Node, isPrefixDirectory bool) error + Snapshot(sn *data.Snapshot) error + Node(path string, node *data.Node, isPrefixDirectory bool) error LeaveDir(path string) error Close() error } @@ -99,9 +100,9 @@ type jsonLsPrinter struct { enc *json.Encoder } -func (p *jsonLsPrinter) Snapshot(sn *restic.Snapshot) error { +func (p *jsonLsPrinter) Snapshot(sn *data.Snapshot) error { type lsSnapshot struct { - *restic.Snapshot + *data.Snapshot ID *restic.ID `json:"id"` ShortID string `json:"short_id"` // deprecated MessageType string `json:"message_type"` // "snapshot" @@ -118,14 +119,14 @@ func (p *jsonLsPrinter) Snapshot(sn *restic.Snapshot) error { } // Node formats node in our custom JSON format, followed by a newline. -func (p *jsonLsPrinter) Node(path string, node *restic.Node, isPrefixDirectory bool) error { +func (p *jsonLsPrinter) Node(path string, node *data.Node, isPrefixDirectory bool) error { if isPrefixDirectory { return nil } return lsNodeJSON(p.enc, path, node) } -func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error { +func lsNodeJSON(enc *json.Encoder, path string, node *data.Node) error { n := &struct { Name string `json:"name"` Type string `json:"type"` @@ -161,7 +162,7 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error { } // Always print size for regular files, even when empty, // but never for other types. - if node.Type == restic.NodeTypeFile { + if node.Type == data.NodeTypeFile { n.Size = &n.size } @@ -179,7 +180,7 @@ type ncduLsPrinter struct { // Snapshot prints a restic snapshot in Ncdu save format. // It opens the JSON list. Nodes are added with lsNodeNcdu and the list is closed by lsCloseNcdu. // Format documentation: https://dev.yorhel.nl/ncdu/jsonfmt -func (p *ncduLsPrinter) Snapshot(sn *restic.Snapshot) error { +func (p *ncduLsPrinter) Snapshot(sn *data.Snapshot) error { const NcduMajorVer = 1 const NcduMinorVer = 2 @@ -192,7 +193,7 @@ func (p *ncduLsPrinter) Snapshot(sn *restic.Snapshot) error { return err } -func lsNcduNode(_ string, node *restic.Node) ([]byte, error) { +func lsNcduNode(_ string, node *data.Node) ([]byte, error) { type NcduNode struct { Name string `json:"name"` Asize uint64 `json:"asize"` @@ -217,7 +218,7 @@ func lsNcduNode(_ string, node *restic.Node) ([]byte, error) { Dev: node.DeviceID, Ino: node.Inode, NLink: node.Links, - NotReg: node.Type != restic.NodeTypeDir && node.Type != restic.NodeTypeFile, + NotReg: node.Type != data.NodeTypeDir && node.Type != data.NodeTypeFile, UID: node.UID, GID: node.GID, Mode: uint16(node.Mode & os.ModePerm), @@ -241,13 +242,13 @@ func lsNcduNode(_ string, node *restic.Node) ([]byte, error) { return json.Marshal(outNode) } -func (p *ncduLsPrinter) Node(path string, node *restic.Node, _ bool) error { +func (p *ncduLsPrinter) Node(path string, node *data.Node, _ bool) error { out, err := lsNcduNode(path, node) if err != nil { return err } - if node.Type == restic.NodeTypeDir { + if node.Type == data.NodeTypeDir { _, err = fmt.Fprintf(p.out, ",\n%s[\n%s%s", strings.Repeat(" ", p.depth), strings.Repeat(" ", p.depth+1), string(out)) p.depth++ } else { @@ -277,11 +278,11 @@ type textLsPrinter struct { } } -func (p *textLsPrinter) Snapshot(sn *restic.Snapshot) error { +func (p *textLsPrinter) Snapshot(sn *data.Snapshot) error { p.termPrinter.P("%v filtered by %v:", sn, p.dirs) return nil } -func (p *textLsPrinter) Node(path string, node *restic.Node, isPrefixDirectory bool) error { +func (p *textLsPrinter) Node(path string, node *data.Node, isPrefixDirectory bool) error { if !isPrefixDirectory { p.termPrinter.S("%s", formatNode(path, node, p.ListLong, p.HumanReadable)) } @@ -298,7 +299,7 @@ func (p *textLsPrinter) Close() error { // for ls -l output sorting type toSortOutput struct { nodepath string - node *restic.Node + node *data.Node } func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string, term ui.Terminal) error { @@ -403,7 +404,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri } } - sn, subfolder, err := (&restic.SnapshotFilter{ + sn, subfolder, err := (&data.SnapshotFilter{ Hosts: opts.Hosts, Paths: opts.Paths, Tags: opts.Tags, @@ -412,7 +413,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri return err } - sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder) + sn.Tree, err = data.FindTreeDirectory(ctx, repo, sn.Tree, subfolder) if err != nil { return err } @@ -421,7 +422,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri return err } - processNode := func(_ restic.ID, nodepath string, node *restic.Node, err error) error { + processNode := func(_ restic.ID, nodepath string, node *data.Node, err error) error { if err != nil { return err } @@ -456,7 +457,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri // otherwise, signal the walker to not walk recursively into any // subdirs - if node.Type == restic.NodeTypeDir { + if node.Type == data.NodeTypeDir { // immediately generate leaveDir if the directory is skipped if printedDir { if err := printer.LeaveDir(nodepath); err != nil { @@ -493,10 +494,10 @@ type sortedPrinter struct { reverse bool } -func (p *sortedPrinter) Snapshot(sn *restic.Snapshot) error { +func (p *sortedPrinter) Snapshot(sn *data.Snapshot) error { return p.printer.Snapshot(sn) } -func (p *sortedPrinter) Node(path string, node *restic.Node, isPrefixDirectory bool) error { +func (p *sortedPrinter) Node(path string, node *data.Node, isPrefixDirectory bool) error { if !isPrefixDirectory { p.collector = append(p.collector, toSortOutput{path, node}) } diff --git a/cmd/restic/cmd_ls_integration_test.go b/cmd/restic/cmd_ls_integration_test.go index 4f13b89e3..1f16fec10 100644 --- a/cmd/restic/cmd_ls_integration_test.go +++ b/cmd/restic/cmd_ls_integration_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -129,7 +130,7 @@ func TestRunLsJson(t *testing.T) { // partial copy of snapshot structure from cmd_ls type lsSnapshot struct { - *restic.Snapshot + *data.Snapshot ID *restic.ID `json:"id"` ShortID string `json:"short_id"` // deprecated MessageType string `json:"message_type"` // "snapshot" diff --git a/cmd/restic/cmd_ls_test.go b/cmd/restic/cmd_ls_test.go index 3d4e1dbc7..95d13a9b9 100644 --- a/cmd/restic/cmd_ls_test.go +++ b/cmd/restic/cmd_ls_test.go @@ -7,13 +7,13 @@ import ( "testing" "time" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/data" rtest "github.com/restic/restic/internal/test" ) type lsTestNode struct { path string - restic.Node + data.Node } var lsTestNodes = []lsTestNode{ @@ -21,9 +21,9 @@ var lsTestNodes = []lsTestNode{ // Permissions, by convention is "-" per mode bit { path: "/bar/baz", - Node: restic.Node{ + Node: data.Node{ Name: "baz", - Type: restic.NodeTypeFile, + Type: data.NodeTypeFile, Size: 12345, UID: 10000000, GID: 20000000, @@ -37,9 +37,9 @@ var lsTestNodes = []lsTestNode{ // Even empty files get an explicit size. { path: "/foo/empty", - Node: restic.Node{ + Node: data.Node{ Name: "empty", - Type: restic.NodeTypeFile, + Type: data.NodeTypeFile, Size: 0, UID: 1001, GID: 1001, @@ -54,9 +54,9 @@ var lsTestNodes = []lsTestNode{ // Mode is printed in decimal, including the type bits. { path: "/foo/link", - Node: restic.Node{ + Node: data.Node{ Name: "link", - Type: restic.NodeTypeSymlink, + Type: data.NodeTypeSymlink, Mode: os.ModeSymlink | 0777, LinkTarget: "not printed", }, @@ -64,9 +64,9 @@ var lsTestNodes = []lsTestNode{ { path: "/some/directory", - Node: restic.Node{ + Node: data.Node{ Name: "directory", - Type: restic.NodeTypeDir, + Type: data.NodeTypeDir, Mode: os.ModeDir | 0755, ModTime: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC), AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC), @@ -77,9 +77,9 @@ var lsTestNodes = []lsTestNode{ // Test encoding of setuid/setgid/sticky bit { path: "/some/sticky", - Node: restic.Node{ + Node: data.Node{ Name: "sticky", - Type: restic.NodeTypeDir, + Type: data.NodeTypeDir, Mode: os.ModeDir | 0755 | os.ModeSetuid | os.ModeSetgid | os.ModeSticky, }, }, @@ -134,24 +134,24 @@ func TestLsNcdu(t *testing.T) { } modTime := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) - rtest.OK(t, printer.Snapshot(&restic.Snapshot{ + rtest.OK(t, printer.Snapshot(&data.Snapshot{ Hostname: "host", Paths: []string{"/example"}, })) - rtest.OK(t, printer.Node("/directory", &restic.Node{ - Type: restic.NodeTypeDir, + rtest.OK(t, printer.Node("/directory", &data.Node{ + Type: data.NodeTypeDir, Name: "directory", ModTime: modTime, }, false)) - rtest.OK(t, printer.Node("/directory/data", &restic.Node{ - Type: restic.NodeTypeFile, + rtest.OK(t, printer.Node("/directory/data", &data.Node{ + Type: data.NodeTypeFile, Name: "data", Size: 42, ModTime: modTime, }, false)) rtest.OK(t, printer.LeaveDir("/directory")) - rtest.OK(t, printer.Node("/file", &restic.Node{ - Type: restic.NodeTypeFile, + rtest.OK(t, printer.Node("/file", &data.Node{ + Type: data.NodeTypeFile, Name: "file", Size: 12345, ModTime: modTime, diff --git a/cmd/restic/cmd_mount.go b/cmd/restic/cmd_mount.go index 84e28f86f..f94f999f2 100644 --- a/cmd/restic/cmd_mount.go +++ b/cmd/restic/cmd_mount.go @@ -13,9 +13,9 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui" "github.com/restic/restic/internal/fuse" @@ -95,7 +95,7 @@ type MountOptions struct { OwnerRoot bool AllowOther bool NoDefaultPermissions bool - restic.SnapshotFilter + data.SnapshotFilter TimeTemplate string PathTemplates []string } diff --git a/cmd/restic/cmd_mount_integration_test.go b/cmd/restic/cmd_mount_integration_test.go index 9c76b1b9a..d4747c721 100644 --- a/cmd/restic/cmd_mount_integration_test.go +++ b/cmd/restic/cmd_mount_integration_test.go @@ -13,6 +13,7 @@ import ( "time" systemFuse "github.com/anacrolix/fuse" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -137,7 +138,7 @@ func checkSnapshots(t testing.TB, gopts GlobalOptions, mountpoint string, snapsh defer unlock() for _, id := range snapshotIDs { - snapshot, err := restic.LoadSnapshot(ctx, repo, id) + snapshot, err := data.LoadSnapshot(ctx, repo, id) rtest.OK(t, err) ts := snapshot.Time.Format(time.RFC3339) diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index dabcc8293..778fce207 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" @@ -278,8 +279,8 @@ func printPruneStats(printer progress.Printer, stats repository.PruneStats) erro func getUsedBlobs(ctx context.Context, repo restic.Repository, usedBlobs restic.FindBlobSet, ignoreSnapshots restic.IDSet, printer progress.Printer) error { var snapshotTrees restic.IDs printer.P("loading all snapshots...\n") - err := restic.ForAllSnapshots(ctx, repo, repo, ignoreSnapshots, - func(id restic.ID, sn *restic.Snapshot, err error) error { + err := data.ForAllSnapshots(ctx, repo, repo, ignoreSnapshots, + func(id restic.ID, sn *data.Snapshot, err error) error { if err != nil { debug.Log("failed to load snapshot %v (error %v)", id, err) return err @@ -298,5 +299,5 @@ func getUsedBlobs(ctx context.Context, repo restic.Repository, usedBlobs restic. bar.SetMax(uint64(len(snapshotTrees))) defer bar.Done() - return restic.FindUsedBlobs(ctx, repo, snapshotTrees, usedBlobs, bar) + return data.FindUsedBlobs(ctx, repo, snapshotTrees, usedBlobs, bar) } diff --git a/cmd/restic/cmd_recover.go b/cmd/restic/cmd_recover.go index 3d1b7aec0..6d70961c0 100644 --- a/cmd/restic/cmd_recover.go +++ b/cmd/restic/cmd_recover.go @@ -5,6 +5,7 @@ import ( "os" "time" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -87,7 +88,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions, term ui.Terminal) erro bar := printer.NewCounter("trees loaded") bar.SetMax(uint64(len(trees))) for id := range trees { - tree, err := restic.LoadTree(ctx, repo, id) + tree, err := data.LoadTree(ctx, repo, id) if ctx.Err() != nil { return ctx.Err() } @@ -97,7 +98,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions, term ui.Terminal) erro } for _, node := range tree.Nodes { - if node.Type == restic.NodeTypeDir && node.Subtree != nil { + if node.Type == data.NodeTypeDir && node.Subtree != nil { trees[*node.Subtree] = true } } @@ -106,7 +107,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions, term ui.Terminal) erro bar.Done() printer.P("load snapshots\n") - err = restic.ForAllSnapshots(ctx, snapshotLister, repo, nil, func(_ restic.ID, sn *restic.Snapshot, _ error) error { + err = data.ForAllSnapshots(ctx, snapshotLister, repo, nil, func(_ restic.ID, sn *data.Snapshot, _ error) error { trees[*sn.Tree] = true return nil }) @@ -133,11 +134,11 @@ func runRecover(ctx context.Context, gopts GlobalOptions, term ui.Terminal) erro return ctx.Err() } - tree := restic.NewTree(len(roots)) + tree := data.NewTree(len(roots)) for id := range roots { var subtreeID = id - node := restic.Node{ - Type: restic.NodeTypeDir, + node := data.Node{ + Type: data.NodeTypeDir, Name: id.Str(), Mode: 0755, Subtree: &subtreeID, @@ -157,7 +158,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions, term ui.Terminal) erro var treeID restic.ID wg.Go(func() error { var err error - treeID, err = restic.SaveTree(wgCtx, repo, tree) + treeID, err = data.SaveTree(wgCtx, repo, tree) if err != nil { return errors.Fatalf("unable to save new tree to the repository: %v", err) } @@ -178,14 +179,14 @@ func runRecover(ctx context.Context, gopts GlobalOptions, term ui.Terminal) erro } func createSnapshot(ctx context.Context, printer progress.Printer, name, hostname string, tags []string, repo restic.SaverUnpacked[restic.WriteableFileType], tree *restic.ID) error { - sn, err := restic.NewSnapshot([]string{name}, tags, hostname, time.Now()) + sn, err := data.NewSnapshot([]string{name}, tags, hostname, time.Now()) if err != nil { return errors.Fatalf("unable to save snapshot: %v", err) } sn.Tree = tree - id, err := restic.SaveSnapshot(ctx, repo, sn) + id, err := data.SaveSnapshot(ctx, repo, sn) if err != nil { return errors.Fatalf("unable to save snapshot: %v", err) } diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index 0c7608e7f..d0c9663c6 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -3,6 +3,7 @@ package main import ( "context" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui" @@ -63,7 +64,7 @@ type RepairOptions struct { DryRun bool Forget bool - restic.SnapshotFilter + data.SnapshotFilter } func (opts *RepairOptions) AddFlags(f *pflag.FlagSet) { @@ -96,12 +97,12 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt // - trees which cannot be loaded (-> the tree contents will be removed) // - files whose contents are not fully available (-> file will be modified) rewriter := walker.NewTreeRewriter(walker.RewriteOpts{ - RewriteNode: func(node *restic.Node, path string) *restic.Node { - if node.Type == restic.NodeTypeIrregular || node.Type == restic.NodeTypeInvalid { + RewriteNode: func(node *data.Node, path string) *data.Node { + if node.Type == data.NodeTypeIrregular || node.Type == data.NodeTypeInvalid { printer.P(" file %q: removed node with invalid type %q", path, node.Type) return nil } - if node.Type != restic.NodeTypeFile { + if node.Type != data.NodeTypeFile { return node } @@ -135,7 +136,7 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt } // If a subtree fails to load, remove it printer.P(" dir %q: replaced with empty directory", path) - emptyID, err := restic.SaveTree(ctx, repo, &restic.Tree{}) + emptyID, err := data.SaveTree(ctx, repo, &data.Tree{}) if err != nil { return restic.ID{}, err } @@ -148,7 +149,7 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args, printer) { printer.P("\n%v", sn) changed, err := filterAndReplaceSnapshot(ctx, repo, sn, - func(ctx context.Context, sn *restic.Snapshot) (restic.ID, *restic.SnapshotSummary, error) { + func(ctx context.Context, sn *data.Snapshot) (restic.ID, *data.SnapshotSummary, error) { id, err := rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) return id, nil, err }, opts.DryRun, opts.Forget, nil, "repaired", printer) diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index fab22327e..8641ef75e 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -5,10 +5,10 @@ import ( "path/filepath" "time" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/filter" - "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restorer" "github.com/restic/restic/internal/ui" "github.com/restic/restic/internal/ui/progress" @@ -59,7 +59,7 @@ type RestoreOptions struct { filter.ExcludePatternOptions filter.IncludePatternOptions Target string - restic.SnapshotFilter + data.SnapshotFilter DryRun bool Sparse bool Verify bool @@ -142,7 +142,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, } defer unlock() - sn, subfolder, err := (&restic.SnapshotFilter{ + sn, subfolder, err := (&data.SnapshotFilter{ Hosts: opts.Hosts, Paths: opts.Paths, Tags: opts.Tags, @@ -156,7 +156,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, return err } - sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder) + sn.Tree, err = data.FindTreeDirectory(ctx, repo, sn.Tree, subfolder) if err != nil { return err } diff --git a/cmd/restic/cmd_restore_integration_test.go b/cmd/restic/cmd_restore_integration_test.go index 7716555d4..e7d3e8184 100644 --- a/cmd/restic/cmd_restore_integration_test.go +++ b/cmd/restic/cmd_restore_integration_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -37,7 +38,7 @@ func testRunRestoreAssumeFailure(t testing.TB, snapshotID string, opts RestoreOp func testRunRestoreLatest(t testing.TB, gopts GlobalOptions, dir string, paths []string, hosts []string) { opts := RestoreOptions{ Target: dir, - SnapshotFilter: restic.SnapshotFilter{ + SnapshotFilter: data.SnapshotFilter{ Hosts: hosts, Paths: paths, }, diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index fb6f66b32..d8671ddc2 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/pflag" "golang.org/x/sync/errgroup" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/filter" @@ -105,7 +106,7 @@ type RewriteOptions struct { SnapshotSummary bool Metadata snapshotMetadataArgs - restic.SnapshotFilter + data.SnapshotFilter filter.ExcludePatternOptions } @@ -122,9 +123,9 @@ func (opts *RewriteOptions) AddFlags(f *pflag.FlagSet) { // rewriteFilterFunc returns the filtered tree ID or an error. If a snapshot summary is returned, the snapshot will // be updated accordingly. -type rewriteFilterFunc func(ctx context.Context, sn *restic.Snapshot) (restic.ID, *restic.SnapshotSummary, error) +type rewriteFilterFunc func(ctx context.Context, sn *data.Snapshot) (restic.ID, *data.SnapshotSummary, error) -func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, opts RewriteOptions, printer progress.Printer) (bool, error) { +func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *data.Snapshot, opts RewriteOptions, printer progress.Printer) (bool, error) { if sn.Tree == nil { return false, errors.Errorf("snapshot %v has nil tree", sn.ID().Str()) } @@ -152,7 +153,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti return true } - rewriteNode := func(node *restic.Node, path string) *restic.Node { + rewriteNode := func(node *data.Node, path string) *data.Node { if selectByName(path) { return node } @@ -162,13 +163,13 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti rewriter, querySize := walker.NewSnapshotSizeRewriter(rewriteNode) - filter = func(ctx context.Context, sn *restic.Snapshot) (restic.ID, *restic.SnapshotSummary, error) { + filter = func(ctx context.Context, sn *data.Snapshot) (restic.ID, *data.SnapshotSummary, error) { id, err := rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) if err != nil { return restic.ID{}, nil, err } ss := querySize() - summary := &restic.SnapshotSummary{} + summary := &data.SnapshotSummary{} if sn.Summary != nil { *summary = *sn.Summary } @@ -178,7 +179,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti } } else { - filter = func(_ context.Context, sn *restic.Snapshot) (restic.ID, *restic.SnapshotSummary, error) { + filter = func(_ context.Context, sn *data.Snapshot) (restic.ID, *data.SnapshotSummary, error) { return *sn.Tree, nil, nil } } @@ -187,14 +188,14 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti filter, opts.DryRun, opts.Forget, metadata, "rewrite", printer) } -func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *restic.Snapshot, +func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *data.Snapshot, filter rewriteFilterFunc, dryRun bool, forget bool, newMetadata *snapshotMetadata, addTag string, printer progress.Printer) (bool, error) { wg, wgCtx := errgroup.WithContext(ctx) repo.StartPackUploader(wgCtx, wg) var filteredTree restic.ID - var summary *restic.SnapshotSummary + var summary *data.SnapshotSummary wg.Go(func() error { var err error filteredTree, summary, err = filter(ctx, sn) @@ -273,7 +274,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r } // Save the new snapshot. - id, err := restic.SaveSnapshot(ctx, repo, sn) + id, err := data.SaveSnapshot(ctx, repo, sn) if err != nil { return false, err } diff --git a/cmd/restic/cmd_rewrite_integration_test.go b/cmd/restic/cmd_rewrite_integration_test.go index b43f80dd0..1a0dd0abe 100644 --- a/cmd/restic/cmd_rewrite_integration_test.go +++ b/cmd/restic/cmd_rewrite_integration_test.go @@ -5,6 +5,7 @@ import ( "path/filepath" "testing" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -37,17 +38,17 @@ func createBasicRewriteRepo(t testing.TB, env *testEnvironment) restic.ID { return snapshotIDs[0] } -func getSnapshot(t testing.TB, snapshotID restic.ID, env *testEnvironment) *restic.Snapshot { +func getSnapshot(t testing.TB, snapshotID restic.ID, env *testEnvironment) *data.Snapshot { t.Helper() - var snapshots []*restic.Snapshot + var snapshots []*data.Snapshot err := withTermStatus(t, env.gopts, func(ctx context.Context, gopts GlobalOptions) error { printer := ui.NewProgressPrinter(gopts.JSON, gopts.verbosity, gopts.term) ctx, repo, unlock, err := openWithReadLock(ctx, gopts, false, printer) rtest.OK(t, err) defer unlock() - snapshots, err = restic.TestLoadAllSnapshots(ctx, repo, nil) + snapshots, err = data.TestLoadAllSnapshots(ctx, repo, nil) return err }) rtest.OK(t, err) @@ -116,14 +117,14 @@ func testRewriteMetadata(t *testing.T, metadata snapshotMetadataArgs) { createBasicRewriteRepo(t, env) testRunRewriteExclude(t, env.gopts, []string{}, true, metadata) - var snapshots []*restic.Snapshot + var snapshots []*data.Snapshot err := withTermStatus(t, env.gopts, func(ctx context.Context, gopts GlobalOptions) error { printer := ui.NewProgressPrinter(gopts.JSON, gopts.verbosity, gopts.term) ctx, repo, unlock, err := openWithReadLock(ctx, gopts, false, printer) rtest.OK(t, err) defer unlock() - snapshots, err = restic.TestLoadAllSnapshots(ctx, repo, nil) + snapshots, err = data.TestLoadAllSnapshots(ctx, repo, nil) return err }) rtest.OK(t, err) @@ -164,19 +165,19 @@ func TestRewriteSnaphotSummary(t *testing.T) { snapshots := testListSnapshots(t, env.gopts, 1) // replace snapshot by one without a summary - var oldSummary *restic.SnapshotSummary + var oldSummary *data.SnapshotSummary err := withTermStatus(t, env.gopts, func(ctx context.Context, gopts GlobalOptions) error { printer := ui.NewProgressPrinter(gopts.JSON, gopts.verbosity, gopts.term) _, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false, printer) rtest.OK(t, err) defer unlock() - sn, err := restic.LoadSnapshot(ctx, repo, snapshots[0]) + sn, err := data.LoadSnapshot(ctx, repo, snapshots[0]) rtest.OK(t, err) oldSummary = sn.Summary sn.Summary = nil rtest.OK(t, repo.RemoveUnpacked(ctx, restic.WriteableSnapshotFile, snapshots[0])) - snapshots[0], err = restic.SaveSnapshot(ctx, repo, sn) + snapshots[0], err = data.SaveSnapshot(ctx, repo, sn) return err }) rtest.OK(t, err) diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index fc2930e85..67b458d1b 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -8,6 +8,7 @@ import ( "sort" "strings" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui" "github.com/restic/restic/internal/ui/table" @@ -46,11 +47,11 @@ Exit status is 12 if the password is incorrect. // SnapshotOptions bundles all options for the snapshots command. type SnapshotOptions struct { - restic.SnapshotFilter + data.SnapshotFilter Compact bool Last bool // This option should be removed in favour of Latest. Latest int - GroupBy restic.SnapshotGroupByOptions + GroupBy data.SnapshotGroupByOptions } func (opts *SnapshotOptions) AddFlags(f *pflag.FlagSet) { @@ -74,14 +75,14 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions } defer unlock() - var snapshots restic.Snapshots + var snapshots data.Snapshots for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args, printer) { snapshots = append(snapshots, sn) } if ctx.Err() != nil { return ctx.Err() } - snapshotGroups, grouped, err := restic.GroupSnapshots(snapshots, opts.GroupBy) + snapshotGroups, grouped, err := data.GroupSnapshots(snapshots, opts.GroupBy) if err != nil { return err } @@ -137,7 +138,7 @@ type filterLastSnapshotsKey struct { } // newFilterLastSnapshotsKey initializes a filterLastSnapshotsKey from a Snapshot -func newFilterLastSnapshotsKey(sn *restic.Snapshot) filterLastSnapshotsKey { +func newFilterLastSnapshotsKey(sn *data.Snapshot) filterLastSnapshotsKey { // Shallow slice copy var paths = make([]string, len(sn.Paths)) copy(paths, sn.Paths) @@ -149,13 +150,13 @@ func newFilterLastSnapshotsKey(sn *restic.Snapshot) filterLastSnapshotsKey { // the limit last entries for each hostname and path. If the snapshot // contains multiple paths, they will be joined and treated as one // item. -func FilterLatestSnapshots(list restic.Snapshots, limit int) restic.Snapshots { +func FilterLatestSnapshots(list data.Snapshots, limit int) data.Snapshots { // Sort the snapshots so that the newer ones are listed first sort.SliceStable(list, func(i, j int) bool { return list[i].Time.After(list[j].Time) }) - var results restic.Snapshots + var results data.Snapshots seen := make(map[filterLastSnapshotsKey]int) for _, sn := range list { key := newFilterLastSnapshotsKey(sn) @@ -168,10 +169,10 @@ func FilterLatestSnapshots(list restic.Snapshots, limit int) restic.Snapshots { } // PrintSnapshots prints a text table of the snapshots in list to stdout. -func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.KeepReason, compact bool) error { +func PrintSnapshots(stdout io.Writer, list data.Snapshots, reasons []data.KeepReason, compact bool) error { // keep the reasons a snasphot is being kept in a map, so that it doesn't // get lost when the list of snapshots is sorted - keepReasons := make(map[restic.ID]restic.KeepReason, len(reasons)) + keepReasons := make(map[restic.ID]data.KeepReason, len(reasons)) if len(reasons) > 0 { for i, sn := range list { id := sn.ID() @@ -287,7 +288,7 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke // following snapshots belong to. // Prints nothing, if we did not group at all. func PrintSnapshotGroupHeader(stdout io.Writer, groupKeyJSON string) error { - var key restic.SnapshotGroupKey + var key data.SnapshotGroupKey err := json.Unmarshal([]byte(groupKeyJSON), &key) if err != nil { @@ -320,7 +321,7 @@ func PrintSnapshotGroupHeader(stdout io.Writer, groupKeyJSON string) error { // Snapshot helps to print Snapshots as JSON with their ID included. type Snapshot struct { - *restic.Snapshot + *data.Snapshot ID *restic.ID `json:"id"` ShortID string `json:"short_id"` // deprecated @@ -328,17 +329,17 @@ type Snapshot struct { // SnapshotGroup helps to print SnapshotGroups as JSON with their GroupReasons included. type SnapshotGroup struct { - GroupKey restic.SnapshotGroupKey `json:"group_key"` - Snapshots []Snapshot `json:"snapshots"` + GroupKey data.SnapshotGroupKey `json:"group_key"` + Snapshots []Snapshot `json:"snapshots"` } // printSnapshotGroupJSON writes the JSON representation of list to stdout. -func printSnapshotGroupJSON(stdout io.Writer, snGroups map[string]restic.Snapshots, grouped bool) error { +func printSnapshotGroupJSON(stdout io.Writer, snGroups map[string]data.Snapshots, grouped bool) error { if grouped { snapshotGroups := []SnapshotGroup{} for k, list := range snGroups { - var key restic.SnapshotGroupKey + var key data.SnapshotGroupKey var err error var snapshots []Snapshot diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index 604805e67..dcddbb6af 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -10,6 +10,7 @@ import ( "github.com/restic/chunker" "github.com/restic/restic/internal/crypto" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restorer" @@ -79,7 +80,7 @@ type StatsOptions struct { // the mode of counting to perform (see consts for available modes) countMode string - restic.SnapshotFilter + data.SnapshotFilter } func (opts *StatsOptions) AddFlags(f *pflag.FlagSet) { @@ -200,7 +201,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args return nil } -func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo restic.Loader, opts StatsOptions, stats *statsContainer) error { +func statsWalkSnapshot(ctx context.Context, snapshot *data.Snapshot, repo restic.Loader, opts StatsOptions, stats *statsContainer) error { if snapshot.Tree == nil { return fmt.Errorf("snapshot %s has nil tree", snapshot.ID().Str()) } @@ -210,7 +211,7 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest if opts.countMode == countModeRawData { // count just the sizes of unique blobs; we don't need to walk the tree // ourselves in this case, since a nifty function does it for us - return restic.FindUsedBlobs(ctx, repo, restic.IDs{*snapshot.Tree}, stats.blobs, nil) + return data.FindUsedBlobs(ctx, repo, restic.IDs{*snapshot.Tree}, stats.blobs, nil) } hardLinkIndex := restorer.NewHardlinkIndex[struct{}]() @@ -225,7 +226,7 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest } func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer, hardLinkIndex *restorer.HardlinkIndex[struct{}]) walker.WalkFunc { - return func(parentTreeID restic.ID, npath string, node *restic.Node, nodeErr error) error { + return func(parentTreeID restic.ID, npath string, node *data.Node, nodeErr error) error { if nodeErr != nil { return nodeErr } @@ -281,7 +282,7 @@ func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer, // will still be restored stats.TotalFileCount++ - if node.Links == 1 || node.Type == restic.NodeTypeDir { + if node.Links == 1 || node.Type == data.NodeTypeDir { stats.TotalSize += node.Size } else { // if hardlinks are present only count each deviceID+inode once @@ -298,7 +299,7 @@ func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer, // makeFileIDByContents returns a hash of the blob IDs of the // node's Content in sequence. -func makeFileIDByContents(node *restic.Node) fileID { +func makeFileIDByContents(node *data.Node) fileID { var bb []byte for _, c := range node.Content { bb = append(bb, c[:]...) diff --git a/cmd/restic/cmd_tag.go b/cmd/restic/cmd_tag.go index 46ded63a1..bc1ca4905 100644 --- a/cmd/restic/cmd_tag.go +++ b/cmd/restic/cmd_tag.go @@ -6,6 +6,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" @@ -49,10 +50,10 @@ Exit status is 12 if the password is incorrect. // TagOptions bundles all options for the 'tag' command. type TagOptions struct { - restic.SnapshotFilter - SetTags restic.TagLists - AddTags restic.TagLists - RemoveTags restic.TagLists + data.SnapshotFilter + SetTags data.TagLists + AddTags data.TagLists + RemoveTags data.TagLists } func (opts *TagOptions) AddFlags(f *pflag.FlagSet) { @@ -73,7 +74,7 @@ type changedSnapshotsSummary struct { ChangedSnapshots int `json:"changed_snapshots"` } -func changeTags(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, setTags, addTags, removeTags []string, printFunc func(changedSnapshot)) (bool, error) { +func changeTags(ctx context.Context, repo *repository.Repository, sn *data.Snapshot, setTags, addTags, removeTags []string, printFunc func(changedSnapshot)) (bool, error) { var changed bool if len(setTags) != 0 { @@ -97,7 +98,7 @@ func changeTags(ctx context.Context, repo *repository.Repository, sn *restic.Sna } // Save the new snapshot. - id, err := restic.SaveSnapshot(ctx, repo, sn) + id, err := data.SaveSnapshot(ctx, repo, sn) if err != nil { return false, err } diff --git a/cmd/restic/cmd_tag_integration_test.go b/cmd/restic/cmd_tag_integration_test.go index 3a92cc7b0..58664fcab 100644 --- a/cmd/restic/cmd_tag_integration_test.go +++ b/cmd/restic/cmd_tag_integration_test.go @@ -4,7 +4,7 @@ import ( "context" "testing" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/data" rtest "github.com/restic/restic/internal/test" ) @@ -32,7 +32,7 @@ func TestTag(t *testing.T) { "expected original ID to be nil, got %v", newest.Original) originalID := *newest.ID - testRunTag(t, TagOptions{SetTags: restic.TagLists{[]string{"NL"}}}, env.gopts) + testRunTag(t, TagOptions{SetTags: data.TagLists{[]string{"NL"}}}, env.gopts) testRunCheck(t, env.gopts) newest, _ = testRunSnapshots(t, env.gopts) if newest == nil { @@ -44,7 +44,7 @@ func TestTag(t *testing.T) { rtest.Assert(t, *newest.Original == originalID, "expected original ID to be set to the first snapshot id") - testRunTag(t, TagOptions{AddTags: restic.TagLists{[]string{"CH"}}}, env.gopts) + testRunTag(t, TagOptions{AddTags: data.TagLists{[]string{"CH"}}}, env.gopts) testRunCheck(t, env.gopts) newest, _ = testRunSnapshots(t, env.gopts) if newest == nil { @@ -56,7 +56,7 @@ func TestTag(t *testing.T) { rtest.Assert(t, *newest.Original == originalID, "expected original ID to be set to the first snapshot id") - testRunTag(t, TagOptions{RemoveTags: restic.TagLists{[]string{"NL"}}}, env.gopts) + testRunTag(t, TagOptions{RemoveTags: data.TagLists{[]string{"NL"}}}, env.gopts) testRunCheck(t, env.gopts) newest, _ = testRunSnapshots(t, env.gopts) if newest == nil { @@ -68,8 +68,8 @@ func TestTag(t *testing.T) { rtest.Assert(t, *newest.Original == originalID, "expected original ID to be set to the first snapshot id") - testRunTag(t, TagOptions{AddTags: restic.TagLists{[]string{"US", "RU"}}}, env.gopts) - testRunTag(t, TagOptions{RemoveTags: restic.TagLists{[]string{"CH", "US", "RU"}}}, env.gopts) + testRunTag(t, TagOptions{AddTags: data.TagLists{[]string{"US", "RU"}}}, env.gopts) + testRunTag(t, TagOptions{RemoveTags: data.TagLists{[]string{"CH", "US", "RU"}}}, env.gopts) testRunCheck(t, env.gopts) newest, _ = testRunSnapshots(t, env.gopts) if newest == nil { @@ -82,7 +82,7 @@ func TestTag(t *testing.T) { "expected original ID to be set to the first snapshot id") // Check special case of removing all tags. - testRunTag(t, TagOptions{SetTags: restic.TagLists{[]string{""}}}, env.gopts) + testRunTag(t, TagOptions{SetTags: data.TagLists{[]string{""}}}, env.gopts) testRunCheck(t, env.gopts) newest, _ = testRunSnapshots(t, env.gopts) if newest == nil { diff --git a/cmd/restic/find.go b/cmd/restic/find.go index eab8b5d90..9d820b66a 100644 --- a/cmd/restic/find.go +++ b/cmd/restic/find.go @@ -4,6 +4,7 @@ import ( "context" "os" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" "github.com/spf13/pflag" @@ -11,7 +12,7 @@ import ( // initMultiSnapshotFilter is used for commands that work on multiple snapshots // MUST be combined with restic.FindFilteredSnapshots or FindFilteredSnapshots -func initMultiSnapshotFilter(flags *pflag.FlagSet, filt *restic.SnapshotFilter, addHostShorthand bool) { +func initMultiSnapshotFilter(flags *pflag.FlagSet, filt *data.SnapshotFilter, addHostShorthand bool) { hostShorthand := "H" if !addHostShorthand { hostShorthand = "" @@ -28,7 +29,7 @@ func initMultiSnapshotFilter(flags *pflag.FlagSet, filt *restic.SnapshotFilter, // initSingleSnapshotFilter is used for commands that work on a single snapshot // MUST be combined with restic.FindFilteredSnapshot -func initSingleSnapshotFilter(flags *pflag.FlagSet, filt *restic.SnapshotFilter) { +func initSingleSnapshotFilter(flags *pflag.FlagSet, filt *data.SnapshotFilter) { flags.StringArrayVarP(&filt.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when snapshot ID \"latest\" is given (can be specified multiple times) (default: $RESTIC_HOST)") flags.Var(&filt.Tags, "tag", "only consider snapshots including `tag[,tag,...]`, when snapshot ID \"latest\" is given (can be specified multiple times)") flags.StringArrayVar(&filt.Paths, "path", nil, "only consider snapshots including this (absolute) `path`, when snapshot ID \"latest\" is given (can be specified multiple times, snapshots must include all specified paths)") @@ -40,8 +41,8 @@ func initSingleSnapshotFilter(flags *pflag.FlagSet, filt *restic.SnapshotFilter) } // FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots. -func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic.LoaderUnpacked, f *restic.SnapshotFilter, snapshotIDs []string, printer progress.Printer) <-chan *restic.Snapshot { - out := make(chan *restic.Snapshot) +func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic.LoaderUnpacked, f *data.SnapshotFilter, snapshotIDs []string, printer progress.Printer) <-chan *data.Snapshot { + out := make(chan *data.Snapshot) go func() { defer close(out) be, err := restic.MemorizeList(ctx, be, restic.SnapshotFile) @@ -50,7 +51,7 @@ func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic. return } - err = f.FindAll(ctx, be, loader, snapshotIDs, func(id string, sn *restic.Snapshot, err error) error { + err = f.FindAll(ctx, be, loader, snapshotIDs, func(id string, sn *data.Snapshot, err error) error { if err != nil { printer.E("Ignoring %q: %v", id, err) } else { diff --git a/cmd/restic/find_test.go b/cmd/restic/find_test.go index a98a14f04..3fffddcd8 100644 --- a/cmd/restic/find_test.go +++ b/cmd/restic/find_test.go @@ -3,7 +3,7 @@ package main import ( "testing" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/data" rtest "github.com/restic/restic/internal/test" "github.com/spf13/pflag" ) @@ -45,7 +45,7 @@ func TestSnapshotFilter(t *testing.T) { for _, mode := range []bool{false, true} { set := pflag.NewFlagSet("test", pflag.PanicOnError) - flt := &restic.SnapshotFilter{} + flt := &data.SnapshotFilter{} if mode { initMultiSnapshotFilter(set, flt, false) } else { diff --git a/cmd/restic/format.go b/cmd/restic/format.go index e7d178b4e..213d76aeb 100644 --- a/cmd/restic/format.go +++ b/cmd/restic/format.go @@ -4,11 +4,11 @@ import ( "fmt" "os" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/ui" ) -func formatNode(path string, n *restic.Node, long bool, human bool) string { +func formatNode(path string, n *data.Node, long bool, human bool) string { if !long { return path } @@ -24,20 +24,20 @@ func formatNode(path string, n *restic.Node, long bool, human bool) string { } switch n.Type { - case restic.NodeTypeFile: + case data.NodeTypeFile: mode = 0 - case restic.NodeTypeDir: + case data.NodeTypeDir: mode = os.ModeDir - case restic.NodeTypeSymlink: + case data.NodeTypeSymlink: mode = os.ModeSymlink target = fmt.Sprintf(" -> %v", n.LinkTarget) - case restic.NodeTypeDev: + case data.NodeTypeDev: mode = os.ModeDevice - case restic.NodeTypeCharDev: + case data.NodeTypeCharDev: mode = os.ModeDevice | os.ModeCharDevice - case restic.NodeTypeFifo: + case data.NodeTypeFifo: mode = os.ModeNamedPipe - case restic.NodeTypeSocket: + case data.NodeTypeSocket: mode = os.ModeSocket } diff --git a/cmd/restic/format_test.go b/cmd/restic/format_test.go index e232a200b..9d19a70f4 100644 --- a/cmd/restic/format_test.go +++ b/cmd/restic/format_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/data" rtest "github.com/restic/restic/internal/test" ) @@ -17,9 +17,9 @@ func TestFormatNode(t *testing.T) { }() testPath := "/test/path" - node := restic.Node{ + node := data.Node{ Name: "baz", - Type: restic.NodeTypeFile, + Type: data.NodeTypeFile, Size: 14680064, UID: 1000, GID: 2000, @@ -28,7 +28,7 @@ func TestFormatNode(t *testing.T) { for _, c := range []struct { path string - restic.Node + data.Node long bool human bool expect string diff --git a/cmd/restic/integration_helpers_test.go b/cmd/restic/integration_helpers_test.go index 892f61e10..2a8fec61a 100644 --- a/cmd/restic/integration_helpers_test.go +++ b/cmd/restic/integration_helpers_test.go @@ -15,6 +15,7 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/retry" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" "github.com/restic/restic/internal/repository" @@ -365,15 +366,15 @@ func lastSnapshot(old, new map[string]struct{}) (map[string]struct{}, string) { return old, "" } -func testLoadSnapshot(t testing.TB, gopts GlobalOptions, id restic.ID) *restic.Snapshot { - var snapshot *restic.Snapshot +func testLoadSnapshot(t testing.TB, gopts GlobalOptions, id restic.ID) *data.Snapshot { + var snapshot *data.Snapshot err := withTermStatus(t, gopts, func(ctx context.Context, gopts GlobalOptions) error { printer := ui.NewProgressPrinter(gopts.JSON, gopts.verbosity, gopts.term) _, repo, unlock, err := openWithReadLock(ctx, gopts, false, printer) rtest.OK(t, err) defer unlock() - snapshot, err = restic.LoadSnapshot(ctx, repo, id) + snapshot, err = data.LoadSnapshot(ctx, repo, id) return err }) rtest.OK(t, err) diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index dcad82c9b..b1cef1ad4 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -170,7 +171,7 @@ func TestFindListOnce(t *testing.T) { snapshotIDs = restic.NewIDSet() // specify the two oldest snapshots explicitly and use "latest" to reference the newest one - for sn := range FindFilteredSnapshots(ctx, repo, repo, &restic.SnapshotFilter{}, []string{ + for sn := range FindFilteredSnapshots(ctx, repo, repo, &data.SnapshotFilter{}, []string{ secondSnapshot[0].String(), secondSnapshot[1].String()[:8], "latest", diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index 7e71a543b..d14cf74e3 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -11,6 +11,7 @@ import ( "sync" "time" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/feature" @@ -66,9 +67,9 @@ func (s *ItemStats) Add(other ItemStats) { s.TreeSizeInRepo += other.TreeSizeInRepo } -// ToNoder returns a restic.Node for a File. +// ToNoder returns a data.Node for a File. type ToNoder interface { - ToNode(ignoreXattrListError bool, warnf func(format string, args ...any)) (*restic.Node, error) + ToNode(ignoreXattrListError bool, warnf func(format string, args ...any)) (*data.Node, error) } type archiverRepo interface { @@ -116,7 +117,7 @@ type Archiver struct { // // CompleteItem may be called asynchronously from several different // goroutines! - CompleteItem func(item string, previous, current *restic.Node, s ItemStats, d time.Duration) + CompleteItem func(item string, previous, current *data.Node, s ItemStats, d time.Duration) // StartFile is called when a file is being processed by a worker. StartFile func(filename string) @@ -193,7 +194,7 @@ func New(repo archiverRepo, filesystem fs.FS, opts Options) *Archiver { FS: filesystem, Options: opts.ApplyDefaults(), - CompleteItem: func(string, *restic.Node, *restic.Node, ItemStats, time.Duration) {}, + CompleteItem: func(string, *data.Node, *data.Node, ItemStats, time.Duration) {}, StartFile: func(string) {}, CompleteBlob: func(uint64) {}, } @@ -223,7 +224,7 @@ func (arch *Archiver) error(item string, err error) error { return errf } -func (arch *Archiver) trackItem(item string, previous, current *restic.Node, s ItemStats, d time.Duration) { +func (arch *Archiver) trackItem(item string, previous, current *data.Node, s ItemStats, d time.Duration) { arch.CompleteItem(item, previous, current, s, d) arch.mu.Lock() @@ -239,7 +240,7 @@ func (arch *Archiver) trackItem(item string, previous, current *restic.Node, s I } switch current.Type { - case restic.NodeTypeDir: + case data.NodeTypeDir: switch { case previous == nil: arch.summary.Dirs.New++ @@ -249,7 +250,7 @@ func (arch *Archiver) trackItem(item string, previous, current *restic.Node, s I arch.summary.Dirs.Changed++ } - case restic.NodeTypeFile: + case data.NodeTypeFile: switch { case previous == nil: arch.summary.Files.New++ @@ -262,7 +263,7 @@ func (arch *Archiver) trackItem(item string, previous, current *restic.Node, s I } // nodeFromFileInfo returns the restic node from an os.FileInfo. -func (arch *Archiver) nodeFromFileInfo(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*restic.Node, error) { +func (arch *Archiver) nodeFromFileInfo(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*data.Node, error) { node, err := meta.ToNode(ignoreXattrListError, func(format string, args ...any) { _ = arch.error(filename, fmt.Errorf(format, args...)) }) @@ -275,7 +276,7 @@ func (arch *Archiver) nodeFromFileInfo(snPath, filename string, meta ToNoder, ig node.AccessTime = node.ModTime } if feature.Flag.Enabled(feature.DeviceIDForHardlinks) { - if node.Links == 1 || node.Type == restic.NodeTypeDir { + if node.Links == 1 || node.Type == data.NodeTypeDir { // the DeviceID is only necessary for hardlinked files // when using subvolumes or snapshots their deviceIDs tend to change which causes // restic to upload new tree blobs @@ -285,7 +286,7 @@ func (arch *Archiver) nodeFromFileInfo(snPath, filename string, meta ToNoder, ig // overwrite name to match that within the snapshot node.Name = path.Base(snPath) // do not filter error for nodes of irregular or invalid type - if node.Type != restic.NodeTypeIrregular && node.Type != restic.NodeTypeInvalid && err != nil { + if node.Type != data.NodeTypeIrregular && node.Type != data.NodeTypeInvalid && err != nil { err = fmt.Errorf("incomplete metadata for %v: %w", filename, err) return node, arch.error(filename, err) } @@ -294,12 +295,12 @@ func (arch *Archiver) nodeFromFileInfo(snPath, filename string, meta ToNoder, ig // loadSubtree tries to load the subtree referenced by node. In case of an error, nil is returned. // If there is no node to load, then nil is returned without an error. -func (arch *Archiver) loadSubtree(ctx context.Context, node *restic.Node) (*restic.Tree, error) { - if node == nil || node.Type != restic.NodeTypeDir || node.Subtree == nil { +func (arch *Archiver) loadSubtree(ctx context.Context, node *data.Node) (*data.Tree, error) { + if node == nil || node.Type != data.NodeTypeDir || node.Subtree == nil { return nil, nil } - tree, err := restic.LoadTree(ctx, arch.Repo, *node.Subtree) + tree, err := data.LoadTree(ctx, arch.Repo, *node.Subtree) if err != nil { debug.Log("unable to load tree %v: %v", node.Subtree.Str(), err) // a tree in the repository is not readable -> warn the user @@ -320,7 +321,7 @@ func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error { // saveDir stores a directory in the repo and returns the node. snPath is the // path within the current snapshot. -func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, meta fs.File, previous *restic.Tree, complete fileCompleteFunc) (d futureNode, err error) { +func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, meta fs.File, previous *data.Tree, complete fileCompleteFunc) (d futureNode, err error) { debug.Log("%v %v", snPath, dir) treeNode, names, err := arch.dirToNodeAndEntries(snPath, dir, meta) @@ -365,7 +366,7 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me return fn, nil } -func (arch *Archiver) dirToNodeAndEntries(snPath, dir string, meta fs.File) (node *restic.Node, names []string, err error) { +func (arch *Archiver) dirToNodeAndEntries(snPath, dir string, meta fs.File) (node *data.Node, names []string, err error) { err = meta.MakeReadable() if err != nil { return nil, nil, fmt.Errorf("openfile for readdirnames failed: %w", err) @@ -375,7 +376,7 @@ func (arch *Archiver) dirToNodeAndEntries(snPath, dir string, meta fs.File) (nod if err != nil { return nil, nil, err } - if node.Type != restic.NodeTypeDir { + if node.Type != data.NodeTypeDir { return nil, nil, fmt.Errorf("directory %q changed type, refusing to archive", snPath) } @@ -400,7 +401,7 @@ type futureNode struct { type futureNodeResult struct { snPath, target string - node *restic.Node + node *data.Node stats ItemStats err error } @@ -438,7 +439,7 @@ func (fn *futureNode) take(ctx context.Context) futureNodeResult { // allBlobsPresent checks if all blobs (contents) of the given node are // present in the index. -func (arch *Archiver) allBlobsPresent(previous *restic.Node) bool { +func (arch *Archiver) allBlobsPresent(previous *data.Node) bool { // check if all blobs are contained in index for _, id := range previous.Content { if _, ok := arch.Repo.LookupBlobSize(restic.DataBlob, id); !ok { @@ -455,7 +456,7 @@ func (arch *Archiver) allBlobsPresent(previous *restic.Node) bool { // Errors and completion needs to be handled by the caller. // // snPath is the path within the current snapshot. -func (arch *Archiver) save(ctx context.Context, snPath, target string, previous *restic.Node) (fn futureNode, excluded bool, err error) { +func (arch *Archiver) save(ctx context.Context, snPath, target string, previous *data.Node) (fn futureNode, excluded bool, err error) { start := time.Now() debug.Log("%v target %q, previous %v", snPath, target, previous) @@ -574,7 +575,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous arch.StartFile(snPath) }, func() { arch.trackItem(snPath, nil, nil, ItemStats{}, 0) - }, func(node *restic.Node, stats ItemStats) { + }, func(node *data.Node, stats ItemStats) { arch.trackItem(snPath, previous, node, stats, time.Since(start)) }) @@ -591,7 +592,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous } fn, err = arch.saveDir(ctx, snPath, target, meta, oldSubtree, - func(node *restic.Node, stats ItemStats) { + func(node *data.Node, stats ItemStats) { arch.trackItem(snItem, previous, node, stats, time.Since(start)) }) if err != nil { @@ -625,11 +626,11 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous // fileChanged tries to detect whether a file's content has changed compared // to the contents of node, which describes the same path in the parent backup. // It should only be run for regular files. -func fileChanged(fi *fs.ExtendedFileInfo, node *restic.Node, ignoreFlags uint) bool { +func fileChanged(fi *fs.ExtendedFileInfo, node *data.Node, ignoreFlags uint) bool { switch { case node == nil: return true - case node.Type != restic.NodeTypeFile: + case node.Type != data.NodeTypeFile: // We're only called for regular files, so this is a type change. return true case uint64(fi.Size) != node.Size: @@ -658,9 +659,9 @@ func join(elem ...string) string { // saveTree stores a Tree in the repo, returned is the tree. snPath is the path // within the current snapshot. -func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, previous *restic.Tree, complete fileCompleteFunc) (futureNode, int, error) { +func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, previous *data.Tree, complete fileCompleteFunc) (futureNode, int, error) { - var node *restic.Node + var node *data.Node if snPath != "/" { if atree.FileInfoPath == "" { return futureNode{}, 0, errors.Errorf("FileInfoPath for %v is empty", snPath) @@ -673,7 +674,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, } } else { // fake root node - node = &restic.Node{} + node = &data.Node{} } debug.Log("%v (%v nodes), parent %v", snPath, len(atree.Nodes), previous) @@ -721,7 +722,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, } // not a leaf node, archive subtree - fn, _, err := arch.saveTree(ctx, join(snPath, name), &subatree, oldSubtree, func(n *restic.Node, is ItemStats) { + fn, _, err := arch.saveTree(ctx, join(snPath, name), &subatree, oldSubtree, func(n *data.Node, is ItemStats) { arch.trackItem(snItem, oldNode, n, is, time.Since(start)) }) if err != nil { @@ -740,7 +741,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, return fn, len(nodes), nil } -func (arch *Archiver) dirPathToNode(snPath, target string) (node *restic.Node, err error) { +func (arch *Archiver) dirPathToNode(snPath, target string) (node *data.Node, err error) { meta, err := arch.FS.OpenFile(target, 0, true) if err != nil { return nil, err @@ -759,7 +760,7 @@ func (arch *Archiver) dirPathToNode(snPath, target string) (node *restic.Node, e if err != nil { return nil, err } - if node.Type != restic.NodeTypeDir { + if node.Type != data.NodeTypeDir { return nil, errors.Errorf("path is not a directory: %v", target) } return node, err @@ -802,19 +803,19 @@ func resolveRelativeTargets(filesys fs.FS, targets []string) ([]string, error) { // SnapshotOptions collect attributes for a new snapshot. type SnapshotOptions struct { - Tags restic.TagList + Tags data.TagList Hostname string Excludes []string BackupStart time.Time Time time.Time - ParentSnapshot *restic.Snapshot + ParentSnapshot *data.Snapshot ProgramVersion string // SkipIfUnchanged omits the snapshot creation if it is identical to the parent snapshot. SkipIfUnchanged bool } // loadParentTree loads a tree referenced by snapshot id. If id is null, nil is returned. -func (arch *Archiver) loadParentTree(ctx context.Context, sn *restic.Snapshot) *restic.Tree { +func (arch *Archiver) loadParentTree(ctx context.Context, sn *data.Snapshot) *data.Tree { if sn == nil { return nil } @@ -825,7 +826,7 @@ func (arch *Archiver) loadParentTree(ctx context.Context, sn *restic.Snapshot) * } debug.Log("load parent tree %v", *sn.Tree) - tree, err := restic.LoadTree(ctx, arch.Repo, *sn.Tree) + tree, err := data.LoadTree(ctx, arch.Repo, *sn.Tree) if err != nil { debug.Log("unable to load tree %v: %v", *sn.Tree, err) _ = arch.error("/", arch.wrapLoadTreeError(*sn.Tree, err)) @@ -858,7 +859,7 @@ func (arch *Archiver) stopWorkers() { } // Snapshot saves several targets and returns a snapshot. -func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts SnapshotOptions) (*restic.Snapshot, restic.ID, *Summary, error) { +func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts SnapshotOptions) (*data.Snapshot, restic.ID, *Summary, error) { arch.summary = &Summary{ BackupStart: opts.BackupStart, } @@ -886,7 +887,7 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps arch.runWorkers(wgCtx, wg) debug.Log("starting snapshot") - fn, nodeCount, err := arch.saveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), func(_ *restic.Node, is ItemStats) { + fn, nodeCount, err := arch.saveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), func(_ *data.Node, is ItemStats) { arch.trackItem("/", nil, nil, is, time.Since(start)) }) if err != nil { @@ -934,7 +935,7 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps } } - sn, err := restic.NewSnapshot(targets, opts.Tags, opts.Hostname, opts.Time) + sn, err := data.NewSnapshot(targets, opts.Tags, opts.Hostname, opts.Time) if err != nil { return nil, restic.ID{}, nil, err } @@ -946,7 +947,7 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps } sn.Tree = &rootTreeID arch.summary.BackupEnd = time.Now() - sn.Summary = &restic.SnapshotSummary{ + sn.Summary = &data.SnapshotSummary{ BackupStart: arch.summary.BackupStart, BackupEnd: arch.summary.BackupEnd, @@ -964,7 +965,7 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps TotalBytesProcessed: arch.summary.ProcessedBytes, } - id, err := restic.SaveSnapshot(ctx, arch.Repo, sn) + id, err := data.SaveSnapshot(ctx, arch.Repo, sn) if err != nil { return nil, restic.ID{}, nil, err } diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go index b52e47b18..c3f72bf72 100644 --- a/internal/archiver/archiver_test.go +++ b/internal/archiver/archiver_test.go @@ -19,6 +19,7 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/checker" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/feature" "github.com/restic/restic/internal/fs" @@ -37,7 +38,7 @@ func prepareTempdirRepoSrc(t testing.TB, src TestDir) (string, restic.Repository return tempdir, repo } -func saveFile(t testing.TB, repo archiverRepo, filename string, filesystem fs.FS) (*restic.Node, ItemStats) { +func saveFile(t testing.TB, repo archiverRepo, filename string, filesystem fs.FS) (*data.Node, ItemStats) { wg, ctx := errgroup.WithContext(context.TODO()) repo.StartPackUploader(ctx, wg) @@ -52,7 +53,7 @@ func saveFile(t testing.TB, repo archiverRepo, filename string, filesystem fs.FS var ( completeReadingCallback bool - completeCallbackNode *restic.Node + completeCallbackNode *data.Node completeCallbackStats ItemStats completeCallback bool @@ -66,7 +67,7 @@ func saveFile(t testing.TB, repo archiverRepo, filename string, filesystem fs.FS } } - complete := func(node *restic.Node, stats ItemStats) { + complete := func(node *data.Node, stats ItemStats) { completeCallback = true completeCallbackNode = node completeCallbackStats = stats @@ -427,8 +428,8 @@ func (repo *blobCountingRepo) SaveBlob(ctx context.Context, t restic.BlobType, b return id, exists, size, err } -func (repo *blobCountingRepo) SaveTree(ctx context.Context, t *restic.Tree) (restic.ID, error) { - id, err := restic.SaveTree(ctx, repo.archiverRepo, t) +func (repo *blobCountingRepo) SaveTree(ctx context.Context, t *data.Tree) (restic.ID, error) { + id, err := data.SaveTree(ctx, repo.archiverRepo, t) h := restic.BlobHandle{ID: id, Type: restic.TreeBlob} repo.m.Lock() repo.saved[h]++ @@ -548,7 +549,7 @@ func rename(t testing.TB, oldname, newname string) { } } -func nodeFromFile(t testing.TB, localFs fs.FS, filename string) *restic.Node { +func nodeFromFile(t testing.TB, localFs fs.FS, filename string) *data.Node { meta, err := localFs.OpenFile(filename, fs.O_NOFOLLOW, true) rtest.OK(t, err) node, err := meta.ToNode(false, t.Logf) @@ -724,7 +725,7 @@ func TestFilChangedSpecialCases(t *testing.T) { t.Run("type-change", func(t *testing.T) { fi := lstat(t, filename) node := nodeFromFile(t, &fs.Local{}, filename) - node.Type = restic.NodeTypeSymlink + node.Type = data.NodeTypeSymlink if !fileChanged(fi, node, 0) { t.Fatal("node with changed type detected as unchanged") } @@ -865,8 +866,8 @@ func TestArchiverSaveDir(t *testing.T) { } node.Name = targetNodeName - tree := &restic.Tree{Nodes: []*restic.Node{node}} - treeID, err := restic.SaveTree(ctx, repo, tree) + tree := &data.Tree{Nodes: []*data.Node{node}} + treeID, err := data.SaveTree(ctx, repo, tree) if err != nil { t.Fatal(err) } @@ -1679,7 +1680,7 @@ func (f MockFile) Read(p []byte) (int, error) { return n, err } -func checkSnapshotStats(t *testing.T, sn *restic.Snapshot, stat Summary) { +func checkSnapshotStats(t *testing.T, sn *data.Snapshot, stat Summary) { t.Helper() rtest.Equals(t, stat.BackupStart, sn.Summary.BackupStart, "BackupStart") // BackupEnd is set to time.Now() and can't be compared to a fixed value @@ -2219,7 +2220,7 @@ func TestArchiverAbortEarlyOnError(t *testing.T) { } } -func snapshot(t testing.TB, repo archiverRepo, fs fs.FS, parent *restic.Snapshot, filename string) (*restic.Snapshot, *restic.Node) { +func snapshot(t testing.TB, repo archiverRepo, fs fs.FS, parent *data.Snapshot, filename string) (*data.Snapshot, *data.Node) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -2234,7 +2235,7 @@ func snapshot(t testing.TB, repo archiverRepo, fs fs.FS, parent *restic.Snapshot t.Fatal(err) } - tree, err := restic.LoadTree(ctx, repo, *snapshot.Tree) + tree, err := data.LoadTree(ctx, repo, *snapshot.Tree) if err != nil { t.Fatal(err) } @@ -2251,7 +2252,7 @@ type overrideFS struct { fs.FS overrideFI *fs.ExtendedFileInfo resetFIOnRead bool - overrideNode *restic.Node + overrideNode *data.Node overrideErr error } @@ -2287,7 +2288,7 @@ func (f overrideFile) MakeReadable() error { return f.File.MakeReadable() } -func (f overrideFile) ToNode(ignoreXattrListError bool, warnf func(format string, args ...any)) (*restic.Node, error) { +func (f overrideFile) ToNode(ignoreXattrListError bool, warnf func(format string, args ...any)) (*data.Node, error) { if f.ofs.overrideNode == nil { return f.File.ToNode(ignoreXattrListError, warnf) } @@ -2328,7 +2329,7 @@ func TestMetadataChanged(t *testing.T) { fs := &overrideFS{ FS: localFS, overrideFI: fi, - overrideNode: &restic.Node{}, + overrideNode: &data.Node{}, } *fs.overrideNode = *want @@ -2451,11 +2452,11 @@ func TestRacyFileTypeSwap(t *testing.T) { } type mockToNoder struct { - node *restic.Node + node *data.Node err error } -func (m *mockToNoder) ToNode(_ bool, _ func(format string, args ...any)) (*restic.Node, error) { +func (m *mockToNoder) ToNode(_ bool, _ func(format string, args ...any)) (*data.Node, error) { return m.node, m.err } @@ -2474,7 +2475,7 @@ func TestMetadataBackupErrorFiltering(t *testing.T) { } nonExistNoder := &mockToNoder{ - node: &restic.Node{Type: restic.NodeTypeFile}, + node: &data.Node{Type: data.NodeTypeFile}, err: fmt.Errorf("not found"), } @@ -2487,7 +2488,7 @@ func TestMetadataBackupErrorFiltering(t *testing.T) { // check that errors from reading irregular file are not filtered filteredErr = nil nonExistNoder = &mockToNoder{ - node: &restic.Node{Type: restic.NodeTypeIrregular}, + node: &data.Node{Type: data.NodeTypeIrregular}, err: fmt.Errorf(`unsupported file type "irregular"`), } node, err = arch.nodeFromFileInfo("file", filename, nonExistNoder, false) @@ -2515,8 +2516,8 @@ func TestIrregularFile(t *testing.T) { override := &overrideFS{ FS: fs.Local{}, overrideFI: fi, - overrideNode: &restic.Node{ - Type: restic.NodeTypeIrregular, + overrideNode: &data.Node{ + Type: data.NodeTypeIrregular, }, overrideErr: fmt.Errorf(`unsupported file type "irregular"`), } diff --git a/internal/archiver/archiver_unix_test.go b/internal/archiver/archiver_unix_test.go index b6cc1ba4e..55f677cd8 100644 --- a/internal/archiver/archiver_unix_test.go +++ b/internal/archiver/archiver_unix_test.go @@ -6,13 +6,13 @@ package archiver import ( "testing" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/feature" "github.com/restic/restic/internal/fs" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) -func statAndSnapshot(t *testing.T, repo archiverRepo, name string) (*restic.Node, *restic.Node) { +func statAndSnapshot(t *testing.T, repo archiverRepo, name string) (*data.Node, *data.Node) { want := nodeFromFile(t, &fs.Local{}, name) _, node := snapshot(t, repo, &fs.Local{}, nil, name) return want, node diff --git a/internal/archiver/file_saver.go b/internal/archiver/file_saver.go index ca8ec2fbb..8370bee4d 100644 --- a/internal/archiver/file_saver.go +++ b/internal/archiver/file_saver.go @@ -7,6 +7,7 @@ import ( "sync" "github.com/restic/chunker" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" @@ -28,7 +29,7 @@ type fileSaver struct { CompleteBlob func(bytes uint64) - NodeFromFileInfo func(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*restic.Node, error) + NodeFromFileInfo func(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*data.Node, error) } // newFileSaver returns a new file saver. A worker pool with fileWorkers is @@ -64,7 +65,7 @@ func (s *fileSaver) TriggerShutdown() { } // fileCompleteFunc is called when the file has been saved. -type fileCompleteFunc func(*restic.Node, ItemStats) +type fileCompleteFunc func(*data.Node, ItemStats) // Save stores the file f and returns the data once it has been completed. The // file is closed by Save. completeReading is only called if the file was read @@ -160,7 +161,7 @@ func (s *fileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat return } - if node.Type != restic.NodeTypeFile { + if node.Type != data.NodeTypeFile { _ = f.Close() completeError(errors.Errorf("node type %q is wrong", node.Type)) return diff --git a/internal/archiver/file_saver_test.go b/internal/archiver/file_saver_test.go index a5d090c05..5aab78558 100644 --- a/internal/archiver/file_saver_test.go +++ b/internal/archiver/file_saver_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/restic/chunker" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" @@ -49,7 +50,7 @@ func startFileSaver(ctx context.Context, t testing.TB, _ fs.FS) (*fileSaver, con } s := newFileSaver(ctx, wg, saveBlob, pol, workers, workers) - s.NodeFromFileInfo = func(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*restic.Node, error) { + s.NodeFromFileInfo = func(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*data.Node, error) { return meta.ToNode(ignoreXattrListError, t.Logf) } @@ -64,7 +65,7 @@ func TestFileSaver(t *testing.T) { startFn := func() {} completeReadingFn := func() {} - completeFn := func(*restic.Node, ItemStats) {} + completeFn := func(*data.Node, ItemStats) {} testFs := fs.Local{} s, ctx, wg := startFileSaver(ctx, t, testFs) diff --git a/internal/archiver/testing.go b/internal/archiver/testing.go index e555a70d6..666a8c556 100644 --- a/internal/archiver/testing.go +++ b/internal/archiver/testing.go @@ -11,13 +11,14 @@ import ( "testing" "time" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" ) // TestSnapshot creates a new snapshot of path. -func TestSnapshot(t testing.TB, repo restic.Repository, path string, parent *restic.ID) *restic.Snapshot { +func TestSnapshot(t testing.TB, repo restic.Repository, path string, parent *restic.ID) *data.Snapshot { arch := New(repo, fs.Local{}, Options{}) opts := SnapshotOptions{ Time: time.Now(), @@ -25,7 +26,7 @@ func TestSnapshot(t testing.TB, repo restic.Repository, path string, parent *res Tags: []string{"test"}, } if parent != nil { - sn, err := restic.LoadSnapshot(context.TODO(), repo, *parent) + sn, err := data.LoadSnapshot(context.TODO(), repo, *parent) if err != nil { t.Fatal(err) } @@ -232,7 +233,7 @@ func TestEnsureFiles(t testing.TB, target string, dir TestDir) { } // TestEnsureFileContent checks if the file in the repo is the same as file. -func TestEnsureFileContent(ctx context.Context, t testing.TB, repo restic.BlobLoader, filename string, node *restic.Node, file TestFile) { +func TestEnsureFileContent(ctx context.Context, t testing.TB, repo restic.BlobLoader, filename string, node *data.Node, file TestFile) { if int(node.Size) != len(file.Content) { t.Fatalf("%v: wrong node size: want %d, got %d", filename, node.Size, len(file.Content)) return @@ -263,7 +264,7 @@ func TestEnsureFileContent(ctx context.Context, t testing.TB, repo restic.BlobLo func TestEnsureTree(ctx context.Context, t testing.TB, prefix string, repo restic.BlobLoader, treeID restic.ID, dir TestDir) { t.Helper() - tree, err := restic.LoadTree(ctx, repo, treeID) + tree, err := data.LoadTree(ctx, repo, treeID) if err != nil { t.Fatal(err) return @@ -289,7 +290,7 @@ func TestEnsureTree(ctx context.Context, t testing.TB, prefix string, repo resti switch e := entry.(type) { case TestDir: - if node.Type != restic.NodeTypeDir { + if node.Type != data.NodeTypeDir { t.Errorf("tree node %v has wrong type %q, want %q", nodePrefix, node.Type, "dir") return } @@ -301,12 +302,12 @@ func TestEnsureTree(ctx context.Context, t testing.TB, prefix string, repo resti TestEnsureTree(ctx, t, path.Join(prefix, node.Name), repo, *node.Subtree, e) case TestFile: - if node.Type != restic.NodeTypeFile { + if node.Type != data.NodeTypeFile { t.Errorf("tree node %v has wrong type %q, want %q", nodePrefix, node.Type, "file") } TestEnsureFileContent(ctx, t, repo, nodePrefix, node, e) case TestSymlink: - if node.Type != restic.NodeTypeSymlink { + if node.Type != data.NodeTypeSymlink { t.Errorf("tree node %v has wrong type %q, want %q", nodePrefix, node.Type, "symlink") } @@ -331,7 +332,7 @@ func TestEnsureSnapshot(t testing.TB, repo restic.Repository, snapshotID restic. ctx, cancel := context.WithCancel(context.Background()) defer cancel() - sn, err := restic.LoadSnapshot(ctx, repo, snapshotID) + sn, err := data.LoadSnapshot(ctx, repo, snapshotID) if err != nil { t.Fatal(err) return diff --git a/internal/archiver/tree_saver.go b/internal/archiver/tree_saver.go index aeedefef5..d0e802765 100644 --- a/internal/archiver/tree_saver.go +++ b/internal/archiver/tree_saver.go @@ -4,6 +4,7 @@ import ( "context" "errors" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" "golang.org/x/sync/errgroup" @@ -42,7 +43,7 @@ func (s *treeSaver) TriggerShutdown() { } // Save stores the dir d and returns the data once it has been completed. -func (s *treeSaver) Save(ctx context.Context, snPath string, target string, node *restic.Node, nodes []futureNode, complete fileCompleteFunc) futureNode { +func (s *treeSaver) Save(ctx context.Context, snPath string, target string, node *data.Node, nodes []futureNode, complete fileCompleteFunc) futureNode { fn, ch := newFutureNode() job := saveTreeJob{ snPath: snPath, @@ -65,22 +66,22 @@ func (s *treeSaver) Save(ctx context.Context, snPath string, target string, node type saveTreeJob struct { snPath string target string - node *restic.Node + node *data.Node nodes []futureNode ch chan<- futureNodeResult complete fileCompleteFunc } // save stores the nodes as a tree in the repo. -func (s *treeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, ItemStats, error) { +func (s *treeSaver) save(ctx context.Context, job *saveTreeJob) (*data.Node, ItemStats, error) { var stats ItemStats node := job.node nodes := job.nodes // allow GC of nodes array once the loop is finished job.nodes = nil - builder := restic.NewTreeJSONBuilder() - var lastNode *restic.Node + builder := data.NewTreeJSONBuilder() + var lastNode *data.Node for i, fn := range nodes { // fn is a copy, so clear the original value explicitly @@ -110,7 +111,7 @@ func (s *treeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, I } err := builder.AddNode(fnr.node) - if err != nil && errors.Is(err, restic.ErrTreeNotOrdered) && lastNode != nil && fnr.node.Equals(*lastNode) { + if err != nil && errors.Is(err, data.ErrTreeNotOrdered) && lastNode != nil && fnr.node.Equals(*lastNode) { debug.Log("insert %v failed: %v", fnr.node.Name, err) // ignore error if an _identical_ node already exists, but nevertheless issue a warning _ = s.errFn(fnr.target, err) diff --git a/internal/archiver/tree_saver_test.go b/internal/archiver/tree_saver_test.go index 4aa4c51f1..2a4826444 100644 --- a/internal/archiver/tree_saver_test.go +++ b/internal/archiver/tree_saver_test.go @@ -6,6 +6,7 @@ import ( "runtime" "testing" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" @@ -46,7 +47,7 @@ func TestTreeSaver(t *testing.T) { var results []futureNode for i := 0; i < 20; i++ { - node := &restic.Node{ + node := &data.Node{ Name: fmt.Sprintf("file-%d", i), } @@ -86,11 +87,11 @@ func TestTreeSaverError(t *testing.T) { var results []futureNode for i := 0; i < test.trees; i++ { - node := &restic.Node{ + node := &data.Node{ Name: fmt.Sprintf("file-%d", i), } nodes := []futureNode{ - newFutureNodeWithResult(futureNodeResult{node: &restic.Node{ + newFutureNodeWithResult(futureNodeResult{node: &data.Node{ Name: fmt.Sprintf("child-%d", i), }}), } @@ -125,20 +126,20 @@ func TestTreeSaverDuplicates(t *testing.T) { ctx, cancel, b, shutdown := setupTreeSaver() defer cancel() - node := &restic.Node{ + node := &data.Node{ Name: "file", } nodes := []futureNode{ - newFutureNodeWithResult(futureNodeResult{node: &restic.Node{ + newFutureNodeWithResult(futureNodeResult{node: &data.Node{ Name: "child", }}), } if identicalNodes { - nodes = append(nodes, newFutureNodeWithResult(futureNodeResult{node: &restic.Node{ + nodes = append(nodes, newFutureNodeWithResult(futureNodeResult{node: &data.Node{ Name: "child", }})) } else { - nodes = append(nodes, newFutureNodeWithResult(futureNodeResult{node: &restic.Node{ + nodes = append(nodes, newFutureNodeWithResult(futureNodeResult{node: &data.Node{ Name: "child", Size: 42, }})) diff --git a/internal/checker/checker.go b/internal/checker/checker.go index 1f5d29a67..56a3eb9bb 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -8,6 +8,7 @@ import ( "sync" "github.com/klauspost/compress/zstd" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" @@ -256,7 +257,7 @@ func (e *TreeError) Error() string { } // checkTreeWorker checks the trees received and sends out errors to errChan. -func (c *Checker) checkTreeWorker(ctx context.Context, trees <-chan restic.TreeItem, out chan<- error) { +func (c *Checker) checkTreeWorker(ctx context.Context, trees <-chan data.TreeItem, out chan<- error) { for job := range trees { debug.Log("check tree %v (tree %v, err %v)", job.ID, job.Tree, job.Error) @@ -281,7 +282,7 @@ func (c *Checker) checkTreeWorker(ctx context.Context, trees <-chan restic.TreeI } func loadSnapshotTreeIDs(ctx context.Context, lister restic.Lister, repo restic.LoaderUnpacked) (ids restic.IDs, errs []error) { - err := restic.ForAllSnapshots(ctx, lister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error { + err := data.ForAllSnapshots(ctx, lister, repo, nil, func(id restic.ID, sn *data.Snapshot, err error) error { if err != nil { errs = append(errs, err) return nil @@ -315,7 +316,7 @@ func (c *Checker) Structure(ctx context.Context, p *progress.Counter, errChan ch } wg, ctx := errgroup.WithContext(ctx) - treeStream := restic.StreamTrees(ctx, wg, c.repo, trees, func(treeID restic.ID) bool { + treeStream := data.StreamTrees(ctx, wg, c.repo, trees, func(treeID restic.ID) bool { // blobRefs may be accessed in parallel by checkTree c.blobRefs.Lock() h := restic.BlobHandle{ID: treeID, Type: restic.TreeBlob} @@ -344,12 +345,12 @@ func (c *Checker) Structure(ctx context.Context, p *progress.Counter, errChan ch } } -func (c *Checker) checkTree(id restic.ID, tree *restic.Tree) (errs []error) { +func (c *Checker) checkTree(id restic.ID, tree *data.Tree) (errs []error) { debug.Log("checking tree %v", id) for _, node := range tree.Nodes { switch node.Type { - case restic.NodeTypeFile: + case data.NodeTypeFile: if node.Content == nil { errs = append(errs, &Error{TreeID: id, Err: errors.Errorf("file %q has nil blob list", node.Name)}) } @@ -385,7 +386,7 @@ func (c *Checker) checkTree(id restic.ID, tree *restic.Tree) (errs []error) { c.blobRefs.Unlock() } - case restic.NodeTypeDir: + case data.NodeTypeDir: if node.Subtree == nil { errs = append(errs, &Error{TreeID: id, Err: errors.Errorf("dir node %q has no subtree", node.Name)}) continue @@ -396,7 +397,7 @@ func (c *Checker) checkTree(id restic.ID, tree *restic.Tree) (errs []error) { continue } - case restic.NodeTypeSymlink, restic.NodeTypeSocket, restic.NodeTypeCharDev, restic.NodeTypeDev, restic.NodeTypeFifo: + case data.NodeTypeSymlink, data.NodeTypeSocket, data.NodeTypeCharDev, data.NodeTypeDev, data.NodeTypeFifo: // nothing to check default: diff --git a/internal/checker/checker_test.go b/internal/checker/checker_test.go index 29538e8ec..ce1542210 100644 --- a/internal/checker/checker_test.go +++ b/internal/checker/checker_test.go @@ -16,6 +16,7 @@ import ( "github.com/restic/restic/internal/archiver" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/checker" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository/hashing" @@ -440,7 +441,7 @@ type loadTreesOnceRepository struct { DuplicateTree bool } -func (r *loadTreesOnceRepository) LoadTree(ctx context.Context, id restic.ID) (*restic.Tree, error) { +func (r *loadTreesOnceRepository) LoadTree(ctx context.Context, id restic.ID) (*data.Tree, error) { r.mutex.Lock() defer r.mutex.Unlock() @@ -450,7 +451,7 @@ func (r *loadTreesOnceRepository) LoadTree(ctx context.Context, id restic.ID) (* return nil, errors.Errorf("trying to load tree with id %v twice", id) } r.loadedTrees.Insert(id) - return restic.LoadTree(ctx, r.Repository, id) + return data.LoadTree(ctx, r.Repository, id) } func TestCheckerNoDuplicateTreeDecodes(t *testing.T) { @@ -481,11 +482,11 @@ type delayRepository struct { Unblocker sync.Once } -func (r *delayRepository) LoadTree(ctx context.Context, id restic.ID) (*restic.Tree, error) { +func (r *delayRepository) LoadTree(ctx context.Context, id restic.ID) (*data.Tree, error) { if id == r.DelayTree { <-r.UnblockChannel } - return restic.LoadTree(ctx, r.Repository, id) + return data.LoadTree(ctx, r.Repository, id) } func (r *delayRepository) LookupBlobSize(t restic.BlobType, id restic.ID) (uint, bool) { @@ -507,20 +508,20 @@ func TestCheckerBlobTypeConfusion(t *testing.T) { repo := repository.TestRepository(t) - damagedNode := &restic.Node{ + damagedNode := &data.Node{ Name: "damaged", - Type: restic.NodeTypeFile, + Type: data.NodeTypeFile, Mode: 0644, Size: 42, Content: restic.IDs{restic.TestParseID("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef")}, } - damagedTree := &restic.Tree{ - Nodes: []*restic.Node{damagedNode}, + damagedTree := &data.Tree{ + Nodes: []*data.Node{damagedNode}, } wg, wgCtx := errgroup.WithContext(ctx) repo.StartPackUploader(wgCtx, wg) - id, err := restic.SaveTree(ctx, repo, damagedTree) + id, err := data.SaveTree(ctx, repo, damagedTree) test.OK(t, repo.Flush(ctx)) test.OK(t, err) @@ -532,35 +533,35 @@ func TestCheckerBlobTypeConfusion(t *testing.T) { _, _, _, err = repo.SaveBlob(ctx, restic.DataBlob, buf, id, false) test.OK(t, err) - malNode := &restic.Node{ + malNode := &data.Node{ Name: "aaaaa", - Type: restic.NodeTypeFile, + Type: data.NodeTypeFile, Mode: 0644, Size: uint64(len(buf)), Content: restic.IDs{id}, } - dirNode := &restic.Node{ + dirNode := &data.Node{ Name: "bbbbb", - Type: restic.NodeTypeDir, + Type: data.NodeTypeDir, Mode: 0755, Subtree: &id, } - rootTree := &restic.Tree{ - Nodes: []*restic.Node{malNode, dirNode}, + rootTree := &data.Tree{ + Nodes: []*data.Node{malNode, dirNode}, } - rootID, err := restic.SaveTree(ctx, repo, rootTree) + rootID, err := data.SaveTree(ctx, repo, rootTree) test.OK(t, err) test.OK(t, repo.Flush(ctx)) - snapshot, err := restic.NewSnapshot([]string{"/damaged"}, []string{"test"}, "foo", time.Now()) + snapshot, err := data.NewSnapshot([]string{"/damaged"}, []string{"test"}, "foo", time.Now()) test.OK(t, err) snapshot.Tree = &rootID - snapID, err := restic.SaveSnapshot(ctx, repo, snapshot) + snapID, err := data.SaveSnapshot(ctx, repo, snapshot) test.OK(t, err) t.Logf("saved snapshot %v", snapID.Str()) @@ -637,7 +638,7 @@ func benchmarkSnapshotScaling(t *testing.B, newSnapshots int) { defer cleanup() snID := restic.TestParseID("51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02") - sn2, err := restic.LoadSnapshot(context.TODO(), repo, snID) + sn2, err := data.LoadSnapshot(context.TODO(), repo, snID) if err != nil { t.Fatal(err) } @@ -645,13 +646,13 @@ func benchmarkSnapshotScaling(t *testing.B, newSnapshots int) { treeID := sn2.Tree for i := 0; i < newSnapshots; i++ { - sn, err := restic.NewSnapshot([]string{"test" + strconv.Itoa(i)}, nil, "", time.Now()) + sn, err := data.NewSnapshot([]string{"test" + strconv.Itoa(i)}, nil, "", time.Now()) if err != nil { t.Fatal(err) } sn.Tree = treeID - _, err = restic.SaveSnapshot(context.TODO(), repo, sn) + _, err = data.SaveSnapshot(context.TODO(), repo, sn) if err != nil { t.Fatal(err) } diff --git a/internal/restic/duration.go b/internal/data/duration.go similarity index 99% rename from internal/restic/duration.go rename to internal/data/duration.go index 6ab4c3c79..9762b70e6 100644 --- a/internal/restic/duration.go +++ b/internal/data/duration.go @@ -1,4 +1,4 @@ -package restic +package data import ( "fmt" diff --git a/internal/restic/duration_test.go b/internal/data/duration_test.go similarity index 99% rename from internal/restic/duration_test.go rename to internal/data/duration_test.go index c5f0390db..bd421bd91 100644 --- a/internal/restic/duration_test.go +++ b/internal/data/duration_test.go @@ -1,4 +1,4 @@ -package restic +package data import ( "testing" diff --git a/internal/restic/find.go b/internal/data/find.go similarity index 68% rename from internal/restic/find.go rename to internal/data/find.go index 135cc9ad5..14d64670e 100644 --- a/internal/restic/find.go +++ b/internal/data/find.go @@ -1,23 +1,24 @@ -package restic +package data import ( "context" "sync" + "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" "golang.org/x/sync/errgroup" ) // FindUsedBlobs traverses the tree ID and adds all seen blobs (trees and data // blobs) to the set blobs. Already seen tree blobs will not be visited again. -func FindUsedBlobs(ctx context.Context, repo Loader, treeIDs IDs, blobs FindBlobSet, p *progress.Counter) error { +func FindUsedBlobs(ctx context.Context, repo restic.Loader, treeIDs restic.IDs, blobs restic.FindBlobSet, p *progress.Counter) error { var lock sync.Mutex wg, ctx := errgroup.WithContext(ctx) - treeStream := StreamTrees(ctx, wg, repo, treeIDs, func(treeID ID) bool { + treeStream := StreamTrees(ctx, wg, repo, treeIDs, func(treeID restic.ID) bool { // locking is necessary the goroutine below concurrently adds data blobs lock.Lock() - h := BlobHandle{ID: treeID, Type: TreeBlob} + h := restic.BlobHandle{ID: treeID, Type: restic.TreeBlob} blobReferenced := blobs.Has(h) // noop if already referenced blobs.Insert(h) @@ -36,7 +37,7 @@ func FindUsedBlobs(ctx context.Context, repo Loader, treeIDs IDs, blobs FindBlob switch node.Type { case NodeTypeFile: for _, blob := range node.Content { - blobs.Insert(BlobHandle{ID: blob, Type: DataBlob}) + blobs.Insert(restic.BlobHandle{ID: blob, Type: restic.DataBlob}) } } } diff --git a/internal/restic/find_test.go b/internal/data/find_test.go similarity index 84% rename from internal/restic/find_test.go rename to internal/data/find_test.go index 9b8315ad4..466e553b4 100644 --- a/internal/restic/find_test.go +++ b/internal/data/find_test.go @@ -1,4 +1,4 @@ -package restic_test +package data_test import ( "bufio" @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -86,9 +87,9 @@ var findTestTime = time.Unix(1469960361, 23) func TestFindUsedBlobs(t *testing.T) { repo := repository.TestRepository(t) - var snapshots []*restic.Snapshot + var snapshots []*data.Snapshot for i := 0; i < findTestSnapshots; i++ { - sn := restic.TestCreateSnapshot(t, repo, findTestTime.Add(time.Duration(i)*time.Second), findTestDepth) + sn := data.TestCreateSnapshot(t, repo, findTestTime.Add(time.Duration(i)*time.Second), findTestDepth) t.Logf("snapshot %v saved, tree %v", sn.ID().Str(), sn.Tree.Str()) snapshots = append(snapshots, sn) } @@ -98,7 +99,7 @@ func TestFindUsedBlobs(t *testing.T) { for i, sn := range snapshots { usedBlobs := restic.NewBlobSet() - err := restic.FindUsedBlobs(context.TODO(), repo, restic.IDs{*sn.Tree}, usedBlobs, p) + err := data.FindUsedBlobs(context.TODO(), repo, restic.IDs{*sn.Tree}, usedBlobs, p) if err != nil { t.Errorf("FindUsedBlobs returned error: %v", err) continue @@ -131,7 +132,7 @@ func TestMultiFindUsedBlobs(t *testing.T) { var snapshotTrees restic.IDs for i := 0; i < findTestSnapshots; i++ { - sn := restic.TestCreateSnapshot(t, repo, findTestTime.Add(time.Duration(i)*time.Second), findTestDepth) + sn := data.TestCreateSnapshot(t, repo, findTestTime.Add(time.Duration(i)*time.Second), findTestDepth) t.Logf("snapshot %v saved, tree %v", sn.ID().Str(), sn.Tree.Str()) snapshotTrees = append(snapshotTrees, *sn.Tree) } @@ -148,7 +149,7 @@ func TestMultiFindUsedBlobs(t *testing.T) { // run twice to check progress bar handling of duplicate tree roots usedBlobs := restic.NewBlobSet() for i := 1; i < 3; i++ { - err := restic.FindUsedBlobs(context.TODO(), repo, snapshotTrees, usedBlobs, p) + err := data.FindUsedBlobs(context.TODO(), repo, snapshotTrees, usedBlobs, p) test.OK(t, err) v, _ := p.Get() test.Equals(t, v, uint64(i*len(snapshotTrees))) @@ -177,16 +178,16 @@ func (r ForbiddenRepo) Connections() uint { func TestFindUsedBlobsSkipsSeenBlobs(t *testing.T) { repo := repository.TestRepository(t) - snapshot := restic.TestCreateSnapshot(t, repo, findTestTime, findTestDepth) + snapshot := data.TestCreateSnapshot(t, repo, findTestTime, findTestDepth) t.Logf("snapshot %v saved, tree %v", snapshot.ID().Str(), snapshot.Tree.Str()) usedBlobs := restic.NewBlobSet() - err := restic.FindUsedBlobs(context.TODO(), repo, restic.IDs{*snapshot.Tree}, usedBlobs, nil) + err := data.FindUsedBlobs(context.TODO(), repo, restic.IDs{*snapshot.Tree}, usedBlobs, nil) if err != nil { t.Fatalf("FindUsedBlobs returned error: %v", err) } - err = restic.FindUsedBlobs(context.TODO(), ForbiddenRepo{}, restic.IDs{*snapshot.Tree}, usedBlobs, nil) + err = data.FindUsedBlobs(context.TODO(), ForbiddenRepo{}, restic.IDs{*snapshot.Tree}, usedBlobs, nil) if err != nil { t.Fatalf("FindUsedBlobs returned error: %v", err) } @@ -195,13 +196,13 @@ func TestFindUsedBlobsSkipsSeenBlobs(t *testing.T) { func BenchmarkFindUsedBlobs(b *testing.B) { repo := repository.TestRepository(b) - sn := restic.TestCreateSnapshot(b, repo, findTestTime, findTestDepth) + sn := data.TestCreateSnapshot(b, repo, findTestTime, findTestDepth) b.ResetTimer() for i := 0; i < b.N; i++ { blobs := restic.NewBlobSet() - err := restic.FindUsedBlobs(context.TODO(), repo, restic.IDs{*sn.Tree}, blobs, nil) + err := data.FindUsedBlobs(context.TODO(), repo, restic.IDs{*sn.Tree}, blobs, nil) if err != nil { b.Error(err) } diff --git a/internal/restic/node.go b/internal/data/node.go similarity index 98% rename from internal/restic/node.go rename to internal/data/node.go index 640368768..f97224364 100644 --- a/internal/restic/node.go +++ b/internal/data/node.go @@ -1,4 +1,4 @@ -package restic +package data import ( "bytes" @@ -13,6 +13,7 @@ import ( "unicode/utf8" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/debug" ) @@ -105,8 +106,8 @@ type Node struct { ExtendedAttributes []ExtendedAttribute `json:"extended_attributes,omitempty"` GenericAttributes map[GenericAttributeType]json.RawMessage `json:"generic_attributes,omitempty"` Device uint64 `json:"device,omitempty"` // in case of Type == "dev", stat.st_rdev - Content IDs `json:"content"` - Subtree *ID `json:"subtree,omitempty"` + Content restic.IDs `json:"content"` + Subtree *restic.ID `json:"subtree,omitempty"` Error string `json:"error,omitempty"` diff --git a/internal/restic/node_test.go b/internal/data/node_test.go similarity index 99% rename from internal/restic/node_test.go rename to internal/data/node_test.go index b544b2afd..f1b651efe 100644 --- a/internal/restic/node_test.go +++ b/internal/data/node_test.go @@ -1,4 +1,4 @@ -package restic +package data import ( "encoding/json" diff --git a/internal/restic/node_windows.go b/internal/data/node_windows.go similarity index 98% rename from internal/restic/node_windows.go rename to internal/data/node_windows.go index 09316f725..5270d7f48 100644 --- a/internal/restic/node_windows.go +++ b/internal/data/node_windows.go @@ -1,4 +1,4 @@ -package restic +package data import ( "encoding/json" diff --git a/internal/restic/snapshot.go b/internal/data/snapshot.go similarity index 80% rename from internal/restic/snapshot.go rename to internal/data/snapshot.go index f9cdf4daf..39368dba2 100644 --- a/internal/restic/snapshot.go +++ b/internal/data/snapshot.go @@ -1,4 +1,4 @@ -package restic +package data import ( "context" @@ -9,26 +9,27 @@ import ( "time" "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/restic" ) // Snapshot is the state of a resource at one point in time. type Snapshot struct { - Time time.Time `json:"time"` - Parent *ID `json:"parent,omitempty"` - Tree *ID `json:"tree"` - Paths []string `json:"paths"` - Hostname string `json:"hostname,omitempty"` - Username string `json:"username,omitempty"` - UID uint32 `json:"uid,omitempty"` - GID uint32 `json:"gid,omitempty"` - Excludes []string `json:"excludes,omitempty"` - Tags []string `json:"tags,omitempty"` - Original *ID `json:"original,omitempty"` + Time time.Time `json:"time"` + Parent *restic.ID `json:"parent,omitempty"` + Tree *restic.ID `json:"tree"` + Paths []string `json:"paths"` + Hostname string `json:"hostname,omitempty"` + Username string `json:"username,omitempty"` + UID uint32 `json:"uid,omitempty"` + GID uint32 `json:"gid,omitempty"` + Excludes []string `json:"excludes,omitempty"` + Tags []string `json:"tags,omitempty"` + Original *restic.ID `json:"original,omitempty"` ProgramVersion string `json:"program_version,omitempty"` Summary *SnapshotSummary `json:"summary,omitempty"` - id *ID // plaintext ID, used during restore + id *restic.ID // plaintext ID, used during restore } type SnapshotSummary struct { @@ -79,9 +80,9 @@ func NewSnapshot(paths []string, tags []string, hostname string, time time.Time) } // LoadSnapshot loads the snapshot with the id and returns it. -func LoadSnapshot(ctx context.Context, loader LoaderUnpacked, id ID) (*Snapshot, error) { +func LoadSnapshot(ctx context.Context, loader restic.LoaderUnpacked, id restic.ID) (*Snapshot, error) { sn := &Snapshot{id: &id} - err := LoadJSONUnpacked(ctx, loader, SnapshotFile, id, sn) + err := restic.LoadJSONUnpacked(ctx, loader, restic.SnapshotFile, id, sn) if err != nil { return nil, fmt.Errorf("failed to load snapshot %v: %w", id.Str(), err) } @@ -90,8 +91,8 @@ func LoadSnapshot(ctx context.Context, loader LoaderUnpacked, id ID) (*Snapshot, } // SaveSnapshot saves the snapshot sn and returns its ID. -func SaveSnapshot(ctx context.Context, repo SaverUnpacked[WriteableFileType], sn *Snapshot) (ID, error) { - return SaveJSONUnpacked(ctx, repo, WriteableSnapshotFile, sn) +func SaveSnapshot(ctx context.Context, repo restic.SaverUnpacked[restic.WriteableFileType], sn *Snapshot) (restic.ID, error) { + return restic.SaveJSONUnpacked(ctx, repo, restic.WriteableSnapshotFile, sn) } // ForAllSnapshots reads all snapshots in parallel and calls the @@ -99,11 +100,11 @@ func SaveSnapshot(ctx context.Context, repo SaverUnpacked[WriteableFileType], sn // If the called function returns an error, this function is cancelled and // also returns this error. // If a snapshot ID is in excludeIDs, it will be ignored. -func ForAllSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked, excludeIDs IDSet, fn func(ID, *Snapshot, error) error) error { +func ForAllSnapshots(ctx context.Context, be restic.Lister, loader restic.LoaderUnpacked, excludeIDs restic.IDSet, fn func(restic.ID, *Snapshot, error) error) error { var m sync.Mutex // For most snapshots decoding is nearly for free, thus just assume were only limited by IO - return ParallelList(ctx, be, SnapshotFile, loader.Connections(), func(ctx context.Context, id ID, _ int64) error { + return restic.ParallelList(ctx, be, restic.SnapshotFile, loader.Connections(), func(ctx context.Context, id restic.ID, _ int64) error { if excludeIDs.Has(id) { return nil } @@ -121,7 +122,7 @@ func (sn Snapshot) String() string { } // ID returns the snapshot's ID. -func (sn Snapshot) ID() *ID { +func (sn Snapshot) ID() *restic.ID { return sn.id } @@ -133,7 +134,7 @@ func (sn *Snapshot) fillUserInfo() error { sn.Username = usr.Username // set userid and groupid - sn.UID, sn.GID, err = uidGidInt(usr) + sn.UID, sn.GID, err = restic.UidGidInt(usr) return err } diff --git a/internal/restic/snapshot_find.go b/internal/data/snapshot_find.go similarity index 82% rename from internal/restic/snapshot_find.go rename to internal/data/snapshot_find.go index 6eb51b237..a0fdb8074 100644 --- a/internal/restic/snapshot_find.go +++ b/internal/data/snapshot_find.go @@ -1,4 +1,4 @@ -package restic +package data import ( "context" @@ -8,6 +8,7 @@ import ( "time" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" ) // ErrNoSnapshotFound is returned when no snapshot for the given criteria could be found. @@ -34,7 +35,7 @@ func (f *SnapshotFilter) matches(sn *Snapshot) bool { // findLatest finds the latest snapshot with optional target/directory, // tags, hostname, and timestamp filters. -func (f *SnapshotFilter) findLatest(ctx context.Context, be Lister, loader LoaderUnpacked) (*Snapshot, error) { +func (f *SnapshotFilter) findLatest(ctx context.Context, be restic.Lister, loader restic.LoaderUnpacked) (*Snapshot, error) { var err error absTargets := make([]string, 0, len(f.Paths)) @@ -51,7 +52,7 @@ func (f *SnapshotFilter) findLatest(ctx context.Context, be Lister, loader Loade var latest *Snapshot - err = ForAllSnapshots(ctx, be, loader, nil, func(id ID, snapshot *Snapshot, err error) error { + err = ForAllSnapshots(ctx, be, loader, nil, func(id restic.ID, snapshot *Snapshot, err error) error { if err != nil { return errors.Errorf("Error loading snapshot %v: %v", id.Str(), err) } @@ -90,14 +91,14 @@ func splitSnapshotID(s string) (id, subfolder string) { // FindSnapshot takes a string and tries to find a snapshot whose ID matches // the string as closely as possible. -func FindSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, s string) (*Snapshot, string, error) { +func FindSnapshot(ctx context.Context, be restic.Lister, loader restic.LoaderUnpacked, s string) (*Snapshot, string, error) { s, subfolder := splitSnapshotID(s) // no need to list snapshots if `s` is already a full id - id, err := ParseID(s) + id, err := restic.ParseID(s) if err != nil { // find snapshot id with prefix - id, err = Find(ctx, be, SnapshotFile, s) + id, err = restic.Find(ctx, be, restic.SnapshotFile, s) if err != nil { return nil, "", err } @@ -108,7 +109,7 @@ func FindSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, s strin // FindLatest returns either the latest of a filtered list of all snapshots // or a snapshot specified by `snapshotID`. -func (f *SnapshotFilter) FindLatest(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotID string) (*Snapshot, string, error) { +func (f *SnapshotFilter) FindLatest(ctx context.Context, be restic.Lister, loader restic.LoaderUnpacked, snapshotID string) (*Snapshot, string, error) { id, subfolder := splitSnapshotID(snapshotID) if id == "latest" { sn, err := f.findLatest(ctx, be, loader) @@ -126,12 +127,12 @@ type SnapshotFindCb func(string, *Snapshot, error) error var ErrInvalidSnapshotSyntax = errors.New(": syntax not allowed") // FindAll yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots. -func (f *SnapshotFilter) FindAll(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotIDs []string, fn SnapshotFindCb) error { +func (f *SnapshotFilter) FindAll(ctx context.Context, be restic.Lister, loader restic.LoaderUnpacked, snapshotIDs []string, fn SnapshotFindCb) error { if len(snapshotIDs) != 0 { var err error usedFilter := false - ids := NewIDSet() + ids := restic.NewIDSet() // Process all snapshot IDs given as arguments. for _, s := range snapshotIDs { if ctx.Err() != nil { @@ -183,7 +184,7 @@ func (f *SnapshotFilter) FindAll(ctx context.Context, be Lister, loader LoaderUn return nil } - return ForAllSnapshots(ctx, be, loader, nil, func(id ID, sn *Snapshot, err error) error { + return ForAllSnapshots(ctx, be, loader, nil, func(id restic.ID, sn *Snapshot, err error) error { if err == nil && !f.matches(sn) { return nil } diff --git a/internal/restic/snapshot_find_test.go b/internal/data/snapshot_find_test.go similarity index 61% rename from internal/restic/snapshot_find_test.go rename to internal/data/snapshot_find_test.go index 84bffd694..eec20b2ab 100644 --- a/internal/restic/snapshot_find_test.go +++ b/internal/data/snapshot_find_test.go @@ -1,21 +1,21 @@ -package restic_test +package data_test import ( "context" "testing" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/repository" - "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) func TestFindLatestSnapshot(t *testing.T) { repo := repository.TestRepository(t) - restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1) - restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1) - latestSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1) + data.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1) + data.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1) + latestSnapshot := data.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1) - f := restic.SnapshotFilter{Hosts: []string{"foo"}} + f := data.SnapshotFilter{Hosts: []string{"foo"}} sn, _, err := f.FindLatest(context.TODO(), repo, repo, "latest") if err != nil { t.Fatalf("FindLatest returned error: %v", err) @@ -28,11 +28,11 @@ func TestFindLatestSnapshot(t *testing.T) { func TestFindLatestSnapshotWithMaxTimestamp(t *testing.T) { repo := repository.TestRepository(t) - restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1) - desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1) - restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1) + data.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1) + desiredSnapshot := data.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1) + data.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1) - sn, _, err := (&restic.SnapshotFilter{ + sn, _, err := (&data.SnapshotFilter{ Hosts: []string{"foo"}, TimestampLimit: parseTimeUTC("2018-08-08 08:08:08"), }).FindLatest(context.TODO(), repo, repo, "latest") @@ -47,8 +47,8 @@ func TestFindLatestSnapshotWithMaxTimestamp(t *testing.T) { func TestFindLatestWithSubpath(t *testing.T) { repo := repository.TestRepository(t) - restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1) - desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1) + data.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1) + desiredSnapshot := data.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1) for _, exp := range []struct { query string @@ -62,7 +62,7 @@ func TestFindLatestWithSubpath(t *testing.T) { {desiredSnapshot.ID().String() + ":subfolder", "subfolder"}, } { t.Run("", func(t *testing.T) { - sn, subfolder, err := (&restic.SnapshotFilter{}).FindLatest(context.TODO(), repo, repo, exp.query) + sn, subfolder, err := (&data.SnapshotFilter{}).FindLatest(context.TODO(), repo, repo, exp.query) if err != nil { t.Fatalf("FindLatest returned error: %v", err) } @@ -75,13 +75,13 @@ func TestFindLatestWithSubpath(t *testing.T) { func TestFindAllSubpathError(t *testing.T) { repo := repository.TestRepository(t) - desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1) + desiredSnapshot := data.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1) count := 0 - test.OK(t, (&restic.SnapshotFilter{}).FindAll(context.TODO(), repo, repo, + test.OK(t, (&data.SnapshotFilter{}).FindAll(context.TODO(), repo, repo, []string{"latest:subfolder", desiredSnapshot.ID().Str() + ":subfolder"}, - func(id string, sn *restic.Snapshot, err error) error { - if err == restic.ErrInvalidSnapshotSyntax { + func(id string, sn *data.Snapshot, err error) error { + if err == data.ErrInvalidSnapshotSyntax { count++ return nil } diff --git a/internal/restic/snapshot_group.go b/internal/data/snapshot_group.go similarity index 99% rename from internal/restic/snapshot_group.go rename to internal/data/snapshot_group.go index f4e1ed384..ca4b0aabc 100644 --- a/internal/restic/snapshot_group.go +++ b/internal/data/snapshot_group.go @@ -1,4 +1,4 @@ -package restic +package data import ( "encoding/json" diff --git a/internal/restic/snapshot_group_test.go b/internal/data/snapshot_group_test.go similarity index 65% rename from internal/restic/snapshot_group_test.go rename to internal/data/snapshot_group_test.go index f9d6ff460..3cbbc7862 100644 --- a/internal/restic/snapshot_group_test.go +++ b/internal/data/snapshot_group_test.go @@ -1,41 +1,41 @@ -package restic_test +package data_test import ( "testing" "github.com/google/go-cmp/cmp" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/test" ) func TestGroupByOptions(t *testing.T) { for _, exp := range []struct { from string - opts restic.SnapshotGroupByOptions + opts data.SnapshotGroupByOptions normalized string }{ { from: "", - opts: restic.SnapshotGroupByOptions{}, + opts: data.SnapshotGroupByOptions{}, normalized: "", }, { from: "host,paths", - opts: restic.SnapshotGroupByOptions{Host: true, Path: true}, + opts: data.SnapshotGroupByOptions{Host: true, Path: true}, normalized: "host,paths", }, { from: "host,path,tag", - opts: restic.SnapshotGroupByOptions{Host: true, Path: true, Tag: true}, + opts: data.SnapshotGroupByOptions{Host: true, Path: true, Tag: true}, normalized: "host,paths,tags", }, { from: "hosts,paths,tags", - opts: restic.SnapshotGroupByOptions{Host: true, Path: true, Tag: true}, + opts: data.SnapshotGroupByOptions{Host: true, Path: true, Tag: true}, normalized: "host,paths,tags", }, } { - var opts restic.SnapshotGroupByOptions + var opts data.SnapshotGroupByOptions test.OK(t, opts.Set(exp.from)) if !cmp.Equal(opts, exp.opts) { t.Errorf("unexpected opts %s", cmp.Diff(opts, exp.opts)) @@ -43,7 +43,7 @@ func TestGroupByOptions(t *testing.T) { test.Equals(t, opts.String(), exp.normalized) } - var opts restic.SnapshotGroupByOptions + var opts data.SnapshotGroupByOptions err := opts.Set("tags,invalid") test.Assert(t, err != nil, "missing error on invalid tags") test.Assert(t, !opts.Host && !opts.Path && !opts.Tag, "unexpected opts %s %s %s", opts.Host, opts.Path, opts.Tag) diff --git a/internal/restic/snapshot_policy.go b/internal/data/snapshot_policy.go similarity index 99% rename from internal/restic/snapshot_policy.go rename to internal/data/snapshot_policy.go index 28f871f4a..1ee5af984 100644 --- a/internal/restic/snapshot_policy.go +++ b/internal/data/snapshot_policy.go @@ -1,4 +1,4 @@ -package restic +package data import ( "fmt" diff --git a/internal/restic/snapshot_policy_test.go b/internal/data/snapshot_policy_test.go similarity index 82% rename from internal/restic/snapshot_policy_test.go rename to internal/data/snapshot_policy_test.go index 75f0f18f4..7cb943361 100644 --- a/internal/restic/snapshot_policy_test.go +++ b/internal/data/snapshot_policy_test.go @@ -1,4 +1,4 @@ -package restic_test +package data_test import ( "encoding/json" @@ -10,7 +10,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/data" ) func parseTimeUTC(s string) time.Time { @@ -24,7 +24,7 @@ func parseTimeUTC(s string) time.Time { // Returns the maximum number of snapshots to be kept according to this policy. // If any of the counts is -1 it will return 0. -func policySum(e *restic.ExpirePolicy) int { +func policySum(e *data.ExpirePolicy) int { if e.Last == -1 || e.Hourly == -1 || e.Daily == -1 || e.Weekly == -1 || e.Monthly == -1 || e.Yearly == -1 { return 0 } @@ -36,11 +36,11 @@ func TestExpireSnapshotOps(t *testing.T) { data := []struct { expectEmpty bool expectSum int - p *restic.ExpirePolicy + p *data.ExpirePolicy }{ - {true, 0, &restic.ExpirePolicy{}}, - {true, 0, &restic.ExpirePolicy{Tags: []restic.TagList{}}}, - {false, 22, &restic.ExpirePolicy{Daily: 7, Weekly: 2, Monthly: 3, Yearly: 10}}, + {true, 0, &data.ExpirePolicy{}}, + {true, 0, &data.ExpirePolicy{Tags: []data.TagList{}}}, + {false, 22, &data.ExpirePolicy{Daily: 7, Weekly: 2, Monthly: 3, Yearly: 10}}, } for i, d := range data { isEmpty := d.p.Empty() @@ -57,8 +57,8 @@ func TestExpireSnapshotOps(t *testing.T) { // ApplyPolicyResult is used to marshal/unmarshal the golden files for // TestApplyPolicy. type ApplyPolicyResult struct { - Keep restic.Snapshots `json:"keep"` - Reasons []restic.KeepReason `json:"reasons,omitempty"` + Keep data.Snapshots `json:"keep"` + Reasons []data.KeepReason `json:"reasons,omitempty"` } func loadGoldenFile(t testing.TB, filename string) (res ApplyPolicyResult) { @@ -75,7 +75,7 @@ func loadGoldenFile(t testing.TB, filename string) (res ApplyPolicyResult) { return res } -func saveGoldenFile(t testing.TB, filename string, keep restic.Snapshots, reasons []restic.KeepReason) { +func saveGoldenFile(t testing.TB, filename string, keep data.Snapshots, reasons []data.KeepReason) { res := ApplyPolicyResult{ Keep: keep, Reasons: reasons, @@ -92,7 +92,7 @@ func saveGoldenFile(t testing.TB, filename string, keep restic.Snapshots, reason } func TestApplyPolicy(t *testing.T) { - var testExpireSnapshots = restic.Snapshots{ + var testExpireSnapshots = data.Snapshots{ {Time: parseTimeUTC("2014-09-01 10:20:30")}, {Time: parseTimeUTC("2014-09-02 10:20:30")}, {Time: parseTimeUTC("2014-09-05 10:20:30")}, @@ -198,7 +198,7 @@ func TestApplyPolicy(t *testing.T) { {Time: parseTimeUTC("2016-01-18 12:02:03")}, } - var tests = []restic.ExpirePolicy{ + var tests = []data.ExpirePolicy{ {}, {Last: 10}, {Last: 15}, @@ -217,29 +217,29 @@ func TestApplyPolicy(t *testing.T) { {Daily: 2, Weekly: 2, Monthly: 6}, {Yearly: 10}, {Daily: 7, Weekly: 2, Monthly: 3, Yearly: 10}, - {Tags: []restic.TagList{{"foo"}}}, - {Tags: []restic.TagList{{"foo", "bar"}}}, - {Tags: []restic.TagList{{"foo"}, {"bar"}}}, - {Within: restic.ParseDurationOrPanic("1d")}, - {Within: restic.ParseDurationOrPanic("2d")}, - {Within: restic.ParseDurationOrPanic("7d")}, - {Within: restic.ParseDurationOrPanic("1m")}, - {Within: restic.ParseDurationOrPanic("1m14d")}, - {Within: restic.ParseDurationOrPanic("1y1d1m")}, - {Within: restic.ParseDurationOrPanic("13d23h")}, - {Within: restic.ParseDurationOrPanic("2m2h")}, - {Within: restic.ParseDurationOrPanic("1y2m3d3h")}, - {WithinHourly: restic.ParseDurationOrPanic("1y2m3d3h")}, - {WithinDaily: restic.ParseDurationOrPanic("1y2m3d3h")}, - {WithinWeekly: restic.ParseDurationOrPanic("1y2m3d3h")}, - {WithinMonthly: restic.ParseDurationOrPanic("1y2m3d3h")}, - {WithinYearly: restic.ParseDurationOrPanic("1y2m3d3h")}, - {Within: restic.ParseDurationOrPanic("1h"), - WithinHourly: restic.ParseDurationOrPanic("1d"), - WithinDaily: restic.ParseDurationOrPanic("7d"), - WithinWeekly: restic.ParseDurationOrPanic("1m"), - WithinMonthly: restic.ParseDurationOrPanic("1y"), - WithinYearly: restic.ParseDurationOrPanic("9999y")}, + {Tags: []data.TagList{{"foo"}}}, + {Tags: []data.TagList{{"foo", "bar"}}}, + {Tags: []data.TagList{{"foo"}, {"bar"}}}, + {Within: data.ParseDurationOrPanic("1d")}, + {Within: data.ParseDurationOrPanic("2d")}, + {Within: data.ParseDurationOrPanic("7d")}, + {Within: data.ParseDurationOrPanic("1m")}, + {Within: data.ParseDurationOrPanic("1m14d")}, + {Within: data.ParseDurationOrPanic("1y1d1m")}, + {Within: data.ParseDurationOrPanic("13d23h")}, + {Within: data.ParseDurationOrPanic("2m2h")}, + {Within: data.ParseDurationOrPanic("1y2m3d3h")}, + {WithinHourly: data.ParseDurationOrPanic("1y2m3d3h")}, + {WithinDaily: data.ParseDurationOrPanic("1y2m3d3h")}, + {WithinWeekly: data.ParseDurationOrPanic("1y2m3d3h")}, + {WithinMonthly: data.ParseDurationOrPanic("1y2m3d3h")}, + {WithinYearly: data.ParseDurationOrPanic("1y2m3d3h")}, + {Within: data.ParseDurationOrPanic("1h"), + WithinHourly: data.ParseDurationOrPanic("1d"), + WithinDaily: data.ParseDurationOrPanic("7d"), + WithinWeekly: data.ParseDurationOrPanic("1m"), + WithinMonthly: data.ParseDurationOrPanic("1y"), + WithinYearly: data.ParseDurationOrPanic("9999y")}, {Last: -1}, // keep all {Last: -1, Hourly: -1}, // keep all (Last overrides Hourly) {Hourly: -1}, // keep all hourlies @@ -249,7 +249,7 @@ func TestApplyPolicy(t *testing.T) { for i, p := range tests { t.Run("", func(t *testing.T) { - keep, remove, reasons := restic.ApplyPolicy(testExpireSnapshots, p) + keep, remove, reasons := data.ApplyPolicy(testExpireSnapshots, p) if len(keep)+len(remove) != len(testExpireSnapshots) { t.Errorf("len(keep)+len(remove) = %d != len(testExpireSnapshots) = %d", @@ -273,7 +273,7 @@ func TestApplyPolicy(t *testing.T) { want := loadGoldenFile(t, goldenFilename) - cmpOpts := cmpopts.IgnoreUnexported(restic.Snapshot{}) + cmpOpts := cmpopts.IgnoreUnexported(data.Snapshot{}) if !cmp.Equal(want.Keep, keep, cmpOpts) { t.Error(cmp.Diff(want.Keep, keep, cmpOpts)) diff --git a/internal/restic/snapshot_test.go b/internal/data/snapshot_test.go similarity index 71% rename from internal/restic/snapshot_test.go rename to internal/data/snapshot_test.go index 68016287a..cf5a7aab5 100644 --- a/internal/restic/snapshot_test.go +++ b/internal/data/snapshot_test.go @@ -1,19 +1,19 @@ -package restic_test +package data_test import ( "context" "testing" "time" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/repository" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) func TestNewSnapshot(t *testing.T) { paths := []string{"/home/foobar"} - _, err := restic.NewSnapshot(paths, nil, "foo", time.Now()) + _, err := data.NewSnapshot(paths, nil, "foo", time.Now()) rtest.OK(t, err) } @@ -21,7 +21,7 @@ func TestTagList(t *testing.T) { paths := []string{"/home/foobar"} tags := []string{""} - sn, _ := restic.NewSnapshot(paths, nil, "foo", time.Now()) + sn, _ := data.NewSnapshot(paths, nil, "foo", time.Now()) r := sn.HasTags(tags) rtest.Assert(t, r, "Failed to match untagged snapshot") @@ -35,15 +35,15 @@ func testLoadJSONUnpacked(t *testing.T, version uint) { repo, _, _ := repository.TestRepositoryWithVersion(t, version) // archive a snapshot - sn := restic.Snapshot{} + sn := data.Snapshot{} sn.Hostname = "foobar" sn.Username = "test!" - id, err := restic.SaveSnapshot(context.TODO(), repo, &sn) + id, err := data.SaveSnapshot(context.TODO(), repo, &sn) rtest.OK(t, err) // restore - sn2, err := restic.LoadSnapshot(context.TODO(), repo, id) + sn2, err := data.LoadSnapshot(context.TODO(), repo, id) rtest.OK(t, err) rtest.Equals(t, sn.Hostname, sn2.Hostname) diff --git a/internal/restic/tag_list.go b/internal/data/tag_list.go similarity index 98% rename from internal/restic/tag_list.go rename to internal/data/tag_list.go index b293b14f7..b31072d94 100644 --- a/internal/restic/tag_list.go +++ b/internal/data/tag_list.go @@ -1,4 +1,4 @@ -package restic +package data import ( "fmt" diff --git a/internal/restic/tag_list_test.go b/internal/data/tag_list_test.go similarity index 97% rename from internal/restic/tag_list_test.go rename to internal/data/tag_list_test.go index d1cf6a9c3..1c406b8c9 100644 --- a/internal/restic/tag_list_test.go +++ b/internal/data/tag_list_test.go @@ -1,4 +1,4 @@ -package restic +package data import ( "testing" diff --git a/internal/restic/testdata/filter_snapshots_0 b/internal/data/testdata/filter_snapshots_0 similarity index 100% rename from internal/restic/testdata/filter_snapshots_0 rename to internal/data/testdata/filter_snapshots_0 diff --git a/internal/restic/testdata/filter_snapshots_1 b/internal/data/testdata/filter_snapshots_1 similarity index 100% rename from internal/restic/testdata/filter_snapshots_1 rename to internal/data/testdata/filter_snapshots_1 diff --git a/internal/restic/testdata/filter_snapshots_2 b/internal/data/testdata/filter_snapshots_2 similarity index 100% rename from internal/restic/testdata/filter_snapshots_2 rename to internal/data/testdata/filter_snapshots_2 diff --git a/internal/restic/testdata/filter_snapshots_3 b/internal/data/testdata/filter_snapshots_3 similarity index 100% rename from internal/restic/testdata/filter_snapshots_3 rename to internal/data/testdata/filter_snapshots_3 diff --git a/internal/restic/testdata/filter_snapshots_4 b/internal/data/testdata/filter_snapshots_4 similarity index 100% rename from internal/restic/testdata/filter_snapshots_4 rename to internal/data/testdata/filter_snapshots_4 diff --git a/internal/restic/testdata/filter_snapshots_5 b/internal/data/testdata/filter_snapshots_5 similarity index 100% rename from internal/restic/testdata/filter_snapshots_5 rename to internal/data/testdata/filter_snapshots_5 diff --git a/internal/restic/testdata/filter_snapshots_6 b/internal/data/testdata/filter_snapshots_6 similarity index 100% rename from internal/restic/testdata/filter_snapshots_6 rename to internal/data/testdata/filter_snapshots_6 diff --git a/internal/restic/testdata/filter_snapshots_7 b/internal/data/testdata/filter_snapshots_7 similarity index 100% rename from internal/restic/testdata/filter_snapshots_7 rename to internal/data/testdata/filter_snapshots_7 diff --git a/internal/restic/testdata/filter_snapshots_8 b/internal/data/testdata/filter_snapshots_8 similarity index 100% rename from internal/restic/testdata/filter_snapshots_8 rename to internal/data/testdata/filter_snapshots_8 diff --git a/internal/restic/testdata/filter_snapshots_9 b/internal/data/testdata/filter_snapshots_9 similarity index 100% rename from internal/restic/testdata/filter_snapshots_9 rename to internal/data/testdata/filter_snapshots_9 diff --git a/internal/restic/testdata/policy_keep_snapshots_0 b/internal/data/testdata/policy_keep_snapshots_0 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_0 rename to internal/data/testdata/policy_keep_snapshots_0 diff --git a/internal/restic/testdata/policy_keep_snapshots_1 b/internal/data/testdata/policy_keep_snapshots_1 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_1 rename to internal/data/testdata/policy_keep_snapshots_1 diff --git a/internal/restic/testdata/policy_keep_snapshots_10 b/internal/data/testdata/policy_keep_snapshots_10 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_10 rename to internal/data/testdata/policy_keep_snapshots_10 diff --git a/internal/restic/testdata/policy_keep_snapshots_11 b/internal/data/testdata/policy_keep_snapshots_11 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_11 rename to internal/data/testdata/policy_keep_snapshots_11 diff --git a/internal/restic/testdata/policy_keep_snapshots_12 b/internal/data/testdata/policy_keep_snapshots_12 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_12 rename to internal/data/testdata/policy_keep_snapshots_12 diff --git a/internal/restic/testdata/policy_keep_snapshots_13 b/internal/data/testdata/policy_keep_snapshots_13 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_13 rename to internal/data/testdata/policy_keep_snapshots_13 diff --git a/internal/restic/testdata/policy_keep_snapshots_14 b/internal/data/testdata/policy_keep_snapshots_14 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_14 rename to internal/data/testdata/policy_keep_snapshots_14 diff --git a/internal/restic/testdata/policy_keep_snapshots_15 b/internal/data/testdata/policy_keep_snapshots_15 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_15 rename to internal/data/testdata/policy_keep_snapshots_15 diff --git a/internal/restic/testdata/policy_keep_snapshots_16 b/internal/data/testdata/policy_keep_snapshots_16 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_16 rename to internal/data/testdata/policy_keep_snapshots_16 diff --git a/internal/restic/testdata/policy_keep_snapshots_17 b/internal/data/testdata/policy_keep_snapshots_17 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_17 rename to internal/data/testdata/policy_keep_snapshots_17 diff --git a/internal/restic/testdata/policy_keep_snapshots_18 b/internal/data/testdata/policy_keep_snapshots_18 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_18 rename to internal/data/testdata/policy_keep_snapshots_18 diff --git a/internal/restic/testdata/policy_keep_snapshots_19 b/internal/data/testdata/policy_keep_snapshots_19 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_19 rename to internal/data/testdata/policy_keep_snapshots_19 diff --git a/internal/restic/testdata/policy_keep_snapshots_2 b/internal/data/testdata/policy_keep_snapshots_2 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_2 rename to internal/data/testdata/policy_keep_snapshots_2 diff --git a/internal/restic/testdata/policy_keep_snapshots_20 b/internal/data/testdata/policy_keep_snapshots_20 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_20 rename to internal/data/testdata/policy_keep_snapshots_20 diff --git a/internal/restic/testdata/policy_keep_snapshots_21 b/internal/data/testdata/policy_keep_snapshots_21 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_21 rename to internal/data/testdata/policy_keep_snapshots_21 diff --git a/internal/restic/testdata/policy_keep_snapshots_22 b/internal/data/testdata/policy_keep_snapshots_22 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_22 rename to internal/data/testdata/policy_keep_snapshots_22 diff --git a/internal/restic/testdata/policy_keep_snapshots_23 b/internal/data/testdata/policy_keep_snapshots_23 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_23 rename to internal/data/testdata/policy_keep_snapshots_23 diff --git a/internal/restic/testdata/policy_keep_snapshots_24 b/internal/data/testdata/policy_keep_snapshots_24 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_24 rename to internal/data/testdata/policy_keep_snapshots_24 diff --git a/internal/restic/testdata/policy_keep_snapshots_25 b/internal/data/testdata/policy_keep_snapshots_25 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_25 rename to internal/data/testdata/policy_keep_snapshots_25 diff --git a/internal/restic/testdata/policy_keep_snapshots_26 b/internal/data/testdata/policy_keep_snapshots_26 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_26 rename to internal/data/testdata/policy_keep_snapshots_26 diff --git a/internal/restic/testdata/policy_keep_snapshots_27 b/internal/data/testdata/policy_keep_snapshots_27 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_27 rename to internal/data/testdata/policy_keep_snapshots_27 diff --git a/internal/restic/testdata/policy_keep_snapshots_28 b/internal/data/testdata/policy_keep_snapshots_28 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_28 rename to internal/data/testdata/policy_keep_snapshots_28 diff --git a/internal/restic/testdata/policy_keep_snapshots_29 b/internal/data/testdata/policy_keep_snapshots_29 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_29 rename to internal/data/testdata/policy_keep_snapshots_29 diff --git a/internal/restic/testdata/policy_keep_snapshots_3 b/internal/data/testdata/policy_keep_snapshots_3 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_3 rename to internal/data/testdata/policy_keep_snapshots_3 diff --git a/internal/restic/testdata/policy_keep_snapshots_30 b/internal/data/testdata/policy_keep_snapshots_30 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_30 rename to internal/data/testdata/policy_keep_snapshots_30 diff --git a/internal/restic/testdata/policy_keep_snapshots_31 b/internal/data/testdata/policy_keep_snapshots_31 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_31 rename to internal/data/testdata/policy_keep_snapshots_31 diff --git a/internal/restic/testdata/policy_keep_snapshots_32 b/internal/data/testdata/policy_keep_snapshots_32 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_32 rename to internal/data/testdata/policy_keep_snapshots_32 diff --git a/internal/restic/testdata/policy_keep_snapshots_33 b/internal/data/testdata/policy_keep_snapshots_33 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_33 rename to internal/data/testdata/policy_keep_snapshots_33 diff --git a/internal/restic/testdata/policy_keep_snapshots_34 b/internal/data/testdata/policy_keep_snapshots_34 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_34 rename to internal/data/testdata/policy_keep_snapshots_34 diff --git a/internal/restic/testdata/policy_keep_snapshots_35 b/internal/data/testdata/policy_keep_snapshots_35 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_35 rename to internal/data/testdata/policy_keep_snapshots_35 diff --git a/internal/restic/testdata/policy_keep_snapshots_36 b/internal/data/testdata/policy_keep_snapshots_36 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_36 rename to internal/data/testdata/policy_keep_snapshots_36 diff --git a/internal/restic/testdata/policy_keep_snapshots_37 b/internal/data/testdata/policy_keep_snapshots_37 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_37 rename to internal/data/testdata/policy_keep_snapshots_37 diff --git a/internal/restic/testdata/policy_keep_snapshots_38 b/internal/data/testdata/policy_keep_snapshots_38 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_38 rename to internal/data/testdata/policy_keep_snapshots_38 diff --git a/internal/restic/testdata/policy_keep_snapshots_39 b/internal/data/testdata/policy_keep_snapshots_39 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_39 rename to internal/data/testdata/policy_keep_snapshots_39 diff --git a/internal/restic/testdata/policy_keep_snapshots_4 b/internal/data/testdata/policy_keep_snapshots_4 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_4 rename to internal/data/testdata/policy_keep_snapshots_4 diff --git a/internal/restic/testdata/policy_keep_snapshots_5 b/internal/data/testdata/policy_keep_snapshots_5 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_5 rename to internal/data/testdata/policy_keep_snapshots_5 diff --git a/internal/restic/testdata/policy_keep_snapshots_6 b/internal/data/testdata/policy_keep_snapshots_6 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_6 rename to internal/data/testdata/policy_keep_snapshots_6 diff --git a/internal/restic/testdata/policy_keep_snapshots_7 b/internal/data/testdata/policy_keep_snapshots_7 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_7 rename to internal/data/testdata/policy_keep_snapshots_7 diff --git a/internal/restic/testdata/policy_keep_snapshots_8 b/internal/data/testdata/policy_keep_snapshots_8 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_8 rename to internal/data/testdata/policy_keep_snapshots_8 diff --git a/internal/restic/testdata/policy_keep_snapshots_9 b/internal/data/testdata/policy_keep_snapshots_9 similarity index 100% rename from internal/restic/testdata/policy_keep_snapshots_9 rename to internal/data/testdata/policy_keep_snapshots_9 diff --git a/internal/restic/testdata/used_blobs_snapshot0 b/internal/data/testdata/used_blobs_snapshot0 similarity index 100% rename from internal/restic/testdata/used_blobs_snapshot0 rename to internal/data/testdata/used_blobs_snapshot0 diff --git a/internal/restic/testdata/used_blobs_snapshot1 b/internal/data/testdata/used_blobs_snapshot1 similarity index 100% rename from internal/restic/testdata/used_blobs_snapshot1 rename to internal/data/testdata/used_blobs_snapshot1 diff --git a/internal/restic/testdata/used_blobs_snapshot2 b/internal/data/testdata/used_blobs_snapshot2 similarity index 100% rename from internal/restic/testdata/used_blobs_snapshot2 rename to internal/data/testdata/used_blobs_snapshot2 diff --git a/internal/data/testing.go b/internal/data/testing.go new file mode 100644 index 000000000..0c5fdb6d7 --- /dev/null +++ b/internal/data/testing.go @@ -0,0 +1,194 @@ +package data + +import ( + "context" + "fmt" + "io" + "math/rand" + "testing" + "time" + + "github.com/restic/chunker" + "github.com/restic/restic/internal/restic" + "golang.org/x/sync/errgroup" +) + +// fakeFile returns a reader which yields deterministic pseudo-random data. +func fakeFile(seed, size int64) io.Reader { + return io.LimitReader(rand.New(rand.NewSource(seed)), size) +} + +type fakeFileSystem struct { + t testing.TB + repo restic.Repository + buf []byte + chunker *chunker.Chunker + rand *rand.Rand +} + +// saveFile reads from rd and saves the blobs in the repository. The list of +// IDs is returned. +func (fs *fakeFileSystem) saveFile(ctx context.Context, rd io.Reader) (blobs restic.IDs) { + if fs.buf == nil { + fs.buf = make([]byte, chunker.MaxSize) + } + + if fs.chunker == nil { + fs.chunker = chunker.New(rd, fs.repo.Config().ChunkerPolynomial) + } else { + fs.chunker.Reset(rd, fs.repo.Config().ChunkerPolynomial) + } + + blobs = restic.IDs{} + for { + chunk, err := fs.chunker.Next(fs.buf) + if err == io.EOF { + break + } + + if err != nil { + fs.t.Fatalf("unable to save chunk in repo: %v", err) + } + + id, _, _, err := fs.repo.SaveBlob(ctx, restic.DataBlob, chunk.Data, restic.ID{}, false) + if err != nil { + fs.t.Fatalf("error saving chunk: %v", err) + } + + blobs = append(blobs, id) + } + + return blobs +} + +const ( + maxFileSize = 20000 + maxSeed = 32 + maxNodes = 15 +) + +// saveTree saves a tree of fake files in the repo and returns the ID. +func (fs *fakeFileSystem) saveTree(ctx context.Context, seed int64, depth int) restic.ID { + rnd := rand.NewSource(seed) + numNodes := int(rnd.Int63() % maxNodes) + + var tree Tree + for i := 0; i < numNodes; i++ { + + // randomly select the type of the node, either tree (p = 1/4) or file (p = 3/4). + if depth > 1 && rnd.Int63()%4 == 0 { + treeSeed := rnd.Int63() % maxSeed + id := fs.saveTree(ctx, treeSeed, depth-1) + + node := &Node{ + Name: fmt.Sprintf("dir-%v", treeSeed), + Type: NodeTypeDir, + Mode: 0755, + Subtree: &id, + } + + tree.Nodes = append(tree.Nodes, node) + continue + } + + fileSeed := rnd.Int63() % maxSeed + fileSize := (maxFileSize / maxSeed) * fileSeed + + node := &Node{ + Name: fmt.Sprintf("file-%v", fileSeed), + Type: NodeTypeFile, + Mode: 0644, + Size: uint64(fileSize), + } + + node.Content = fs.saveFile(ctx, fakeFile(fileSeed, fileSize)) + tree.Nodes = append(tree.Nodes, node) + } + + tree.Sort() + + id, err := SaveTree(ctx, fs.repo, &tree) + if err != nil { + fs.t.Fatalf("SaveTree returned error: %v", err) + } + return id +} + +// TestCreateSnapshot creates a snapshot filled with fake data. The +// fake data is generated deterministically from the timestamp `at`, which is +// also used as the snapshot's timestamp. The tree's depth can be specified +// with the parameter depth. The parameter duplication is a probability that +// the same blob will saved again. +func TestCreateSnapshot(t testing.TB, repo restic.Repository, at time.Time, depth int) *Snapshot { + seed := at.Unix() + t.Logf("create fake snapshot at %s with seed %d", at, seed) + + fakedir := fmt.Sprintf("fakedir-at-%v", at.Format("2006-01-02 15:04:05")) + snapshot, err := NewSnapshot([]string{fakedir}, []string{"test"}, "foo", at) + if err != nil { + t.Fatal(err) + } + + fs := fakeFileSystem{ + t: t, + repo: repo, + rand: rand.New(rand.NewSource(seed)), + } + + var wg errgroup.Group + repo.StartPackUploader(context.TODO(), &wg) + + treeID := fs.saveTree(context.TODO(), seed, depth) + snapshot.Tree = &treeID + + err = repo.Flush(context.Background()) + if err != nil { + t.Fatal(err) + } + + id, err := SaveSnapshot(context.TODO(), repo, snapshot) + if err != nil { + t.Fatal(err) + } + + snapshot.id = &id + + t.Logf("saved snapshot %v", id.Str()) + + return snapshot +} + +// TestSetSnapshotID sets the snapshot's ID. +func TestSetSnapshotID(_ testing.TB, sn *Snapshot, id restic.ID) { + sn.id = &id +} + +// ParseDurationOrPanic parses a duration from a string or panics if string is invalid. +// The format is `6y5m234d37h`. +func ParseDurationOrPanic(s string) Duration { + d, err := ParseDuration(s) + if err != nil { + panic(err) + } + + return d +} + +// TestLoadAllSnapshots returns a list of all snapshots in the repo. +// If a snapshot ID is in excludeIDs, it will not be included in the result. +func TestLoadAllSnapshots(ctx context.Context, repo restic.ListerLoaderUnpacked, excludeIDs restic.IDSet) (snapshots Snapshots, err error) { + err = ForAllSnapshots(ctx, repo, repo, excludeIDs, func(id restic.ID, sn *Snapshot, err error) error { + if err != nil { + return err + } + + snapshots = append(snapshots, sn) + return nil + }) + + if err != nil { + return nil, err + } + + return snapshots, nil +} diff --git a/internal/restic/testing_test.go b/internal/data/testing_test.go similarity index 75% rename from internal/restic/testing_test.go rename to internal/data/testing_test.go index 0a0c43892..6f6dec05b 100644 --- a/internal/restic/testing_test.go +++ b/internal/data/testing_test.go @@ -1,4 +1,4 @@ -package restic_test +package data_test import ( "context" @@ -6,6 +6,7 @@ import ( "time" "github.com/restic/restic/internal/checker" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" ) @@ -20,10 +21,10 @@ const ( func TestCreateSnapshot(t *testing.T) { repo := repository.TestRepository(t) for i := 0; i < testCreateSnapshots; i++ { - restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth) + data.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth) } - snapshots, err := restic.TestLoadAllSnapshots(context.TODO(), repo, restic.NewIDSet()) + snapshots, err := data.TestLoadAllSnapshots(context.TODO(), repo, restic.NewIDSet()) if err != nil { t.Fatal(err) } @@ -54,6 +55,6 @@ func BenchmarkTestCreateSnapshot(t *testing.B) { t.ResetTimer() for i := 0; i < t.N; i++ { - restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth) + data.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth) } } diff --git a/internal/restic/tree.go b/internal/data/tree.go similarity index 88% rename from internal/restic/tree.go rename to internal/data/tree.go index 70b718e64..9031f3bf5 100644 --- a/internal/restic/tree.go +++ b/internal/data/tree.go @@ -1,4 +1,4 @@ -package restic +package data import ( "bytes" @@ -10,6 +10,7 @@ import ( "strings" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/debug" ) @@ -94,7 +95,7 @@ func (t *Tree) Sort() { } // Subtrees returns a slice of all subtree IDs of the tree. -func (t *Tree) Subtrees() (trees IDs) { +func (t *Tree) Subtrees() (trees restic.IDs) { for _, node := range t.Nodes { if node.Type == NodeTypeDir && node.Subtree != nil { trees = append(trees, *node.Subtree) @@ -105,10 +106,10 @@ func (t *Tree) Subtrees() (trees IDs) { } // LoadTree loads a tree from the repository. -func LoadTree(ctx context.Context, r BlobLoader, id ID) (*Tree, error) { +func LoadTree(ctx context.Context, r restic.BlobLoader, id restic.ID) (*Tree, error) { debug.Log("load tree %v", id) - buf, err := r.LoadBlob(ctx, TreeBlob, id, nil) + buf, err := r.LoadBlob(ctx, restic.TreeBlob, id, nil) if err != nil { return nil, err } @@ -125,17 +126,17 @@ func LoadTree(ctx context.Context, r BlobLoader, id ID) (*Tree, error) { // SaveTree stores a tree into the repository and returns the ID. The ID is // checked against the index. The tree is only stored when the index does not // contain the ID. -func SaveTree(ctx context.Context, r BlobSaver, t *Tree) (ID, error) { +func SaveTree(ctx context.Context, r restic.BlobSaver, t *Tree) (restic.ID, error) { buf, err := json.Marshal(t) if err != nil { - return ID{}, errors.Wrap(err, "MarshalJSON") + return restic.ID{}, errors.Wrap(err, "MarshalJSON") } // append a newline so that the data is always consistent (json.Encoder // adds a newline after each object) buf = append(buf, '\n') - id, _, _, err := r.SaveBlob(ctx, TreeBlob, buf, ID{}, false) + id, _, _, err := r.SaveBlob(ctx, restic.TreeBlob, buf, restic.ID{}, false) return id, err } @@ -179,7 +180,7 @@ func (builder *TreeJSONBuilder) Finalize() ([]byte, error) { return buf, nil } -func FindTreeDirectory(ctx context.Context, repo BlobLoader, id *ID, dir string) (*ID, error) { +func FindTreeDirectory(ctx context.Context, repo restic.BlobLoader, id *restic.ID, dir string) (*restic.ID, error) { if id == nil { return nil, errors.New("tree id is null") } diff --git a/internal/restic/tree_stream.go b/internal/data/tree_stream.go similarity index 89% rename from internal/restic/tree_stream.go rename to internal/data/tree_stream.go index 123295533..c7d3588b5 100644 --- a/internal/restic/tree_stream.go +++ b/internal/data/tree_stream.go @@ -1,4 +1,4 @@ -package restic +package data import ( "context" @@ -7,13 +7,14 @@ import ( "sync" "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" "golang.org/x/sync/errgroup" ) // TreeItem is used to return either an error or the tree for a tree id type TreeItem struct { - ID + restic.ID Error error *Tree } @@ -24,12 +25,12 @@ type trackedTreeItem struct { } type trackedID struct { - ID + restic.ID rootIdx int } // loadTreeWorker loads trees from repo and sends them to out. -func loadTreeWorker(ctx context.Context, repo Loader, +func loadTreeWorker(ctx context.Context, repo restic.Loader, in <-chan trackedID, out chan<- trackedTreeItem) { for treeID := range in { @@ -45,8 +46,8 @@ func loadTreeWorker(ctx context.Context, repo Loader, } } -func filterTrees(ctx context.Context, repo Loader, trees IDs, loaderChan chan<- trackedID, hugeTreeLoaderChan chan<- trackedID, - in <-chan trackedTreeItem, out chan<- TreeItem, skip func(tree ID) bool, p *progress.Counter) { +func filterTrees(ctx context.Context, repo restic.Loader, trees restic.IDs, loaderChan chan<- trackedID, hugeTreeLoaderChan chan<- trackedID, + in <-chan trackedTreeItem, out chan<- TreeItem, skip func(tree restic.ID) bool, p *progress.Counter) { var ( inCh = in @@ -77,7 +78,7 @@ func filterTrees(ctx context.Context, repo Loader, trees IDs, loaderChan chan<- continue } - treeSize, found := repo.LookupBlobSize(TreeBlob, nextTreeID.ID) + treeSize, found := repo.LookupBlobSize(restic.TreeBlob, nextTreeID.ID) if found && treeSize > 50*1024*1024 { loadCh = hugeTreeLoaderChan } else { @@ -154,7 +155,7 @@ func filterTrees(ctx context.Context, repo Loader, trees IDs, loaderChan chan<- // is guaranteed to always be called from the same goroutine. To shutdown the started // goroutines, either read all items from the channel or cancel the context. Then `Wait()` // on the errgroup until all goroutines were stopped. -func StreamTrees(ctx context.Context, wg *errgroup.Group, repo Loader, trees IDs, skip func(tree ID) bool, p *progress.Counter) <-chan TreeItem { +func StreamTrees(ctx context.Context, wg *errgroup.Group, repo restic.Loader, trees restic.IDs, skip func(tree restic.ID) bool, p *progress.Counter) <-chan TreeItem { loaderChan := make(chan trackedID) hugeTreeChan := make(chan trackedID, 10) loadedTreeChan := make(chan trackedTreeItem) diff --git a/internal/restic/tree_test.go b/internal/data/tree_test.go similarity index 85% rename from internal/restic/tree_test.go rename to internal/data/tree_test.go index 08ebe3984..92949628b 100644 --- a/internal/restic/tree_test.go +++ b/internal/data/tree_test.go @@ -1,4 +1,4 @@ -package restic_test +package data_test import ( "context" @@ -10,6 +10,7 @@ import ( "testing" "github.com/restic/restic/internal/archiver" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -60,7 +61,7 @@ func TestTree(t *testing.T) { }() } -var testNodes = []restic.Node{ +var testNodes = []data.Node{ {Name: "normal"}, {Name: "with backslashes \\zzz"}, {Name: "test utf-8 föbärß"}, @@ -70,11 +71,11 @@ var testNodes = []restic.Node{ func TestNodeMarshal(t *testing.T) { for i, n := range testNodes { - data, err := json.Marshal(&n) + nodeData, err := json.Marshal(&n) rtest.OK(t, err) - var node restic.Node - err = json.Unmarshal(data, &node) + var node data.Node + err = json.Unmarshal(nodeData, &node) rtest.OK(t, err) if n.Name != node.Name { @@ -83,7 +84,7 @@ func TestNodeMarshal(t *testing.T) { } } -func nodeForFile(t *testing.T, name string) *restic.Node { +func nodeForFile(t *testing.T, name string) *data.Node { f, err := (&fs.Local{}).OpenFile(name, fs.O_NOFOLLOW, true) rtest.OK(t, err) node, err := f.ToNode(false, t.Logf) @@ -108,15 +109,15 @@ func TestEmptyLoadTree(t *testing.T) { var wg errgroup.Group repo.StartPackUploader(context.TODO(), &wg) // save tree - tree := restic.NewTree(0) - id, err := restic.SaveTree(context.TODO(), repo, tree) + tree := data.NewTree(0) + id, err := data.SaveTree(context.TODO(), repo, tree) rtest.OK(t, err) // save packs rtest.OK(t, repo.Flush(context.Background())) // load tree again - tree2, err := restic.LoadTree(context.TODO(), repo, id) + tree2, err := data.LoadTree(context.TODO(), repo, id) rtest.OK(t, err) rtest.Assert(t, tree.Equals(tree2), @@ -127,8 +128,8 @@ func TestEmptyLoadTree(t *testing.T) { func TestTreeEqualSerialization(t *testing.T) { files := []string{"node.go", "tree.go", "tree_test.go"} for i := 1; i <= len(files); i++ { - tree := restic.NewTree(i) - builder := restic.NewTreeJSONBuilder() + tree := data.NewTree(i) + builder := data.NewTreeJSONBuilder() for _, fn := range files[:i] { node := nodeForFile(t, fn) @@ -138,7 +139,7 @@ func TestTreeEqualSerialization(t *testing.T) { rtest.Assert(t, tree.Insert(node) != nil, "no error on duplicate node") rtest.Assert(t, builder.AddNode(node) != nil, "no error on duplicate node") - rtest.Assert(t, errors.Is(builder.AddNode(node), restic.ErrTreeNotOrdered), "wrong error returned") + rtest.Assert(t, errors.Is(builder.AddNode(node), data.ErrTreeNotOrdered), "wrong error returned") } treeBytes, err := json.Marshal(tree) @@ -156,7 +157,7 @@ func TestTreeEqualSerialization(t *testing.T) { func BenchmarkBuildTree(b *testing.B) { const size = 100 // Directories of this size are not uncommon. - nodes := make([]restic.Node, size) + nodes := make([]data.Node, size) for i := range nodes { // Archiver.SaveTree inputs in sorted order, so do that here too. nodes[i].Name = strconv.Itoa(i) @@ -166,7 +167,7 @@ func BenchmarkBuildTree(b *testing.B) { b.ReportAllocs() for i := 0; i < b.N; i++ { - t := restic.NewTree(size) + t := data.NewTree(size) for i := range nodes { _ = t.Insert(&nodes[i]) @@ -188,7 +189,7 @@ func testLoadTree(t *testing.T, version uint) { sn := archiver.TestSnapshot(t, repo, rtest.BenchArchiveDirectory, nil) rtest.OK(t, repo.Flush(context.Background())) - _, err := restic.LoadTree(context.TODO(), repo, *sn.Tree) + _, err := data.LoadTree(context.TODO(), repo, *sn.Tree) rtest.OK(t, err) } @@ -209,14 +210,14 @@ func benchmarkLoadTree(t *testing.B, version uint) { t.ResetTimer() for i := 0; i < t.N; i++ { - _, err := restic.LoadTree(context.TODO(), repo, *sn.Tree) + _, err := data.LoadTree(context.TODO(), repo, *sn.Tree) rtest.OK(t, err) } } func TestFindTreeDirectory(t *testing.T) { repo := repository.TestRepository(t) - sn := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:08"), 3) + sn := data.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:08"), 3) for _, exp := range []struct { subfolder string @@ -234,7 +235,7 @@ func TestFindTreeDirectory(t *testing.T) { {"dir-21/dir-24", restic.TestParseID("74626b3fb2bd4b3e572b81a4059b3e912bcf2a8f69fecd9c187613b7173f13b1"), nil}, } { t.Run("", func(t *testing.T) { - id, err := restic.FindTreeDirectory(context.TODO(), repo, sn.Tree, exp.subfolder) + id, err := data.FindTreeDirectory(context.TODO(), repo, sn.Tree, exp.subfolder) if exp.err == nil { rtest.OK(t, err) rtest.Assert(t, exp.id == *id, "unexpected id, expected %v, got %v", exp.id, id) @@ -244,6 +245,6 @@ func TestFindTreeDirectory(t *testing.T) { }) } - _, err := restic.FindTreeDirectory(context.TODO(), repo, nil, "") + _, err := data.FindTreeDirectory(context.TODO(), repo, nil, "") rtest.Assert(t, err != nil, "missing error on null tree id") } diff --git a/internal/dump/common.go b/internal/dump/common.go index 8c02af04b..aea5c1291 100644 --- a/internal/dump/common.go +++ b/internal/dump/common.go @@ -6,6 +6,7 @@ import ( "path" "github.com/restic/restic/internal/bloblru" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/walker" "golang.org/x/sync/errgroup" @@ -29,12 +30,12 @@ func New(format string, repo restic.Loader, w io.Writer) *Dumper { } } -func (d *Dumper) DumpTree(ctx context.Context, tree *restic.Tree, rootPath string) error { +func (d *Dumper) DumpTree(ctx context.Context, tree *data.Tree, rootPath string) error { ctx, cancel := context.WithCancel(ctx) defer cancel() // ch is buffered to deal with variable download/write speeds. - ch := make(chan *restic.Node, 10) + ch := make(chan *data.Node, 10) go sendTrees(ctx, d.repo, tree, rootPath, ch) switch d.format { @@ -47,7 +48,7 @@ func (d *Dumper) DumpTree(ctx context.Context, tree *restic.Tree, rootPath strin } } -func sendTrees(ctx context.Context, repo restic.BlobLoader, tree *restic.Tree, rootPath string, ch chan *restic.Node) { +func sendTrees(ctx context.Context, repo restic.BlobLoader, tree *data.Tree, rootPath string, ch chan *data.Node) { defer close(ch) for _, root := range tree.Nodes { @@ -58,7 +59,7 @@ func sendTrees(ctx context.Context, repo restic.BlobLoader, tree *restic.Tree, r } } -func sendNodes(ctx context.Context, repo restic.BlobLoader, root *restic.Node, ch chan *restic.Node) error { +func sendNodes(ctx context.Context, repo restic.BlobLoader, root *data.Node, ch chan *data.Node) error { select { case ch <- root: case <-ctx.Done(): @@ -66,11 +67,11 @@ func sendNodes(ctx context.Context, repo restic.BlobLoader, root *restic.Node, c } // If this is no directory we are finished - if root.Type != restic.NodeTypeDir { + if root.Type != data.NodeTypeDir { return nil } - err := walker.Walk(ctx, repo, *root.Subtree, walker.WalkVisitor{ProcessNode: func(_ restic.ID, nodepath string, node *restic.Node, err error) error { + err := walker.Walk(ctx, repo, *root.Subtree, walker.WalkVisitor{ProcessNode: func(_ restic.ID, nodepath string, node *data.Node, err error) error { if err != nil { return err } @@ -80,7 +81,7 @@ func sendNodes(ctx context.Context, repo restic.BlobLoader, root *restic.Node, c node.Path = path.Join(root.Path, nodepath) - if node.Type != restic.NodeTypeFile && node.Type != restic.NodeTypeDir && node.Type != restic.NodeTypeSymlink { + if node.Type != data.NodeTypeFile && node.Type != data.NodeTypeDir && node.Type != data.NodeTypeSymlink { return nil } @@ -98,11 +99,11 @@ func sendNodes(ctx context.Context, repo restic.BlobLoader, root *restic.Node, c // WriteNode writes a file node's contents directly to d's Writer, // without caring about d's format. -func (d *Dumper) WriteNode(ctx context.Context, node *restic.Node) error { +func (d *Dumper) WriteNode(ctx context.Context, node *data.Node) error { return d.writeNode(ctx, d.w, node) } -func (d *Dumper) writeNode(ctx context.Context, w io.Writer, node *restic.Node) error { +func (d *Dumper) writeNode(ctx context.Context, w io.Writer, node *data.Node) error { wg, ctx := errgroup.WithContext(ctx) limit := int(d.repo.Connections()) wg.SetLimit(1 + limit) // +1 for the writer. diff --git a/internal/dump/common_test.go b/internal/dump/common_test.go index afd19df63..5599e2717 100644 --- a/internal/dump/common_test.go +++ b/internal/dump/common_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/restic/restic/internal/archiver" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -81,7 +82,7 @@ func WriteTest(t *testing.T, format string, cd CheckDump) { sn, _, _, err := arch.Snapshot(ctx, []string{"."}, archiver.SnapshotOptions{}) rtest.OK(t, err) - tree, err := restic.LoadTree(ctx, repo, *sn.Tree) + tree, err := data.LoadTree(ctx, repo, *sn.Tree) rtest.OK(t, err) dst := &bytes.Buffer{} diff --git a/internal/dump/tar.go b/internal/dump/tar.go index c5933d4f8..fe61c361b 100644 --- a/internal/dump/tar.go +++ b/internal/dump/tar.go @@ -7,12 +7,12 @@ import ( "os" "path/filepath" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" ) -func (d *Dumper) dumpTar(ctx context.Context, ch <-chan *restic.Node) (err error) { +func (d *Dumper) dumpTar(ctx context.Context, ch <-chan *data.Node) (err error) { w := tar.NewWriter(d.w) defer func() { @@ -48,7 +48,7 @@ func tarIdentifier(id uint32) int { return int(id) } -func (d *Dumper) dumpNodeTar(ctx context.Context, node *restic.Node, w *tar.Writer) error { +func (d *Dumper) dumpNodeTar(ctx context.Context, node *data.Node, w *tar.Writer) error { relPath, err := filepath.Rel("/", node.Path) if err != nil { return err @@ -79,16 +79,16 @@ func (d *Dumper) dumpNodeTar(ctx context.Context, node *restic.Node, w *tar.Writ header.Mode |= cISVTX } - if node.Type == restic.NodeTypeFile { + if node.Type == data.NodeTypeFile { header.Typeflag = tar.TypeReg } - if node.Type == restic.NodeTypeSymlink { + if node.Type == data.NodeTypeSymlink { header.Typeflag = tar.TypeSymlink header.Linkname = node.LinkTarget } - if node.Type == restic.NodeTypeDir { + if node.Type == data.NodeTypeDir { header.Typeflag = tar.TypeDir header.Name += "/" } @@ -100,7 +100,7 @@ func (d *Dumper) dumpNodeTar(ctx context.Context, node *restic.Node, w *tar.Writ return d.writeNode(ctx, w, node) } -func parseXattrs(xattrs []restic.ExtendedAttribute) map[string]string { +func parseXattrs(xattrs []data.ExtendedAttribute) map[string]string { tmpMap := make(map[string]string) for _, attr := range xattrs { diff --git a/internal/dump/tar_test.go b/internal/dump/tar_test.go index cb3cb08c4..06e49c828 100644 --- a/internal/dump/tar_test.go +++ b/internal/dump/tar_test.go @@ -13,7 +13,7 @@ import ( "testing" "time" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/data" rtest "github.com/restic/restic/internal/test" ) @@ -120,12 +120,12 @@ func checkTar(t *testing.T, testDir string, srcTar *bytes.Buffer) error { func TestFieldTooLong(t *testing.T) { const maxSpecialFileSize = 1 << 20 // Unexported limit in archive/tar. - node := restic.Node{ + node := data.Node{ Name: "file_with_xattr", Path: "/file_with_xattr", - Type: restic.NodeTypeFile, + Type: data.NodeTypeFile, Mode: 0644, - ExtendedAttributes: []restic.ExtendedAttribute{ + ExtendedAttributes: []data.ExtendedAttribute{ { Name: "user.way_too_large", Value: make([]byte, 2*maxSpecialFileSize), diff --git a/internal/dump/zip.go b/internal/dump/zip.go index 17aeb4829..30ea585a3 100644 --- a/internal/dump/zip.go +++ b/internal/dump/zip.go @@ -5,11 +5,11 @@ import ( "context" "path/filepath" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" ) -func (d *Dumper) dumpZip(ctx context.Context, ch <-chan *restic.Node) (err error) { +func (d *Dumper) dumpZip(ctx context.Context, ch <-chan *data.Node) (err error) { w := zip.NewWriter(d.w) defer func() { @@ -27,7 +27,7 @@ func (d *Dumper) dumpZip(ctx context.Context, ch <-chan *restic.Node) (err error return nil } -func (d *Dumper) dumpNodeZip(ctx context.Context, node *restic.Node, zw *zip.Writer) error { +func (d *Dumper) dumpNodeZip(ctx context.Context, node *data.Node, zw *zip.Writer) error { relPath, err := filepath.Rel("/", node.Path) if err != nil { return err @@ -39,11 +39,11 @@ func (d *Dumper) dumpNodeZip(ctx context.Context, node *restic.Node, zw *zip.Wri Modified: node.ModTime, } header.SetMode(node.Mode) - if node.Type == restic.NodeTypeFile { + if node.Type == data.NodeTypeFile { header.Method = zip.Deflate } - if node.Type == restic.NodeTypeDir { + if node.Type == data.NodeTypeDir { header.Name += "/" } @@ -52,7 +52,7 @@ func (d *Dumper) dumpNodeZip(ctx context.Context, node *restic.Node, zw *zip.Wri return errors.Wrap(err, "ZipHeader") } - if node.Type == restic.NodeTypeSymlink { + if node.Type == data.NodeTypeSymlink { if _, err = w.Write([]byte(node.LinkTarget)); err != nil { return errors.Wrap(err, "Write") } diff --git a/internal/fs/file_windows.go b/internal/fs/file_windows.go index 2e8dad4fa..f1ff733fa 100644 --- a/internal/fs/file_windows.go +++ b/internal/fs/file_windows.go @@ -8,7 +8,7 @@ import ( "strings" "time" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/data" "golang.org/x/sys/windows" ) @@ -115,7 +115,7 @@ func clearAttribute(path string, attribute uint32) error { } // openHandleForEA return a file handle for file or dir for setting/getting EAs -func openHandleForEA(nodeType restic.NodeType, path string, writeAccess bool) (handle windows.Handle, err error) { +func openHandleForEA(nodeType data.NodeType, path string, writeAccess bool) (handle windows.Handle, err error) { path = fixpath(path) fileAccess := windows.FILE_READ_EA if writeAccess { @@ -123,10 +123,10 @@ func openHandleForEA(nodeType restic.NodeType, path string, writeAccess bool) (h } switch nodeType { - case restic.NodeTypeFile: + case data.NodeTypeFile: utf16Path := windows.StringToUTF16Ptr(path) handle, err = windows.CreateFile(utf16Path, uint32(fileAccess), 0, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL, 0) - case restic.NodeTypeDir: + case data.NodeTypeDir: utf16Path := windows.StringToUTF16Ptr(path) handle, err = windows.CreateFile(utf16Path, uint32(fileAccess), 0, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_BACKUP_SEMANTICS, 0) default: diff --git a/internal/fs/fs_local.go b/internal/fs/fs_local.go index dfbdab3b0..9d9ba9035 100644 --- a/internal/fs/fs_local.go +++ b/internal/fs/fs_local.go @@ -4,7 +4,7 @@ import ( "os" "path/filepath" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/data" ) // Local is the local file system. Most methods are just passed on to the stdlib. @@ -152,7 +152,7 @@ func (f *localFile) Stat() (*ExtendedFileInfo, error) { return f.fi, err } -func (f *localFile) ToNode(ignoreXattrListError bool, warnf func(format string, args ...any)) (*restic.Node, error) { +func (f *localFile) ToNode(ignoreXattrListError bool, warnf func(format string, args ...any)) (*data.Node, error) { if err := f.cacheFI(); err != nil { return nil, err } diff --git a/internal/fs/fs_local_test.go b/internal/fs/fs_local_test.go index aff0b7a50..f980d1c72 100644 --- a/internal/fs/fs_local_test.go +++ b/internal/fs/fs_local_test.go @@ -7,7 +7,7 @@ import ( "slices" "testing" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/data" rtest "github.com/restic/restic/internal/test" ) @@ -15,7 +15,7 @@ type fsLocalMetadataTestcase struct { name string follow bool setup func(t *testing.T, path string) - nodeType restic.NodeType + nodeType data.NodeType } func TestFSLocalMetadata(t *testing.T) { @@ -25,21 +25,21 @@ func TestFSLocalMetadata(t *testing.T) { setup: func(t *testing.T, path string) { rtest.OK(t, os.WriteFile(path, []byte("example"), 0o600)) }, - nodeType: restic.NodeTypeFile, + nodeType: data.NodeTypeFile, }, { name: "directory", setup: func(t *testing.T, path string) { rtest.OK(t, os.Mkdir(path, 0o600)) }, - nodeType: restic.NodeTypeDir, + nodeType: data.NodeTypeDir, }, { name: "symlink", setup: func(t *testing.T, path string) { rtest.OK(t, os.Symlink(path+"old", path)) }, - nodeType: restic.NodeTypeSymlink, + nodeType: data.NodeTypeSymlink, }, { name: "symlink file", @@ -48,7 +48,7 @@ func TestFSLocalMetadata(t *testing.T) { rtest.OK(t, os.WriteFile(path+"file", []byte("example"), 0o600)) rtest.OK(t, os.Symlink(path+"file", path)) }, - nodeType: restic.NodeTypeFile, + nodeType: data.NodeTypeFile, }, } { runFSLocalTestcase(t, test) @@ -74,7 +74,7 @@ func runFSLocalTestcase(t *testing.T, test fsLocalMetadataTestcase) { } -func checkMetadata(t *testing.T, f File, path string, follow bool, nodeType restic.NodeType) { +func checkMetadata(t *testing.T, f File, path string, follow bool, nodeType data.NodeType) { fi, err := f.Stat() rtest.OK(t, err) var fi2 os.FileInfo @@ -114,7 +114,7 @@ func testFSLocalRead(t *testing.T, makeReadable bool) { rtest.OK(t, os.WriteFile(path, []byte(testdata), 0o600)) f := openReadable(t, path, makeReadable) - checkMetadata(t, f, path, false, restic.NodeTypeFile) + checkMetadata(t, f, path, false, data.NodeTypeFile) data, err := io.ReadAll(f) rtest.OK(t, err) @@ -147,7 +147,7 @@ func testFSLocalReaddir(t *testing.T, makeReadable bool) { rtest.OK(t, os.WriteFile(filepath.Join(path, entries[0]), []byte("example"), 0o600)) f := openReadable(t, path, makeReadable) - checkMetadata(t, f, path, false, restic.NodeTypeDir) + checkMetadata(t, f, path, false, data.NodeTypeDir) names, err := f.Readdirnames(-1) rtest.OK(t, err) @@ -173,7 +173,7 @@ func TestFSLocalReadableRace(t *testing.T) { err = f.MakeReadable() if err == nil { // a file handle based implementation should still work - checkMetadata(t, f, pathNew, false, restic.NodeTypeFile) + checkMetadata(t, f, pathNew, false, data.NodeTypeFile) data, err := io.ReadAll(f) rtest.OK(t, err) @@ -207,7 +207,7 @@ func TestFSLocalTypeChange(t *testing.T) { rtest.OK(t, err) if !fi.Mode.IsDir() { // a file handle based implementation should still reference the file - checkMetadata(t, f, pathNew, false, restic.NodeTypeFile) + checkMetadata(t, f, pathNew, false, data.NodeTypeFile) data, err := io.ReadAll(f) rtest.OK(t, err) diff --git a/internal/fs/fs_local_unix_test.go b/internal/fs/fs_local_unix_test.go index 5bcb5efd0..c38081caf 100644 --- a/internal/fs/fs_local_unix_test.go +++ b/internal/fs/fs_local_unix_test.go @@ -6,7 +6,7 @@ import ( "syscall" "testing" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/data" rtest "github.com/restic/restic/internal/test" ) @@ -24,14 +24,14 @@ func TestFSLocalMetadataUnix(t *testing.T) { addr := &syscall.SockaddrUnix{Name: path} rtest.OK(t, syscall.Bind(fd, addr)) }, - nodeType: restic.NodeTypeSocket, + nodeType: data.NodeTypeSocket, }, { name: "fifo", setup: func(t *testing.T, path string) { rtest.OK(t, mkfifo(path, 0o600)) }, - nodeType: restic.NodeTypeFifo, + nodeType: data.NodeTypeFifo, }, // device files can only be created as root } { diff --git a/internal/fs/fs_reader.go b/internal/fs/fs_reader.go index 32a4e8fee..a98a44de3 100644 --- a/internal/fs/fs_reader.go +++ b/internal/fs/fs_reader.go @@ -10,8 +10,8 @@ import ( "syscall" "time" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" ) // Reader is a file system which provides a directory with a single file. When @@ -267,7 +267,7 @@ func (f fakeFile) Stat() (*ExtendedFileInfo, error) { return f.fi, nil } -func (f fakeFile) ToNode(_ bool, _ func(format string, args ...any)) (*restic.Node, error) { +func (f fakeFile) ToNode(_ bool, _ func(format string, args ...any)) (*data.Node, error) { node := buildBasicNode(f.name, f.fi) // fill minimal info with current values for uid, gid diff --git a/internal/fs/interface.go b/internal/fs/interface.go index 19d2eb1f7..dec902abf 100644 --- a/internal/fs/interface.go +++ b/internal/fs/interface.go @@ -3,7 +3,7 @@ package fs import ( "io" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/data" ) // FS bundles all methods needed for a file system. @@ -45,8 +45,8 @@ type File interface { Readdirnames(n int) ([]string, error) Stat() (*ExtendedFileInfo, error) - // ToNode returns a restic.Node for the File. The internally used os.FileInfo + // ToNode returns a data.Node for the File. The internally used os.FileInfo // must be consistent with that returned by Stat(). In particular, the metadata // returned by consecutive calls to Stat() and ToNode() must match. - ToNode(ignoreXattrListError bool, warnf func(format string, args ...any)) (*restic.Node, error) + ToNode(ignoreXattrListError bool, warnf func(format string, args ...any)) (*data.Node, error) } diff --git a/internal/fs/node.go b/internal/fs/node.go index ac8e58203..5d4c35546 100644 --- a/internal/fs/node.go +++ b/internal/fs/node.go @@ -8,14 +8,14 @@ import ( "sync" "syscall" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" ) // nodeFromFileInfo returns a new node from the given path and FileInfo. It // returns the first error that is encountered, together with a node. -func nodeFromFileInfo(path string, fi *ExtendedFileInfo, ignoreXattrListError bool, warnf func(format string, args ...any)) (*restic.Node, error) { +func nodeFromFileInfo(path string, fi *ExtendedFileInfo, ignoreXattrListError bool, warnf func(format string, args ...any)) (*data.Node, error) { node := buildBasicNode(path, fi) if err := nodeFillExtendedStat(node, path, fi); err != nil { @@ -27,9 +27,9 @@ func nodeFromFileInfo(path string, fi *ExtendedFileInfo, ignoreXattrListError bo return node, err } -func buildBasicNode(path string, fi *ExtendedFileInfo) *restic.Node { +func buildBasicNode(path string, fi *ExtendedFileInfo) *data.Node { mask := os.ModePerm | os.ModeType | os.ModeSetuid | os.ModeSetgid | os.ModeSticky - node := &restic.Node{ + node := &data.Node{ Path: path, Name: fi.Name, Mode: fi.Mode & mask, @@ -37,36 +37,36 @@ func buildBasicNode(path string, fi *ExtendedFileInfo) *restic.Node { } node.Type = nodeTypeFromFileInfo(fi.Mode) - if node.Type == restic.NodeTypeFile { + if node.Type == data.NodeTypeFile { node.Size = uint64(fi.Size) } return node } -func nodeTypeFromFileInfo(mode os.FileMode) restic.NodeType { +func nodeTypeFromFileInfo(mode os.FileMode) data.NodeType { switch mode & os.ModeType { case 0: - return restic.NodeTypeFile + return data.NodeTypeFile case os.ModeDir: - return restic.NodeTypeDir + return data.NodeTypeDir case os.ModeSymlink: - return restic.NodeTypeSymlink + return data.NodeTypeSymlink case os.ModeDevice | os.ModeCharDevice: - return restic.NodeTypeCharDev + return data.NodeTypeCharDev case os.ModeDevice: - return restic.NodeTypeDev + return data.NodeTypeDev case os.ModeNamedPipe: - return restic.NodeTypeFifo + return data.NodeTypeFifo case os.ModeSocket: - return restic.NodeTypeSocket + return data.NodeTypeSocket case os.ModeIrregular: - return restic.NodeTypeIrregular + return data.NodeTypeIrregular } - return restic.NodeTypeInvalid + return data.NodeTypeInvalid } -func nodeFillExtendedStat(node *restic.Node, path string, stat *ExtendedFileInfo) error { +func nodeFillExtendedStat(node *data.Node, path string, stat *ExtendedFileInfo) error { node.Inode = stat.Inode node.DeviceID = stat.DeviceID node.ChangeTime = stat.ChangeTime @@ -78,25 +78,25 @@ func nodeFillExtendedStat(node *restic.Node, path string, stat *ExtendedFileInfo node.Group = lookupGroup(stat.GID) switch node.Type { - case restic.NodeTypeFile: + case data.NodeTypeFile: node.Size = uint64(stat.Size) node.Links = stat.Links - case restic.NodeTypeDir: - case restic.NodeTypeSymlink: + case data.NodeTypeDir: + case data.NodeTypeSymlink: var err error node.LinkTarget, err = os.Readlink(fixpath(path)) node.Links = stat.Links if err != nil { return errors.WithStack(err) } - case restic.NodeTypeDev: + case data.NodeTypeDev: node.Device = stat.Device node.Links = stat.Links - case restic.NodeTypeCharDev: + case data.NodeTypeCharDev: node.Device = stat.Device node.Links = stat.Links - case restic.NodeTypeFifo: - case restic.NodeTypeSocket: + case data.NodeTypeFifo: + case data.NodeTypeSocket: default: return errors.Errorf("unsupported file type %q", node.Type) } @@ -158,23 +158,23 @@ func lookupGroup(gid uint32) string { } // NodeCreateAt creates the node at the given path but does NOT restore node meta data. -func NodeCreateAt(node *restic.Node, path string) (err error) { +func NodeCreateAt(node *data.Node, path string) (err error) { debug.Log("create node %v at %v", node.Name, path) switch node.Type { - case restic.NodeTypeDir: + case data.NodeTypeDir: err = nodeCreateDirAt(node, path) - case restic.NodeTypeFile: + case data.NodeTypeFile: err = nodeCreateFileAt(path) - case restic.NodeTypeSymlink: + case data.NodeTypeSymlink: err = nodeCreateSymlinkAt(node, path) - case restic.NodeTypeDev: + case data.NodeTypeDev: err = nodeCreateDevAt(node, path) - case restic.NodeTypeCharDev: + case data.NodeTypeCharDev: err = nodeCreateCharDevAt(node, path) - case restic.NodeTypeFifo: + case data.NodeTypeFifo: err = nodeCreateFifoAt(path) - case restic.NodeTypeSocket: + case data.NodeTypeSocket: err = nil default: err = errors.Errorf("filetype %q not implemented", node.Type) @@ -183,7 +183,7 @@ func NodeCreateAt(node *restic.Node, path string) (err error) { return err } -func nodeCreateDirAt(node *restic.Node, path string) error { +func nodeCreateDirAt(node *data.Node, path string) error { err := os.Mkdir(fixpath(path), node.Mode) if err != nil && !os.IsExist(err) { return errors.WithStack(err) @@ -205,7 +205,7 @@ func nodeCreateFileAt(path string) error { return nil } -func nodeCreateSymlinkAt(node *restic.Node, path string) error { +func nodeCreateSymlinkAt(node *data.Node, path string) error { if err := os.Symlink(node.LinkTarget, fixpath(path)); err != nil { return errors.WithStack(err) } @@ -213,11 +213,11 @@ func nodeCreateSymlinkAt(node *restic.Node, path string) error { return nil } -func nodeCreateDevAt(node *restic.Node, path string) error { +func nodeCreateDevAt(node *data.Node, path string) error { return mknod(path, syscall.S_IFBLK|0600, node.Device) } -func nodeCreateCharDevAt(node *restic.Node, path string) error { +func nodeCreateCharDevAt(node *data.Node, path string) error { return mknod(path, syscall.S_IFCHR|0600, node.Device) } @@ -230,7 +230,7 @@ func mkfifo(path string, mode uint32) (err error) { } // NodeRestoreMetadata restores node metadata -func NodeRestoreMetadata(node *restic.Node, path string, warn func(msg string), xattrSelectFilter func(xattrName string) bool) error { +func NodeRestoreMetadata(node *data.Node, path string, warn func(msg string), xattrSelectFilter func(xattrName string) bool) error { err := nodeRestoreMetadata(node, path, warn, xattrSelectFilter) if err != nil { // It is common to have permission errors for folders like /home @@ -246,7 +246,7 @@ func NodeRestoreMetadata(node *restic.Node, path string, warn func(msg string), return err } -func nodeRestoreMetadata(node *restic.Node, path string, warn func(msg string), xattrSelectFilter func(xattrName string) bool) error { +func nodeRestoreMetadata(node *data.Node, path string, warn func(msg string), xattrSelectFilter func(xattrName string) bool) error { var firsterr error if err := lchown(path, int(node.UID), int(node.GID)); err != nil { @@ -277,7 +277,7 @@ func nodeRestoreMetadata(node *restic.Node, path string, warn func(msg string), // Moving RestoreTimestamps and restoreExtendedAttributes calls above as for readonly files in windows // calling Chmod below will no longer allow any modifications to be made on the file and the // calls above would fail. - if node.Type != restic.NodeTypeSymlink { + if node.Type != data.NodeTypeSymlink { if err := chmod(path, node.Mode); err != nil { if firsterr == nil { firsterr = errors.WithStack(err) @@ -288,7 +288,7 @@ func nodeRestoreMetadata(node *restic.Node, path string, warn func(msg string), return firsterr } -func nodeRestoreTimestamps(node *restic.Node, path string) error { +func nodeRestoreTimestamps(node *data.Node, path string) error { atime := node.AccessTime.UnixNano() mtime := node.ModTime.UnixNano() diff --git a/internal/fs/node_linux.go b/internal/fs/node_linux.go index ee13e0a9e..71b180d74 100644 --- a/internal/fs/node_linux.go +++ b/internal/fs/node_linux.go @@ -1,12 +1,12 @@ package fs import ( - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/data" "golang.org/x/sys/unix" ) // utimesNano is like syscall.UtimesNano, except that it does not follow symlinks. -func utimesNano(path string, atime, mtime int64, _ restic.NodeType) error { +func utimesNano(path string, atime, mtime int64, _ data.NodeType) error { times := []unix.Timespec{ unix.NsecToTimespec(atime), unix.NsecToTimespec(mtime), diff --git a/internal/fs/node_linux_test.go b/internal/fs/node_linux_test.go index e9f1cf860..e3ea10db7 100644 --- a/internal/fs/node_linux_test.go +++ b/internal/fs/node_linux_test.go @@ -5,14 +5,14 @@ import ( "strings" "testing" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) func TestRestoreSymlinkTimestampsError(t *testing.T) { d := t.TempDir() - node := restic.Node{Type: restic.NodeTypeSymlink} + node := data.Node{Type: data.NodeTypeSymlink} err := nodeRestoreTimestamps(&node, d+"/nosuchfile") rtest.Assert(t, errors.Is(err, fs.ErrNotExist), "want ErrNotExist, got %q", err) rtest.Assert(t, strings.Contains(err.Error(), d), "filename not in %q", err) diff --git a/internal/fs/node_noxattr.go b/internal/fs/node_noxattr.go index 2923aea74..c061d1777 100644 --- a/internal/fs/node_noxattr.go +++ b/internal/fs/node_noxattr.go @@ -3,16 +3,14 @@ package fs -import ( - "github.com/restic/restic/internal/restic" -) +import "github.com/restic/restic/internal/data" // nodeRestoreExtendedAttributes is a no-op -func nodeRestoreExtendedAttributes(_ *restic.Node, _ string, _ func(xattrName string) bool) error { +func nodeRestoreExtendedAttributes(_ *data.Node, _ string, _ func(xattrName string) bool) error { return nil } // nodeFillExtendedAttributes is a no-op -func nodeFillExtendedAttributes(_ *restic.Node, _ string, _ bool, _ func(format string, args ...any)) error { +func nodeFillExtendedAttributes(_ *data.Node, _ string, _ bool, _ func(format string, args ...any)) error { return nil } diff --git a/internal/fs/node_test.go b/internal/fs/node_test.go index 2b4c76c92..41d951657 100644 --- a/internal/fs/node_test.go +++ b/internal/fs/node_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -47,10 +48,10 @@ func parseTime(s string) time.Time { return t.Local() } -var nodeTests = []restic.Node{ +var nodeTests = []data.Node{ { Name: "testFile", - Type: restic.NodeTypeFile, + Type: data.NodeTypeFile, Content: restic.IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), @@ -61,7 +62,7 @@ var nodeTests = []restic.Node{ }, { Name: "testSuidFile", - Type: restic.NodeTypeFile, + Type: data.NodeTypeFile, Content: restic.IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), @@ -72,7 +73,7 @@ var nodeTests = []restic.Node{ }, { Name: "testSuidFile2", - Type: restic.NodeTypeFile, + Type: data.NodeTypeFile, Content: restic.IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), @@ -83,7 +84,7 @@ var nodeTests = []restic.Node{ }, { Name: "testSticky", - Type: restic.NodeTypeFile, + Type: data.NodeTypeFile, Content: restic.IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), @@ -94,7 +95,7 @@ var nodeTests = []restic.Node{ }, { Name: "testDir", - Type: restic.NodeTypeDir, + Type: data.NodeTypeDir, Subtree: nil, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), @@ -105,7 +106,7 @@ var nodeTests = []restic.Node{ }, { Name: "testSymlink", - Type: restic.NodeTypeSymlink, + Type: data.NodeTypeSymlink, LinkTarget: "invalid", UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), @@ -119,7 +120,7 @@ var nodeTests = []restic.Node{ // metadata, so we can test if CreateAt works with pre-existing files. { Name: "testFile", - Type: restic.NodeTypeFile, + Type: data.NodeTypeFile, Content: restic.IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), @@ -130,7 +131,7 @@ var nodeTests = []restic.Node{ }, { Name: "testDir", - Type: restic.NodeTypeDir, + Type: data.NodeTypeDir, Subtree: nil, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), @@ -141,7 +142,7 @@ var nodeTests = []restic.Node{ }, { Name: "testXattrFile", - Type: restic.NodeTypeFile, + Type: data.NodeTypeFile, Content: restic.IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), @@ -149,13 +150,13 @@ var nodeTests = []restic.Node{ ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []restic.ExtendedAttribute{ + ExtendedAttributes: []data.ExtendedAttribute{ {Name: "user.foo", Value: []byte("bar")}, }, }, { Name: "testXattrDir", - Type: restic.NodeTypeDir, + Type: data.NodeTypeDir, Subtree: nil, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), @@ -163,13 +164,13 @@ var nodeTests = []restic.Node{ ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []restic.ExtendedAttribute{ + ExtendedAttributes: []data.ExtendedAttribute{ {Name: "user.foo", Value: []byte("bar")}, }, }, { Name: "testXattrFileMacOSResourceFork", - Type: restic.NodeTypeFile, + Type: data.NodeTypeFile, Content: restic.IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), @@ -177,7 +178,7 @@ var nodeTests = []restic.Node{ ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []restic.ExtendedAttribute{ + ExtendedAttributes: []data.ExtendedAttribute{ {Name: "com.apple.ResourceFork", Value: []byte("bar")}, }, }, @@ -242,7 +243,7 @@ func TestNodeRestoreAt(t *testing.T) { "%v: UID doesn't match (%v != %v)", test.Type, test.UID, n2.UID) rtest.Assert(t, test.GID == n2.GID, "%v: GID doesn't match (%v != %v)", test.Type, test.GID, n2.GID) - if test.Type != restic.NodeTypeSymlink { + if test.Type != data.NodeTypeSymlink { // On OpenBSD only root can set sticky bit (see sticky(8)). if runtime.GOOS != "openbsd" && runtime.GOOS != "netbsd" && runtime.GOOS != "solaris" && test.Name == "testSticky" { rtest.Assert(t, test.Mode == n2.Mode, @@ -262,11 +263,11 @@ func TestNodeRestoreAt(t *testing.T) { } } -func AssertFsTimeEqual(t *testing.T, label string, nodeType restic.NodeType, t1 time.Time, t2 time.Time) { +func AssertFsTimeEqual(t *testing.T, label string, nodeType data.NodeType, t1 time.Time, t2 time.Time) { var equal bool // Go currently doesn't support setting timestamps of symbolic links on darwin and bsd - if nodeType == restic.NodeTypeSymlink { + if nodeType == data.NodeTypeSymlink { switch runtime.GOOS { case "darwin", "freebsd", "openbsd", "netbsd", "solaris": return diff --git a/internal/fs/node_unix.go b/internal/fs/node_unix.go index e88e54251..436742a4f 100644 --- a/internal/fs/node_unix.go +++ b/internal/fs/node_unix.go @@ -6,7 +6,7 @@ package fs import ( "os" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/data" ) func lchown(name string, uid, gid int) error { @@ -14,11 +14,11 @@ func lchown(name string, uid, gid int) error { } // nodeRestoreGenericAttributes is no-op. -func nodeRestoreGenericAttributes(node *restic.Node, _ string, warn func(msg string)) error { - return restic.HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) +func nodeRestoreGenericAttributes(node *data.Node, _ string, warn func(msg string)) error { + return data.HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) } // nodeFillGenericAttributes is a no-op. -func nodeFillGenericAttributes(_ *restic.Node, _ string, _ *ExtendedFileInfo) error { +func nodeFillGenericAttributes(_ *data.Node, _ string, _ *ExtendedFileInfo) error { return nil } diff --git a/internal/fs/node_unix_notlinux.go b/internal/fs/node_unix_notlinux.go index f8846638c..451015c3f 100644 --- a/internal/fs/node_unix_notlinux.go +++ b/internal/fs/node_unix_notlinux.go @@ -5,12 +5,12 @@ package fs import ( "syscall" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/data" ) // utimesNano is like syscall.UtimesNano, except that it skips symlinks. -func utimesNano(path string, atime, mtime int64, typ restic.NodeType) error { - if typ == restic.NodeTypeSymlink { +func utimesNano(path string, atime, mtime int64, typ data.NodeType) error { + if typ == data.NodeTypeSymlink { return nil } diff --git a/internal/fs/node_unix_test.go b/internal/fs/node_unix_test.go index 7d5f7fa98..0c9588038 100644 --- a/internal/fs/node_unix_test.go +++ b/internal/fs/node_unix_test.go @@ -12,8 +12,8 @@ import ( "syscall" "testing" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -30,7 +30,7 @@ func stat(t testing.TB, filename string) (fi os.FileInfo, ok bool) { return fi, true } -func checkFile(t testing.TB, fi fs.FileInfo, node *restic.Node) { +func checkFile(t testing.TB, fi fs.FileInfo, node *data.Node) { t.Helper() stat := fi.Sys().(*syscall.Stat_t) @@ -47,7 +47,7 @@ func checkFile(t testing.TB, fi fs.FileInfo, node *restic.Node) { t.Errorf("Dev does not match, want %v, got %v", stat.Dev, node.DeviceID) } - if node.Size != uint64(stat.Size) && node.Type != restic.NodeTypeSymlink { + if node.Size != uint64(stat.Size) && node.Type != data.NodeTypeSymlink { t.Errorf("Size does not match, want %v, got %v", stat.Size, node.Size) } @@ -76,7 +76,7 @@ func checkFile(t testing.TB, fi fs.FileInfo, node *restic.Node) { } } -func checkDevice(t testing.TB, fi fs.FileInfo, node *restic.Node) { +func checkDevice(t testing.TB, fi fs.FileInfo, node *data.Node) { stat := fi.Sys().(*syscall.Stat_t) if node.Device != uint64(stat.Rdev) { t.Errorf("Rdev does not match, want %v, got %v", stat.Rdev, node.Device) @@ -124,9 +124,9 @@ func TestNodeFromFileInfo(t *testing.T) { rtest.OK(t, err) switch node.Type { - case restic.NodeTypeFile, restic.NodeTypeSymlink: + case data.NodeTypeFile, data.NodeTypeSymlink: checkFile(t, fi, node) - case restic.NodeTypeDev, restic.NodeTypeCharDev: + case data.NodeTypeDev, data.NodeTypeCharDev: checkFile(t, fi, node) checkDevice(t, fi, node) default: diff --git a/internal/fs/node_windows.go b/internal/fs/node_windows.go index 416b5c7f8..af0bd8256 100644 --- a/internal/fs/node_windows.go +++ b/internal/fs/node_windows.go @@ -10,9 +10,9 @@ import ( "syscall" "unsafe" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" "golang.org/x/sys/windows" ) @@ -43,7 +43,7 @@ func lchown(_ string, _ int, _ int) (err error) { } // utimesNano is like syscall.UtimesNano, except that it sets FILE_FLAG_OPEN_REPARSE_POINT. -func utimesNano(path string, atime, mtime int64, _ restic.NodeType) error { +func utimesNano(path string, atime, mtime int64, _ data.NodeType) error { // tweaked version of UtimesNano from go/src/syscall/syscall_windows.go pathp, e := syscall.UTF16PtrFromString(fixpath(path)) if e != nil { @@ -69,7 +69,7 @@ func utimesNano(path string, atime, mtime int64, _ restic.NodeType) error { } // restore extended attributes for windows -func nodeRestoreExtendedAttributes(node *restic.Node, path string, xattrSelectFilter func(xattrName string) bool) error { +func nodeRestoreExtendedAttributes(node *data.Node, path string, xattrSelectFilter func(xattrName string) bool) error { count := len(node.ExtendedAttributes) if count > 0 { eas := []extendedAttribute{} @@ -91,14 +91,14 @@ func nodeRestoreExtendedAttributes(node *restic.Node, path string, xattrSelectFi // fill extended attributes in the node // It also checks if the volume supports extended attributes and stores the result in a map // so that it does not have to be checked again for subsequent calls for paths in the same volume. -func nodeFillExtendedAttributes(node *restic.Node, path string, _ bool, _ func(format string, args ...any)) (err error) { +func nodeFillExtendedAttributes(node *data.Node, path string, _ bool, _ func(format string, args ...any)) (err error) { if strings.Contains(filepath.Base(path), ":") { // Do not process for Alternate Data Streams in Windows return nil } // only capture xattrs for file/dir - if node.Type != restic.NodeTypeFile && node.Type != restic.NodeTypeDir { + if node.Type != data.NodeTypeFile && node.Type != data.NodeTypeDir { return nil } @@ -131,7 +131,7 @@ func nodeFillExtendedAttributes(node *restic.Node, path string, _ bool, _ func(f //Fill the ExtendedAttributes in the node using the name/value pairs in the windows EA for _, attr := range extAtts { - extendedAttr := restic.ExtendedAttribute{ + extendedAttr := data.ExtendedAttribute{ Name: attr.Name, Value: attr.Value, } @@ -151,7 +151,7 @@ func closeFileHandle(fileHandle windows.Handle, path string) { // restoreExtendedAttributes handles restore of the Windows Extended Attributes to the specified path. // The Windows API requires setting of all the Extended Attributes in one call. -func restoreExtendedAttributes(nodeType restic.NodeType, path string, eas []extendedAttribute) (err error) { +func restoreExtendedAttributes(nodeType data.NodeType, path string, eas []extendedAttribute) (err error) { var fileHandle windows.Handle if fileHandle, err = openHandleForEA(nodeType, path, true); fileHandle == 0 { return nil @@ -188,7 +188,7 @@ func restoreExtendedAttributes(nodeType restic.NodeType, path string, eas []exte } // restoreGenericAttributes restores generic attributes for Windows -func nodeRestoreGenericAttributes(node *restic.Node, path string, warn func(msg string)) (err error) { +func nodeRestoreGenericAttributes(node *data.Node, path string, warn func(msg string)) (err error) { if len(node.GenericAttributes) == 0 { return nil } @@ -213,14 +213,14 @@ func nodeRestoreGenericAttributes(node *restic.Node, path string, warn func(msg } } - restic.HandleUnknownGenericAttributesFound(unknownAttribs, warn) + data.HandleUnknownGenericAttributesFound(unknownAttribs, warn) return errors.Join(errs...) } // genericAttributesToWindowsAttrs converts the generic attributes map to a WindowsAttributes and also returns a string of unknown attributes that it could not convert. -func genericAttributesToWindowsAttrs(attrs map[restic.GenericAttributeType]json.RawMessage) (windowsAttributes restic.WindowsAttributes, unknownAttribs []restic.GenericAttributeType, err error) { +func genericAttributesToWindowsAttrs(attrs map[data.GenericAttributeType]json.RawMessage) (windowsAttributes data.WindowsAttributes, unknownAttribs []data.GenericAttributeType, err error) { waValue := reflect.ValueOf(&windowsAttributes).Elem() - unknownAttribs, err = restic.GenericAttributesToOSAttrs(attrs, reflect.TypeOf(windowsAttributes), &waValue, "windows") + unknownAttribs, err = data.GenericAttributesToOSAttrs(attrs, reflect.TypeOf(windowsAttributes), &waValue, "windows") return windowsAttributes, unknownAttribs, err } @@ -341,7 +341,7 @@ func decryptFile(pathPointer *uint16) error { // nodeFillGenericAttributes fills in the generic attributes for windows like File Attributes, // Created time and Security Descriptors. -func nodeFillGenericAttributes(node *restic.Node, path string, stat *ExtendedFileInfo) error { +func nodeFillGenericAttributes(node *data.Node, path string, stat *ExtendedFileInfo) error { if strings.Contains(filepath.Base(path), ":") { // Do not process for Alternate Data Streams in Windows return nil @@ -360,7 +360,7 @@ func nodeFillGenericAttributes(node *restic.Node, path string, stat *ExtendedFil } var sd *[]byte - if node.Type == restic.NodeTypeFile || node.Type == restic.NodeTypeDir { + if node.Type == data.NodeTypeFile || node.Type == data.NodeTypeDir { if sd, err = getSecurityDescriptor(path); err != nil { return err } @@ -369,7 +369,7 @@ func nodeFillGenericAttributes(node *restic.Node, path string, stat *ExtendedFil winFI := stat.sys.(*syscall.Win32FileAttributeData) // Add Windows attributes - node.GenericAttributes, err = restic.WindowsAttrsToGenericAttributes(restic.WindowsAttributes{ + node.GenericAttributes, err = data.WindowsAttrsToGenericAttributes(data.WindowsAttributes{ CreationTime: &winFI.CreationTime, FileAttributes: &winFI.FileAttributes, SecurityDescriptor: sd, diff --git a/internal/fs/node_windows_test.go b/internal/fs/node_windows_test.go index 2d71e178e..426a074d5 100644 --- a/internal/fs/node_windows_test.go +++ b/internal/fs/node_windows_test.go @@ -14,8 +14,8 @@ import ( "testing" "time" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" "golang.org/x/sys/windows" ) @@ -24,19 +24,19 @@ func TestRestoreSecurityDescriptors(t *testing.T) { t.Parallel() tempDir := t.TempDir() for i, sd := range testFileSDs { - testRestoreSecurityDescriptor(t, sd, tempDir, restic.NodeTypeFile, fmt.Sprintf("testfile%d", i)) + testRestoreSecurityDescriptor(t, sd, tempDir, data.NodeTypeFile, fmt.Sprintf("testfile%d", i)) } for i, sd := range testDirSDs { - testRestoreSecurityDescriptor(t, sd, tempDir, restic.NodeTypeDir, fmt.Sprintf("testdir%d", i)) + testRestoreSecurityDescriptor(t, sd, tempDir, data.NodeTypeDir, fmt.Sprintf("testdir%d", i)) } } -func testRestoreSecurityDescriptor(t *testing.T, sd string, tempDir string, fileType restic.NodeType, fileName string) { +func testRestoreSecurityDescriptor(t *testing.T, sd string, tempDir string, fileType data.NodeType, fileName string) { // Decode the encoded string SD to get the security descriptor input in bytes. sdInputBytes, err := base64.StdEncoding.DecodeString(sd) test.OK(t, errors.Wrapf(err, "Error decoding SD for: %s", fileName)) // Wrap the security descriptor bytes in windows attributes and convert to generic attributes. - genericAttributes, err := restic.WindowsAttrsToGenericAttributes(restic.WindowsAttributes{CreationTime: nil, FileAttributes: nil, SecurityDescriptor: &sdInputBytes}) + genericAttributes, err := data.WindowsAttrsToGenericAttributes(data.WindowsAttributes{CreationTime: nil, FileAttributes: nil, SecurityDescriptor: &sdInputBytes}) test.OK(t, errors.Wrapf(err, "Error constructing windows attributes for: %s", fileName)) // Construct a Node with the generic attributes. expectedNode := getNode(fileName, fileType, genericAttributes) @@ -56,8 +56,8 @@ func testRestoreSecurityDescriptor(t *testing.T, sd string, tempDir string, file compareSecurityDescriptors(t, testPath, *sdByteFromRestoredNode, *sdBytesFromRestoredPath) } -func getNode(name string, fileType restic.NodeType, genericAttributes map[restic.GenericAttributeType]json.RawMessage) restic.Node { - return restic.Node{ +func getNode(name string, fileType data.NodeType, genericAttributes map[data.GenericAttributeType]json.RawMessage) data.Node { + return data.Node{ Name: name, Type: fileType, Mode: 0644, @@ -68,7 +68,7 @@ func getNode(name string, fileType restic.NodeType, genericAttributes map[restic } } -func getWindowsAttr(t *testing.T, testPath string, node *restic.Node) restic.WindowsAttributes { +func getWindowsAttr(t *testing.T, testPath string, node *data.Node) data.WindowsAttributes { windowsAttributes, unknownAttribs, err := genericAttributesToWindowsAttrs(node.GenericAttributes) test.OK(t, errors.Wrapf(err, "Error getting windows attr from generic attr: %s", testPath)) test.Assert(t, len(unknownAttribs) == 0, "Unknown attribs found: %s for: %s", unknownAttribs, testPath) @@ -83,19 +83,19 @@ func TestRestoreCreationTime(t *testing.T) { attr := fi.Sys().(*syscall.Win32FileAttributeData) creationTimeAttribute := attr.CreationTime //Using the temp dir creation time as the test creation time for the test file and folder - runGenericAttributesTest(t, path, restic.TypeCreationTime, restic.WindowsAttributes{CreationTime: &creationTimeAttribute}, false) + runGenericAttributesTest(t, path, data.TypeCreationTime, data.WindowsAttributes{CreationTime: &creationTimeAttribute}, false) } func TestRestoreFileAttributes(t *testing.T) { t.Parallel() - genericAttributeName := restic.TypeFileAttributes + genericAttributeName := data.TypeFileAttributes tempDir := t.TempDir() normal := uint32(syscall.FILE_ATTRIBUTE_NORMAL) hidden := uint32(syscall.FILE_ATTRIBUTE_HIDDEN) system := uint32(syscall.FILE_ATTRIBUTE_SYSTEM) archive := uint32(syscall.FILE_ATTRIBUTE_ARCHIVE) encrypted := uint32(windows.FILE_ATTRIBUTE_ENCRYPTED) - fileAttributes := []restic.WindowsAttributes{ + fileAttributes := []data.WindowsAttributes{ //normal {FileAttributes: &normal}, //hidden @@ -108,12 +108,12 @@ func TestRestoreFileAttributes(t *testing.T) { {FileAttributes: &encrypted}, } for i, fileAttr := range fileAttributes { - genericAttrs, err := restic.WindowsAttrsToGenericAttributes(fileAttr) + genericAttrs, err := data.WindowsAttrsToGenericAttributes(fileAttr) test.OK(t, err) - expectedNodes := []restic.Node{ + expectedNodes := []data.Node{ { Name: fmt.Sprintf("testfile%d", i), - Type: restic.NodeTypeFile, + Type: data.NodeTypeFile, Mode: 0655, ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), @@ -128,7 +128,7 @@ func TestRestoreFileAttributes(t *testing.T) { system = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY | windows.FILE_ATTRIBUTE_SYSTEM) archive = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY | windows.FILE_ATTRIBUTE_ARCHIVE) encrypted = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY | windows.FILE_ATTRIBUTE_ENCRYPTED) - folderAttributes := []restic.WindowsAttributes{ + folderAttributes := []data.WindowsAttributes{ //normal {FileAttributes: &normal}, //hidden @@ -141,12 +141,12 @@ func TestRestoreFileAttributes(t *testing.T) { {FileAttributes: &encrypted}, } for i, folderAttr := range folderAttributes { - genericAttrs, err := restic.WindowsAttrsToGenericAttributes(folderAttr) + genericAttrs, err := data.WindowsAttrsToGenericAttributes(folderAttr) test.OK(t, err) - expectedNodes := []restic.Node{ + expectedNodes := []data.Node{ { Name: fmt.Sprintf("testdirectory%d", i), - Type: restic.NodeTypeDir, + Type: data.NodeTypeDir, Mode: 0755, ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), @@ -158,13 +158,13 @@ func TestRestoreFileAttributes(t *testing.T) { } } -func runGenericAttributesTest(t *testing.T, tempDir string, genericAttributeName restic.GenericAttributeType, genericAttributeExpected restic.WindowsAttributes, warningExpected bool) { - genericAttributes, err := restic.WindowsAttrsToGenericAttributes(genericAttributeExpected) +func runGenericAttributesTest(t *testing.T, tempDir string, genericAttributeName data.GenericAttributeType, genericAttributeExpected data.WindowsAttributes, warningExpected bool) { + genericAttributes, err := data.WindowsAttrsToGenericAttributes(genericAttributeExpected) test.OK(t, err) - expectedNodes := []restic.Node{ + expectedNodes := []data.Node{ { Name: "testfile", - Type: restic.NodeTypeFile, + Type: data.NodeTypeFile, Mode: 0644, ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), @@ -173,7 +173,7 @@ func runGenericAttributesTest(t *testing.T, tempDir string, genericAttributeName }, { Name: "testdirectory", - Type: restic.NodeTypeDir, + Type: data.NodeTypeDir, Mode: 0755, ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), @@ -183,29 +183,29 @@ func runGenericAttributesTest(t *testing.T, tempDir string, genericAttributeName } runGenericAttributesTestForNodes(t, expectedNodes, tempDir, genericAttributeName, genericAttributeExpected, warningExpected) } -func runGenericAttributesTestForNodes(t *testing.T, expectedNodes []restic.Node, tempDir string, genericAttr restic.GenericAttributeType, genericAttributeExpected restic.WindowsAttributes, warningExpected bool) { +func runGenericAttributesTestForNodes(t *testing.T, expectedNodes []data.Node, tempDir string, genericAttr data.GenericAttributeType, genericAttributeExpected data.WindowsAttributes, warningExpected bool) { for _, testNode := range expectedNodes { testPath, node := restoreAndGetNode(t, tempDir, &testNode, warningExpected) rawMessage := node.GenericAttributes[genericAttr] - genericAttrsExpected, err := restic.WindowsAttrsToGenericAttributes(genericAttributeExpected) + genericAttrsExpected, err := data.WindowsAttrsToGenericAttributes(genericAttributeExpected) test.OK(t, err) rawMessageExpected := genericAttrsExpected[genericAttr] test.Equals(t, rawMessageExpected, rawMessage, "Generic attribute: %s got from NodeFromFileInfo not equal for path: %s", string(genericAttr), testPath) } } -func restoreAndGetNode(t *testing.T, tempDir string, testNode *restic.Node, warningExpected bool) (string, *restic.Node) { +func restoreAndGetNode(t *testing.T, tempDir string, testNode *data.Node, warningExpected bool) (string, *data.Node) { testPath := filepath.Join(tempDir, "001", testNode.Name) err := os.MkdirAll(filepath.Dir(testPath), testNode.Mode) test.OK(t, errors.Wrapf(err, "Failed to create parent directories for: %s", testPath)) - if testNode.Type == restic.NodeTypeFile { + if testNode.Type == data.NodeTypeFile { testFile, err := os.Create(testPath) test.OK(t, errors.Wrapf(err, "Failed to create test file: %s", testPath)) testFile.Close() - } else if testNode.Type == restic.NodeTypeDir { + } else if testNode.Type == data.NodeTypeDir { err := os.Mkdir(testPath, testNode.Mode) test.OK(t, errors.Wrapf(err, "Failed to create test directory: %s", testPath)) @@ -231,19 +231,19 @@ func restoreAndGetNode(t *testing.T, tempDir string, testNode *restic.Node, warn return testPath, nodeFromFileInfo } -const TypeSomeNewAttribute restic.GenericAttributeType = "MockAttributes.SomeNewAttribute" +const TypeSomeNewAttribute data.GenericAttributeType = "MockAttributes.SomeNewAttribute" func TestNewGenericAttributeType(t *testing.T) { t.Parallel() - newGenericAttribute := map[restic.GenericAttributeType]json.RawMessage{} + newGenericAttribute := map[data.GenericAttributeType]json.RawMessage{} newGenericAttribute[TypeSomeNewAttribute] = []byte("any value") tempDir := t.TempDir() - expectedNodes := []restic.Node{ + expectedNodes := []data.Node{ { Name: "testfile", - Type: restic.NodeTypeFile, + Type: data.NodeTypeFile, Mode: 0644, ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), @@ -252,7 +252,7 @@ func TestNewGenericAttributeType(t *testing.T) { }, { Name: "testdirectory", - Type: restic.NodeTypeDir, + Type: data.NodeTypeDir, Mode: 0755, ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), @@ -272,26 +272,26 @@ func TestNewGenericAttributeType(t *testing.T) { func TestRestoreExtendedAttributes(t *testing.T) { t.Parallel() tempDir := t.TempDir() - expectedNodes := []restic.Node{ + expectedNodes := []data.Node{ { Name: "testfile", - Type: restic.NodeTypeFile, + Type: data.NodeTypeFile, Mode: 0644, ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []restic.ExtendedAttribute{ + ExtendedAttributes: []data.ExtendedAttribute{ {"user.foo", []byte("bar")}, }, }, { Name: "testdirectory", - Type: restic.NodeTypeDir, + Type: data.NodeTypeDir, Mode: 0755, ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []restic.ExtendedAttribute{ + ExtendedAttributes: []data.ExtendedAttribute{ {"user.foo", []byte("bar")}, }, }, @@ -302,9 +302,9 @@ func TestRestoreExtendedAttributes(t *testing.T) { var handle windows.Handle var err error utf16Path := windows.StringToUTF16Ptr(testPath) - if node.Type == restic.NodeTypeFile { + if node.Type == data.NodeTypeFile { handle, err = windows.CreateFile(utf16Path, windows.FILE_READ_EA, 0, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL, 0) - } else if node.Type == restic.NodeTypeDir { + } else if node.Type == data.NodeTypeDir { handle, err = windows.CreateFile(utf16Path, windows.FILE_READ_EA, 0, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_BACKUP_SEMANTICS, 0) } test.OK(t, errors.Wrapf(err, "Error opening file/directory for: %s", testPath)) diff --git a/internal/fs/node_xattr.go b/internal/fs/node_xattr.go index 9ebd3524b..546df1f37 100644 --- a/internal/fs/node_xattr.go +++ b/internal/fs/node_xattr.go @@ -7,9 +7,9 @@ import ( "os" "syscall" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" "github.com/pkg/xattr" ) @@ -63,7 +63,7 @@ func handleXattrErr(err error) error { } } -func nodeRestoreExtendedAttributes(node *restic.Node, path string, xattrSelectFilter func(xattrName string) bool) error { +func nodeRestoreExtendedAttributes(node *data.Node, path string, xattrSelectFilter func(xattrName string) bool) error { expectedAttrs := map[string]struct{}{} for _, attr := range node.ExtendedAttributes { // Only restore xattrs that match the filter @@ -96,7 +96,7 @@ func nodeRestoreExtendedAttributes(node *restic.Node, path string, xattrSelectFi return nil } -func nodeFillExtendedAttributes(node *restic.Node, path string, ignoreListError bool, warnf func(format string, args ...any)) error { +func nodeFillExtendedAttributes(node *data.Node, path string, ignoreListError bool, warnf func(format string, args ...any)) error { xattrs, err := listxattr(path) debug.Log("fillExtendedAttributes(%v) %v %v", path, xattrs, err) if err != nil { @@ -106,14 +106,14 @@ func nodeFillExtendedAttributes(node *restic.Node, path string, ignoreListError return err } - node.ExtendedAttributes = make([]restic.ExtendedAttribute, 0, len(xattrs)) + node.ExtendedAttributes = make([]data.ExtendedAttribute, 0, len(xattrs)) for _, attr := range xattrs { attrVal, err := getxattr(path, attr) if err != nil { warnf("can not obtain extended attribute %v for %v:\n", attr, path) continue } - attr := restic.ExtendedAttribute{ + attr := data.ExtendedAttribute{ Name: attr, Value: attrVal, } diff --git a/internal/fs/node_xattr_all_test.go b/internal/fs/node_xattr_all_test.go index 9e35582ba..dd26c97b3 100644 --- a/internal/fs/node_xattr_all_test.go +++ b/internal/fs/node_xattr_all_test.go @@ -11,12 +11,12 @@ import ( "strings" "testing" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/filter" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) -func setAndVerifyXattr(t *testing.T, file string, attrs []restic.ExtendedAttribute) { +func setAndVerifyXattr(t *testing.T, file string, attrs []data.ExtendedAttribute) { if runtime.GOOS == "windows" { // windows seems to convert the xattr name to upper case for i := range attrs { @@ -24,15 +24,15 @@ func setAndVerifyXattr(t *testing.T, file string, attrs []restic.ExtendedAttribu } } - node := &restic.Node{ - Type: restic.NodeTypeFile, + node := &data.Node{ + Type: data.NodeTypeFile, ExtendedAttributes: attrs, } /* restore all xattrs */ rtest.OK(t, nodeRestoreExtendedAttributes(node, file, func(_ string) bool { return true })) - nodeActual := &restic.Node{ - Type: restic.NodeTypeFile, + nodeActual := &data.Node{ + Type: data.NodeTypeFile, } rtest.OK(t, nodeFillExtendedAttributes(nodeActual, file, false, t.Logf)) @@ -40,7 +40,7 @@ func setAndVerifyXattr(t *testing.T, file string, attrs []restic.ExtendedAttribu } func setAndVerifyXattrWithSelectFilter(t *testing.T, file string, testAttr []testXattrToRestore, xattrSelectFilter func(_ string) bool) { - attrs := make([]restic.ExtendedAttribute, len(testAttr)) + attrs := make([]data.ExtendedAttribute, len(testAttr)) for i := range testAttr { // windows seems to convert the xattr name to upper case if runtime.GOOS == "windows" { @@ -49,15 +49,15 @@ func setAndVerifyXattrWithSelectFilter(t *testing.T, file string, testAttr []tes attrs[i] = testAttr[i].xattr } - node := &restic.Node{ - Type: restic.NodeTypeFile, + node := &data.Node{ + Type: data.NodeTypeFile, ExtendedAttributes: attrs, } rtest.OK(t, nodeRestoreExtendedAttributes(node, file, xattrSelectFilter)) - nodeActual := &restic.Node{ - Type: restic.NodeTypeFile, + nodeActual := &data.Node{ + Type: data.NodeTypeFile, } rtest.OK(t, nodeFillExtendedAttributes(nodeActual, file, false, t.Logf)) @@ -82,7 +82,7 @@ func setAndVerifyXattrWithSelectFilter(t *testing.T, file string, testAttr []tes } type testXattrToRestore struct { - xattr restic.ExtendedAttribute + xattr data.ExtendedAttribute shouldRestore bool } @@ -91,14 +91,14 @@ func TestOverwriteXattr(t *testing.T) { file := filepath.Join(dir, "file") rtest.OK(t, os.WriteFile(file, []byte("hello world"), 0o600)) - setAndVerifyXattr(t, file, []restic.ExtendedAttribute{ + setAndVerifyXattr(t, file, []data.ExtendedAttribute{ { Name: "user.foo", Value: []byte("bar"), }, }) - setAndVerifyXattr(t, file, []restic.ExtendedAttribute{ + setAndVerifyXattr(t, file, []data.ExtendedAttribute{ { Name: "user.other", Value: []byte("some"), @@ -133,21 +133,21 @@ func TestOverwriteXattrWithSelectFilter(t *testing.T) { setAndVerifyXattrWithSelectFilter(t, file, []testXattrToRestore{ { - xattr: restic.ExtendedAttribute{ + xattr: data.ExtendedAttribute{ Name: "user.foo", Value: []byte("bar"), }, shouldRestore: true, }, { - xattr: restic.ExtendedAttribute{ + xattr: data.ExtendedAttribute{ Name: "user.test", Value: []byte("testxattr"), }, shouldRestore: true, }, { - xattr: restic.ExtendedAttribute{ + xattr: data.ExtendedAttribute{ Name: "security.other", Value: []byte("testing"), }, @@ -163,35 +163,35 @@ func TestOverwriteXattrWithSelectFilter(t *testing.T) { setAndVerifyXattrWithSelectFilter(t, file, []testXattrToRestore{ { - xattr: restic.ExtendedAttribute{ + xattr: data.ExtendedAttribute{ Name: "user.other", Value: []byte("some"), }, shouldRestore: true, }, { - xattr: restic.ExtendedAttribute{ + xattr: data.ExtendedAttribute{ Name: "security.other", Value: []byte("testing"), }, shouldRestore: false, }, { - xattr: restic.ExtendedAttribute{ + xattr: data.ExtendedAttribute{ Name: "user.open", Value: []byte("door"), }, shouldRestore: true, }, { - xattr: restic.ExtendedAttribute{ + xattr: data.ExtendedAttribute{ Name: "user.common", Value: []byte("testing"), }, shouldRestore: true, }, { - xattr: restic.ExtendedAttribute{ + xattr: data.ExtendedAttribute{ Name: "user.bad", Value: []byte("dontincludeme"), }, diff --git a/internal/fuse/dir.go b/internal/fuse/dir.go index a0317a757..75fee3d17 100644 --- a/internal/fuse/dir.go +++ b/internal/fuse/dir.go @@ -14,6 +14,7 @@ import ( "github.com/anacrolix/fuse" "github.com/anacrolix/fuse/fs" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" ) @@ -28,10 +29,10 @@ var _ = fs.NodeStringLookuper(&dir{}) type dir struct { root *Root forget forgetFn - items map[string]*restic.Node + items map[string]*data.Node inode uint64 parentInode uint64 - node *restic.Node + node *data.Node m sync.Mutex cache treeCache } @@ -40,7 +41,7 @@ func cleanupNodeName(name string) string { return filepath.Base(name) } -func newDir(root *Root, forget forgetFn, inode, parentInode uint64, node *restic.Node) (*dir, error) { +func newDir(root *Root, forget forgetFn, inode, parentInode uint64, node *data.Node) (*dir, error) { debug.Log("new dir for %v (%v)", node.Name, node.Subtree) return &dir{ @@ -65,16 +66,16 @@ func unwrapCtxCanceled(err error) error { // replaceSpecialNodes replaces nodes with name "." and "/" by their contents. // Otherwise, the node is returned. -func replaceSpecialNodes(ctx context.Context, repo restic.BlobLoader, node *restic.Node) ([]*restic.Node, error) { - if node.Type != restic.NodeTypeDir || node.Subtree == nil { - return []*restic.Node{node}, nil +func replaceSpecialNodes(ctx context.Context, repo restic.BlobLoader, node *data.Node) ([]*data.Node, error) { + if node.Type != data.NodeTypeDir || node.Subtree == nil { + return []*data.Node{node}, nil } if node.Name != "." && node.Name != "/" { - return []*restic.Node{node}, nil + return []*data.Node{node}, nil } - tree, err := restic.LoadTree(ctx, repo, *node.Subtree) + tree, err := data.LoadTree(ctx, repo, *node.Subtree) if err != nil { return nil, unwrapCtxCanceled(err) } @@ -82,12 +83,12 @@ func replaceSpecialNodes(ctx context.Context, repo restic.BlobLoader, node *rest return tree.Nodes, nil } -func newDirFromSnapshot(root *Root, forget forgetFn, inode uint64, snapshot *restic.Snapshot) (*dir, error) { +func newDirFromSnapshot(root *Root, forget forgetFn, inode uint64, snapshot *data.Snapshot) (*dir, error) { debug.Log("new dir for snapshot %v (%v)", snapshot.ID(), snapshot.Tree) return &dir{ root: root, forget: forget, - node: &restic.Node{ + node: &data.Node{ AccessTime: snapshot.Time, ModTime: snapshot.Time, ChangeTime: snapshot.Time, @@ -109,12 +110,12 @@ func (d *dir) open(ctx context.Context) error { debug.Log("open dir %v (%v)", d.node.Name, d.node.Subtree) - tree, err := restic.LoadTree(ctx, d.root.repo, *d.node.Subtree) + tree, err := data.LoadTree(ctx, d.root.repo, *d.node.Subtree) if err != nil { debug.Log(" error loading tree %v: %v", d.node.Subtree, err) return unwrapCtxCanceled(err) } - items := make(map[string]*restic.Node) + items := make(map[string]*data.Node) for _, n := range tree.Nodes { if ctx.Err() != nil { return ctx.Err() @@ -156,7 +157,7 @@ func (d *dir) calcNumberOfLinks() uint32 { // of directories contained by d count := uint32(2) for _, node := range d.items { - if node.Type == restic.NodeTypeDir { + if node.Type == data.NodeTypeDir { count++ } } @@ -191,11 +192,11 @@ func (d *dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { name := cleanupNodeName(node.Name) var typ fuse.DirentType switch node.Type { - case restic.NodeTypeDir: + case data.NodeTypeDir: typ = fuse.DT_Dir - case restic.NodeTypeFile: + case data.NodeTypeFile: typ = fuse.DT_File - case restic.NodeTypeSymlink: + case data.NodeTypeSymlink: typ = fuse.DT_Link } @@ -225,13 +226,13 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) { } inode := inodeFromNode(d.inode, node) switch node.Type { - case restic.NodeTypeDir: + case data.NodeTypeDir: return newDir(d.root, forget, inode, d.inode, node) - case restic.NodeTypeFile: + case data.NodeTypeFile: return newFile(d.root, forget, inode, node) - case restic.NodeTypeSymlink: + case data.NodeTypeSymlink: return newLink(d.root, forget, inode, node) - case restic.NodeTypeDev, restic.NodeTypeCharDev, restic.NodeTypeFifo, restic.NodeTypeSocket: + case data.NodeTypeDev, data.NodeTypeCharDev, data.NodeTypeFifo, data.NodeTypeSocket: return newOther(d.root, forget, inode, node) default: debug.Log(" node %v has unknown type %v", name, node.Type) diff --git a/internal/fuse/file.go b/internal/fuse/file.go index a69471f83..8ce90961f 100644 --- a/internal/fuse/file.go +++ b/internal/fuse/file.go @@ -7,6 +7,7 @@ import ( "context" "sort" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -28,7 +29,7 @@ var _ = fs.NodeOpener(&file{}) type file struct { root *Root forget forgetFn - node *restic.Node + node *data.Node inode uint64 } @@ -38,7 +39,7 @@ type openFile struct { cumsize []uint64 } -func newFile(root *Root, forget forgetFn, inode uint64, node *restic.Node) (fusefile *file, err error) { +func newFile(root *Root, forget forgetFn, inode uint64, node *data.Node) (fusefile *file, err error) { debug.Log("create new file for %v with %d blobs", node.Name, len(node.Content)) return &file{ inode: inode, diff --git a/internal/fuse/fuse_test.go b/internal/fuse/fuse_test.go index 3c0648bc6..dc8128b65 100644 --- a/internal/fuse/fuse_test.go +++ b/internal/fuse/fuse_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/restic/restic/internal/bloblru" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -52,15 +53,15 @@ func firstSnapshotID(t testing.TB, repo restic.Lister) (first restic.ID) { return first } -func loadFirstSnapshot(t testing.TB, repo restic.ListerLoaderUnpacked) *restic.Snapshot { +func loadFirstSnapshot(t testing.TB, repo restic.ListerLoaderUnpacked) *data.Snapshot { id := firstSnapshotID(t, repo) - sn, err := restic.LoadSnapshot(context.TODO(), repo, id) + sn, err := data.LoadSnapshot(context.TODO(), repo, id) rtest.OK(t, err) return sn } -func loadTree(t testing.TB, repo restic.Loader, id restic.ID) *restic.Tree { - tree, err := restic.LoadTree(context.TODO(), repo, id) +func loadTree(t testing.TB, repo restic.Loader, id restic.ID) *data.Tree { + tree, err := data.LoadTree(context.TODO(), repo, id) rtest.OK(t, err) return tree } @@ -73,7 +74,7 @@ func TestFuseFile(t *testing.T) { timestamp, err := time.Parse(time.RFC3339, "2017-01-24T10:42:56+01:00") rtest.OK(t, err) - restic.TestCreateSnapshot(t, repo, timestamp, 2) + data.TestCreateSnapshot(t, repo, timestamp, 2) sn := loadFirstSnapshot(t, repo) tree := loadTree(t, repo, *sn.Tree) @@ -109,7 +110,7 @@ func TestFuseFile(t *testing.T) { t.Logf("filesize is %v, memfile has size %v", filesize, len(memfile)) - node := &restic.Node{ + node := &data.Node{ Name: "foo", Inode: 23, Mode: 0742, @@ -152,7 +153,7 @@ func TestFuseDir(t *testing.T) { root := &Root{repo: repo, blobCache: bloblru.New(blobCacheSize)} - node := &restic.Node{ + node := &data.Node{ Mode: 0755, UID: 42, GID: 43, @@ -180,7 +181,7 @@ func TestFuseDir(t *testing.T) { // Test top-level directories for their UID and GID. func TestTopUIDGID(t *testing.T) { repo := repository.TestRepository(t) - restic.TestCreateSnapshot(t, repo, time.Unix(1460289341, 207401672), 0) + data.TestCreateSnapshot(t, repo, time.Unix(1460289341, 207401672), 0) testTopUIDGID(t, Config{}, repo, uint32(os.Getuid()), uint32(os.Getgid())) testTopUIDGID(t, Config{OwnerIsRoot: true}, repo, 0, 0) @@ -210,7 +211,7 @@ func testTopUIDGID(t *testing.T, cfg Config, repo restic.Repository, uid, gid ui snapshotdir, err := idsdir.(fs.NodeStringLookuper).Lookup(ctx, snapID) rtest.OK(t, err) - // restic.TestCreateSnapshot does not set the UID/GID thus it must be always zero + // data.TestCreateSnapshot does not set the UID/GID thus it must be always zero err = snapshotdir.Attr(ctx, &attr) rtest.OK(t, err) rtest.Equals(t, uint32(0), attr.Uid) @@ -235,7 +236,7 @@ func testStableLookup(t *testing.T, node fs.Node, path string) fs.Node { func TestStableNodeObjects(t *testing.T) { repo := repository.TestRepository(t) - restic.TestCreateSnapshot(t, repo, time.Unix(1460289341, 207401672), 2) + data.TestCreateSnapshot(t, repo, time.Unix(1460289341, 207401672), 2) root := NewRoot(repo, Config{}) idsdir := testStableLookup(t, root, "ids") @@ -264,9 +265,9 @@ func TestBlocks(t *testing.T) { target := strings.Repeat("x", int(c.size)) for _, n := range []fs.Node{ - &file{root: root, node: &restic.Node{Size: uint64(c.size)}}, - &link{root: root, node: &restic.Node{LinkTarget: target}}, - &snapshotLink{root: root, snapshot: &restic.Snapshot{}, target: target}, + &file{root: root, node: &data.Node{Size: uint64(c.size)}}, + &link{root: root, node: &data.Node{LinkTarget: target}}, + &snapshotLink{root: root, snapshot: &data.Snapshot{}, target: target}, } { var a fuse.Attr err := n.Attr(context.TODO(), &a) @@ -277,7 +278,7 @@ func TestBlocks(t *testing.T) { } func TestInodeFromNode(t *testing.T) { - node := &restic.Node{Name: "foo.txt", Type: restic.NodeTypeCharDev, Links: 2} + node := &data.Node{Name: "foo.txt", Type: data.NodeTypeCharDev, Links: 2} ino1 := inodeFromNode(1, node) ino2 := inodeFromNode(2, node) rtest.Assert(t, ino1 == ino2, "inodes %d, %d of hard links differ", ino1, ino2) @@ -289,9 +290,9 @@ func TestInodeFromNode(t *testing.T) { // Regression test: in a path a/b/b, the grandchild should not get the // same inode as the grandparent. - a := &restic.Node{Name: "a", Type: restic.NodeTypeDir, Links: 2} - ab := &restic.Node{Name: "b", Type: restic.NodeTypeDir, Links: 2} - abb := &restic.Node{Name: "b", Type: restic.NodeTypeDir, Links: 2} + a := &data.Node{Name: "a", Type: data.NodeTypeDir, Links: 2} + ab := &data.Node{Name: "b", Type: data.NodeTypeDir, Links: 2} + abb := &data.Node{Name: "b", Type: data.NodeTypeDir, Links: 2} inoA := inodeFromNode(1, a) inoAb := inodeFromNode(inoA, ab) inoAbb := inodeFromNode(inoAb, abb) @@ -300,7 +301,7 @@ func TestInodeFromNode(t *testing.T) { } func TestLink(t *testing.T) { - node := &restic.Node{Name: "foo.txt", Type: restic.NodeTypeSymlink, Links: 1, LinkTarget: "dst", ExtendedAttributes: []restic.ExtendedAttribute{ + node := &data.Node{Name: "foo.txt", Type: data.NodeTypeSymlink, Links: 1, LinkTarget: "dst", ExtendedAttributes: []data.ExtendedAttribute{ {Name: "foo", Value: []byte("bar")}, }} @@ -329,15 +330,15 @@ var sink uint64 func BenchmarkInode(b *testing.B) { for _, sub := range []struct { name string - node restic.Node + node data.Node }{ { name: "no_hard_links", - node: restic.Node{Name: "a somewhat long-ish filename.svg.bz2", Type: restic.NodeTypeFifo}, + node: data.Node{Name: "a somewhat long-ish filename.svg.bz2", Type: data.NodeTypeFifo}, }, { name: "hard_link", - node: restic.Node{Name: "some other filename", Type: restic.NodeTypeFile, Links: 2}, + node: data.Node{Name: "some other filename", Type: data.NodeTypeFile, Links: 2}, }, } { b.Run(sub.name, func(b *testing.B) { diff --git a/internal/fuse/inode.go b/internal/fuse/inode.go index 88d5b8bb8..b70180445 100644 --- a/internal/fuse/inode.go +++ b/internal/fuse/inode.go @@ -7,7 +7,7 @@ import ( "encoding/binary" "github.com/cespare/xxhash/v2" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/data" ) const prime = 11400714785074694791 // prime1 from xxhash. @@ -24,8 +24,8 @@ func inodeFromName(parent uint64, name string) uint64 { } // inodeFromNode generates an inode number for a file within a snapshot. -func inodeFromNode(parent uint64, node *restic.Node) (inode uint64) { - if node.Links > 1 && node.Type != restic.NodeTypeDir { +func inodeFromNode(parent uint64, node *data.Node) (inode uint64) { + if node.Links > 1 && node.Type != data.NodeTypeDir { // If node has hard links, give them all the same inode, // irrespective of the parent. var buf [16]byte diff --git a/internal/fuse/link.go b/internal/fuse/link.go index f8bf8d3ee..e37cb48f3 100644 --- a/internal/fuse/link.go +++ b/internal/fuse/link.go @@ -8,7 +8,7 @@ import ( "github.com/anacrolix/fuse" "github.com/anacrolix/fuse/fs" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/data" ) // Statically ensure that *link implements the given interface @@ -20,11 +20,11 @@ var _ = fs.NodeReadlinker(&link{}) type link struct { root *Root forget forgetFn - node *restic.Node + node *data.Node inode uint64 } -func newLink(root *Root, forget forgetFn, inode uint64, node *restic.Node) (*link, error) { +func newLink(root *Root, forget forgetFn, inode uint64, node *data.Node) (*link, error) { return &link{root: root, forget: forget, inode: inode, node: node}, nil } diff --git a/internal/fuse/other.go b/internal/fuse/other.go index cbd9667cc..07ba0fe50 100644 --- a/internal/fuse/other.go +++ b/internal/fuse/other.go @@ -8,7 +8,7 @@ import ( "github.com/anacrolix/fuse" "github.com/anacrolix/fuse/fs" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/data" ) // Statically ensure that *other implements the given interface @@ -18,11 +18,11 @@ var _ = fs.NodeReadlinker(&other{}) type other struct { root *Root forget forgetFn - node *restic.Node + node *data.Node inode uint64 } -func newOther(root *Root, forget forgetFn, inode uint64, node *restic.Node) (*other, error) { +func newOther(root *Root, forget forgetFn, inode uint64, node *data.Node) (*other, error) { return &other{root: root, forget: forget, inode: inode, node: node}, nil } diff --git a/internal/fuse/root.go b/internal/fuse/root.go index 72a0634fc..40f52ae5a 100644 --- a/internal/fuse/root.go +++ b/internal/fuse/root.go @@ -7,6 +7,7 @@ import ( "os" "github.com/restic/restic/internal/bloblru" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" @@ -16,7 +17,7 @@ import ( // Config holds settings for the fuse mount. type Config struct { OwnerIsRoot bool - Filter restic.SnapshotFilter + Filter data.SnapshotFilter TimeTemplate string PathTemplates []string } diff --git a/internal/fuse/snapshots_dir.go b/internal/fuse/snapshots_dir.go index bcab16084..4883b9ed8 100644 --- a/internal/fuse/snapshots_dir.go +++ b/internal/fuse/snapshots_dir.go @@ -8,8 +8,8 @@ import ( "os" "syscall" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/restic" "github.com/anacrolix/fuse" "github.com/anacrolix/fuse/fs" @@ -138,14 +138,14 @@ type snapshotLink struct { forget forgetFn inode uint64 target string - snapshot *restic.Snapshot + snapshot *data.Snapshot } var _ = fs.NodeForgetter(&snapshotLink{}) var _ = fs.NodeReadlinker(&snapshotLink{}) // newSnapshotLink -func newSnapshotLink(root *Root, forget forgetFn, inode uint64, target string, snapshot *restic.Snapshot) (*snapshotLink, error) { +func newSnapshotLink(root *Root, forget forgetFn, inode uint64, target string, snapshot *data.Snapshot) (*snapshotLink, error) { return &snapshotLink{root: root, forget: forget, inode: inode, target: target, snapshot: snapshot}, nil } diff --git a/internal/fuse/snapshots_dirstruct.go b/internal/fuse/snapshots_dirstruct.go index 03ff53193..141132abc 100644 --- a/internal/fuse/snapshots_dirstruct.go +++ b/internal/fuse/snapshots_dirstruct.go @@ -14,14 +14,14 @@ import ( "sync" "time" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/restic" ) type MetaDirData struct { // set if this is a symlink or a snapshot mount point linkTarget string - snapshot *restic.Snapshot + snapshot *data.Snapshot // names is set if this is a pseudo directory names map[string]*MetaDirData } @@ -57,7 +57,7 @@ func NewSnapshotsDirStructure(root *Root, pathTemplates []string, timeTemplate s // pathsFromSn generates the paths from pathTemplate and timeTemplate // where the variables are replaced by the snapshot data. // The time is given as suffix if the pathTemplate ends with "%T". -func pathsFromSn(pathTemplate string, timeTemplate string, sn *restic.Snapshot) (paths []string, timeSuffix string) { +func pathsFromSn(pathTemplate string, timeTemplate string, sn *data.Snapshot) (paths []string, timeSuffix string) { timeformat := sn.Time.Format(timeTemplate) inVerb := false @@ -207,11 +207,11 @@ func uniqueName(entries map[string]*MetaDirData, prefix, name string) string { // makeDirs inserts all paths generated from pathTemplates and // TimeTemplate for all given snapshots into d.names. // Also adds d.latest links if "%T" is at end of a path template -func (d *SnapshotsDirStructure) makeDirs(snapshots restic.Snapshots) { +func (d *SnapshotsDirStructure) makeDirs(snapshots data.Snapshots) { entries := make(map[string]*MetaDirData) type mountData struct { - sn *restic.Snapshot + sn *data.Snapshot linkTarget string // if linkTarget!= "", this is a symlink childFn string child *MetaDirData @@ -293,8 +293,8 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error { return nil } - var snapshots restic.Snapshots - err := d.root.cfg.Filter.FindAll(ctx, d.root.repo, d.root.repo, nil, func(_ string, sn *restic.Snapshot, _ error) error { + var snapshots data.Snapshots + err := d.root.cfg.Filter.FindAll(ctx, d.root.repo, d.root.repo, nil, func(_ string, sn *data.Snapshot, _ error) error { if sn != nil { snapshots = append(snapshots, sn) } diff --git a/internal/fuse/snapshots_dirstruct_test.go b/internal/fuse/snapshots_dirstruct_test.go index 1e823a475..0295bca13 100644 --- a/internal/fuse/snapshots_dirstruct_test.go +++ b/internal/fuse/snapshots_dirstruct_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) @@ -15,8 +16,8 @@ import ( func TestPathsFromSn(t *testing.T) { id1, _ := restic.ParseID("1234567812345678123456781234567812345678123456781234567812345678") time1, _ := time.Parse("2006-01-02T15:04:05", "2021-01-01T00:00:01") - sn1 := &restic.Snapshot{Hostname: "host", Username: "user", Tags: []string{"tag1", "tag2"}, Time: time1} - restic.TestSetSnapshotID(t, sn1, id1) + sn1 := &data.Snapshot{Hostname: "host", Username: "user", Tags: []string{"tag1", "tag2"}, Time: time1} + data.TestSetSnapshotID(t, sn1, id1) var p []string var s string @@ -67,27 +68,27 @@ func TestMakeDirs(t *testing.T) { id0, _ := restic.ParseID("0000000012345678123456781234567812345678123456781234567812345678") time0, _ := time.Parse("2006-01-02T15:04:05", "2020-12-31T00:00:01") - sn0 := &restic.Snapshot{Hostname: "host", Username: "user", Tags: []string{"tag1", "tag2"}, Time: time0} - restic.TestSetSnapshotID(t, sn0, id0) + sn0 := &data.Snapshot{Hostname: "host", Username: "user", Tags: []string{"tag1", "tag2"}, Time: time0} + data.TestSetSnapshotID(t, sn0, id0) id1, _ := restic.ParseID("1234567812345678123456781234567812345678123456781234567812345678") time1, _ := time.Parse("2006-01-02T15:04:05", "2021-01-01T00:00:01") - sn1 := &restic.Snapshot{Hostname: "host", Username: "user", Tags: []string{"tag1", "tag2"}, Time: time1} - restic.TestSetSnapshotID(t, sn1, id1) + sn1 := &data.Snapshot{Hostname: "host", Username: "user", Tags: []string{"tag1", "tag2"}, Time: time1} + data.TestSetSnapshotID(t, sn1, id1) id2, _ := restic.ParseID("8765432112345678123456781234567812345678123456781234567812345678") time2, _ := time.Parse("2006-01-02T15:04:05", "2021-01-01T01:02:03") - sn2 := &restic.Snapshot{Hostname: "host2", Username: "user2", Tags: []string{"tag2", "tag3", "tag4"}, Time: time2} - restic.TestSetSnapshotID(t, sn2, id2) + sn2 := &data.Snapshot{Hostname: "host2", Username: "user2", Tags: []string{"tag2", "tag3", "tag4"}, Time: time2} + data.TestSetSnapshotID(t, sn2, id2) id3, _ := restic.ParseID("aaaaaaaa12345678123456781234567812345678123456781234567812345678") time3, _ := time.Parse("2006-01-02T15:04:05", "2021-01-01T01:02:03") - sn3 := &restic.Snapshot{Hostname: "host", Username: "user2", Tags: []string{}, Time: time3} - restic.TestSetSnapshotID(t, sn3, id3) + sn3 := &data.Snapshot{Hostname: "host", Username: "user2", Tags: []string{}, Time: time3} + data.TestSetSnapshotID(t, sn3, id3) - sds.makeDirs(restic.Snapshots{sn0, sn1, sn2, sn3}) + sds.makeDirs(data.Snapshots{sn0, sn1, sn2, sn3}) - expNames := make(map[string]*restic.Snapshot) + expNames := make(map[string]*data.Snapshot) expLatest := make(map[string]string) // entries for sn0 @@ -214,8 +215,8 @@ func TestMakeDirs(t *testing.T) { verifyEntries(t, expNames, expLatest, sds.entries) } -func verifyEntries(t *testing.T, expNames map[string]*restic.Snapshot, expLatest map[string]string, entries map[string]*MetaDirData) { - actNames := make(map[string]*restic.Snapshot) +func verifyEntries(t *testing.T, expNames map[string]*data.Snapshot, expLatest map[string]string, entries map[string]*MetaDirData) { + actNames := make(map[string]*data.Snapshot) actLatest := make(map[string]string) for path, entry := range entries { actNames[path] = entry.snapshot @@ -257,9 +258,9 @@ func TestMakeEmptyDirs(t *testing.T) { pathTemplates: pathTemplates, timeTemplate: timeTemplate, } - sds.makeDirs(restic.Snapshots{}) + sds.makeDirs(data.Snapshots{}) - expNames := make(map[string]*restic.Snapshot) + expNames := make(map[string]*data.Snapshot) expLatest := make(map[string]string) // empty entries for dir structure diff --git a/internal/fuse/xattr.go b/internal/fuse/xattr.go index e534c3c0e..f219c2a4b 100644 --- a/internal/fuse/xattr.go +++ b/internal/fuse/xattr.go @@ -5,18 +5,18 @@ package fuse import ( "github.com/anacrolix/fuse" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/restic" ) -func nodeToXattrList(node *restic.Node, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) { +func nodeToXattrList(node *data.Node, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) { debug.Log("Listxattr(%v, %v)", node.Name, req.Size) for _, attr := range node.ExtendedAttributes { resp.Append(attr.Name) } } -func nodeGetXattr(node *restic.Node, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error { +func nodeGetXattr(node *data.Node, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error { debug.Log("Getxattr(%v, %v, %v)", node.Name, req.Name, req.Size) attrval := node.GetExtendedAttribute(req.Name) if attrval != nil { diff --git a/internal/repository/index/master_index_test.go b/internal/repository/index/master_index_test.go index 063af862d..ad991b308 100644 --- a/internal/repository/index/master_index_test.go +++ b/internal/repository/index/master_index_test.go @@ -10,6 +10,7 @@ import ( "github.com/restic/restic/internal/checker" "github.com/restic/restic/internal/crypto" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository/index" "github.com/restic/restic/internal/restic" @@ -350,7 +351,7 @@ func createFilledRepo(t testing.TB, snapshots int, version uint) (restic.Reposit repo, unpacked, _ := repository.TestRepositoryWithVersion(t, version) for i := 0; i < snapshots; i++ { - restic.TestCreateSnapshot(t, repo, snapshotTime.Add(time.Duration(i)*time.Second), depth) + data.TestCreateSnapshot(t, repo, snapshotTime.Add(time.Duration(i)*time.Second), depth) } return repo, unpacked } @@ -423,7 +424,7 @@ func testIndexSavePartial(t *testing.T, version uint) { // add+remove new snapshot and track its pack files packsBefore := listPacks(t, repo) - sn := restic.TestCreateSnapshot(t, repo, snapshotTime.Add(time.Duration(4)*time.Second), depth) + sn := data.TestCreateSnapshot(t, repo, snapshotTime.Add(time.Duration(4)*time.Second), depth) rtest.OK(t, repo.RemoveUnpacked(context.TODO(), restic.WriteableSnapshotFile, *sn.ID())) packsAfter := listPacks(t, repo) newPacks := packsAfter.Sub(packsBefore) diff --git a/internal/restic/lock.go b/internal/restic/lock.go index 8616ca7e7..7b7f04af8 100644 --- a/internal/restic/lock.go +++ b/internal/restic/lock.go @@ -147,7 +147,7 @@ func (l *Lock) fillUserInfo() error { } l.Username = usr.Username - l.UID, l.GID, err = uidGidInt(usr) + l.UID, l.GID, err = UidGidInt(usr) return err } diff --git a/internal/restic/lock_unix.go b/internal/restic/lock_unix.go index 3f426ae32..4cbbceb6c 100644 --- a/internal/restic/lock_unix.go +++ b/internal/restic/lock_unix.go @@ -13,8 +13,8 @@ import ( "github.com/restic/restic/internal/errors" ) -// uidGidInt returns uid, gid of the user as a number. -func uidGidInt(u *user.User) (uid, gid uint32, err error) { +// UidGidInt returns uid, gid of the user as a number. +func UidGidInt(u *user.User) (uid, gid uint32, err error) { ui, err := strconv.ParseUint(u.Uid, 10, 32) if err != nil { return 0, 0, errors.Errorf("invalid UID %q", u.Uid) diff --git a/internal/restic/lock_windows.go b/internal/restic/lock_windows.go index 60ded23d0..3cd7c3517 100644 --- a/internal/restic/lock_windows.go +++ b/internal/restic/lock_windows.go @@ -7,8 +7,8 @@ import ( "github.com/restic/restic/internal/debug" ) -// uidGidInt always returns 0 on Windows, since uid isn't numbers -func uidGidInt(_ *user.User) (uid, gid uint32, err error) { +// UidGidInt always returns 0 on Windows, since uid isn't numbers +func UidGidInt(_ *user.User) (uid, gid uint32, err error) { return 0, 0, nil } diff --git a/internal/restic/testing.go b/internal/restic/testing.go index 3e0563430..73e00be70 100644 --- a/internal/restic/testing.go +++ b/internal/restic/testing.go @@ -1,162 +1,9 @@ package restic import ( - "context" "fmt" - "io" - "math/rand" - "testing" - "time" - - "github.com/restic/chunker" - "golang.org/x/sync/errgroup" ) -// fakeFile returns a reader which yields deterministic pseudo-random data. -func fakeFile(seed, size int64) io.Reader { - return io.LimitReader(rand.New(rand.NewSource(seed)), size) -} - -type fakeFileSystem struct { - t testing.TB - repo Repository - buf []byte - chunker *chunker.Chunker - rand *rand.Rand -} - -// saveFile reads from rd and saves the blobs in the repository. The list of -// IDs is returned. -func (fs *fakeFileSystem) saveFile(ctx context.Context, rd io.Reader) (blobs IDs) { - if fs.buf == nil { - fs.buf = make([]byte, chunker.MaxSize) - } - - if fs.chunker == nil { - fs.chunker = chunker.New(rd, fs.repo.Config().ChunkerPolynomial) - } else { - fs.chunker.Reset(rd, fs.repo.Config().ChunkerPolynomial) - } - - blobs = IDs{} - for { - chunk, err := fs.chunker.Next(fs.buf) - if err == io.EOF { - break - } - - if err != nil { - fs.t.Fatalf("unable to save chunk in repo: %v", err) - } - - id, _, _, err := fs.repo.SaveBlob(ctx, DataBlob, chunk.Data, ID{}, false) - if err != nil { - fs.t.Fatalf("error saving chunk: %v", err) - } - - blobs = append(blobs, id) - } - - return blobs -} - -const ( - maxFileSize = 20000 - maxSeed = 32 - maxNodes = 15 -) - -// saveTree saves a tree of fake files in the repo and returns the ID. -func (fs *fakeFileSystem) saveTree(ctx context.Context, seed int64, depth int) ID { - rnd := rand.NewSource(seed) - numNodes := int(rnd.Int63() % maxNodes) - - var tree Tree - for i := 0; i < numNodes; i++ { - - // randomly select the type of the node, either tree (p = 1/4) or file (p = 3/4). - if depth > 1 && rnd.Int63()%4 == 0 { - treeSeed := rnd.Int63() % maxSeed - id := fs.saveTree(ctx, treeSeed, depth-1) - - node := &Node{ - Name: fmt.Sprintf("dir-%v", treeSeed), - Type: NodeTypeDir, - Mode: 0755, - Subtree: &id, - } - - tree.Nodes = append(tree.Nodes, node) - continue - } - - fileSeed := rnd.Int63() % maxSeed - fileSize := (maxFileSize / maxSeed) * fileSeed - - node := &Node{ - Name: fmt.Sprintf("file-%v", fileSeed), - Type: NodeTypeFile, - Mode: 0644, - Size: uint64(fileSize), - } - - node.Content = fs.saveFile(ctx, fakeFile(fileSeed, fileSize)) - tree.Nodes = append(tree.Nodes, node) - } - - tree.Sort() - - id, err := SaveTree(ctx, fs.repo, &tree) - if err != nil { - fs.t.Fatalf("SaveTree returned error: %v", err) - } - return id -} - -// TestCreateSnapshot creates a snapshot filled with fake data. The -// fake data is generated deterministically from the timestamp `at`, which is -// also used as the snapshot's timestamp. The tree's depth can be specified -// with the parameter depth. The parameter duplication is a probability that -// the same blob will saved again. -func TestCreateSnapshot(t testing.TB, repo Repository, at time.Time, depth int) *Snapshot { - seed := at.Unix() - t.Logf("create fake snapshot at %s with seed %d", at, seed) - - fakedir := fmt.Sprintf("fakedir-at-%v", at.Format("2006-01-02 15:04:05")) - snapshot, err := NewSnapshot([]string{fakedir}, []string{"test"}, "foo", at) - if err != nil { - t.Fatal(err) - } - - fs := fakeFileSystem{ - t: t, - repo: repo, - rand: rand.New(rand.NewSource(seed)), - } - - var wg errgroup.Group - repo.StartPackUploader(context.TODO(), &wg) - - treeID := fs.saveTree(context.TODO(), seed, depth) - snapshot.Tree = &treeID - - err = repo.Flush(context.Background()) - if err != nil { - t.Fatal(err) - } - - id, err := SaveSnapshot(context.TODO(), repo, snapshot) - if err != nil { - t.Fatal(err) - } - - snapshot.id = &id - - t.Logf("saved snapshot %v", id.Str()) - - return snapshot -} - // TestParseID parses s as a ID and panics if that fails. func TestParseID(s string) ID { id, err := ParseID(s) @@ -171,38 +18,3 @@ func TestParseID(s string) ID { func TestParseHandle(s string, t BlobType) BlobHandle { return BlobHandle{ID: TestParseID(s), Type: t} } - -// TestSetSnapshotID sets the snapshot's ID. -func TestSetSnapshotID(_ testing.TB, sn *Snapshot, id ID) { - sn.id = &id -} - -// ParseDurationOrPanic parses a duration from a string or panics if string is invalid. -// The format is `6y5m234d37h`. -func ParseDurationOrPanic(s string) Duration { - d, err := ParseDuration(s) - if err != nil { - panic(err) - } - - return d -} - -// TestLoadAllSnapshots returns a list of all snapshots in the repo. -// If a snapshot ID is in excludeIDs, it will not be included in the result. -func TestLoadAllSnapshots(ctx context.Context, repo ListerLoaderUnpacked, excludeIDs IDSet) (snapshots Snapshots, err error) { - err = ForAllSnapshots(ctx, repo, repo, excludeIDs, func(id ID, sn *Snapshot, err error) error { - if err != nil { - return err - } - - snapshots = append(snapshots, sn) - return nil - }) - - if err != nil { - return nil, err - } - - return snapshots, nil -} diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index 8ba323754..da70c7ef0 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -8,6 +8,7 @@ import ( "path/filepath" "sync/atomic" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" @@ -21,7 +22,7 @@ import ( // Restorer is used to restore a snapshot to a directory. type Restorer struct { repo restic.Repository - sn *restic.Snapshot + sn *data.Snapshot opts Options fileList map[string]bool @@ -98,7 +99,7 @@ func (c *OverwriteBehavior) Type() string { } // NewRestorer creates a restorer preloaded with the content from the snapshot id. -func NewRestorer(repo restic.Repository, sn *restic.Snapshot, opts Options) *Restorer { +func NewRestorer(repo restic.Repository, sn *data.Snapshot, opts Options) *Restorer { r := &Restorer{ repo: repo, opts: opts, @@ -113,11 +114,11 @@ func NewRestorer(repo restic.Repository, sn *restic.Snapshot, opts Options) *Res } type treeVisitor struct { - enterDir func(node *restic.Node, target, location string) error - visitNode func(node *restic.Node, target, location string) error + enterDir func(node *data.Node, target, location string) error + visitNode func(node *data.Node, target, location string) error // 'entries' contains all files the snapshot contains for this node. This also includes files // ignored by the SelectFilter. - leaveDir func(node *restic.Node, target, location string, entries []string) error + leaveDir func(node *data.Node, target, location string, entries []string) error } func (res *Restorer) sanitizeError(location string, err error) error { @@ -154,7 +155,7 @@ func (res *Restorer) traverseTree(ctx context.Context, target string, treeID res func (res *Restorer) traverseTreeInner(ctx context.Context, target, location string, treeID restic.ID, visitor treeVisitor) (filenames []string, hasRestored bool, err error) { debug.Log("%v %v %v", target, location, treeID) - tree, err := restic.LoadTree(ctx, res.repo, treeID) + tree, err := data.LoadTree(ctx, res.repo, treeID) if err != nil { debug.Log("error loading tree %v: %v", treeID, err) return nil, hasRestored, res.sanitizeError(location, err) @@ -206,18 +207,18 @@ func (res *Restorer) traverseTreeInner(ctx context.Context, target, location str } // sockets cannot be restored - if node.Type == restic.NodeTypeSocket { + if node.Type == data.NodeTypeSocket { continue } - selectedForRestore, childMayBeSelected := res.SelectFilter(nodeLocation, node.Type == restic.NodeTypeDir) + selectedForRestore, childMayBeSelected := res.SelectFilter(nodeLocation, node.Type == data.NodeTypeDir) debug.Log("SelectFilter returned %v %v for %q", selectedForRestore, childMayBeSelected, nodeLocation) if selectedForRestore { hasRestored = true } - if node.Type == restic.NodeTypeDir { + if node.Type == data.NodeTypeDir { if node.Subtree == nil { return nil, hasRestored, errors.Errorf("Dir without subtree in tree %v", treeID.Str()) } @@ -269,7 +270,7 @@ func (res *Restorer) traverseTreeInner(ctx context.Context, target, location str return filenames, hasRestored, nil } -func (res *Restorer) restoreNodeTo(node *restic.Node, target, location string) error { +func (res *Restorer) restoreNodeTo(node *data.Node, target, location string) error { if !res.opts.DryRun { debug.Log("restoreNode %v %v %v", node.Name, target, location) if err := fs.Remove(target); err != nil && !errors.Is(err, os.ErrNotExist) { @@ -287,7 +288,7 @@ func (res *Restorer) restoreNodeTo(node *restic.Node, target, location string) e return res.restoreNodeMetadataTo(node, target, location) } -func (res *Restorer) restoreNodeMetadataTo(node *restic.Node, target, location string) error { +func (res *Restorer) restoreNodeMetadataTo(node *data.Node, target, location string) error { if res.opts.DryRun { return nil } @@ -299,7 +300,7 @@ func (res *Restorer) restoreNodeMetadataTo(node *restic.Node, target, location s return err } -func (res *Restorer) restoreHardlinkAt(node *restic.Node, target, path, location string) error { +func (res *Restorer) restoreHardlinkAt(node *data.Node, target, path, location string) error { if !res.opts.DryRun { if err := fs.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { return errors.Wrap(err, "RemoveCreateHardlink") @@ -368,7 +369,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) (uint64, error) // first tree pass: create directories and collect all files to restore err = res.traverseTree(ctx, dst, *res.sn.Tree, treeVisitor{ - enterDir: func(_ *restic.Node, target, location string) error { + enterDir: func(_ *data.Node, target, location string) error { debug.Log("first pass, enterDir: mkdir %q, leaveDir should restore metadata", location) if location != string(filepath.Separator) { res.opts.Progress.AddFile(0) @@ -376,13 +377,13 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) (uint64, error) return res.ensureDir(target) }, - visitNode: func(node *restic.Node, target, location string) error { + visitNode: func(node *data.Node, target, location string) error { debug.Log("first pass, visitNode: mkdir %q, leaveDir on second pass should restore metadata", location) if err := res.ensureDir(filepath.Dir(target)); err != nil { return err } - if node.Type != restic.NodeTypeFile { + if node.Type != data.NodeTypeFile { res.opts.Progress.AddFile(0) return nil } @@ -436,9 +437,9 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) (uint64, error) // second tree pass: restore special files and filesystem metadata err = res.traverseTree(ctx, dst, *res.sn.Tree, treeVisitor{ - visitNode: func(node *restic.Node, target, location string) error { + visitNode: func(node *data.Node, target, location string) error { debug.Log("second pass, visitNode: restore node %q", location) - if node.Type != restic.NodeTypeFile { + if node.Type != data.NodeTypeFile { _, err := res.withOverwriteCheck(ctx, node, target, location, false, nil, func(_ bool, _ *fileState) error { return res.restoreNodeTo(node, target, location) }) @@ -458,7 +459,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) (uint64, error) // don't touch skipped files return nil }, - leaveDir: func(node *restic.Node, target, location string, expectedFilenames []string) error { + leaveDir: func(node *data.Node, target, location string, expectedFilenames []string) error { if res.opts.Delete { if err := res.removeUnexpectedFiles(ctx, target, location, expectedFilenames); err != nil { return err @@ -555,7 +556,7 @@ func (res *Restorer) hasRestoredFile(location string) (metadataOnly bool, ok boo return metadataOnly, ok } -func (res *Restorer) withOverwriteCheck(ctx context.Context, node *restic.Node, target, location string, isHardlink bool, buf []byte, cb func(updateMetadataOnly bool, matches *fileState) error) ([]byte, error) { +func (res *Restorer) withOverwriteCheck(ctx context.Context, node *data.Node, target, location string, isHardlink bool, buf []byte, cb func(updateMetadataOnly bool, matches *fileState) error) ([]byte, error) { overwrite, err := shouldOverwrite(res.opts.Overwrite, node, target) if err != nil { return buf, err @@ -570,7 +571,7 @@ func (res *Restorer) withOverwriteCheck(ctx context.Context, node *restic.Node, var matches *fileState updateMetadataOnly := false - if node.Type == restic.NodeTypeFile && !isHardlink { + if node.Type == data.NodeTypeFile && !isHardlink { // if a file fails to verify, then matches is nil which results in restoring from scratch matches, buf, _ = res.verifyFile(ctx, target, node, false, res.opts.Overwrite == OverwriteIfChanged, buf) // skip files that are already correct completely @@ -580,7 +581,7 @@ func (res *Restorer) withOverwriteCheck(ctx context.Context, node *restic.Node, return buf, cb(updateMetadataOnly, matches) } -func shouldOverwrite(overwrite OverwriteBehavior, node *restic.Node, destination string) (bool, error) { +func shouldOverwrite(overwrite OverwriteBehavior, node *data.Node, destination string) (bool, error) { if overwrite == OverwriteAlways || overwrite == OverwriteIfChanged { return true, nil } @@ -605,7 +606,7 @@ func shouldOverwrite(overwrite OverwriteBehavior, node *restic.Node, destination } // Snapshot returns the snapshot this restorer is configured to use. -func (res *Restorer) Snapshot() *restic.Snapshot { +func (res *Restorer) Snapshot() *data.Snapshot { return res.sn } @@ -618,7 +619,7 @@ const nVerifyWorkers = 8 // verified. func (res *Restorer) VerifyFiles(ctx context.Context, dst string, countRestoredFiles uint64, p *progress.Counter) (int, error) { type mustCheck struct { - node *restic.Node + node *data.Node path string } @@ -639,8 +640,8 @@ func (res *Restorer) VerifyFiles(ctx context.Context, dst string, countRestoredF defer close(work) err := res.traverseTree(ctx, dst, *res.sn.Tree, treeVisitor{ - visitNode: func(node *restic.Node, target, location string) error { - if node.Type != restic.NodeTypeFile { + visitNode: func(node *data.Node, target, location string) error { + if node.Type != data.NodeTypeFile { return nil } if metadataOnly, ok := res.hasRestoredFile(location); !ok || metadataOnly { @@ -708,7 +709,7 @@ func (s *fileState) HasMatchingBlob(i int) bool { // buf and the first return value are scratch space, passed around for reuse. // Reusing buffers prevents the verifier goroutines allocating all of RAM and // flushing the filesystem cache (at least on Linux). -func (res *Restorer) verifyFile(ctx context.Context, target string, node *restic.Node, failFast bool, trustMtime bool, buf []byte) (*fileState, []byte, error) { +func (res *Restorer) verifyFile(ctx context.Context, target string, node *data.Node, failFast bool, trustMtime bool, buf []byte) (*fileState, []byte, error) { f, err := fs.OpenFile(target, fs.O_RDONLY|fs.O_NOFOLLOW, 0) if err != nil { return nil, buf, err diff --git a/internal/restorer/restorer_test.go b/internal/restorer/restorer_test.go index 39e5411f5..ee9f7ef58 100644 --- a/internal/restorer/restorer_test.go +++ b/internal/restorer/restorer_test.go @@ -17,6 +17,7 @@ import ( "time" "github.com/restic/restic/internal/archiver" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" @@ -75,11 +76,11 @@ func saveFile(t testing.TB, repo restic.BlobSaver, data string) restic.ID { return id } -func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode uint64, getGenericAttributes func(attr *FileAttributes, isDir bool) (genericAttributes map[restic.GenericAttributeType]json.RawMessage)) restic.ID { +func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode uint64, getGenericAttributes func(attr *FileAttributes, isDir bool) (genericAttributes map[data.GenericAttributeType]json.RawMessage)) restic.ID { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - tree := &restic.Tree{} + tree := &data.Tree{} for name, n := range nodes { inode++ switch node := n.(type) { @@ -107,8 +108,8 @@ func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode u if mode == 0 { mode = 0644 } - err := tree.Insert(&restic.Node{ - Type: restic.NodeTypeFile, + err := tree.Insert(&data.Node{ + Type: data.NodeTypeFile, Mode: mode, ModTime: node.ModTime, Name: name, @@ -122,8 +123,8 @@ func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode u }) rtest.OK(t, err) case Symlink: - err := tree.Insert(&restic.Node{ - Type: restic.NodeTypeSymlink, + err := tree.Insert(&data.Node{ + Type: data.NodeTypeSymlink, Mode: os.ModeSymlink | 0o777, ModTime: node.ModTime, Name: name, @@ -142,8 +143,8 @@ func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode u mode = 0755 } - err := tree.Insert(&restic.Node{ - Type: restic.NodeTypeDir, + err := tree.Insert(&data.Node{ + Type: data.NodeTypeDir, Mode: mode, ModTime: node.ModTime, Name: name, @@ -158,7 +159,7 @@ func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode u } } - id, err := restic.SaveTree(ctx, repo, tree) + id, err := data.SaveTree(ctx, repo, tree) if err != nil { t.Fatal(err) } @@ -166,7 +167,7 @@ func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode u return id } -func saveSnapshot(t testing.TB, repo restic.Repository, snapshot Snapshot, getGenericAttributes func(attr *FileAttributes, isDir bool) (genericAttributes map[restic.GenericAttributeType]json.RawMessage)) (*restic.Snapshot, restic.ID) { +func saveSnapshot(t testing.TB, repo restic.Repository, snapshot Snapshot, getGenericAttributes func(attr *FileAttributes, isDir bool) (genericAttributes map[data.GenericAttributeType]json.RawMessage)) (*data.Snapshot, restic.ID) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -178,13 +179,13 @@ func saveSnapshot(t testing.TB, repo restic.Repository, snapshot Snapshot, getGe t.Fatal(err) } - sn, err := restic.NewSnapshot([]string{"test"}, nil, "", time.Now()) + sn, err := data.NewSnapshot([]string{"test"}, nil, "", time.Now()) if err != nil { t.Fatal(err) } sn.Tree = &treeID - id, err := restic.SaveSnapshot(ctx, repo, sn) + id, err := data.SaveSnapshot(ctx, repo, sn) if err != nil { t.Fatal(err) } @@ -192,7 +193,7 @@ func saveSnapshot(t testing.TB, repo restic.Repository, snapshot Snapshot, getGe return sn, id } -var noopGetGenericAttributes = func(attr *FileAttributes, isDir bool) (genericAttributes map[restic.GenericAttributeType]json.RawMessage) { +var noopGetGenericAttributes = func(attr *FileAttributes, isDir bool) (genericAttributes map[data.GenericAttributeType]json.RawMessage) { // No-op return nil } @@ -553,8 +554,8 @@ func checkVisitOrder(list []TreeVisit) TraverseTreeCheck { var pos int return func(t testing.TB) treeVisitor { - check := func(funcName string) func(*restic.Node, string, string, []string) error { - return func(node *restic.Node, target, location string, expectedFilenames []string) error { + check := func(funcName string) func(*data.Node, string, string, []string) error { + return func(node *data.Node, target, location string, expectedFilenames []string) error { if pos >= len(list) { t.Errorf("step %v, %v(%v): expected no more than %d function calls", pos, funcName, location, len(list)) pos++ @@ -580,9 +581,9 @@ func checkVisitOrder(list []TreeVisit) TraverseTreeCheck { return nil } } - checkNoFilename := func(funcName string) func(*restic.Node, string, string) error { + checkNoFilename := func(funcName string) func(*data.Node, string, string) error { f := check(funcName) - return func(node *restic.Node, target, location string) error { + return func(node *data.Node, target, location string) error { return f(node, target, location, nil) } } diff --git a/internal/restorer/restorer_windows_test.go b/internal/restorer/restorer_windows_test.go index 4764bed2d..804378975 100644 --- a/internal/restorer/restorer_windows_test.go +++ b/internal/restorer/restorer_windows_test.go @@ -15,9 +15,9 @@ import ( "time" "unsafe" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" - "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test" "golang.org/x/sys/windows" @@ -252,7 +252,7 @@ func createEncryptedFileWriteData(filepath string, fileInfo NodeInfo) (err error func setup(t *testing.T, nodesMap map[string]Node) *Restorer { repo := repository.TestRepository(t) - getFileAttributes := func(attr *FileAttributes, isDir bool) (genericAttributes map[restic.GenericAttributeType]json.RawMessage) { + getFileAttributes := func(attr *FileAttributes, isDir bool) (genericAttributes map[data.GenericAttributeType]json.RawMessage) { if attr == nil { return } @@ -263,7 +263,7 @@ func setup(t *testing.T, nodesMap map[string]Node) *Restorer { //If the node is a directory add FILE_ATTRIBUTE_DIRECTORY to attributes fileattr |= windows.FILE_ATTRIBUTE_DIRECTORY } - attrs, err := restic.WindowsAttrsToGenericAttributes(restic.WindowsAttributes{FileAttributes: &fileattr}) + attrs, err := data.WindowsAttrsToGenericAttributes(data.WindowsAttributes{FileAttributes: &fileattr}) test.OK(t, err) return attrs } diff --git a/internal/ui/backup/progress.go b/internal/ui/backup/progress.go index 0feaf64eb..17ccfa8f4 100644 --- a/internal/ui/backup/progress.go +++ b/internal/ui/backup/progress.go @@ -5,6 +5,7 @@ import ( "time" "github.com/restic/restic/internal/archiver" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" ) @@ -113,7 +114,7 @@ func (p *Progress) CompleteBlob(bytes uint64) { // CompleteItem is the status callback function for the archiver when a // file/dir has been saved successfully. -func (p *Progress) CompleteItem(item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration) { +func (p *Progress) CompleteItem(item string, previous, current *data.Node, s archiver.ItemStats, d time.Duration) { if current == nil { // error occurred, tell the status display to remove the line p.mu.Lock() @@ -123,7 +124,7 @@ func (p *Progress) CompleteItem(item string, previous, current *restic.Node, s a } switch current.Type { - case restic.NodeTypeDir: + case data.NodeTypeDir: p.mu.Lock() p.addProcessed(Counter{Dirs: 1}) p.mu.Unlock() @@ -137,7 +138,7 @@ func (p *Progress) CompleteItem(item string, previous, current *restic.Node, s a p.printer.CompleteItem("dir modified", item, s, d) } - case restic.NodeTypeFile: + case data.NodeTypeFile: p.mu.Lock() p.addProcessed(Counter{Files: 1}) delete(p.currentFiles, item) diff --git a/internal/ui/backup/progress_test.go b/internal/ui/backup/progress_test.go index 5c088336d..7b53f2116 100644 --- a/internal/ui/backup/progress_test.go +++ b/internal/ui/backup/progress_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/restic/restic/internal/archiver" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" ) @@ -54,10 +55,10 @@ func TestProgress(t *testing.T) { prog.CompleteBlob(1024) // "dir unchanged" - node := restic.Node{Type: restic.NodeTypeDir} + node := data.Node{Type: data.NodeTypeDir} prog.CompleteItem("foo", &node, &node, archiver.ItemStats{}, 0) // "file new" - node.Type = restic.NodeTypeFile + node.Type = data.NodeTypeFile prog.CompleteItem("foo", nil, &node, archiver.ItemStats{}, 0) time.Sleep(10 * time.Millisecond) diff --git a/internal/walker/rewriter.go b/internal/walker/rewriter.go index 968ef44f3..4d70e4d09 100644 --- a/internal/walker/rewriter.go +++ b/internal/walker/rewriter.go @@ -5,11 +5,12 @@ import ( "fmt" "path" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" ) -type NodeRewriteFunc func(node *restic.Node, path string) *restic.Node +type NodeRewriteFunc func(node *data.Node, path string) *data.Node type FailedTreeRewriteFunc func(nodeID restic.ID, path string, err error) (restic.ID, error) type QueryRewrittenSizeFunc func() SnapshotSize @@ -45,7 +46,7 @@ func NewTreeRewriter(opts RewriteOpts) *TreeRewriter { } // setup default implementations if rw.opts.RewriteNode == nil { - rw.opts.RewriteNode = func(node *restic.Node, _ string) *restic.Node { + rw.opts.RewriteNode = func(node *data.Node, _ string) *data.Node { return node } } @@ -63,9 +64,9 @@ func NewSnapshotSizeRewriter(rewriteNode NodeRewriteFunc) (*TreeRewriter, QueryR var size uint64 t := NewTreeRewriter(RewriteOpts{ - RewriteNode: func(node *restic.Node, path string) *restic.Node { + RewriteNode: func(node *data.Node, path string) *data.Node { node = rewriteNode(node, path) - if node != nil && node.Type == restic.NodeTypeFile { + if node != nil && node.Type == data.NodeTypeFile { count++ size += node.Size } @@ -94,7 +95,7 @@ func (t *TreeRewriter) RewriteTree(ctx context.Context, repo BlobLoadSaver, node } // a nil nodeID will lead to a load error - curTree, err := restic.LoadTree(ctx, repo, nodeID) + curTree, err := data.LoadTree(ctx, repo, nodeID) if err != nil { return t.opts.RewriteFailedTree(nodeID, nodepath, err) } @@ -103,7 +104,7 @@ func (t *TreeRewriter) RewriteTree(ctx context.Context, repo BlobLoadSaver, node // check that we can properly encode this tree without losing information // The alternative of using json/Decoder.DisallowUnknownFields() doesn't work as we use // a custom UnmarshalJSON to decode trees, see also https://github.com/golang/go/issues/41144 - testID, err := restic.SaveTree(ctx, repo, curTree) + testID, err := data.SaveTree(ctx, repo, curTree) if err != nil { return restic.ID{}, err } @@ -114,7 +115,7 @@ func (t *TreeRewriter) RewriteTree(ctx context.Context, repo BlobLoadSaver, node debug.Log("filterTree: %s, nodeId: %s\n", nodepath, nodeID.Str()) - tb := restic.NewTreeJSONBuilder() + tb := data.NewTreeJSONBuilder() for _, node := range curTree.Nodes { if ctx.Err() != nil { return restic.ID{}, ctx.Err() @@ -126,7 +127,7 @@ func (t *TreeRewriter) RewriteTree(ctx context.Context, repo BlobLoadSaver, node continue } - if node.Type != restic.NodeTypeDir { + if node.Type != data.NodeTypeDir { err = tb.AddNode(node) if err != nil { return restic.ID{}, err diff --git a/internal/walker/rewriter_test.go b/internal/walker/rewriter_test.go index 7c613e4ae..a21af36c6 100644 --- a/internal/walker/rewriter_test.go +++ b/internal/walker/rewriter_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/pkg/errors" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) @@ -44,7 +45,7 @@ func checkRewriteItemOrder(want []string) checkRewriteFunc { pos := 0 return func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB)) { rewriter = NewTreeRewriter(RewriteOpts{ - RewriteNode: func(node *restic.Node, path string) *restic.Node { + RewriteNode: func(node *data.Node, path string) *data.Node { if pos >= len(want) { t.Errorf("additional unexpected path found: %v", path) return nil @@ -74,7 +75,7 @@ func checkRewriteSkips(skipFor map[string]struct{}, want []string, disableCache return func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB)) { rewriter = NewTreeRewriter(RewriteOpts{ - RewriteNode: func(node *restic.Node, path string) *restic.Node { + RewriteNode: func(node *data.Node, path string) *data.Node { if pos >= len(want) { t.Errorf("additional unexpected path found: %v", path) return nil @@ -108,8 +109,8 @@ func checkRewriteSkips(skipFor map[string]struct{}, want []string, disableCache func checkIncreaseNodeSize(increase uint64) checkRewriteFunc { return func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB)) { rewriter = NewTreeRewriter(RewriteOpts{ - RewriteNode: func(node *restic.Node, path string) *restic.Node { - if node.Type == restic.NodeTypeFile { + RewriteNode: func(node *data.Node, path string) *data.Node { + if node.Type == data.NodeTypeFile { node.Size += increase } return node @@ -324,11 +325,11 @@ func TestSnapshotSizeQuery(t *testing.T) { ctx, cancel := context.WithCancel(context.TODO()) defer cancel() - rewriteNode := func(node *restic.Node, path string) *restic.Node { + rewriteNode := func(node *data.Node, path string) *data.Node { if path == "/bar" { return nil } - if node.Type == restic.NodeTypeFile { + if node.Type == data.NodeTypeFile { node.Size += 21 } return node @@ -366,7 +367,7 @@ func TestRewriterFailOnUnknownFields(t *testing.T) { defer cancel() rewriter := NewTreeRewriter(RewriteOpts{ - RewriteNode: func(node *restic.Node, path string) *restic.Node { + RewriteNode: func(node *data.Node, path string) *data.Node { // tree loading must not succeed t.Fail() return node diff --git a/internal/walker/walker.go b/internal/walker/walker.go index 0d69106bf..67c4a9d03 100644 --- a/internal/walker/walker.go +++ b/internal/walker/walker.go @@ -7,6 +7,7 @@ import ( "github.com/pkg/errors" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/restic" ) @@ -21,7 +22,7 @@ var ErrSkipNode = errors.New("skip this node") // When the special value ErrSkipNode is returned and node is a dir node, it is // not walked. When the node is not a dir node, the remaining items in this // tree are skipped. -type WalkFunc func(parentTreeID restic.ID, path string, node *restic.Node, nodeErr error) (err error) +type WalkFunc func(parentTreeID restic.ID, path string, node *data.Node, nodeErr error) (err error) type WalkVisitor struct { // If the node is a `dir`, it will be entered afterwards unless `ErrSkipNode` @@ -35,7 +36,7 @@ type WalkVisitor struct { // error, it is passed up the call stack. The trees in ignoreTrees are not // walked. If walkFn ignores trees, these are added to the set. func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, visitor WalkVisitor) error { - tree, err := restic.LoadTree(ctx, repo, root) + tree, err := data.LoadTree(ctx, repo, root) err = visitor.ProcessNode(root, "/", nil, err) if err != nil { @@ -51,7 +52,7 @@ func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, visitor W // walk recursively traverses the tree, ignoring subtrees when the ID of the // subtree is in ignoreTrees. If err is nil and ignore is true, the subtree ID // will be added to ignoreTrees by walk. -func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree *restic.Tree, visitor WalkVisitor) (err error) { +func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree *data.Tree, visitor WalkVisitor) (err error) { sort.Slice(tree.Nodes, func(i, j int) bool { return tree.Nodes[i].Name < tree.Nodes[j].Name }) @@ -63,11 +64,11 @@ func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTree p := path.Join(prefix, node.Name) - if node.Type == restic.NodeTypeInvalid { + if node.Type == data.NodeTypeInvalid { return errors.Errorf("node type is empty for node %q", node.Name) } - if node.Type != restic.NodeTypeDir { + if node.Type != data.NodeTypeDir { err := visitor.ProcessNode(parentTreeID, p, node, nil) if err != nil { if err == ErrSkipNode { @@ -85,7 +86,7 @@ func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTree return errors.Errorf("subtree for node %v in tree %v is nil", node.Name, p) } - subtree, err := restic.LoadTree(ctx, repo, *node.Subtree) + subtree, err := data.LoadTree(ctx, repo, *node.Subtree) err = visitor.ProcessNode(parentTreeID, p, node, err) if err != nil { if err == ErrSkipNode { diff --git a/internal/walker/walker_test.go b/internal/walker/walker_test.go index 47d872d97..fa561bf19 100644 --- a/internal/walker/walker_test.go +++ b/internal/walker/walker_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/pkg/errors" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -26,7 +27,7 @@ func BuildTreeMap(tree TestTree) (m TreeMap, root restic.ID) { } func buildTreeMap(tree TestTree, m TreeMap) restic.ID { - tb := restic.NewTreeJSONBuilder() + tb := data.NewTreeJSONBuilder() var names []string for name := range tree { names = append(names, name) @@ -37,9 +38,9 @@ func buildTreeMap(tree TestTree, m TreeMap) restic.ID { item := tree[name] switch elem := item.(type) { case TestFile: - err := tb.AddNode(&restic.Node{ + err := tb.AddNode(&data.Node{ Name: name, - Type: restic.NodeTypeFile, + Type: data.NodeTypeFile, Size: elem.Size, }) if err != nil { @@ -47,10 +48,10 @@ func buildTreeMap(tree TestTree, m TreeMap) restic.ID { } case TestTree: id := buildTreeMap(elem, m) - err := tb.AddNode(&restic.Node{ + err := tb.AddNode(&data.Node{ Name: name, Subtree: &id, - Type: restic.NodeTypeDir, + Type: data.NodeTypeDir, }) if err != nil { panic(err) @@ -100,7 +101,7 @@ type checkFunc func(t testing.TB) (walker WalkFunc, leaveDir func(path string) e func checkItemOrder(want []string) checkFunc { pos := 0 return func(t testing.TB) (walker WalkFunc, leaveDir func(path string) error, final func(testing.TB, error)) { - walker = func(treeID restic.ID, path string, node *restic.Node, err error) error { + walker = func(treeID restic.ID, path string, node *data.Node, err error) error { if err != nil { t.Errorf("error walking %v: %v", path, err) return err @@ -137,7 +138,7 @@ func checkItemOrder(want []string) checkFunc { func checkParentTreeOrder(want []string) checkFunc { pos := 0 return func(t testing.TB) (walker WalkFunc, leaveDir func(path string) error, final func(testing.TB, error)) { - walker = func(treeID restic.ID, path string, node *restic.Node, err error) error { + walker = func(treeID restic.ID, path string, node *data.Node, err error) error { if err != nil { t.Errorf("error walking %v: %v", path, err) return err @@ -172,7 +173,7 @@ func checkSkipFor(skipFor map[string]struct{}, wantPaths []string) checkFunc { var pos int return func(t testing.TB) (walker WalkFunc, leaveDir func(path string) error, final func(testing.TB, error)) { - walker = func(treeID restic.ID, path string, node *restic.Node, err error) error { + walker = func(treeID restic.ID, path string, node *data.Node, err error) error { if err != nil { t.Errorf("error walking %v: %v", path, err) return err @@ -214,7 +215,7 @@ func checkErrorReturned(errForPath string) checkFunc { expectedErr := fmt.Errorf("error for %v", errForPath) return func(t testing.TB) (walker WalkFunc, leaveDir func(path string) error, final func(testing.TB, error)) { - walker = func(treeID restic.ID, path string, node *restic.Node, err error) error { + walker = func(treeID restic.ID, path string, node *data.Node, err error) error { if path == errForPath { return expectedErr }