TheArchive/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2025-12-26 23:16:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4550 from ndecker/ls-ncdu

Browse Source 
Ls ncdu
This commit is contained in:
Michael Eischer
2024-01-27 12:27:35 +00:00
committed by GitHub
parent c90f24a06c 10e71af759
commit e44e4b00a6
11 changed files with 492 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
cmd/restic/cmd_find.go
View File

@@ -260,7 +260,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
}
f.out.newsn = sn
return walker.Walk(ctx, f.repo, *sn.Tree, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
if err != nil {
debug.Log("Error loading tree %v: %v", parentTreeID, err)
@@ -327,7 +327,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
debug.Log(" found match\n")
f.out.PrintPattern(nodepath, node)
return nil
})
}})
}
func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
@@ -338,7 +338,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
}
f.out.newsn = sn
return walker.Walk(ctx, f.repo, *sn.Tree, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
if err != nil {
debug.Log("Error loading tree %v: %v", parentTreeID, err)
@@ -388,7 +388,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
}
return nil
})
}})
}
var errAllPacksFound = errors.New("all packs found")

205
cmd/restic/cmd_ls.go
View File

@@ -3,6 +3,8 @@ package main
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"strings"
"time"
@@ -51,6 +53,7 @@ type LsOptions struct {
restic.SnapshotFilter
Recursive bool
HumanReadable bool
Ncdu bool
}
var lsOptions LsOptions
@@ -63,16 +66,47 @@ func init() {
flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories")
flags.BoolVar(&lsOptions.HumanReadable, "human-readable", false, "print sizes in human readable format")
flags.BoolVar(&lsOptions.Ncdu, "ncdu", false, "output NCDU export format (pipe into 'ncdu -f -')")
}
type lsSnapshot struct {
*restic.Snapshot
ID *restic.ID `json:"id"`
ShortID string `json:"short_id"`
StructType string `json:"struct_type"` // "snapshot"
type lsPrinter interface {
Snapshot(sn *restic.Snapshot)
Node(path string, node *restic.Node)
LeaveDir(path string)
Close()
}
type jsonLsPrinter struct {
enc *json.Encoder
}
func (p *jsonLsPrinter) Snapshot(sn *restic.Snapshot) {
type lsSnapshot struct {
*restic.Snapshot
ID *restic.ID `json:"id"`
ShortID string `json:"short_id"`
StructType string `json:"struct_type"` // "snapshot"
}
err := p.enc.Encode(lsSnapshot{
Snapshot: sn,
ID: sn.ID(),
ShortID: sn.ID().Str(),
StructType: "snapshot",
})
if err != nil {
Warnf("JSON encode failed: %v\n", err)
}
}
// Print node in our custom JSON format, followed by a newline.
func (p *jsonLsPrinter) Node(path string, node *restic.Node) {
err := lsNodeJSON(p.enc, path, node)
if err != nil {
Warnf("JSON encode failed: %v\n", err)
}
}
func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
n := &struct {
Name string `json:"name"`
@@ -114,10 +148,117 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
return enc.Encode(n)
}
func (p *jsonLsPrinter) LeaveDir(_ string) {}
func (p *jsonLsPrinter) Close() {}
type ncduLsPrinter struct {
out io.Writer
depth int
}
// lsSnapshotNcdu prints a restic snapshot in Ncdu save format.
// It opens the JSON list. Nodes are added with lsNodeNcdu and the list is closed by lsCloseNcdu.
// Format documentation: https://dev.yorhel.nl/ncdu/jsonfmt
func (p *ncduLsPrinter) Snapshot(sn *restic.Snapshot) {
const NcduMajorVer = 1
const NcduMinorVer = 2
snapshotBytes, err := json.Marshal(sn)
if err != nil {
Warnf("JSON encode failed: %v\n", err)
}
p.depth++
fmt.Fprintf(p.out, "[%d, %d, %s", NcduMajorVer, NcduMinorVer, string(snapshotBytes))
}
func lsNcduNode(_ string, node *restic.Node) ([]byte, error) {
type NcduNode struct {
Name string `json:"name"`
Asize uint64 `json:"asize"`
Dsize uint64 `json:"dsize"`
Dev uint64 `json:"dev"`
Ino uint64 `json:"ino"`
NLink uint64 `json:"nlink"`
NotReg bool `json:"notreg"`
UID uint32 `json:"uid"`
GID uint32 `json:"gid"`
Mode uint16 `json:"mode"`
Mtime int64 `json:"mtime"`
}
outNode := NcduNode{
Name: node.Name,
Asize: node.Size,
Dsize: node.Size,
Dev: node.DeviceID,
Ino: node.Inode,
NLink: node.Links,
NotReg: node.Type != "dir" && node.Type != "file",
UID: node.UID,
GID: node.GID,
Mode: uint16(node.Mode & os.ModePerm),
Mtime: node.ModTime.Unix(),
}
// bits according to inode(7) manpage
if node.Mode&os.ModeSetuid != 0 {
outNode.Mode |= 0o4000
}
if node.Mode&os.ModeSetgid != 0 {
outNode.Mode |= 0o2000
}
if node.Mode&os.ModeSticky != 0 {
outNode.Mode |= 0o1000
}
return json.Marshal(outNode)
}
func (p *ncduLsPrinter) Node(path string, node *restic.Node) {
out, err := lsNcduNode(path, node)
if err != nil {
Warnf("JSON encode failed: %v\n", err)
}
if node.Type == "dir" {
fmt.Fprintf(p.out, ",\n%s[\n%s%s", strings.Repeat(" ", p.depth), strings.Repeat(" ", p.depth+1), string(out))
p.depth++
} else {
fmt.Fprintf(p.out, ",\n%s%s", strings.Repeat(" ", p.depth), string(out))
}
}
func (p *ncduLsPrinter) LeaveDir(_ string) {
p.depth--
fmt.Fprintf(p.out, "\n%s]", strings.Repeat(" ", p.depth))
}
func (p *ncduLsPrinter) Close() {
fmt.Fprint(p.out, "\n]\n")
}
type textLsPrinter struct {
dirs []string
ListLong bool
HumanReadable bool
}
func (p *textLsPrinter) Snapshot(sn *restic.Snapshot) {
Verbosef("%v filtered by %v:\n", sn, p.dirs)
}
func (p *textLsPrinter) Node(path string, node *restic.Node) {
Printf("%s\n", formatNode(path, node, p.ListLong, p.HumanReadable))
}
func (p *textLsPrinter) LeaveDir(_ string) {}
func (p *textLsPrinter) Close() {}
func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error {
if len(args) == 0 {
return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'")
}
if opts.Ncdu && gopts.JSON {
return errors.Fatal("only either '--json' or '--ncdu' can be specified")
}
// extract any specific directories to walk
var dirs []string
@@ -179,38 +320,21 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
return err
}
var (
printSnapshot func(sn *restic.Snapshot)
printNode func(path string, node *restic.Node)
)
var printer lsPrinter
if gopts.JSON {
enc := json.NewEncoder(globalOptions.stdout)
printSnapshot = func(sn *restic.Snapshot) {
err := enc.Encode(lsSnapshot{
Snapshot: sn,
ID: sn.ID(),
ShortID: sn.ID().Str(),
StructType: "snapshot",
})
if err != nil {
Warnf("JSON encode failed: %v\n", err)
}
printer = &jsonLsPrinter{
enc: json.NewEncoder(globalOptions.stdout),
}
printNode = func(path string, node *restic.Node) {
err := lsNodeJSON(enc, path, node)
if err != nil {
Warnf("JSON encode failed: %v\n", err)
}
} else if opts.Ncdu {
printer = &ncduLsPrinter{
out: globalOptions.stdout,
}
} else {
printSnapshot = func(sn *restic.Snapshot) {
Verbosef("%v filtered by %v:\n", sn, dirs)
}
printNode = func(path string, node *restic.Node) {
Printf("%s\n", formatNode(path, node, opts.ListLong, opts.HumanReadable))
printer = &textLsPrinter{
dirs: dirs,
ListLong: opts.ListLong,
HumanReadable: opts.HumanReadable,
}
}
@@ -228,9 +352,9 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
return err
}
printSnapshot(sn)
printer.Snapshot(sn)
err = walker.Walk(ctx, repo, *sn.Tree, func(_ restic.ID, nodepath string, node *restic.Node, err error) error {
processNode := func(_ restic.ID, nodepath string, node *restic.Node, err error) error {
if err != nil {
return err
}
@@ -240,7 +364,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
if withinDir(nodepath) {
// if we're within a dir, print the node
printNode(nodepath, node)
printer.Node(nodepath, node)
// if recursive listing is requested, signal the walker that it
// should continue walking recursively
@@ -261,11 +385,22 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
return walker.ErrSkipNode
}
return nil
}
err = walker.Walk(ctx, repo, *sn.Tree, walker.WalkVisitor{
ProcessNode: processNode,
LeaveDir: func(path string) {
// the root path `/` has no corresponding node and is thus also skipped by processNode
if withinDir(path) && path != "/" {
printer.LeaveDir(path)
}
},
})
if err != nil {
return err
}
printer.Close()
return nil
}

36
cmd/restic/cmd_ls_integration_test.go
View File

@@ -2,18 +2,46 @@ package main
import (
"context"
"encoding/json"
"path/filepath"
"strings"
"testing"
rtest "github.com/restic/restic/internal/test"
)
func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
func testRunLsWithOpts(t testing.TB, gopts GlobalOptions, opts LsOptions, args []string) []byte {
buf, err := withCaptureStdout(func() error {
gopts.Quiet = true
opts := LsOptions{}
return runLs(context.TODO(), opts, gopts, []string{snapshotID})
return runLs(context.TODO(), opts, gopts, args)
})
rtest.OK(t, err)
return strings.Split(buf.String(), "\n")
return buf.Bytes()
}
func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
out := testRunLsWithOpts(t, gopts, LsOptions{}, []string{snapshotID})
return strings.Split(string(out), "\n")
}
func assertIsValidJSON(t *testing.T, data []byte) {
// Sanity check: output must be valid JSON.
var v interface{}
err := json.Unmarshal(data, &v)
rtest.OK(t, err)
}
func TestRunLsNcdu(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
testRunInit(t, env.gopts)
opts := BackupOptions{}
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
ncdu := testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest"})
assertIsValidJSON(t, ncdu)
ncdu = testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest", "/testdata"})
assertIsValidJSON(t, ncdu)
}

199
cmd/restic/cmd_ls_test.go
View File

@@ -11,78 +11,94 @@ import (
rtest "github.com/restic/restic/internal/test"
)
type lsTestNode struct {
path string
restic.Node
}
var lsTestNodes = []lsTestNode{
// Mode is omitted when zero.
// Permissions, by convention is "-" per mode bit
{
path: "/bar/baz",
Node: restic.Node{
Name: "baz",
Type: "file",
Size: 12345,
UID: 10000000,
GID: 20000000,
User: "nobody",
Group: "nobodies",
Links: 1,
},
},
// Even empty files get an explicit size.
{
path: "/foo/empty",
Node: restic.Node{
Name: "empty",
Type: "file",
Size: 0,
UID: 1001,
GID: 1001,
User: "not printed",
Group: "not printed",
Links: 0xF00,
},
},
// Non-regular files do not get a size.
// Mode is printed in decimal, including the type bits.
{
path: "/foo/link",
Node: restic.Node{
Name: "link",
Type: "symlink",
Mode: os.ModeSymlink | 0777,
LinkTarget: "not printed",
},
},
{
path: "/some/directory",
Node: restic.Node{
Name: "directory",
Type: "dir",
Mode: os.ModeDir | 0755,
ModTime: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC),
AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC),
ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC),
},
},
// Test encoding of setuid/setgid/sticky bit
{
path: "/some/sticky",
Node: restic.Node{
Name: "sticky",
Type: "dir",
Mode: os.ModeDir | 0755 | os.ModeSetuid | os.ModeSetgid | os.ModeSticky,
},
},
}
func TestLsNodeJSON(t *testing.T) {
for _, c := range []struct {
path string
restic.Node
expect string
}{
// Mode is omitted when zero.
// Permissions, by convention is "-" per mode bit
{
path: "/bar/baz",
Node: restic.Node{
Name: "baz",
Type: "file",
Size: 12345,
UID: 10000000,
GID: 20000000,
User: "nobody",
Group: "nobodies",
Links: 1,
},
expect: `{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
},
// Even empty files get an explicit size.
{
path: "/foo/empty",
Node: restic.Node{
Name: "empty",
Type: "file",
Size: 0,
UID: 1001,
GID: 1001,
User: "not printed",
Group: "not printed",
Links: 0xF00,
},
expect: `{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
},
// Non-regular files do not get a size.
// Mode is printed in decimal, including the type bits.
{
path: "/foo/link",
Node: restic.Node{
Name: "link",
Type: "symlink",
Mode: os.ModeSymlink | 0777,
LinkTarget: "not printed",
},
expect: `{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
},
{
path: "/some/directory",
Node: restic.Node{
Name: "directory",
Type: "dir",
Mode: os.ModeDir | 0755,
ModTime: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC),
AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC),
ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC),
},
expect: `{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`,
},
for i, expect := range []string{
`{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
`{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
`{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
`{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`,
`{"name":"sticky","type":"dir","path":"/some/sticky","uid":0,"gid":0,"mode":2161115629,"permissions":"dugtrwxr-xr-x","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
} {
c := lsTestNodes[i]
buf := new(bytes.Buffer)
enc := json.NewEncoder(buf)
err := lsNodeJSON(enc, c.path, &c.Node)
rtest.OK(t, err)
rtest.Equals(t, c.expect+"\n", buf.String())
rtest.Equals(t, expect+"\n", buf.String())
// Sanity check: output must be valid JSON.
var v interface{}
@@ -90,3 +106,54 @@ func TestLsNodeJSON(t *testing.T) {
rtest.OK(t, err)
}
}
func TestLsNcduNode(t *testing.T) {
for i, expect := range []string{
`{"name":"baz","asize":12345,"dsize":12345,"dev":0,"ino":0,"nlink":1,"notreg":false,"uid":10000000,"gid":20000000,"mode":0,"mtime":-62135596800}`,
`{"name":"empty","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":3840,"notreg":false,"uid":1001,"gid":1001,"mode":0,"mtime":-62135596800}`,
`{"name":"link","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":true,"uid":0,"gid":0,"mode":511,"mtime":-62135596800}`,
`{"name":"directory","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":493,"mtime":1577934245}`,
`{"name":"sticky","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":4077,"mtime":-62135596800}`,
} {
c := lsTestNodes[i]
out, err := lsNcduNode(c.path, &c.Node)
rtest.OK(t, err)
rtest.Equals(t, expect, string(out))
// Sanity check: output must be valid JSON.
var v interface{}
err = json.Unmarshal(out, &v)
rtest.OK(t, err)
}
}
func TestLsNcdu(t *testing.T) {
var buf bytes.Buffer
printer := &ncduLsPrinter{
out: &buf,
}
printer.Snapshot(&restic.Snapshot{
Hostname: "host",
Paths: []string{"/example"},
})
printer.Node("/directory", &restic.Node{
Type: "dir",
Name: "directory",
})
printer.Node("/directory/data", &restic.Node{
Type: "file",
Name: "data",
Size: 42,
})
printer.LeaveDir("/directory")
printer.Close()
rtest.Equals(t, `[1, 2, {"time":"0001-01-01T00:00:00Z","tree":null,"paths":["/example"],"hostname":"host"},
[
{"name":"directory","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":-62135596800},
{"name":"data","asize":42,"dsize":42,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":-62135596800}
]
]
`, buf.String())
}

4
cmd/restic/cmd_stats.go
View File

@@ -203,7 +203,9 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest
}
hardLinkIndex := restorer.NewHardlinkIndex[struct{}]()
err := walker.Walk(ctx, repo, *snapshot.Tree, statsWalkTree(repo, opts, stats, hardLinkIndex))
err := walker.Walk(ctx, repo, *snapshot.Tree, walker.WalkVisitor{
ProcessNode: statsWalkTree(repo, opts, stats, hardLinkIndex),
})
if err != nil {
return fmt.Errorf("walking tree %s: %v", *snapshot.Tree, err)
}