Add exclude filter to archiver and 'backup' command

cmd/restic/cmd_backup.go

@@ -10,13 +10,15 @@ import (
 
 	"github.com/restic/restic"
 	"github.com/restic/restic/backend"
+	"github.com/restic/restic/filter"
 	"github.com/restic/restic/repository"
 	"golang.org/x/crypto/ssh/terminal"
 )
 
 type CmdBackup struct {
-	Parent string `short:"p" long:"parent"    description:"use this parent snapshot (default: last snapshot in repo that has the same target)"`
-	Force  bool   `short:"f" long:"force" description:"Force re-reading the target. Overrides the \"parent\" flag"`
+	Parent  string   `short:"p" long:"parent"  description:"use this parent snapshot (default: last snapshot in repo that has the same target)"`
+	Force   bool     `short:"f" long:"force"   description:"Force re-reading the target. Overrides the \"parent\" flag"`
+	Exclude []string `short:"e" long:"exclude" description:"Exclude a pattern (can be specified multiple times)"`
 
 	global *GlobalOptions
 }
@@ -282,14 +284,22 @@ func (cmd CmdBackup) Execute(args []string) error {
 
 	cmd.global.Verbosef("scan %v\n", target)
 
-	stat, err := restic.Scan(target, cmd.newScanProgress())
+	selectFilter := func(item string, fi os.FileInfo) bool {
+		matched, err := filter.List(cmd.Exclude, item)
+		if err != nil {
+			cmd.global.Warnf("error for exclude pattern: %v", err)
+		}
 
-	// TODO: add filter
-	// arch.Filter = func(dir string, fi os.FileInfo) bool {
-	// 	return true
-	// }
+		return !matched
+	}
+
+	stat, err := restic.Scan(target, selectFilter, cmd.newScanProgress())
+	if err != nil {
+		return err
+	}
 
 	arch := restic.NewArchiver(repo)
+	arch.SelectFilter = selectFilter
 
 	arch.Error = func(dir string, fi os.FileInfo, err error) error {
 		// TODO: make ignoring errors configurable

cmd/restic/cmd_ls.go

@@ -11,6 +11,8 @@ import (
 )
 
 type CmdLs struct {
+	Long bool `short:"l" long:"long" description:"Use a long listing format showing size and mode"`
+
 	global *GlobalOptions
 }
 
@@ -24,7 +26,11 @@ func init() {
 	}
 }
 
-func printNode(prefix string, n *restic.Node) string {
+func (cmd CmdLs) printNode(prefix string, n *restic.Node) string {
+	if !cmd.Long {
+		return filepath.Join(prefix, n.Name)
+	}
+
 	switch n.Type {
 	case "file":
 		return fmt.Sprintf("%s %5d %5d %6d %s %s",
@@ -40,17 +46,17 @@ func printNode(prefix string, n *restic.Node) string {
 	}
 }
 
-func printTree(prefix string, repo *repository.Repository, id backend.ID) error {
+func (cmd CmdLs) printTree(prefix string, repo *repository.Repository, id backend.ID) error {
 	tree, err := restic.LoadTree(repo, id)
 	if err != nil {
 		return err
 	}
 
 	for _, entry := range tree.Nodes {
-		fmt.Println(printNode(prefix, entry))
+		cmd.global.Printf(cmd.printNode(prefix, entry) + "\n")
 
 		if entry.Type == "dir" && entry.Subtree != nil {
-			err = printTree(filepath.Join(prefix, entry.Name), repo, entry.Subtree)
+			err = cmd.printTree(filepath.Join(prefix, entry.Name), repo, entry.Subtree)
 			if err != nil {
 				return err
 			}
@@ -89,7 +95,7 @@ func (cmd CmdLs) Execute(args []string) error {
 		return err
 	}
 
-	fmt.Printf("snapshot of %v at %s:\n", sn.Paths, sn.Time)
+	cmd.global.Verbosef("snapshot of %v at %s:\n", sn.Paths, sn.Time)
 
-	return printTree("", repo, sn.Tree)
+	return cmd.printTree("", repo, sn.Tree)
 }

cmd/restic/integration_test.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"strings"
 	"syscall"
 	"testing"
 	"time"
@@ -44,7 +45,11 @@ func cmdInit(t testing.TB, global GlobalOptions) {
 }
 
 func cmdBackup(t testing.TB, global GlobalOptions, target []string, parentID backend.ID) {
-	cmd := &CmdBackup{global: &global}
+	cmdBackupExcludes(t, global, target, parentID, nil)
+}
+
+func cmdBackupExcludes(t testing.TB, global GlobalOptions, target []string, parentID backend.ID, excludes []string) {
+	cmd := &CmdBackup{global: &global, Exclude: excludes}
 	cmd.Parent = parentID.String()
 
 	t.Logf("backing up %v", target)
@@ -73,6 +78,16 @@ func cmdCheck(t testing.TB, global GlobalOptions) {
 	OK(t, cmd.Execute(nil))
 }
 
+func cmdLs(t testing.TB, global GlobalOptions, snapshotID string) []string {
+	var buf bytes.Buffer
+	global.stdout = &buf
+
+	cmd := &CmdLs{global: &global}
+	OK(t, cmd.Execute([]string{snapshotID}))
+
+	return strings.Split(string(buf.Bytes()), "\n")
+}
+
 func TestBackup(t *testing.T) {
 	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
 		datafile := filepath.Join("testdata", "backup-data.tar.gz")
@@ -237,6 +252,86 @@ func TestBackupMissingFile2(t *testing.T) {
 	})
 }
 
+func includes(haystack []string, needle string) bool {
+	for _, s := range haystack {
+		if s == needle {
+			return true
+		}
+	}
+
+	return false
+}
+
+func loadSnapshotMap(t testing.TB, global GlobalOptions) map[string]struct{} {
+	snapshotIDs := cmdList(t, global, "snapshots")
+
+	m := make(map[string]struct{})
+	for _, id := range snapshotIDs {
+		m[id.String()] = struct{}{}
+	}
+
+	return m
+}
+
+func lastSnapshot(old, new map[string]struct{}) (map[string]struct{}, string) {
+	for k := range new {
+		if _, ok := old[k]; !ok {
+			old[k] = struct{}{}
+			return old, k
+		}
+	}
+
+	return old, ""
+}
+
+var backupExcludeFilenames = []string{
+	"testfile1",
+	"foo.tar.gz",
+	"private/secret/passwords.txt",
+	"work/source/test.c",
+}
+
+func TestBackupExclude(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		cmdInit(t, global)
+
+		datadir := filepath.Join(env.base, "testdata")
+
+		for _, filename := range backupExcludeFilenames {
+			fp := filepath.Join(datadir, filename)
+			OK(t, os.MkdirAll(filepath.Dir(fp), 0755))
+
+			f, err := os.Create(fp)
+			OK(t, err)
+
+			fmt.Fprintf(f, filename)
+			OK(t, f.Close())
+		}
+
+		snapshots := make(map[string]struct{})
+
+		cmdBackup(t, global, []string{datadir}, nil)
+		snapshots, snapshotID := lastSnapshot(snapshots, loadSnapshotMap(t, global))
+		files := cmdLs(t, global, snapshotID)
+		Assert(t, includes(files, filepath.Join("testdata", "foo.tar.gz")),
+			"expected file %q in first snapshot, but it's not included", "foo.tar.gz")
+
+		cmdBackupExcludes(t, global, []string{datadir}, nil, []string{"*.tar.gz"})
+		snapshots, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, global))
+		files = cmdLs(t, global, snapshotID)
+		Assert(t, !includes(files, filepath.Join("testdata", "foo.tar.gz")),
+			"expected file %q not in first snapshot, but it's included", "foo.tar.gz")
+
+		cmdBackupExcludes(t, global, []string{datadir}, nil, []string{"*.tar.gz", "private/secret"})
+		snapshots, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, global))
+		files = cmdLs(t, global, snapshotID)
+		Assert(t, !includes(files, filepath.Join("testdata", "foo.tar.gz")),
+			"expected file %q not in first snapshot, but it's included", "foo.tar.gz")
+		Assert(t, !includes(files, filepath.Join("testdata", "private", "secret", "passwords.txt")),
+			"expected file %q not in first snapshot, but it's included", "passwords.txt")
+	})
+}
+
 const (
 	incrementalFirstWrite  = 20 * 1042 * 1024
 	incrementalSecondWrite = 12 * 1042 * 1024