Use new archiver code for backup

This commit is contained in:
Alexander Neumann
2018-04-22 11:57:20 +02:00
parent c703d21d55
commit 0e78ac92d8
9 changed files with 863 additions and 337 deletions

View File

@@ -2,21 +2,24 @@ package main
import (
"bufio"
"fmt"
"context"
"io"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/spf13/cobra"
tomb "gopkg.in/tomb.v2"
"github.com/restic/restic/internal/archiver"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/termstatus"
)
var cmdBackup = &cobra.Command{
@@ -42,11 +45,16 @@ given as the arguments.
return errors.Fatal("cannot use both `--stdin` and `--files-from -`")
}
if backupOptions.Stdin {
return readBackupFromStdin(backupOptions, globalOptions, args)
}
var t tomb.Tomb
term := termstatus.New(globalOptions.stdout, globalOptions.stderr)
t.Go(func() error { term.Run(t.Context(globalOptions.ctx)); return nil })
return runBackup(backupOptions, globalOptions, args)
err := runBackup(backupOptions, globalOptions, term, args)
if err != nil {
return err
}
t.Kill(nil)
return t.Wait()
},
}
@@ -90,127 +98,6 @@ func init() {
f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories")
}
func newScanProgress(gopts GlobalOptions) *restic.Progress {
if gopts.Quiet {
return nil
}
p := restic.NewProgress()
p.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
if IsProcessBackground() {
return
}
PrintProgress("[%s] %d directories, %d files, %s", formatDuration(d), s.Dirs, s.Files, formatBytes(s.Bytes))
}
p.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
PrintProgress("scanned %d directories, %d files in %s\n", s.Dirs, s.Files, formatDuration(d))
}
return p
}
func newArchiveProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress {
if gopts.Quiet {
return nil
}
archiveProgress := restic.NewProgress()
var bps, eta uint64
itemsTodo := todo.Files + todo.Dirs
archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
if IsProcessBackground() {
return
}
sec := uint64(d / time.Second)
if todo.Bytes > 0 && sec > 0 && ticker {
bps = s.Bytes / sec
if s.Bytes >= todo.Bytes {
eta = 0
} else if bps > 0 {
eta = (todo.Bytes - s.Bytes) / bps
}
}
itemsDone := s.Files + s.Dirs
status1 := fmt.Sprintf("[%s] %s %s / %s %d / %d items %d errors ",
formatDuration(d),
formatPercent(s.Bytes, todo.Bytes),
formatBytes(s.Bytes), formatBytes(todo.Bytes),
itemsDone, itemsTodo,
s.Errors)
status2 := fmt.Sprintf("ETA %s ", formatSeconds(eta))
if w := stdoutTerminalWidth(); w > 0 {
maxlen := w - len(status2) - 1
if maxlen < 4 {
status1 = ""
} else if len(status1) > maxlen {
status1 = status1[:maxlen-4]
status1 += "... "
}
}
PrintProgress("%s%s", status1, status2)
}
archiveProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
fmt.Printf("\nduration: %s\n", formatDuration(d))
}
return archiveProgress
}
func newArchiveStdinProgress(gopts GlobalOptions) *restic.Progress {
if gopts.Quiet {
return nil
}
archiveProgress := restic.NewProgress()
var bps uint64
archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
if IsProcessBackground() {
return
}
sec := uint64(d / time.Second)
if s.Bytes > 0 && sec > 0 && ticker {
bps = s.Bytes / sec
}
status1 := fmt.Sprintf("[%s] %s %s/s", formatDuration(d),
formatBytes(s.Bytes),
formatBytes(bps))
if w := stdoutTerminalWidth(); w > 0 {
maxlen := w - len(status1)
if maxlen < 4 {
status1 = ""
} else if len(status1) > maxlen {
status1 = status1[:maxlen-4]
status1 += "... "
}
}
PrintProgress("%s", status1)
}
archiveProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
fmt.Printf("\nduration: %s\n", formatDuration(d))
}
return archiveProgress
}
// filterExisting returns a slice of all existing items, or an error if no
// items exist at all.
func filterExisting(items []string) (result []string, err error) {
@@ -231,72 +118,10 @@ func filterExisting(items []string) (result []string, err error) {
return
}
func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string) error {
if len(args) != 0 {
return errors.Fatal("when reading from stdin, no additional files can be specified")
}
fn := opts.StdinFilename
if fn == "" {
return errors.Fatal("filename for backup from stdin must not be empty")
}
if filepath.Base(fn) != fn || path.Base(fn) != fn {
return errors.Fatal("filename is invalid (may not contain a directory, slash or backslash)")
}
var t time.Time
if opts.TimeStamp != "" {
parsedT, err := time.Parse("2006-01-02 15:04:05", opts.TimeStamp)
if err != nil {
return err
}
t = parsedT
} else {
t = time.Now()
}
if gopts.password == "" {
return errors.Fatal("unable to read password from stdin when data is to be read from stdin, use --password-file or $RESTIC_PASSWORD")
}
repo, err := OpenRepository(gopts)
if err != nil {
return err
}
lock, err := lockRepo(repo)
defer unlockRepo(lock)
if err != nil {
return err
}
err = repo.LoadIndex(gopts.ctx)
if err != nil {
return err
}
r := &archiver.Reader{
Repository: repo,
Tags: opts.Tags,
Hostname: opts.Hostname,
TimeStamp: t,
}
_, id, err := r.Archive(gopts.ctx, fn, os.Stdin, newArchiveStdinProgress(gopts))
if err != nil {
return err
}
Verbosef("archived as %v\n", id.Str())
return nil
}
// readFromFile will read all lines from the given filename and write them to a
// string array, if filename is empty readFromFile returns and empty string
// array. If filename is a dash (-), readFromFile will read the lines from
// the standard input.
// readFromFile will read all lines from the given filename and return them as
// a string array, if filename is empty readFromFile returns and empty string
// array. If filename is a dash (-), readFromFile will read the lines from the
// standard input.
func readLinesFromFile(filename string) ([]string, error) {
if filename == "" {
return nil, nil
@@ -335,47 +160,45 @@ func readLinesFromFile(filename string) ([]string, error) {
return lines, nil
}
func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
// Check returns an error when an invalid combination of options was set.
func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
if opts.FilesFrom == "-" && gopts.password == "" {
return errors.Fatal("unable to read password from stdin when data is to be read from stdin, use --password-file or $RESTIC_PASSWORD")
}
fromfile, err := readLinesFromFile(opts.FilesFrom)
if err != nil {
return err
}
// merge files from files-from into normal args so we can reuse the normal
// args checks and have the ability to use both files-from and args at the
// same time
args = append(args, fromfile...)
if len(args) == 0 {
return errors.Fatal("nothing to backup, please specify target files/dirs")
}
target := make([]string, 0, len(args))
for _, d := range args {
if a, err := filepath.Abs(d); err == nil {
d = a
if opts.Stdin {
if opts.FilesFrom != "" {
return errors.Fatal("--stdin and --files-from cannot be used together")
}
if len(args) > 0 {
return errors.Fatal("--stdin was specified and files/dirs were listed as arguments")
}
target = append(target, d)
}
target, err = filterExisting(target)
if err != nil {
return err
}
// rejectFuncs collect functions that can reject items from the backup
var rejectFuncs []RejectFunc
return nil
}
// collectRejectFuncs returns a list of all functions which may reject data
// from being saved in a snapshot
func collectRejectFuncs(opts BackupOptions, repo *repository.Repository, targets []string) (fs []RejectFunc, err error) {
// allowed devices
if opts.ExcludeOtherFS {
f, err := rejectByDevice(target)
f, err := rejectByDevice(targets)
if err != nil {
return err
return nil, err
}
rejectFuncs = append(rejectFuncs, f)
fs = append(fs, f)
}
// exclude restic cache
if repo.Cache != nil {
f, err := rejectResticCache(repo)
if err != nil {
return nil, err
}
fs = append(fs, f)
}
// add patterns from file
@@ -384,7 +207,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
}
if len(opts.Excludes) > 0 {
rejectFuncs = append(rejectFuncs, rejectByPattern(opts.Excludes))
fs = append(fs, rejectByPattern(opts.Excludes))
}
if opts.ExcludeCaches {
@@ -394,111 +217,17 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
for _, spec := range opts.ExcludeIfPresent {
f, err := rejectIfPresent(spec)
if err != nil {
return err
return nil, err
}
rejectFuncs = append(rejectFuncs, f)
fs = append(fs, f)
}
repo, err := OpenRepository(gopts)
if err != nil {
return err
}
lock, err := lockRepo(repo)
defer unlockRepo(lock)
if err != nil {
return err
}
// exclude restic cache
if repo.Cache != nil {
f, err := rejectResticCache(repo)
if err != nil {
return err
}
rejectFuncs = append(rejectFuncs, f)
}
err = repo.LoadIndex(gopts.ctx)
if err != nil {
return err
}
var parentSnapshotID *restic.ID
// Force using a parent
if !opts.Force && opts.Parent != "" {
id, err := restic.FindSnapshot(repo, opts.Parent)
if err != nil {
return errors.Fatalf("invalid id %q: %v", opts.Parent, err)
}
parentSnapshotID = &id
}
// Find last snapshot to set it as parent, if not already set
if !opts.Force && parentSnapshotID == nil {
id, err := restic.FindLatestSnapshot(gopts.ctx, repo, target, []restic.TagList{}, opts.Hostname)
if err == nil {
parentSnapshotID = &id
} else if err != restic.ErrNoSnapshotFound {
return err
}
}
if parentSnapshotID != nil {
Verbosef("using parent snapshot %v\n", parentSnapshotID.Str())
}
Verbosef("scan %v\n", target)
selectFilter := func(item string, fi os.FileInfo) bool {
for _, reject := range rejectFuncs {
if reject(item, fi) {
return false
}
}
return true
}
var stat restic.Stat
if !gopts.Quiet {
stat, err = archiver.Scan(target, selectFilter, newScanProgress(gopts))
if err != nil {
return err
}
}
arch := archiver.New(repo)
arch.Excludes = opts.Excludes
arch.SelectFilter = selectFilter
arch.WithAccessTime = opts.WithAtime
arch.Warn = func(dir string, fi os.FileInfo, err error) {
// TODO: make ignoring errors configurable
Warnf("%s\rwarning for %s: %v\n", ClearLine(), dir, err)
}
timeStamp := time.Now()
if opts.TimeStamp != "" {
timeStamp, err = time.Parse(TimeFormat, opts.TimeStamp)
if err != nil {
return errors.Fatalf("error in time option: %v\n", err)
}
}
_, id, err := arch.Snapshot(gopts.ctx, newArchiveProgress(gopts, stat), target, opts.Tags, opts.Hostname, parentSnapshotID, timeStamp)
if err != nil {
return err
}
Verbosef("snapshot %s saved\n", id.Str())
return nil
return fs, nil
}
// readExcludePatternsFromFiles reads all exclude files and returns the list of
// exclude patterns.
func readExcludePatternsFromFiles(excludeFiles []string) []string {
var excludes []string
for _, filename := range excludeFiles {
@@ -540,3 +269,217 @@ func readExcludePatternsFromFiles(excludeFiles []string) []string {
}
return excludes
}
// collectTargets returns a list of target files/dirs from several sources.
func collectTargets(opts BackupOptions, args []string) (targets []string, err error) {
if opts.Stdin {
return nil, nil
}
fromfile, err := readLinesFromFile(opts.FilesFrom)
if err != nil {
return nil, err
}
// merge files from files-from into normal args so we can reuse the normal
// args checks and have the ability to use both files-from and args at the
// same time
args = append(args, fromfile...)
if len(args) == 0 && !opts.Stdin {
return nil, errors.Fatal("nothing to backup, please specify target files/dirs")
}
targets = args
targets, err = filterExisting(targets)
if err != nil {
return nil, err
}
return targets, nil
}
// parent returns the ID of the parent snapshot. If there is none, nil is
// returned.
func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string) (parentID *restic.ID, err error) {
// Force using a parent
if !opts.Force && opts.Parent != "" {
id, err := restic.FindSnapshot(repo, opts.Parent)
if err != nil {
return nil, errors.Fatalf("invalid id %q: %v", opts.Parent, err)
}
parentID = &id
}
// Find last snapshot to set it as parent, if not already set
if !opts.Force && parentID == nil {
id, err := restic.FindLatestSnapshot(ctx, repo, targets, []restic.TagList{}, opts.Hostname)
if err == nil {
parentID = &id
} else if err != restic.ErrNoSnapshotFound {
return nil, err
}
}
return parentID, nil
}
func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
err := opts.Check(gopts, args)
if err != nil {
return err
}
targets, err := collectTargets(opts, args)
if err != nil {
return err
}
var t tomb.Tomb
p := ui.NewBackup(term, gopts.verbosity)
// use the terminal for stdout/stderr
prevStdout, prevStderr := gopts.stdout, gopts.stderr
defer func() {
gopts.stdout, gopts.stderr = prevStdout, prevStderr
}()
gopts.stdout, gopts.stderr = p.Stdout(), p.Stderr()
if s, ok := os.LookupEnv("RESTIC_PROGRESS_FPS"); ok {
fps, err := strconv.Atoi(s)
if err == nil && fps >= 1 {
if fps > 60 {
fps = 60
}
p.MinUpdatePause = time.Second / time.Duration(fps)
}
}
t.Go(func() error { return p.Run(t.Context(gopts.ctx)) })
p.V("open repository")
repo, err := OpenRepository(gopts)
if err != nil {
return err
}
p.V("lock repository")
lock, err := lockRepo(repo)
defer unlockRepo(lock)
if err != nil {
return err
}
// rejectFuncs collect functions that can reject items from the backup
rejectFuncs, err := collectRejectFuncs(opts, repo, targets)
if err != nil {
return err
}
p.V("load index files")
err = repo.LoadIndex(gopts.ctx)
if err != nil {
return err
}
parentSnapshotID, err := findParentSnapshot(gopts.ctx, repo, opts, targets)
if err != nil {
return err
}
if parentSnapshotID != nil {
p.V("using parent snapshot %v\n", parentSnapshotID.Str())
}
selectFilter := func(item string, fi os.FileInfo) bool {
for _, reject := range rejectFuncs {
if reject(item, fi) {
return false
}
}
return true
}
timeStamp := time.Now()
if opts.TimeStamp != "" {
timeStamp, err = time.Parse(TimeFormat, opts.TimeStamp)
if err != nil {
return errors.Fatalf("error in time option: %v\n", err)
}
}
var targetFS fs.FS = fs.Local{}
if opts.Stdin {
p.V("read data from stdin")
targetFS = &fs.Reader{
ModTime: timeStamp,
Name: opts.StdinFilename,
Mode: 0644,
ReadCloser: os.Stdin,
}
targets = []string{opts.StdinFilename}
}
sc := archiver.NewScanner(targetFS)
sc.Select = selectFilter
sc.Error = p.ScannerError
sc.Result = p.ReportTotal
p.V("start scan")
t.Go(func() error { return sc.Scan(t.Context(gopts.ctx), targets) })
arch := archiver.New(repo, targetFS, archiver.Options{})
arch.Select = selectFilter
arch.WithAtime = opts.WithAtime
arch.Error = p.Error
arch.CompleteItem = p.CompleteItemFn
arch.StartFile = p.StartFile
arch.CompleteBlob = p.CompleteBlob
if parentSnapshotID == nil {
parentSnapshotID = &restic.ID{}
}
snapshotOpts := archiver.SnapshotOptions{
Excludes: opts.Excludes,
Tags: opts.Tags,
Time: timeStamp,
Hostname: opts.Hostname,
ParentSnapshot: *parentSnapshotID,
}
uploader := archiver.IndexUploader{
Repository: repo,
Start: func() {
p.VV("uploading intermediate index")
},
Complete: func(id restic.ID) {
p.V("uploaded intermediate index %v", id.Str())
},
}
t.Go(func() error {
return uploader.Upload(gopts.ctx, t.Context(gopts.ctx), 30*time.Second)
})
p.V("start backup")
_, id, err := arch.Snapshot(gopts.ctx, targets, snapshotOpts)
if err != nil {
return err
}
p.Finish()
p.P("snapshot %s saved\n", id.Str())
// cleanly shutdown all running goroutines
t.Kill(nil)
// let's see if one returned an error
err = t.Wait()
if err != nil {
return err
}
return nil
}

View File

@@ -43,8 +43,7 @@ type GlobalOptions struct {
Repo string
PasswordFile string
Quiet bool
Verbose bool
Debug bool
Verbose int
NoLock bool
JSON bool
CacheDir string
@@ -90,8 +89,7 @@ func init() {
f.StringVarP(&globalOptions.Repo, "repo", "r", os.Getenv("RESTIC_REPOSITORY"), "repository to backup to or restore from (default: $RESTIC_REPOSITORY)")
f.StringVarP(&globalOptions.PasswordFile, "password-file", "p", os.Getenv("RESTIC_PASSWORD_FILE"), "read the repository password from a file (default: $RESTIC_PASSWORD_FILE)")
f.BoolVarP(&globalOptions.Quiet, "quiet", "q", false, "do not output comprehensive progress report")
f.BoolVarP(&globalOptions.Verbose, "verbose", "v", false, "be verbose")
f.BoolVar(&globalOptions.Debug, "debug", false, "be very verbose")
f.CountVarP(&globalOptions.Verbose, "verbose", "v", "be verbose (specify --verbose multiple times or level `n`)")
f.BoolVar(&globalOptions.NoLock, "no-lock", false, "do not lock the repo, this allows some operations on read-only repos")
f.BoolVarP(&globalOptions.JSON, "json", "", false, "set output mode to JSON for commands that support it")
f.StringVar(&globalOptions.CacheDir, "cache-dir", "", "set the cache directory")

View File

@@ -3,6 +3,7 @@ package main
import (
"bufio"
"bytes"
"context"
"crypto/rand"
"encoding/json"
"fmt"
@@ -23,6 +24,8 @@ import (
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/termstatus"
"golang.org/x/sync/errgroup"
)
func parseIDsFromReader(t testing.TB, rd io.Reader) restic.IDs {
@@ -52,13 +55,28 @@ func testRunInit(t testing.TB, opts GlobalOptions) {
}
func testRunBackup(t testing.TB, dir string, target []string, opts BackupOptions, gopts GlobalOptions) {
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
var wg errgroup.Group
term := termstatus.New(gopts.stdout, gopts.stderr)
wg.Go(func() error { term.Run(ctx); return nil })
gopts.stdout = ioutil.Discard
t.Logf("backing up %v in %v", target, dir)
if dir != "" {
cleanup := fs.TestChdir(t, dir)
defer cleanup()
}
rtest.OK(t, runBackup(opts, gopts, target))
rtest.OK(t, runBackup(opts, gopts, term, target))
cancel()
err := wg.Wait()
if err != nil {
t.Fatal(err)
}
}
func testRunList(t testing.TB, tpe string, opts GlobalOptions) restic.IDs {

View File

@@ -30,19 +30,19 @@ directories in an encrypted repository stored on different backends.
DisableAutoGenTag: true,
PersistentPreRunE: func(c *cobra.Command, args []string) error {
// set verbosity
// set verbosity, default is one
globalOptions.verbosity = 1
if globalOptions.Quiet && (globalOptions.Verbose || globalOptions.Debug) {
return errors.Fatal("--quiet and --verbose or --debug cannot be specified at the same time")
if globalOptions.Quiet && (globalOptions.Verbose > 1) {
return errors.Fatal("--quiet and --verbose cannot be specified at the same time")
}
switch {
case globalOptions.Verbose >= 2:
globalOptions.verbosity = 3
case globalOptions.Verbose > 0:
globalOptions.verbosity = 2
case globalOptions.Quiet:
globalOptions.verbosity = 0
case globalOptions.Verbose:
globalOptions.verbosity = 2
case globalOptions.Debug:
globalOptions.verbosity = 3
}
// parse extended options