Merge pull request #5555 from MichaelEischer/extract-globaloptions

Split globalOptions into separate package

internal/backend/all/all.go

@@ -0,0 +1,28 @@
+package all
+
+import (
+	"github.com/restic/restic/internal/backend/azure"
+	"github.com/restic/restic/internal/backend/b2"
+	"github.com/restic/restic/internal/backend/gs"
+	"github.com/restic/restic/internal/backend/local"
+	"github.com/restic/restic/internal/backend/location"
+	"github.com/restic/restic/internal/backend/rclone"
+	"github.com/restic/restic/internal/backend/rest"
+	"github.com/restic/restic/internal/backend/s3"
+	"github.com/restic/restic/internal/backend/sftp"
+	"github.com/restic/restic/internal/backend/swift"
+)
+
+func Backends() *location.Registry {
+	backends := location.NewRegistry()
+	backends.Register(azure.NewFactory())
+	backends.Register(b2.NewFactory())
+	backends.Register(gs.NewFactory())
+	backends.Register(local.NewFactory())
+	backends.Register(rclone.NewFactory())
+	backends.Register(rest.NewFactory())
+	backends.Register(s3.NewFactory())
+	backends.Register(sftp.NewFactory())
+	backends.Register(swift.NewFactory())
+	return backends
+}

internal/global/global.go

@@ -0,0 +1,612 @@
+package global
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/restic/chunker"
+	"github.com/restic/restic/internal/backend"
+	"github.com/restic/restic/internal/backend/cache"
+	"github.com/restic/restic/internal/backend/limiter"
+	"github.com/restic/restic/internal/backend/location"
+	"github.com/restic/restic/internal/backend/logger"
+	"github.com/restic/restic/internal/backend/retry"
+	"github.com/restic/restic/internal/backend/sema"
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/options"
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/textfile"
+	"github.com/restic/restic/internal/ui"
+	"github.com/restic/restic/internal/ui/progress"
+	"github.com/spf13/pflag"
+
+	"github.com/restic/restic/internal/errors"
+)
+
+// ErrNoRepository is used to report if opening a repository failed due
+// to a missing backend storage location or config file
+var ErrNoRepository = errors.New("repository does not exist")
+
+const Version = "0.18.1-dev (compiled manually)"
+
+// TimeFormat is the format used for all timestamps printed by restic.
+const TimeFormat = "2006-01-02 15:04:05"
+
+type BackendWrapper func(r backend.Backend) (backend.Backend, error)
+
+// Options hold all global options for restic.
+type Options struct {
+	Repo               string
+	RepositoryFile     string
+	PasswordFile       string
+	PasswordCommand    string
+	KeyHint            string
+	Quiet              bool
+	Verbose            int
+	NoLock             bool
+	RetryLock          time.Duration
+	JSON               bool
+	CacheDir           string
+	NoCache            bool
+	CleanupCache       bool
+	Compression        repository.CompressionMode
+	PackSize           uint
+	NoExtraVerify      bool
+	InsecureNoPassword bool
+
+	backend.TransportOptions
+	limiter.Limits
+
+	Password string
+	Term     ui.Terminal
+
+	Backends                              *location.Registry
+	BackendTestHook, BackendInnerTestHook BackendWrapper
+
+	// Verbosity is set as follows:
+	//  0 means: don't print any messages except errors, this is used when --quiet is specified
+	//  1 is the default: print essential messages
+	//  2 means: print more messages, report minor things, this is used when --verbose is specified
+	//  3 means: print very detailed debug messages, this is used when --verbose=2 is specified
+	Verbosity uint
+
+	Options []string
+
+	Extended options.Options
+}
+
+func (opts *Options) AddFlags(f *pflag.FlagSet) {
+	f.StringVarP(&opts.Repo, "repo", "r", "", "`repository` to backup to or restore from (default: $RESTIC_REPOSITORY)")
+	f.StringVarP(&opts.RepositoryFile, "repository-file", "", "", "`file` to read the repository location from (default: $RESTIC_REPOSITORY_FILE)")
+	f.StringVarP(&opts.PasswordFile, "password-file", "p", "", "`file` to read the repository password from (default: $RESTIC_PASSWORD_FILE)")
+	f.StringVarP(&opts.KeyHint, "key-hint", "", "", "`key` ID of key to try decrypting first (default: $RESTIC_KEY_HINT)")
+	f.StringVarP(&opts.PasswordCommand, "password-command", "", "", "shell `command` to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND)")
+	f.BoolVarP(&opts.Quiet, "quiet", "q", false, "do not output comprehensive progress report")
+	// use empty parameter name as `-v, --verbose n` instead of the correct `--verbose=n` is confusing
+	f.CountVarP(&opts.Verbose, "verbose", "v", "be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2)")
+	f.BoolVar(&opts.NoLock, "no-lock", false, "do not lock the repository, this allows some operations on read-only repositories")
+	f.DurationVar(&opts.RetryLock, "retry-lock", 0, "retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries)")
+	f.BoolVarP(&opts.JSON, "json", "", false, "set output mode to JSON for commands that support it")
+	f.StringVar(&opts.CacheDir, "cache-dir", "", "set the cache `directory`. (default: use system default cache directory)")
+	f.BoolVar(&opts.NoCache, "no-cache", false, "do not use a local cache")
+	f.StringSliceVar(&opts.RootCertFilenames, "cacert", nil, "`file` to load root certificates from (default: use system certificates or $RESTIC_CACERT)")
+	f.StringVar(&opts.TLSClientCertKeyFilename, "tls-client-cert", "", "path to a `file` containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT)")
+	f.BoolVar(&opts.InsecureNoPassword, "insecure-no-password", false, "use an empty password for the repository, must be passed to every restic command (insecure)")
+	f.BoolVar(&opts.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)")
+	f.BoolVar(&opts.CleanupCache, "cleanup-cache", false, "auto remove old cache directories")
+	f.Var(&opts.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|fastest|better|max) (default: $RESTIC_COMPRESSION)")
+	f.BoolVar(&opts.NoExtraVerify, "no-extra-verify", false, "skip additional verification of data before upload (see documentation)")
+	f.IntVar(&opts.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum `rate` in KiB/s. (default: unlimited)")
+	f.IntVar(&opts.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum `rate` in KiB/s. (default: unlimited)")
+	f.UintVar(&opts.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)")
+	f.StringSliceVarP(&opts.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)")
+	f.StringVar(&opts.HTTPUserAgent, "http-user-agent", "", "set a http user agent for outgoing http requests")
+	f.DurationVar(&opts.StuckRequestTimeout, "stuck-request-timeout", 5*time.Minute, "`duration` after which to retry stuck requests")
+
+	opts.Repo = os.Getenv("RESTIC_REPOSITORY")
+	opts.RepositoryFile = os.Getenv("RESTIC_REPOSITORY_FILE")
+	opts.PasswordFile = os.Getenv("RESTIC_PASSWORD_FILE")
+	opts.KeyHint = os.Getenv("RESTIC_KEY_HINT")
+	opts.PasswordCommand = os.Getenv("RESTIC_PASSWORD_COMMAND")
+	if os.Getenv("RESTIC_CACERT") != "" {
+		opts.RootCertFilenames = strings.Split(os.Getenv("RESTIC_CACERT"), ",")
+	}
+	opts.TLSClientCertKeyFilename = os.Getenv("RESTIC_TLS_CLIENT_CERT")
+	comp := os.Getenv("RESTIC_COMPRESSION")
+	if comp != "" {
+		// ignore error as there's no good way to handle it
+		_ = opts.Compression.Set(comp)
+	}
+	// parse target pack size from env, on error the default value will be used
+	targetPackSize, _ := strconv.ParseUint(os.Getenv("RESTIC_PACK_SIZE"), 10, 32)
+	opts.PackSize = uint(targetPackSize)
+
+	if os.Getenv("RESTIC_HTTP_USER_AGENT") != "" {
+		opts.HTTPUserAgent = os.Getenv("RESTIC_HTTP_USER_AGENT")
+	}
+}
+
+func (opts *Options) PreRun(needsPassword bool) error {
+	// set verbosity, default is one
+	opts.Verbosity = 1
+	if opts.Quiet && opts.Verbose > 0 {
+		return errors.Fatal("--quiet and --verbose cannot be specified at the same time")
+	}
+
+	switch {
+	case opts.Verbose >= 2:
+		opts.Verbosity = 3
+	case opts.Verbose > 0:
+		opts.Verbosity = 2
+	case opts.Quiet:
+		opts.Verbosity = 0
+	}
+
+	// parse extended options
+	extendedOpts, err := options.Parse(opts.Options)
+	if err != nil {
+		return err
+	}
+	opts.Extended = extendedOpts
+	if !needsPassword {
+		return nil
+	}
+	pwd, err := resolvePassword(opts, "RESTIC_PASSWORD")
+	if err != nil {
+		return errors.Fatalf("Resolving password failed: %v", err)
+	}
+	opts.Password = pwd
+	return nil
+}
+
+// resolvePassword determines the password to be used for opening the repository.
+func resolvePassword(opts *Options, envStr string) (string, error) {
+	if opts.PasswordFile != "" && opts.PasswordCommand != "" {
+		return "", errors.Fatalf("Password file and command are mutually exclusive options")
+	}
+	if opts.PasswordCommand != "" {
+		args, err := backend.SplitShellStrings(opts.PasswordCommand)
+		if err != nil {
+			return "", err
+		}
+		cmd := exec.Command(args[0], args[1:]...)
+		cmd.Stderr = os.Stderr
+		output, err := cmd.Output()
+		if err != nil {
+			return "", err
+		}
+		return strings.TrimSpace(string(output)), nil
+	}
+	if opts.PasswordFile != "" {
+		return LoadPasswordFromFile(opts.PasswordFile)
+	}
+
+	if pwd := os.Getenv(envStr); pwd != "" {
+		return pwd, nil
+	}
+
+	return "", nil
+}
+
+// LoadPasswordFromFile loads a password from a file while stripping a BOM and
+// converting the password to UTF-8.
+func LoadPasswordFromFile(pwdFile string) (string, error) {
+	s, err := textfile.Read(pwdFile)
+	if errors.Is(err, os.ErrNotExist) {
+		return "", errors.Fatalf("%s does not exist", pwdFile)
+	}
+	return strings.TrimSpace(string(s)), errors.Wrap(err, "Readfile")
+}
+
+// readPassword reads the password from a password file, the environment
+// variable RESTIC_PASSWORD or prompts the user. If the context is canceled,
+// the function leaks the password reading goroutine.
+func readPassword(ctx context.Context, gopts Options, prompt string) (string, error) {
+	if gopts.InsecureNoPassword {
+		if gopts.Password != "" {
+			return "", errors.Fatal("--insecure-no-password must not be specified together with providing a password via a cli option or environment variable")
+		}
+		return "", nil
+	}
+
+	if gopts.Password != "" {
+		return gopts.Password, nil
+	}
+
+	password, err := gopts.Term.ReadPassword(ctx, prompt)
+	if err != nil {
+		return "", fmt.Errorf("unable to read password: %w", err)
+	}
+
+	if len(password) == 0 {
+		return "", errors.Fatal("an empty password is not allowed by default. Pass the flag `--insecure-no-password` to restic to disable this check")
+	}
+
+	return password, nil
+}
+
+// ReadPasswordTwice calls ReadPassword two times and returns an error when the
+// passwords don't match. If the context is canceled, the function leaks the
+// password reading goroutine.
+func ReadPasswordTwice(ctx context.Context, gopts Options, prompt1, prompt2 string) (string, error) {
+	pw1, err := readPassword(ctx, gopts, prompt1)
+	if err != nil {
+		return "", err
+	}
+	if gopts.Term.InputIsTerminal() {
+		pw2, err := readPassword(ctx, gopts, prompt2)
+		if err != nil {
+			return "", err
+		}
+
+		if pw1 != pw2 {
+			return "", errors.Fatal("passwords do not match")
+		}
+	}
+
+	return pw1, nil
+}
+
+func readRepo(gopts Options) (string, error) {
+	if gopts.Repo == "" && gopts.RepositoryFile == "" {
+		return "", errors.Fatal("Please specify repository location (-r or --repository-file)")
+	}
+
+	repo := gopts.Repo
+	if gopts.RepositoryFile != "" {
+		if repo != "" {
+			return "", errors.Fatal("Options -r and --repository-file are mutually exclusive, please specify only one")
+		}
+
+		s, err := textfile.Read(gopts.RepositoryFile)
+		if errors.Is(err, os.ErrNotExist) {
+			return "", errors.Fatalf("%s does not exist", gopts.RepositoryFile)
+		}
+		if err != nil {
+			return "", err
+		}
+
+		repo = strings.TrimSpace(string(s))
+	}
+
+	return repo, nil
+}
+
+const maxKeys = 20
+
+// OpenRepository reads the password and opens the repository.
+func OpenRepository(ctx context.Context, gopts Options, printer progress.Printer) (*repository.Repository, error) {
+	repo, err := readRepo(gopts)
+	if err != nil {
+		return nil, err
+	}
+
+	be, err := innerOpenBackend(ctx, repo, gopts, gopts.Extended, false, printer)
+	if err != nil {
+		return nil, err
+	}
+
+	err = hasRepositoryConfig(ctx, be, repo, gopts)
+	if err != nil {
+		return nil, err
+	}
+
+	s, err := createRepositoryInstance(be, gopts)
+	if err != nil {
+		return nil, err
+	}
+
+	err = decryptRepository(ctx, s, &gopts, printer)
+	if err != nil {
+		return nil, err
+	}
+
+	printRepositoryInfo(s, gopts, printer)
+
+	if gopts.NoCache {
+		return s, nil
+	}
+
+	err = setupCache(s, gopts, printer)
+	if err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
+// hasRepositoryConfig checks if the repository config file exists and is not empty.
+func hasRepositoryConfig(ctx context.Context, be backend.Backend, repo string, gopts Options) error {
+	fi, err := be.Stat(ctx, backend.Handle{Type: restic.ConfigFile})
+	if be.IsNotExist(err) {
+		//nolint:staticcheck // capitalized error string is intentional
+		return fmt.Errorf("Fatal: %w: unable to open config file: %v\nIs there a repository at the following location?\n%v", ErrNoRepository, err, location.StripPassword(gopts.Backends, repo))
+	}
+	if err != nil {
+		return errors.Fatalf("unable to open config file: %v\n%v", err, location.StripPassword(gopts.Backends, repo))
+	}
+
+	if fi.Size == 0 {
+		return errors.New("config file has zero size, invalid repository?")
+	}
+
+	return nil
+}
+
+// createRepositoryInstance creates a new repository instance with the given options.
+func createRepositoryInstance(be backend.Backend, gopts Options) (*repository.Repository, error) {
+	s, err := repository.New(be, repository.Options{
+		Compression:   gopts.Compression,
+		PackSize:      gopts.PackSize * 1024 * 1024,
+		NoExtraVerify: gopts.NoExtraVerify,
+	})
+	if err != nil {
+		return nil, errors.Fatalf("%s", err)
+	}
+	return s, nil
+}
+
+// decryptRepository handles password reading and decrypts the repository.
+func decryptRepository(ctx context.Context, s *repository.Repository, gopts *Options, printer progress.Printer) error {
+	passwordTriesLeft := 1
+	if gopts.Term.InputIsTerminal() && gopts.Password == "" && !gopts.InsecureNoPassword {
+		passwordTriesLeft = 3
+	}
+
+	var err error
+	for ; passwordTriesLeft > 0; passwordTriesLeft-- {
+		gopts.Password, err = readPassword(ctx, *gopts, "enter password for repository: ")
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		if err != nil && passwordTriesLeft > 1 {
+			gopts.Password = ""
+			printer.E("%s. Try again", err)
+		}
+		if err != nil {
+			continue
+		}
+
+		err = s.SearchKey(ctx, gopts.Password, maxKeys, gopts.KeyHint)
+		if err != nil && passwordTriesLeft > 1 {
+			gopts.Password = ""
+			printer.E("%s. Try again", err)
+		}
+	}
+	if err != nil {
+		if errors.IsFatal(err) || errors.Is(err, repository.ErrNoKeyFound) {
+			return err
+		}
+		return errors.Fatalf("%s", err)
+	}
+
+	return nil
+}
+
+// printRepositoryInfo displays the repository ID, version and compression level.
+func printRepositoryInfo(s *repository.Repository, gopts Options, printer progress.Printer) {
+	id := s.Config().ID
+	if len(id) > 8 {
+		id = id[:8]
+	}
+	extra := ""
+	if s.Config().Version >= 2 {
+		extra = ", compression level " + gopts.Compression.String()
+	}
+	printer.PT("repository %v opened (version %v%s)", id, s.Config().Version, extra)
+}
+
+// setupCache creates a new cache and removes old cache directories if instructed to do so.
+func setupCache(s *repository.Repository, gopts Options, printer progress.Printer) error {
+	c, err := cache.New(s.Config().ID, gopts.CacheDir)
+	if err != nil {
+		printer.E("unable to open cache: %v", err)
+		return err
+	}
+
+	if c.Created {
+		printer.PT("created new cache in %v", c.Base)
+	}
+
+	// start using the cache
+	s.UseCache(c, printer.E)
+
+	oldCacheDirs, err := cache.Old(c.Base)
+	if err != nil {
+		printer.E("unable to find old cache directories: %v", err)
+	}
+
+	// nothing more to do if no old cache dirs could be found
+	if len(oldCacheDirs) == 0 {
+		return nil
+	}
+
+	// cleanup old cache dirs if instructed to do so
+	if gopts.CleanupCache {
+		printer.PT("removing %d old cache dirs from %v", len(oldCacheDirs), c.Base)
+		for _, item := range oldCacheDirs {
+			dir := filepath.Join(c.Base, item.Name())
+			err = os.RemoveAll(dir)
+			if err != nil {
+				printer.E("unable to remove %v: %v", dir, err)
+			}
+		}
+	} else {
+		printer.PT("found %d old cache directories in %v, run `restic cache --cleanup` to remove them",
+			len(oldCacheDirs), c.Base)
+	}
+	return nil
+}
+
+// CreateRepository a repository with the given version and chunker polynomial.
+func CreateRepository(ctx context.Context, gopts Options, version uint, chunkerPolynomial *chunker.Pol, printer progress.Printer) (*repository.Repository, error) {
+	if version < restic.MinRepoVersion || version > restic.MaxRepoVersion {
+		return nil, errors.Fatalf("only repository versions between %v and %v are allowed", restic.MinRepoVersion, restic.MaxRepoVersion)
+	}
+
+	repo, err := readRepo(gopts)
+	if err != nil {
+		return nil, err
+	}
+
+	gopts.Password, err = ReadPasswordTwice(ctx, gopts,
+		"enter password for new repository: ",
+		"enter password again: ")
+	if err != nil {
+		return nil, err
+	}
+
+	be, err := innerOpenBackend(ctx, repo, gopts, gopts.Extended, true, printer)
+	if err != nil {
+		return nil, errors.Fatalf("create repository at %s failed: %v", location.StripPassword(gopts.Backends, repo), err)
+	}
+
+	s, err := createRepositoryInstance(be, gopts)
+	if err != nil {
+		return nil, err
+	}
+
+	err = s.Init(ctx, version, gopts.Password, chunkerPolynomial)
+	if err != nil {
+		return nil, errors.Fatalf("create key in repository at %s failed: %v", location.StripPassword(gopts.Backends, repo), err)
+	}
+
+	return s, nil
+}
+
+func innerOpenBackend(ctx context.Context, s string, gopts Options, opts options.Options, create bool, printer progress.Printer) (backend.Backend, error) {
+	debug.Log("parsing location %v", location.StripPassword(gopts.Backends, s))
+
+	scheme, cfg, err := parseConfig(gopts.Backends, s, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	rt, lim, err := setupTransport(gopts)
+	if err != nil {
+		return nil, err
+	}
+
+	be, err := createOrOpenBackend(ctx, scheme, cfg, rt, lim, gopts, s, create, printer)
+	if err != nil {
+		return nil, err
+	}
+
+	be, err = wrapBackend(be, gopts, printer)
+	if err != nil {
+		return nil, err
+	}
+
+	return be, nil
+}
+
+// parseConfig parses the repository location and extended options and returns the scheme and configuration.
+func parseConfig(backends *location.Registry, s string, opts options.Options) (string, interface{}, error) {
+	loc, err := location.Parse(backends, s)
+	if err != nil {
+		return "", nil, errors.Fatalf("parsing repository location failed: %v", err)
+	}
+
+	cfg := loc.Config
+	if cfg, ok := cfg.(backend.ApplyEnvironmenter); ok {
+		cfg.ApplyEnvironment("")
+	}
+
+	// only apply options for a particular backend here
+	opts = opts.Extract(loc.Scheme)
+	if err := opts.Apply(loc.Scheme, cfg); err != nil {
+		return "", nil, err
+	}
+
+	debug.Log("opening %v repository at %#v", loc.Scheme, cfg)
+	return loc.Scheme, cfg, nil
+}
+
+// setupTransport creates and configures the transport with rate limiting.
+func setupTransport(gopts Options) (http.RoundTripper, limiter.Limiter, error) {
+	rt, err := backend.Transport(gopts.TransportOptions)
+	if err != nil {
+		return nil, nil, errors.Fatalf("%s", err)
+	}
+
+	// wrap the transport so that the throughput via HTTP is limited
+	lim := limiter.NewStaticLimiter(gopts.Limits)
+	rt = lim.Transport(rt)
+
+	return rt, lim, nil
+}
+
+// createOrOpenBackend creates or opens a backend using the appropriate factory method.
+func createOrOpenBackend(ctx context.Context, scheme string, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter, gopts Options, s string, create bool, printer progress.Printer) (backend.Backend, error) {
+	factory := gopts.Backends.Lookup(scheme)
+	if factory == nil {
+		return nil, errors.Fatalf("invalid backend: %q", scheme)
+	}
+
+	var be backend.Backend
+	var err error
+	if create {
+		be, err = factory.Create(ctx, cfg, rt, lim, printer.E)
+	} else {
+		be, err = factory.Open(ctx, cfg, rt, lim, printer.E)
+	}
+
+	if errors.Is(err, backend.ErrNoRepository) {
+		//nolint:staticcheck // capitalized error string is intentional
+		return nil, fmt.Errorf("Fatal: %w at %v: %v", ErrNoRepository, location.StripPassword(gopts.Backends, s), err)
+	}
+	if err != nil {
+		if create {
+			// init already wraps the error message
+			return nil, err
+		}
+		return nil, errors.Fatalf("unable to open repository at %v: %v", location.StripPassword(gopts.Backends, s), err)
+	}
+
+	return be, nil
+}
+
+// wrapBackend applies debug logging, test hooks, and retry wrapper to the backend.
+func wrapBackend(be backend.Backend, gopts Options, printer progress.Printer) (backend.Backend, error) {
+	// wrap with debug logging and connection limiting
+	be = logger.New(sema.NewBackend(be))
+
+	// wrap backend if a test specified an inner hook
+	if gopts.BackendInnerTestHook != nil {
+		var err error
+		be, err = gopts.BackendInnerTestHook(be)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	report := func(msg string, err error, d time.Duration) {
+		if d >= 0 {
+			printer.E("%v returned error, retrying after %v: %v", msg, d, err)
+		} else {
+			printer.E("%v failed: %v", msg, err)
+		}
+	}
+	success := func(msg string, retries int) {
+		printer.E("%v operation successful after %d retries", msg, retries)
+	}
+	be = retry.New(be, 15*time.Minute, report, success)
+
+	// wrap backend if a test specified a hook
+	if gopts.BackendTestHook != nil {
+		var err error
+		be, err = gopts.BackendTestHook(be)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return be, nil
+}

internal/global/global_debug.go

@@ -0,0 +1,126 @@
+//go:build debug || profile
+// +build debug profile
+
+package global
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	_ "net/http/pprof"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/repository"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+
+	"github.com/pkg/profile"
+)
+
+func RegisterProfiling(cmd *cobra.Command, stderr io.Writer) {
+	var profiler Profiler
+
+	origPreRun := cmd.PersistentPreRunE
+	cmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {
+		if origPreRun != nil {
+			if err := origPreRun(cmd, args); err != nil {
+				return err
+			}
+		}
+		return profiler.Start(profiler.opts, stderr)
+	}
+
+	// Once https://github.com/spf13/cobra/issues/1893 is fixed,
+	// this could use PersistentPostRunE instead of OnFinalize,
+	// reverting https://github.com/restic/restic/pull/5373.
+	cobra.OnFinalize(func() {
+		profiler.Stop()
+	})
+
+	profiler.opts.AddFlags(cmd.PersistentFlags())
+}
+
+type Profiler struct {
+	opts ProfileOptions
+	stop interface {
+		Stop()
+	}
+}
+
+type ProfileOptions struct {
+	listen    string
+	memPath   string
+	cpuPath   string
+	tracePath string
+	blockPath string
+	insecure  bool
+}
+
+func (opts *ProfileOptions) AddFlags(f *pflag.FlagSet) {
+	f.StringVar(&opts.listen, "listen-profile", "", "listen on this `address:port` for memory profiling")
+	f.StringVar(&opts.memPath, "mem-profile", "", "write memory profile to `dir`")
+	f.StringVar(&opts.cpuPath, "cpu-profile", "", "write cpu profile to `dir`")
+	f.StringVar(&opts.tracePath, "trace-profile", "", "write trace to `dir`")
+	f.StringVar(&opts.blockPath, "block-profile", "", "write block profile to `dir`")
+	f.BoolVar(&opts.insecure, "insecure-kdf", false, "use insecure KDF settings")
+}
+
+type fakeTestingTB struct {
+	stderr io.Writer
+}
+
+func (t fakeTestingTB) Logf(msg string, args ...interface{}) {
+	fmt.Fprintf(t.stderr, msg, args...)
+}
+
+func (p *Profiler) Start(profileOpts ProfileOptions, stderr io.Writer) error {
+	if profileOpts.listen != "" {
+		fmt.Fprintf(stderr, "running profile HTTP server on %v\n", profileOpts.listen)
+		go func() {
+			err := http.ListenAndServe(profileOpts.listen, nil)
+			if err != nil {
+				fmt.Fprintf(stderr, "profile HTTP server listen failed: %v\n", err)
+			}
+		}()
+	}
+
+	profilesEnabled := 0
+	if profileOpts.memPath != "" {
+		profilesEnabled++
+	}
+	if profileOpts.cpuPath != "" {
+		profilesEnabled++
+	}
+	if profileOpts.tracePath != "" {
+		profilesEnabled++
+	}
+	if profileOpts.blockPath != "" {
+		profilesEnabled++
+	}
+
+	if profilesEnabled > 1 {
+		return errors.Fatal("only one profile (memory, CPU, trace, or block) may be activated at the same time")
+	}
+
+	if profileOpts.memPath != "" {
+		p.stop = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.MemProfile, profile.ProfilePath(profileOpts.memPath))
+	} else if profileOpts.cpuPath != "" {
+		p.stop = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.CPUProfile, profile.ProfilePath(profileOpts.cpuPath))
+	} else if profileOpts.tracePath != "" {
+		p.stop = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.TraceProfile, profile.ProfilePath(profileOpts.tracePath))
+	} else if profileOpts.blockPath != "" {
+		p.stop = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.BlockProfile, profile.ProfilePath(profileOpts.blockPath))
+	}
+
+	if profileOpts.insecure {
+		repository.TestUseLowSecurityKDFParameters(fakeTestingTB{stderr})
+	}
+
+	return nil
+}
+
+func (p *Profiler) Stop() {
+	if p.stop != nil {
+		p.stop.Stop()
+	}
+}

internal/global/global_release.go

@@ -0,0 +1,14 @@
+//go:build !debug && !profile
+// +build !debug,!profile
+
+package global
+
+import (
+	"io"
+
+	"github.com/spf13/cobra"
+)
+
+func RegisterProfiling(_ *cobra.Command, _ io.Writer) {
+	// No profiling in release mode
+}

internal/global/global_test.go

@@ -0,0 +1,51 @@
+package global
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func TestReadRepo(t *testing.T) {
+	tempDir := rtest.TempDir(t)
+
+	// test --repo option
+	var gopts Options
+	gopts.Repo = tempDir
+	repo, err := readRepo(gopts)
+	rtest.OK(t, err)
+	rtest.Equals(t, tempDir, repo)
+
+	// test --repository-file option
+	foo := filepath.Join(tempDir, "foo")
+	err = os.WriteFile(foo, []byte(tempDir+"\n"), 0666)
+	rtest.OK(t, err)
+
+	var gopts2 Options
+	gopts2.RepositoryFile = foo
+	repo, err = readRepo(gopts2)
+	rtest.OK(t, err)
+	rtest.Equals(t, tempDir, repo)
+
+	var gopts3 Options
+	gopts3.RepositoryFile = foo + "-invalid"
+	_, err = readRepo(gopts3)
+	if err == nil {
+		t.Fatal("must not read repository path from invalid file path")
+	}
+}
+
+func TestReadEmptyPassword(t *testing.T) {
+	opts := Options{InsecureNoPassword: true}
+	password, err := readPassword(context.TODO(), opts, "test")
+	rtest.OK(t, err)
+	rtest.Equals(t, "", password, "got unexpected password")
+
+	opts.Password = "invalid"
+	_, err = readPassword(context.TODO(), opts, "test")
+	rtest.Assert(t, strings.Contains(err.Error(), "must not be specified together with providing a password via a cli option or environment variable"), "unexpected error message, got %v", err)
+}

internal/global/secondary_repo.go

@@ -0,0 +1,123 @@
+package global
+
+import (
+	"context"
+	"os"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/spf13/pflag"
+)
+
+type SecondaryRepoOptions struct {
+	Password string
+	// from-repo options
+	Repo               string
+	RepositoryFile     string
+	PasswordFile       string
+	PasswordCommand    string
+	KeyHint            string
+	InsecureNoPassword bool
+	// repo2 options
+	LegacyRepo            string
+	LegacyRepositoryFile  string
+	LegacyPasswordFile    string
+	LegacyPasswordCommand string
+	LegacyKeyHint         string
+}
+
+func (opts *SecondaryRepoOptions) AddFlags(f *pflag.FlagSet, repoPrefix string, repoUsage string) {
+	f.StringVarP(&opts.LegacyRepo, "repo2", "", "", repoPrefix+" `repository` "+repoUsage+" (default: $RESTIC_REPOSITORY2)")
+	f.StringVarP(&opts.LegacyRepositoryFile, "repository-file2", "", "", "`file` from which to read the "+repoPrefix+" repository location "+repoUsage+" (default: $RESTIC_REPOSITORY_FILE2)")
+	f.StringVarP(&opts.LegacyPasswordFile, "password-file2", "", "", "`file` to read the "+repoPrefix+" repository password from (default: $RESTIC_PASSWORD_FILE2)")
+	f.StringVarP(&opts.LegacyKeyHint, "key-hint2", "", "", "key ID of key to try decrypting the "+repoPrefix+" repository first (default: $RESTIC_KEY_HINT2)")
+	f.StringVarP(&opts.LegacyPasswordCommand, "password-command2", "", "", "shell `command` to obtain the "+repoPrefix+" repository password from (default: $RESTIC_PASSWORD_COMMAND2)")
+
+	// hide repo2 options
+	_ = f.MarkDeprecated("repo2", "use --repo or --from-repo instead")
+	_ = f.MarkDeprecated("repository-file2", "use --repository-file or --from-repository-file instead")
+	_ = f.MarkHidden("password-file2")
+	_ = f.MarkHidden("key-hint2")
+	_ = f.MarkHidden("password-command2")
+
+	opts.LegacyRepo = os.Getenv("RESTIC_REPOSITORY2")
+	opts.LegacyRepositoryFile = os.Getenv("RESTIC_REPOSITORY_FILE2")
+	opts.LegacyPasswordFile = os.Getenv("RESTIC_PASSWORD_FILE2")
+	opts.LegacyKeyHint = os.Getenv("RESTIC_KEY_HINT2")
+	opts.LegacyPasswordCommand = os.Getenv("RESTIC_PASSWORD_COMMAND2")
+
+	f.StringVarP(&opts.Repo, "from-repo", "", "", "source `repository` "+repoUsage+" (default: $RESTIC_FROM_REPOSITORY)")
+	f.StringVarP(&opts.RepositoryFile, "from-repository-file", "", "", "`file` from which to read the source repository location "+repoUsage+" (default: $RESTIC_FROM_REPOSITORY_FILE)")
+	f.StringVarP(&opts.PasswordFile, "from-password-file", "", "", "`file` to read the source repository password from (default: $RESTIC_FROM_PASSWORD_FILE)")
+	f.StringVarP(&opts.KeyHint, "from-key-hint", "", "", "key ID of key to try decrypting the source repository first (default: $RESTIC_FROM_KEY_HINT)")
+	f.StringVarP(&opts.PasswordCommand, "from-password-command", "", "", "shell `command` to obtain the source repository password from (default: $RESTIC_FROM_PASSWORD_COMMAND)")
+	f.BoolVar(&opts.InsecureNoPassword, "from-insecure-no-password", false, "use an empty password for the source repository (insecure)")
+
+	opts.Repo = os.Getenv("RESTIC_FROM_REPOSITORY")
+	opts.RepositoryFile = os.Getenv("RESTIC_FROM_REPOSITORY_FILE")
+	opts.PasswordFile = os.Getenv("RESTIC_FROM_PASSWORD_FILE")
+	opts.KeyHint = os.Getenv("RESTIC_FROM_KEY_HINT")
+	opts.PasswordCommand = os.Getenv("RESTIC_FROM_PASSWORD_COMMAND")
+}
+
+func (opts *SecondaryRepoOptions) FillGlobalOpts(ctx context.Context, gopts Options, repoPrefix string) (Options, bool, error) {
+	if opts.Repo == "" && opts.RepositoryFile == "" && opts.LegacyRepo == "" && opts.LegacyRepositoryFile == "" {
+		return Options{}, false, errors.Fatal("Please specify a source repository location (--from-repo or --from-repository-file)")
+	}
+
+	hasFromRepo := opts.Repo != "" || opts.RepositoryFile != "" || opts.PasswordFile != "" ||
+		opts.KeyHint != "" || opts.PasswordCommand != "" || opts.InsecureNoPassword
+	hasRepo2 := opts.LegacyRepo != "" || opts.LegacyRepositoryFile != "" || opts.LegacyPasswordFile != "" ||
+		opts.LegacyKeyHint != "" || opts.LegacyPasswordCommand != ""
+
+	if hasFromRepo && hasRepo2 {
+		return Options{}, false, errors.Fatal("Option groups repo2 and from-repo are mutually exclusive, please specify only one")
+	}
+
+	var err error
+	dstGopts := gopts
+	var pwdEnv string
+
+	if hasFromRepo {
+		if opts.Repo != "" && opts.RepositoryFile != "" {
+			return Options{}, false, errors.Fatal("Options --from-repo and --from-repository-file are mutually exclusive, please specify only one")
+		}
+
+		dstGopts.Repo = opts.Repo
+		dstGopts.RepositoryFile = opts.RepositoryFile
+		dstGopts.PasswordFile = opts.PasswordFile
+		dstGopts.PasswordCommand = opts.PasswordCommand
+		dstGopts.KeyHint = opts.KeyHint
+		dstGopts.InsecureNoPassword = opts.InsecureNoPassword
+
+		pwdEnv = "RESTIC_FROM_PASSWORD"
+		repoPrefix = "source"
+	} else {
+		if opts.LegacyRepo != "" && opts.LegacyRepositoryFile != "" {
+			return Options{}, false, errors.Fatal("Options --repo2 and --repository-file2 are mutually exclusive, please specify only one")
+		}
+
+		dstGopts.Repo = opts.LegacyRepo
+		dstGopts.RepositoryFile = opts.LegacyRepositoryFile
+		dstGopts.PasswordFile = opts.LegacyPasswordFile
+		dstGopts.PasswordCommand = opts.LegacyPasswordCommand
+		dstGopts.KeyHint = opts.LegacyKeyHint
+		// keep existing bevhaior for legacy options
+		dstGopts.InsecureNoPassword = false
+
+		pwdEnv = "RESTIC_PASSWORD2"
+	}
+
+	if opts.Password != "" {
+		dstGopts.Password = opts.Password
+	} else {
+		dstGopts.Password, err = resolvePassword(&dstGopts, pwdEnv)
+		if err != nil {
+			return Options{}, false, err
+		}
+	}
+	dstGopts.Password, err = readPassword(ctx, dstGopts, "enter password for "+repoPrefix+" repository: ")
+	if err != nil {
+		return Options{}, false, err
+	}
+	return dstGopts, hasFromRepo, nil
+}

internal/global/secondary_repo_test.go

@@ -0,0 +1,179 @@
+package global
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	rtest "github.com/restic/restic/internal/test"
+)
+
+// TestFillSecondaryGlobalOpts tests valid and invalid data on fillSecondaryGlobalOpts-function
+func TestFillSecondaryGlobalOpts(t *testing.T) {
+	//secondaryRepoTestCase defines a struct for test cases
+	type secondaryRepoTestCase struct {
+		Opts     SecondaryRepoOptions
+		DstGOpts Options
+		FromRepo bool
+	}
+
+	//validSecondaryRepoTestCases is a list with test cases that must pass
+	var validSecondaryRepoTestCases = []secondaryRepoTestCase{
+		{
+			// Test if Repo and Password are parsed correctly.
+			Opts: SecondaryRepoOptions{
+				Repo:     "backupDst",
+				Password: "secretDst",
+			},
+			DstGOpts: Options{
+				Repo:     "backupDst",
+				Password: "secretDst",
+			},
+			FromRepo: true,
+		},
+		{
+			// Test if RepositoryFile and PasswordFile are parsed correctly.
+			Opts: SecondaryRepoOptions{
+				RepositoryFile: "backupDst",
+				PasswordFile:   "passwordFileDst",
+			},
+			DstGOpts: Options{
+				RepositoryFile: "backupDst",
+				Password:       "secretDst",
+				PasswordFile:   "passwordFileDst",
+			},
+			FromRepo: true,
+		},
+		{
+			// Test if RepositoryFile and PasswordCommand are parsed correctly.
+			Opts: SecondaryRepoOptions{
+				RepositoryFile:  "backupDst",
+				PasswordCommand: "echo secretDst",
+			},
+			DstGOpts: Options{
+				RepositoryFile:  "backupDst",
+				Password:        "secretDst",
+				PasswordCommand: "echo secretDst",
+			},
+			FromRepo: true,
+		},
+		{
+			// Test if LegacyRepo and Password are parsed correctly.
+			Opts: SecondaryRepoOptions{
+				LegacyRepo: "backupDst",
+				Password:   "secretDst",
+			},
+			DstGOpts: Options{
+				Repo:     "backupDst",
+				Password: "secretDst",
+			},
+		},
+		{
+			// Test if LegacyRepositoryFile and LegacyPasswordFile are parsed correctly.
+			Opts: SecondaryRepoOptions{
+				LegacyRepositoryFile: "backupDst",
+				LegacyPasswordFile:   "passwordFileDst",
+			},
+			DstGOpts: Options{
+				RepositoryFile: "backupDst",
+				Password:       "secretDst",
+				PasswordFile:   "passwordFileDst",
+			},
+		},
+		{
+			// Test if LegacyRepositoryFile and LegacyPasswordCommand are parsed correctly.
+			Opts: SecondaryRepoOptions{
+				LegacyRepositoryFile:  "backupDst",
+				LegacyPasswordCommand: "echo secretDst",
+			},
+			DstGOpts: Options{
+				RepositoryFile:  "backupDst",
+				Password:        "secretDst",
+				PasswordCommand: "echo secretDst",
+			},
+		},
+	}
+
+	//invalidSecondaryRepoTestCases is a list with test cases that must fail
+	var invalidSecondaryRepoTestCases = []secondaryRepoTestCase{
+		{
+			// Test must fail on no repo given.
+			Opts: SecondaryRepoOptions{},
+		},
+		{
+			// Test must fail as Repo and RepositoryFile are both given
+			Opts: SecondaryRepoOptions{
+				Repo:           "backupDst",
+				RepositoryFile: "backupDst",
+			},
+		},
+		{
+			// Test must fail as PasswordFile and PasswordCommand are both given
+			Opts: SecondaryRepoOptions{
+				Repo:            "backupDst",
+				PasswordFile:    "passwordFileDst",
+				PasswordCommand: "notEmpty",
+			},
+		},
+		{
+			// Test must fail as PasswordFile does not exist
+			Opts: SecondaryRepoOptions{
+				Repo:         "backupDst",
+				PasswordFile: "NonExistingFile",
+			},
+		},
+		{
+			// Test must fail as PasswordCommand does not exist
+			Opts: SecondaryRepoOptions{
+				Repo:            "backupDst",
+				PasswordCommand: "notEmpty",
+			},
+		},
+		{
+			// Test must fail as current and legacy options are mixed
+			Opts: SecondaryRepoOptions{
+				Repo:       "backupDst",
+				LegacyRepo: "backupDst",
+			},
+		},
+		{
+			// Test must fail as current and legacy options are mixed
+			Opts: SecondaryRepoOptions{
+				Repo:                  "backupDst",
+				LegacyPasswordCommand: "notEmpty",
+			},
+		},
+	}
+
+	//gOpts defines the Global options used in the secondary repository tests
+	var gOpts = Options{
+		Repo:           "backupSrc",
+		RepositoryFile: "backupSrc",
+		Password:       "secretSrc",
+		PasswordFile:   "passwordFileSrc",
+	}
+
+	//Create temp dir to create password file.
+	dir := rtest.TempDir(t)
+	cleanup := rtest.Chdir(t, dir)
+	defer cleanup()
+
+	//Create temporary password file
+	err := os.WriteFile(filepath.Join(dir, "passwordFileDst"), []byte("secretDst"), 0666)
+	rtest.OK(t, err)
+
+	// Test all valid cases
+	for _, testCase := range validSecondaryRepoTestCases {
+		DstGOpts, isFromRepo, err := testCase.Opts.FillGlobalOpts(context.TODO(), gOpts, "destination")
+		rtest.OK(t, err)
+		rtest.Equals(t, DstGOpts, testCase.DstGOpts)
+		rtest.Equals(t, isFromRepo, testCase.FromRepo)
+	}
+
+	// Test all invalid cases
+	for _, testCase := range invalidSecondaryRepoTestCases {
+		_, _, err := testCase.Opts.FillGlobalOpts(context.TODO(), gOpts, "destination")
+		rtest.Assert(t, err != nil, "Expected error, but function did not return an error")
+	}
+}