Add version for 0.15.0

Upgrade dependencies

.github/dependabot.yml

@@ -0,0 +1,13 @@
+version: 2
+updates:
+  # Dependencies listed in go.mod
+  - package-ecosystem: "gomod"
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "monthly"
+
+  # Dependencies listed in .github/workflows/*.yml
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"

.github/workflows/tests.yml

@@ -21,13 +21,11 @@ jobs:
           - job_name: Windows
             go: 1.19.x
             os: windows-latest
-            install_verb: install
 
           - job_name: macOS
             go: 1.19.x
             os: macOS-latest
             test_fuse: false
-            install_verb: install
 
           - job_name: Linux
             go: 1.19.x
@@ -35,31 +33,17 @@ jobs:
             test_cloud_backends: true
             test_fuse: true
             check_changelog: true
-            install_verb: install
+
+          - job_name: Linux (race)
+            go: 1.19.x
+            os: ubuntu-latest
+            test_fuse: true
+            test_opts: "-race"
 
           - job_name: Linux
             go: 1.18.x
             os: ubuntu-latest
             test_fuse: true
-            install_verb: install
-
-          - job_name: Linux
-            go: 1.17.x
-            os: ubuntu-latest
-            test_fuse: true
-            install_verb: install
-
-          - job_name: Linux
-            go: 1.16.x
-            os: ubuntu-latest
-            test_fuse: true
-            install_verb: get
-
-          - job_name: Linux
-            go: 1.15.x
-            os: ubuntu-latest
-            test_fuse: true
-            install_verb: get
 
     name: ${{ matrix.job_name }} Go ${{ matrix.go }}
     runs-on: ${{ matrix.os }}
@@ -69,14 +53,14 @@ jobs:
 
     steps:
       - name: Set up Go ${{ matrix.go }}
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go }}
 
       - name: Get programs (Linux/macOS)
         run: |
           echo "build Go tools"
-          go ${{ matrix.install_verb }} github.com/restic/rest-server/cmd/rest-server@latest
+          go install github.com/restic/rest-server/cmd/rest-server@latest
 
           echo "install minio server"
           mkdir $HOME/bin
@@ -98,7 +82,7 @@ jobs:
           chmod 755 $HOME/bin/rclone
           rm -rf rclone*
 
-          # add $HOME/bin to path ($GOBIN was already added to the path by setup-go@v2)
+          # add $HOME/bin to path ($GOBIN was already added to the path by setup-go@v3)
           echo $HOME/bin >> $GITHUB_PATH
         if: matrix.os == 'ubuntu-latest' || matrix.os == 'macOS-latest'
 
@@ -108,7 +92,7 @@ jobs:
           $ProgressPreference = 'SilentlyContinue'
 
           echo "build Go tools"
-          go ${{ matrix.install_verb }} github.com/restic/rest-server/...
+          go install github.com/restic/rest-server/...
 
           echo "install minio server"
           mkdir $Env:USERPROFILE/bin
@@ -120,7 +104,7 @@ jobs:
           unzip rclone.zip
           copy rclone*/rclone.exe $Env:USERPROFILE/bin
 
-          # add $USERPROFILE/bin to path ($GOBIN was already added to the path by setup-go@v2)
+          # add $USERPROFILE/bin to path ($GOBIN was already added to the path by setup-go@v3)
           echo $Env:USERPROFILE\bin >> $Env:GITHUB_PATH
 
           echo "install tar"
@@ -142,7 +126,7 @@ jobs:
         if: matrix.os == 'windows-latest'
 
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Build with build.go
         run: |
@@ -152,7 +136,7 @@ jobs:
         env:
           RESTIC_TEST_FUSE: ${{ matrix.test_fuse }}
         run: |
-          go test -cover ./...
+          go test -cover ${{matrix.test_opts}} ./...
 
       - name: Test cloud backends
         env:
@@ -193,7 +177,9 @@ jobs:
 
         # only run cloud backend tests for pull requests from and pushes to our
         # own repo, otherwise the secrets are not available
-        if: (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && matrix.test_cloud_backends
+        # Skip for Dependabot pull requests as these are run without secrets
+        # https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#responding-to-events
+        if: (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && (github.actor != 'dependabot[bot]') && matrix.test_cloud_backends
 
       - name: Check changelog files with calens
         run: |
@@ -209,15 +195,16 @@ jobs:
 
       # ATTENTION: the list of architectures must be in sync with helpers/build-release-binaries/main.go!
       matrix:
-        # run cross-compile in two batches parallel so the overall tests run faster
+        # run cross-compile in three batches parallel so the overall tests run faster
         targets:
-          - "linux/386 linux/amd64 linux/arm linux/arm64 linux/ppc64le linux/mips linux/mipsle linux/mips64 linux/mips64le linux/s390x \
-            openbsd/386 openbsd/amd64"
+          - "linux/386 linux/amd64 linux/arm linux/arm64 linux/ppc64le linux/mips linux/mipsle linux/mips64 linux/mips64le linux/s390x"
 
-          - "freebsd/386 freebsd/amd64 freebsd/arm \
+          - "openbsd/386 openbsd/amd64 \
+            freebsd/386 freebsd/amd64 freebsd/arm \
             aix/ppc64 \
-            darwin/amd64 darwin/arm64 \
-            netbsd/386 netbsd/amd64 \
+            darwin/amd64 darwin/arm64"
+
+          - "netbsd/386 netbsd/amd64 \
             windows/386 windows/amd64 \
             solaris/amd64"
 
@@ -230,7 +217,7 @@ jobs:
 
     steps:
       - name: Set up Go ${{ env.latest_go }}
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
           go-version: ${{ env.latest_go }}
 
@@ -239,7 +226,7 @@ jobs:
           go install github.com/mitchellh/gox@latest
 
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Cross-compile with gox for ${{ matrix.targets }}
         env:
@@ -255,22 +242,21 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set up Go ${{ env.latest_go }}
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
           go-version: ${{ env.latest_go }}
 
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v2
+        uses: golangci/golangci-lint-action@v3
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.48
+          version: v1.49
           # Optional: show only new issues if it's a pull request. The default value is `false`.
           only-new-issues: true
           args: --verbose --timeout 5m
-          skip-go-installation: true
 
         # only run golangci-lint for pull requests, otherwise ALL hints get
         # reported. We need to slowly address all issues until we can enable
@@ -288,11 +274,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v4
         with:
           # list of Docker images to use as base name for tags
           images: |
@@ -308,14 +294,14 @@ jobs:
             type=sha
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           push: false
           context: .

.golangci.yml

@@ -55,3 +55,5 @@ issues:
     - exported (function|method|var|type|const) .* should have comment or be unexported
     # revive: ignore constants in all caps
     - don't use ALL_CAPS in Go names; use CamelCase
+    # revive: lots of packages don't have such a comment
+    - "package-comments: should have a package comment"

CONTRIBUTING.md

@@ -76,7 +76,7 @@ Then use the `go` tool to build restic:
 
     $ go build ./cmd/restic
     $ ./restic version
-    restic 0.10.0-dev (compiled manually) compiled with go1.15.2 on linux/amd64
+    restic 0.14.0-dev (compiled manually) compiled with go1.19 on linux/amd64
 
 You can run all tests with the following command:
 

VERSION

@@ -1 +1 @@
-0.14.0
+0.15.0

build.go

@@ -3,8 +3,8 @@
 // This program aims to make building Go programs for end users easier by just
 // calling it with `go run`, without having to setup a GOPATH.
 //
-// This program needs Go >= 1.12. It'll use Go modules for compilation. It
-// builds the package configured as Main in the Config struct.
+// This program checks for a minimum Go version. It will use Go modules for
+// compilation. It builds the package configured as Main in the Config struct.
 
 // BSD 2-Clause License
 //
@@ -43,7 +43,6 @@ package main
 import (
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -59,7 +58,7 @@ var config = Config{
 	Main:             "./cmd/restic",                           // package name for the main package
 	DefaultBuildTags: []string{"selfupdate"},                   // specify build tags which are always used
 	Tests:            []string{"./..."},                        // tests to run
-	MinVersion:       GoVersion{Major: 1, Minor: 14, Patch: 0}, // minimum Go version supported
+	MinVersion:       GoVersion{Major: 1, Minor: 18, Patch: 0}, // minimum Go version supported
 }
 
 // Config configures the build.
@@ -179,7 +178,7 @@ func test(cwd string, env map[string]string, args ...string) error {
 // getVersion returns the version string from the file VERSION in the current
 // directory.
 func getVersionFromFile() string {
-	buf, err := ioutil.ReadFile("VERSION")
+	buf, err := os.ReadFile("VERSION")
 	if err != nil {
 		verbosePrintf("error reading file VERSION: %v\n", err)
 		return ""
@@ -319,12 +318,8 @@ func (v GoVersion) String() string {
 }
 
 func main() {
-	if !goVersion.AtLeast(GoVersion{1, 12, 0}) {
-		die("Go version (%v) is too old, restic requires Go >= 1.12\n", goVersion)
-	}
-
 	if !goVersion.AtLeast(config.MinVersion) {
-		fmt.Fprintf(os.Stderr, "%s detected, this program requires at least %s\n", goVersion, config.MinVersion)
+		fmt.Fprintf(os.Stderr, "Detected version %s is too old, restic requires at least %s\n", goVersion, config.MinVersion)
 		os.Exit(1)
 	}
 

changelog/0.15.0_2023-01-12/issue-14

@@ -0,0 +1,8 @@
+Enhancement: Implement `rewrite` command
+
+Restic now has a `rewrite` command which allows to rewrite existing snapshots
+to remove unwanted files.
+
+https://github.com/restic/restic/issues/14
+https://github.com/restic/restic/pull/2731
+https://github.com/restic/restic/pull/4079

changelog/0.15.0_2023-01-12/issue-1734

@@ -0,0 +1,15 @@
+Enhancement: Inform about successful retries after errors
+
+When a recoverable error is encountered, restic shows a warning message saying
+that it's retrying, e.g.:
+
+`Save(<data/956b9ced99>) returned error, retrying after 357.131936ms: ...`
+
+This message can be confusing in that it never clearly states whether the retry
+is successful or not. This has now been fixed such that restic follows up with
+a message confirming a successful retry, e.g.:
+
+`Save(<data/956b9ced99>) operation successful after 1 retries`
+
+https://github.com/restic/restic/issues/1734
+https://github.com/restic/restic/pull/2661

changelog/0.15.0_2023-01-12/issue-1866

@@ -0,0 +1,12 @@
+Enhancement: Improve handling of directories with duplicate entries
+
+If for some reason a directory contains a duplicate entry, the `backup` command
+would previously fail with a `node "path/to/file" already present` or `nodes
+are not ordered got "path/to/file", last "path/to/file"` error.
+
+The error handling has been improved to only report a warning in this case. Make
+sure to check that the filesystem in question is not damaged if you see this!
+
+https://github.com/restic/restic/issues/1866
+https://github.com/restic/restic/issues/3937
+https://github.com/restic/restic/pull/3880

changelog/0.15.0_2023-01-12/issue-2015

@@ -0,0 +1,10 @@
+Bugfix: Make `mount` return exit code 0 after receiving Ctrl-C / SIGINT
+
+To stop the `mount` command, a user has to press Ctrl-C or send a SIGINT
+signal to restic. This used to cause restic to exit with a non-zero exit code.
+
+The exit code has now been changed to zero as the above is the expected way
+to stop the `mount` command and should therefore be considered successful.
+
+https://github.com/restic/restic/issues/2015
+https://github.com/restic/restic/pull/3894

changelog/0.15.0_2023-01-12/issue-2134

@@ -0,0 +1,19 @@
+Enhancement: Support B2 API keys restricted to hiding but not deleting files
+
+When the B2 backend does not have the necessary permissions to permanently
+delete files, it now automatically falls back to hiding files. This allows
+using restic with an application key which is not allowed to delete files.
+This can prevent an attacker from deleting backups with such an API key.
+
+To use this feature create an application key without the `deleteFiles`
+capability. It is recommended to restrict the key to just one bucket.
+For example using the `b2` command line tool:
+
+`b2 create-key --bucket <bucketName> <keyName> listBuckets,readFiles,writeFiles,listFiles`
+
+Alternatively, you can use the S3 backend to access B2, as described
+in the documentation. In this mode, files are also only hidden instead
+of being deleted permanently.
+
+https://github.com/restic/restic/issues/2134
+https://github.com/restic/restic/pull/2398

changelog/0.15.0_2023-01-12/issue-2152

@@ -0,0 +1,11 @@
+Enhancement: Make `init` open only one connection for the SFTP backend
+
+The `init` command using the SFTP backend used to connect twice to the
+repository. This could be inconvenient if the user must enter a password,
+or cause `init` to fail if the server does not correctly close the first SFTP
+connection.
+
+This has now been fixed by reusing the first/initial SFTP connection opened.
+
+https://github.com/restic/restic/issues/2152
+https://github.com/restic/restic/pull/3882

changelog/0.15.0_2023-01-12/issue-2533

@@ -0,0 +1,13 @@
+Enhancement: Handle cache corruption on disk and in downloads
+
+In rare situations, like for example after a system crash, the data stored
+in the cache might be corrupted. This could cause restic to fail and required
+manually deleting the cache.
+
+Restic now automatically removes broken data from the cache, allowing it
+to recover from such a situation without user intervention. In addition,
+restic retries downloads which return corrupt data in order to also handle
+temporary download problems.
+
+https://github.com/restic/restic/issues/2533
+https://github.com/restic/restic/pull/3521

changelog/0.15.0_2023-01-12/issue-2591

@@ -0,0 +1,17 @@
+Bugfix: Don't read password from stdin for `backup --stdin`
+
+The `backup` command when used with `--stdin` previously tried to read first
+the password, then the data to be backed up from standard input. This meant
+it would often confuse part of the data for the password.
+
+From now on, it will instead exit with the message `Fatal: cannot read both
+password and data from stdin` unless the password is passed in some other
+way (such as `--restic-password-file`, `RESTIC_PASSWORD`, etc).
+
+To enter the password interactively a password command has to be used. For
+example on Linux, `mysqldump somedatabase | restic backup --stdin
+--password-command='sh -c "systemd-ask-password < /dev/tty"'` securely reads
+the password from the terminal.
+
+https://github.com/restic/restic/issues/2591
+https://github.com/restic/restic/pull/4011

changelog/0.15.0_2023-01-12/issue-2699

@@ -0,0 +1,9 @@
+Enhancement: Support restoring symbolic links on Windows
+
+The `restore` command now supports restoring symbolic links on Windows. Because
+of Windows specific restrictions this is only possible when running restic with
+the `SeCreateSymbolicLinkPrivilege` privilege or as an administrator.
+
+https://github.com/restic/restic/issues/1078
+https://github.com/restic/restic/issues/2699
+https://github.com/restic/restic/pull/2875

changelog/0.15.0_2023-01-12/issue-2715

@@ -0,0 +1,20 @@
+Enhancement: Stricter repository lock handling
+
+Previously, restic commands kept running even if they failed to refresh their
+locks in time. This could be a problem e.g. in case the client system running
+a backup entered the standby power mode while the backup was still in progress
+(which would prevent the client from refreshing its lock), and after a short
+delay another host successfully runs `unlock` and `prune` on the repository,
+which would remove all data added by the in-progress backup. If the backup
+client later continues its backup, even though its lock had expired in the
+meantime, this would lead to an incomplete snapshot.
+
+To address this, lock handling is now much stricter. Commands requiring a lock
+are canceled if the lock is not refreshed successfully in time. In addition,
+if a lock file is not readable restic will not allow starting a command. It may
+be necessary to remove invalid lock files manually or use `unlock --remove-all`.
+Please make sure that no other restic processes are running concurrently before
+doing this, however.
+
+https://github.com/restic/restic/issues/2715
+https://github.com/restic/restic/pull/3569

changelog/0.15.0_2023-01-12/issue-2724

@@ -0,0 +1,9 @@
+Change: Include full snapshot ID in JSON output of `backup`
+
+We have changed the JSON output of the backup command to include the full
+snapshot ID instead of just a shortened version, as the latter can be ambiguous
+in some rare cases. To derive the short ID, please truncate the full ID down to
+eight characters.
+
+https://github.com/restic/restic/issues/2724
+https://github.com/restic/restic/pull/3993

changelog/0.15.0_2023-01-12/issue-3029

@@ -0,0 +1,8 @@
+Enhancement: Add support for `credential_process` to S3 backend
+
+Restic now uses a newer library for the S3 backend, which adds support for the
+`credential_process` option in the AWS credential configuration.
+
+https://github.com/restic/restic/issues/3029
+https://github.com/restic/restic/issues/4034
+https://github.com/restic/restic/pull/4025

changelog/0.15.0_2023-01-12/issue-3096

@@ -0,0 +1,8 @@
+Enhancement: Make `mount` command support macOS using macFUSE 4.x
+
+Restic now uses a different FUSE library for mounting snapshots and making them
+available as a FUSE filesystem using the `mount` command. This adds support for
+macFUSE 4.x which can be used to make this work on recent macOS versions.
+
+https://github.com/restic/restic/issues/3096
+https://github.com/restic/restic/pull/4024

changelog/0.15.0_2023-01-12/issue-3124

@@ -0,0 +1,7 @@
+Enhancement: Support JSON output for the `init` command
+
+The `init` command used to ignore the `--json` option, but now outputs a JSON
+message if the repository was created successfully.
+
+https://github.com/restic/restic/issues/3124
+https://github.com/restic/restic/pull/3132

changelog/0.15.0_2023-01-12/issue-3161

@@ -0,0 +1,14 @@
+Bugfix: Delete files on Backblaze B2 more reliably
+
+Restic used to only delete the latest version of files stored in B2. In most
+cases this worked well as there was only a single version of the file. However,
+due to retries while uploading it is possible for multiple file versions to be
+stored at B2. This could lead to various problems for files that should have
+been deleted but still existed.
+
+The implementation has now been changed to delete all versions of files, which
+doubles the amount of Class B transactions necessary to delete files, but
+assures that no file versions are left behind.
+
+https://github.com/restic/restic/issues/3161
+https://github.com/restic/restic/pull/3885

changelog/0.15.0_2023-01-12/issue-3336

@@ -0,0 +1,12 @@
+Bugfix: Make SFTP backend report no space left on device
+
+Backing up to an SFTP backend would spew repeated SSH_FX_FAILURE messages when
+the remote disk was full. Restic now reports "sftp: no space left on device"
+and exits immediately when it detects this condition.
+
+A fix for this issue was implemented in restic 0.12.1, but unfortunately the
+fix itself contained a bug that prevented it from taking effect.
+
+https://github.com/restic/restic/issues/3336
+https://github.com/restic/restic/pull/3345
+https://github.com/restic/restic/pull/4075

changelog/0.15.0_2023-01-12/issue-3567

@@ -0,0 +1,10 @@
+Bugfix: Improve handling of interrupted syscalls in `mount` command
+
+Accessing restic's FUSE mount could result in "input/output" errors when using
+programs in which syscalls can be interrupted. This is for example the case for
+Go programs. This has now been fixed by improved error handling of interrupted
+syscalls.
+
+https://github.com/restic/restic/issues/3567
+https://github.com/restic/restic/issues/3694
+https://github.com/restic/restic/pull/3875

changelog/0.15.0_2023-01-12/issue-3897

@@ -0,0 +1,7 @@
+Bugfix: Fix stuck `copy` command when `-o <backend>.connections=1`
+
+When running the `copy` command with `-o <backend>.connections=1` the
+command would be infinitely stuck. This has now been fixed.
+
+https://github.com/restic/restic/issues/3897
+https://github.com/restic/restic/pull/3898

changelog/0.15.0_2023-01-12/issue-3918

@@ -0,0 +1,9 @@
+Bugfix: Correct prune statistics for partially compressed repositories
+
+In a partially compressed repository, one data blob can exist both in an
+uncompressed and a compressed version. This caused the `prune` statistics to
+become inaccurate and e.g. report a too high value for the unused size, such
+as "unused size after prune: 16777215.991 TiB". This has now been fixed.
+
+https://github.com/restic/restic/issues/3918
+https://github.com/restic/restic/pull/3980

changelog/0.15.0_2023-01-12/issue-3929

@@ -0,0 +1,11 @@
+Change: Make `unlock` display message only when locks were actually removed
+
+The `unlock` command used to print the "successfully removed locks" message
+whenever it was run, regardless of lock files having being removed or not.
+
+This has now been changed such that it only prints the message if any lock
+files were actually removed. In addition, it also reports the number of
+removed lock files.
+
+https://github.com/restic/restic/issues/3929
+https://github.com/restic/restic/pull/3935

changelog/0.15.0_2023-01-12/issue-3932

@@ -0,0 +1,15 @@
+Enhancement: Improve handling of ErrDot errors in rclone and sftp backends
+
+Since Go 1.19, restic can no longer implicitly run relative executables which
+are found in the current directory (e.g. `rclone` if found in `.`). This is a
+security feature of Go to prevent against running unintended and possibly
+harmful executables.
+
+The error message for this was just "cannot run executable found relative to
+current directory". This has now been improved to yield a more specific error
+message, informing the user how to explicitly allow running the executable
+using the `-o rclone.program` and `-o sftp.command` extended options with `./`.
+
+https://github.com/restic/restic/issues/3932
+https://pkg.go.dev/os/exec#hdr-Executables_in_the_current_directory
+https://go.dev/blog/path-security

changelog/0.15.0_2023-01-12/issue-4003

@@ -0,0 +1,8 @@
+Bugfix: Make `backup` no longer hang on Solaris when seeing a FIFO file
+
+The `backup` command used to hang on Solaris whenever it encountered a FIFO
+file (named pipe), due to a bug in the handling of extended attributes. This
+bug has now been fixed.
+
+https://github.com/restic/restic/issues/4003
+https://github.com/restic/restic/pull/4053

changelog/0.15.0_2023-01-12/issue-4016

@@ -0,0 +1,8 @@
+Bugfix: Support ExFAT-formatted local backends on macOS Ventura
+
+ExFAT-formatted disks could not be used as local backends starting from macOS
+Ventura. Restic commands would fail with an "inappropriate ioctl for device"
+error. This has now been fixed.
+
+https://github.com/restic/restic/issues/4016
+https://github.com/restic/restic/pull/4021

changelog/0.15.0_2023-01-12/issue-4033

@@ -0,0 +1,11 @@
+Change: Don't print skipped snapshots by default in `copy` command
+
+The `copy` command used to print each snapshot that was skipped because it
+already existed in the target repository. The amount of this output could
+practically bury the list of snapshots that were actually copied.
+
+From now on, the skipped snapshots are by default not printed at all, but
+this can be re-enabled by increasing the verbosity level of the command.
+
+https://github.com/restic/restic/issues/4033
+https://github.com/restic/restic/pull/4066

changelog/0.15.0_2023-01-12/issue-4085

@@ -0,0 +1,10 @@
+Bugfix: Make `init` ignore "Access Denied" errors when creating S3 buckets
+
+In restic 0.9.0 through 0.13.0, the `init` command ignored some permission
+errors from S3 backends when trying to check for bucket existence, so that
+manually created buckets with custom permissions could be used for backups.
+
+This feature became broken in 0.14.0, but has now been restored again.
+
+https://github.com/restic/restic/issues/4085
+https://github.com/restic/restic/pull/4086

changelog/0.15.0_2023-01-12/issue-4103

@@ -0,0 +1,10 @@
+Bugfix: Don't generate negative UIDs and GIDs in tar files from `dump`
+
+When using a 32-bit build of restic, the `dump` command could in some cases
+create tar files containing negative UIDs and GIDs, which cannot be read by
+GNU tar. This corner case especially applies to backups from stdin on Windows.
+
+This is now fixed such that `dump` creates valid tar files in these cases too.
+
+https://github.com/restic/restic/issues/4103
+https://github.com/restic/restic/pull/4104

changelog/0.15.0_2023-01-12/issue-79

@@ -0,0 +1,17 @@
+Enhancement: Restore files with long runs of zeros as sparse files
+
+When using `restore --sparse`, the restorer may now write files containing long
+runs of zeros as sparse files (also called files with holes), where the zeros
+are not actually written to disk.
+
+How much space is saved by writing sparse files depends on the operating
+system, file system and the distribution of zeros in the file.
+
+During backup restic still reads the whole file including sparse regions, but
+with optimized processing speed of sparse regions.
+
+https://github.com/restic/restic/issues/79
+https://github.com/restic/restic/issues/3903
+https://github.com/restic/restic/pull/2601
+https://github.com/restic/restic/pull/3854
+https://forum.restic.net/t/sparse-file-support/1264

changelog/0.15.0_2023-01-12/pull-2750

@@ -0,0 +1,7 @@
+Enhancement: Make backup file read concurrency configurable
+
+The `backup` command now supports a `--read-concurrency` option which allows
+tuning restic for very fast storage like NVMe disks by controlling the number
+of concurrent file reads during the backup process.
+
+https://github.com/restic/restic/pull/2750

changelog/0.15.0_2023-01-12/pull-3780

@@ -0,0 +1,8 @@
+Bugfix: Make `restore` replace existing symlinks
+
+When restoring a symlink, restic used to report an error if the target path
+already existed. This has now been fixed such that the potentially existing
+target path is first removed before the symlink is restored.
+
+https://github.com/restic/restic/issues/2578
+https://github.com/restic/restic/pull/3780

changelog/0.15.0_2023-01-12/pull-3899

@@ -0,0 +1,6 @@
+Enhancement: Optimize prune memory usage
+
+The `prune` command needs large amounts of memory in order to determine what to
+keep and what to remove. This is now optimized to use up to 30% less memory.
+
+https://github.com/restic/restic/pull/3899

changelog/0.15.0_2023-01-12/pull-3905

@@ -0,0 +1,6 @@
+Enhancement: Improve speed of parent snapshot detection in `backup` command
+
+Backing up a large number of files using `--files-from-verbatim` or `--files-from-raw`
+options could require a long time to find the parent snapshot. This has been improved.
+
+https://github.com/restic/restic/pull/3905

changelog/0.15.0_2023-01-12/pull-3915

@@ -0,0 +1,12 @@
+Enhancement: Add compression statistics to the `stats` command
+
+When executed with `--mode raw-data` on a repository that supports compression,
+the `stats` command now calculates and displays, for the selected repository or
+snapshots: the uncompressed size of the data; the compression progress
+(percentage of data that has been compressed); the compression ratio of the
+compressed data; the total space saving.
+
+It also takes into account both the compressed and uncompressed data if the
+repository is only partially compressed.
+
+https://github.com/restic/restic/pull/3915

changelog/0.15.0_2023-01-12/pull-3925

@@ -0,0 +1,6 @@
+Enhancement: Provide command completion for PowerShell
+
+Restic already provided generation of completion files for bash, fish and zsh.
+Now powershell is supported, too.
+
+https://github.com/restic/restic/pull/3925/files

changelog/0.15.0_2023-01-12/pull-3931

@@ -0,0 +1,10 @@
+Enhancement: Allow `backup` file tree scanner to be disabled
+
+The `backup` command walks the file tree in a separate scanner process to find
+the total size and file/directory count, and uses this to provide an ETA. This
+can slow down backups, especially of network filesystems.
+
+The command now has a new option `--no-scan` which can be used to disable this
+scanning in order to speed up backups when needed.
+
+https://github.com/restic/restic/pull/3931

changelog/0.15.0_2023-01-12/pull-3943

@@ -0,0 +1,9 @@
+Enhancement: Ignore additional/unknown files in repository
+
+If a restic repository had additional files in it (not created by restic),
+commands like `find` and `restore` could become confused and fail with an
+`multiple IDs with prefix "12345678" found` error. These commands now
+ignore such additional files.
+
+https://github.com/restic/restic/pull/3943
+https://forum.restic.net/t/which-protocol-should-i-choose-for-remote-linux-backups/5446/17

changelog/0.15.0_2023-01-12/pull-3951

@@ -0,0 +1,7 @@
+Bugfix: Make `ls` return exit code 1 if snapshot cannot be loaded
+
+The `ls` command used to show a warning and return exit code 0 when failing
+to load a snapshot. This has now been fixed such that it instead returns exit
+code 1 (still showing a warning).
+
+https://github.com/restic/restic/pull/3951

changelog/0.15.0_2023-01-12/pull-3955

@@ -0,0 +1,9 @@
+Enhancement: Improve `backup` performance for small files
+
+When backing up small files restic was slower than it could be. In particular
+this affected backups using maximum compression.
+
+This has been fixed by reworking the internal parallelism of the backup
+command, making it back up small files around two times faster.
+
+https://github.com/restic/restic/pull/3955

changelog/0.15.0_2023-01-12/pull-4041

@@ -0,0 +1,7 @@
+Change: Update dependencies and require Go 1.18 or newer
+
+Most dependencies have been updated. Since some libraries require newer language
+features, support for Go 1.15-1.17 has been dropped, which means that restic now
+requires at least Go 1.18 to build.
+
+https://github.com/restic/restic/pull/4041

changelog/0.15.0_2023-01-12/pull-4100

@@ -0,0 +1,11 @@
+Bugfix: Make `self-update` enabled by default only in release builds
+
+The `self-update` command was previously included by default in all builds of
+restic as opposed to only in official release builds, even if the `selfupdate`
+tag was not explicitly enabled when building.
+
+This has now been corrected, and the `self-update` command is only available
+if restic was built with `-tags selfupdate` (as done for official release
+builds by `build.go`).
+
+https://github.com/restic/restic/pull/4100

cmd/restic/cleanup.go

@@ -11,7 +11,7 @@ import (
 
 var cleanupHandlers struct {
 	sync.Mutex
-	list []func() error
+	list []func(code int) (int, error)
 	done bool
 	ch   chan os.Signal
 }
@@ -25,7 +25,7 @@ func init() {
 // AddCleanupHandler adds the function f to the list of cleanup handlers so
 // that it is executed when all the cleanup handlers are run, e.g. when SIGINT
 // is received.
-func AddCleanupHandler(f func() error) {
+func AddCleanupHandler(f func(code int) (int, error)) {
 	cleanupHandlers.Lock()
 	defer cleanupHandlers.Unlock()
 
@@ -36,22 +36,24 @@ func AddCleanupHandler(f func() error) {
 }
 
 // RunCleanupHandlers runs all registered cleanup handlers
-func RunCleanupHandlers() {
+func RunCleanupHandlers(code int) int {
 	cleanupHandlers.Lock()
 	defer cleanupHandlers.Unlock()
 
 	if cleanupHandlers.done {
-		return
+		return code
 	}
 	cleanupHandlers.done = true
 
 	for _, f := range cleanupHandlers.list {
-		err := f()
+		var err error
+		code, err = f(code)
 		if err != nil {
 			Warnf("error in cleanup handler: %v\n", err)
 		}
 	}
 	cleanupHandlers.list = nil
+	return code
 }
 
 // CleanupHandler handles the SIGINT signals.
@@ -75,6 +77,6 @@ func CleanupHandler(c <-chan os.Signal) {
 // Exit runs the cleanup handlers and then terminates the process with the
 // given exit code.
 func Exit(code int) {
-	RunCleanupHandlers()
+	code = RunCleanupHandlers(code)
 	os.Exit(code)
 }

cmd/restic/cmd_backup.go

@@ -6,11 +6,11 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -21,7 +21,6 @@ import (
 	"github.com/restic/restic/internal/archiver"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/filter"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
@@ -56,8 +55,9 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
 	},
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
+		ctx := cmd.Context()
 		var wg sync.WaitGroup
-		cancelCtx, cancel := context.WithCancel(globalOptions.ctx)
+		cancelCtx, cancel := context.WithCancel(ctx)
 		defer func() {
 			// shutdown termstatus
 			cancel()
@@ -71,35 +71,35 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
 			term.Run(cancelCtx)
 		}()
 
-		return runBackup(backupOptions, globalOptions, term, args)
+		return runBackup(ctx, backupOptions, globalOptions, term, args)
 	},
 }
 
 // BackupOptions bundles all options for the backup command.
 type BackupOptions struct {
-	Parent                  string
-	Force                   bool
-	Excludes                []string
-	InsensitiveExcludes     []string
-	ExcludeFiles            []string
-	InsensitiveExcludeFiles []string
-	ExcludeOtherFS          bool
-	ExcludeIfPresent        []string
-	ExcludeCaches           bool
-	ExcludeLargerThan       string
-	Stdin                   bool
-	StdinFilename           string
-	Tags                    restic.TagLists
-	Host                    string
-	FilesFrom               []string
-	FilesFromVerbatim       []string
-	FilesFromRaw            []string
-	TimeStamp               string
-	WithAtime               bool
-	IgnoreInode             bool
-	IgnoreCtime             bool
-	UseFsSnapshot           bool
-	DryRun                  bool
+	excludePatternOptions
+
+	Parent            string
+	Force             bool
+	ExcludeOtherFS    bool
+	ExcludeIfPresent  []string
+	ExcludeCaches     bool
+	ExcludeLargerThan string
+	Stdin             bool
+	StdinFilename     string
+	Tags              restic.TagLists
+	Host              string
+	FilesFrom         []string
+	FilesFromVerbatim []string
+	FilesFromRaw      []string
+	TimeStamp         string
+	WithAtime         bool
+	IgnoreInode       bool
+	IgnoreCtime       bool
+	UseFsSnapshot     bool
+	DryRun            bool
+	ReadConcurrency   uint
+	NoScan            bool
 }
 
 var backupOptions BackupOptions
@@ -113,10 +113,9 @@ func init() {
 	f := cmdBackup.Flags()
 	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: last snapshot in the repository that has the same target files/directories, and is not newer than the snapshot time)")
 	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
-	f.StringArrayVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
-	f.StringArrayVar(&backupOptions.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
-	f.StringArrayVar(&backupOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
-	f.StringArrayVar(&backupOptions.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns")
+
+	initExcludePatternOptions(f, &backupOptions.excludePatternOptions)
+
 	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems, don't cross filesystem boundaries and subvolumes")
 	f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes `filename[:header]`, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)")
 	f.BoolVar(&backupOptions.ExcludeCaches, "exclude-caches", false, `excludes cache directories that are marked with a CACHEDIR.TAG file. See https://bford.info/cachedir/ for the Cache Directory Tagging Standard`)
@@ -124,7 +123,7 @@ func init() {
 	f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
 	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "`filename` to use when reading from stdin")
 	f.Var(&backupOptions.Tags, "tag", "add `tags` for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times)")
-
+	f.UintVar(&backupOptions.ReadConcurrency, "read-concurrency", 0, "read `n` files concurrently (default: $RESTIC_READ_CONCURRENCY or 2)")
 	f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag")
 	f.StringVar(&backupOptions.Host, "hostname", "", "set the `hostname` for the snapshot manually")
 	err := f.MarkDeprecated("hostname", "use --host")
@@ -132,7 +131,6 @@ func init() {
 		// MarkDeprecated only returns an error when the flag could not be found
 		panic(err)
 	}
-
 	f.StringArrayVar(&backupOptions.FilesFrom, "files-from", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
 	f.StringArrayVar(&backupOptions.FilesFromVerbatim, "files-from-verbatim", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
 	f.StringArrayVar(&backupOptions.FilesFromRaw, "files-from-raw", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
@@ -141,9 +139,14 @@ func init() {
 	f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number changes when checking for modified files")
 	f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files")
 	f.BoolVarP(&backupOptions.DryRun, "dry-run", "n", false, "do not upload or write any data, just show what would be done")
+	f.BoolVar(&backupOptions.NoScan, "no-scan", false, "do not run scanner to estimate size of backup")
 	if runtime.GOOS == "windows" {
 		f.BoolVar(&backupOptions.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)")
 	}
+
+	// parse read concurrency from env, on error the default value will be used
+	readConcurrency, _ := strconv.ParseUint(os.Getenv("RESTIC_READ_CONCURRENCY"), 10, 32)
+	backupOptions.ReadConcurrency = uint(readConcurrency)
 }
 
 // filterExisting returns a slice of all existing items, or an error if no
@@ -183,7 +186,7 @@ func readLines(filename string) ([]string, error) {
 	)
 
 	if filename == "-" {
-		data, err = ioutil.ReadAll(os.Stdin)
+		data, err = io.ReadAll(os.Stdin)
 	} else {
 		data, err = textfile.Read(filename)
 	}
@@ -260,6 +263,10 @@ func readFilenamesRaw(r io.Reader) (names []string, err error) {
 // Check returns an error when an invalid combination of options was set.
 func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
 	if gopts.password == "" {
+		if opts.Stdin {
+			return errors.Fatal("cannot read both password and data from stdin")
+		}
+
 		filesFrom := append(append(opts.FilesFrom, opts.FilesFromVerbatim...), opts.FilesFromRaw...)
 		for _, filename := range filesFrom {
 			if filename == "-" {
@@ -300,48 +307,11 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, t
 		fs = append(fs, f)
 	}
 
-	// add patterns from file
-	if len(opts.ExcludeFiles) > 0 {
-		excludes, err := readExcludePatternsFromFiles(opts.ExcludeFiles)
-		if err != nil {
-			return nil, err
-		}
-
-		if valid, invalidPatterns := filter.ValidatePatterns(excludes); !valid {
-			return nil, errors.Fatalf("--exclude-file: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
-		}
-
-		opts.Excludes = append(opts.Excludes, excludes...)
-	}
-
-	if len(opts.InsensitiveExcludeFiles) > 0 {
-		excludes, err := readExcludePatternsFromFiles(opts.InsensitiveExcludeFiles)
-		if err != nil {
-			return nil, err
-		}
-
-		if valid, invalidPatterns := filter.ValidatePatterns(excludes); !valid {
-			return nil, errors.Fatalf("--iexclude-file: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
-		}
-
-		opts.InsensitiveExcludes = append(opts.InsensitiveExcludes, excludes...)
-	}
-
-	if len(opts.InsensitiveExcludes) > 0 {
-		if valid, invalidPatterns := filter.ValidatePatterns(opts.InsensitiveExcludes); !valid {
-			return nil, errors.Fatalf("--iexclude: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
-		}
-
-		fs = append(fs, rejectByInsensitivePattern(opts.InsensitiveExcludes))
-	}
-
-	if len(opts.Excludes) > 0 {
-		if valid, invalidPatterns := filter.ValidatePatterns(opts.Excludes); !valid {
-			return nil, errors.Fatalf("--exclude: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
-		}
-
-		fs = append(fs, rejectByPattern(opts.Excludes))
+	fsPatterns, err := opts.excludePatternOptions.CollectPatterns()
+	if err != nil {
+		return nil, err
 	}
+	fs = append(fs, fsPatterns...)
 
 	if opts.ExcludeCaches {
 		opts.ExcludeIfPresent = append(opts.ExcludeIfPresent, "CACHEDIR.TAG:Signature: 8a477f597d28d172789f06886806bc55")
@@ -382,53 +352,6 @@ func collectRejectFuncs(opts BackupOptions, repo *repository.Repository, targets
 	return fs, nil
 }
 
-// readExcludePatternsFromFiles reads all exclude files and returns the list of
-// exclude patterns. For each line, leading and trailing white space is removed
-// and comment lines are ignored. For each remaining pattern, environment
-// variables are resolved. For adding a literal dollar sign ($), write $$ to
-// the file.
-func readExcludePatternsFromFiles(excludeFiles []string) ([]string, error) {
-	getenvOrDollar := func(s string) string {
-		if s == "$" {
-			return "$"
-		}
-		return os.Getenv(s)
-	}
-
-	var excludes []string
-	for _, filename := range excludeFiles {
-		err := func() (err error) {
-			data, err := textfile.Read(filename)
-			if err != nil {
-				return err
-			}
-
-			scanner := bufio.NewScanner(bytes.NewReader(data))
-			for scanner.Scan() {
-				line := strings.TrimSpace(scanner.Text())
-
-				// ignore empty lines
-				if line == "" {
-					continue
-				}
-
-				// strip comments
-				if strings.HasPrefix(line, "#") {
-					continue
-				}
-
-				line = os.Expand(line, getenvOrDollar)
-				excludes = append(excludes, line)
-			}
-			return scanner.Err()
-		}()
-		if err != nil {
-			return nil, err
-		}
-	}
-	return excludes, nil
-}
-
 // collectTargets returns a list of target files/dirs from several sources.
 func collectTargets(opts BackupOptions, args []string) (targets []string, err error) {
 	if opts.Stdin {
@@ -451,7 +374,7 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 			var expanded []string
 			expanded, err := filepath.Glob(line)
 			if err != nil {
-				return nil, errors.WithMessage(err, fmt.Sprintf("pattern: %s", line))
+				return nil, fmt.Errorf("pattern: %s: %w", line, err)
 			}
 			if len(expanded) == 0 {
 				Warnf("pattern %q does not match any files, skipping\n", line)
@@ -498,31 +421,24 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 
 // parent returns the ID of the parent snapshot. If there is none, nil is
 // returned.
-func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string, timeStampLimit time.Time) (parentID *restic.ID, err error) {
-	// Force using a parent
-	if !opts.Force && opts.Parent != "" {
-		id, err := restic.FindSnapshot(ctx, repo.Backend(), opts.Parent)
-		if err != nil {
-			return nil, errors.Fatalf("invalid id %q: %v", opts.Parent, err)
-		}
-
-		parentID = &id
+func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string, timeStampLimit time.Time) (*restic.Snapshot, error) {
+	if opts.Force {
+		return nil, nil
 	}
 
-	// Find last snapshot to set it as parent, if not already set
-	if !opts.Force && parentID == nil {
-		id, err := restic.FindLatestSnapshot(ctx, repo.Backend(), repo, targets, []restic.TagList{}, []string{opts.Host}, &timeStampLimit)
-		if err == nil {
-			parentID = &id
-		} else if err != restic.ErrNoSnapshotFound {
-			return nil, err
-		}
+	snName := opts.Parent
+	if snName == "" {
+		snName = "latest"
 	}
-
-	return parentID, nil
+	sn, err := restic.FindFilteredSnapshot(ctx, repo.Backend(), repo, []string{opts.Host}, []restic.TagList{}, targets, &timeStampLimit, snName)
+	// Snapshot not found is ok if no explicit parent was set
+	if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) {
+		err = nil
+	}
+	return sn, err
 }
 
-func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
+func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
 	err := opts.Check(gopts, args)
 	if err != nil {
 		return err
@@ -545,7 +461,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		Verbosef("open repository\n")
 	}
 
-	repo, err := OpenRepository(gopts)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
@@ -556,11 +472,11 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	} else {
 		progressPrinter = backup.NewTextProgress(term, gopts.verbosity)
 	}
-	progressReporter := backup.NewProgress(progressPrinter)
+	progressReporter := backup.NewProgress(progressPrinter,
+		calculateProgressInterval(!gopts.Quiet, gopts.JSON))
 
 	if opts.DryRun {
 		repo.SetDryRun()
-		progressReporter.SetDryRun()
 	}
 
 	// use the terminal for stdout/stderr
@@ -570,17 +486,15 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}()
 	gopts.stdout, gopts.stderr = progressPrinter.Stdout(), progressPrinter.Stderr()
 
-	progressReporter.SetMinUpdatePause(calculateProgressInterval(!gopts.Quiet, gopts.JSON))
-
-	wg, wgCtx := errgroup.WithContext(gopts.ctx)
+	wg, wgCtx := errgroup.WithContext(ctx)
 	cancelCtx, cancel := context.WithCancel(wgCtx)
 	defer cancel()
-	wg.Go(func() error { return progressReporter.Run(cancelCtx) })
+	wg.Go(func() error { progressReporter.Run(cancelCtx); return nil })
 
 	if !gopts.JSON {
 		progressPrinter.V("lock repository")
 	}
-	lock, err := lockRepo(gopts.ctx, repo)
+	lock, ctx, err := lockRepo(ctx, repo)
 	defer unlockRepo(lock)
 	if err != nil {
 		return err
@@ -598,16 +512,16 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		return err
 	}
 
-	var parentSnapshotID *restic.ID
+	var parentSnapshot *restic.Snapshot
 	if !opts.Stdin {
-		parentSnapshotID, err = findParentSnapshot(gopts.ctx, repo, opts, targets, timeStamp)
+		parentSnapshot, err = findParentSnapshot(ctx, repo, opts, targets, timeStamp)
 		if err != nil {
 			return err
 		}
 
 		if !gopts.JSON {
-			if parentSnapshotID != nil {
-				progressPrinter.P("using parent snapshot %v\n", parentSnapshotID.Str())
+			if parentSnapshot != nil {
+				progressPrinter.P("using parent snapshot %v\n", parentSnapshot.ID().Str())
 			} else {
 				progressPrinter.P("no parent snapshot found, will read all files\n")
 			}
@@ -617,7 +531,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	if !gopts.JSON {
 		progressPrinter.V("load index files")
 	}
-	err = repo.LoadIndex(gopts.ctx)
+	err = repo.LoadIndex(ctx)
 	if err != nil {
 		return err
 	}
@@ -674,18 +588,20 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		targets = []string{filename}
 	}
 
-	sc := archiver.NewScanner(targetFS)
-	sc.SelectByName = selectByNameFilter
-	sc.Select = selectFilter
-	sc.Error = progressReporter.ScannerError
-	sc.Result = progressReporter.ReportTotal
+	if !opts.NoScan {
+		sc := archiver.NewScanner(targetFS)
+		sc.SelectByName = selectByNameFilter
+		sc.Select = selectFilter
+		sc.Error = progressPrinter.ScannerError
+		sc.Result = progressReporter.ReportTotal
 
-	if !gopts.JSON {
-		progressPrinter.V("start scan on %v", targets)
+		if !gopts.JSON {
+			progressPrinter.V("start scan on %v", targets)
+		}
+		wg.Go(func() error { return sc.Scan(cancelCtx, targets) })
 	}
-	wg.Go(func() error { return sc.Scan(cancelCtx, targets) })
 
-	arch := archiver.New(repo, targetFS, archiver.Options{})
+	arch := archiver.New(repo, targetFS, archiver.Options{ReadConcurrency: backupOptions.ReadConcurrency})
 	arch.SelectByName = selectByNameFilter
 	arch.Select = selectFilter
 	arch.WithAtime = opts.WithAtime
@@ -707,22 +623,18 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		arch.ChangeIgnoreFlags |= archiver.ChangeIgnoreCtime
 	}
 
-	if parentSnapshotID == nil {
-		parentSnapshotID = &restic.ID{}
-	}
-
 	snapshotOpts := archiver.SnapshotOptions{
 		Excludes:       opts.Excludes,
 		Tags:           opts.Tags.Flatten(),
 		Time:           timeStamp,
 		Hostname:       opts.Host,
-		ParentSnapshot: *parentSnapshotID,
+		ParentSnapshot: parentSnapshot,
 	}
 
 	if !gopts.JSON {
 		progressPrinter.V("start backup on %v", targets)
 	}
-	_, id, err := arch.Snapshot(gopts.ctx, targets, snapshotOpts)
+	_, id, err := arch.Snapshot(ctx, targets, snapshotOpts)
 
 	// cleanly shutdown all running goroutines
 	cancel()
@@ -736,7 +648,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}
 
 	// Report finished execution
-	progressReporter.Finish(id)
+	progressReporter.Finish(id, opts.DryRun)
 	if !gopts.JSON && !opts.DryRun {
 		progressPrinter.P("snapshot %s saved\n", id.Str())
 	}

cmd/restic/cmd_backup_test.go

@@ -14,8 +14,7 @@ import (
 )
 
 func TestCollectTargets(t *testing.T) {
-	dir, cleanup := rtest.TempDir(t)
-	defer cleanup()
+	dir := rtest.TempDir(t)
 
 	fooSpace := "foo "
 	barStar := "bar*"              // Must sort before the others, below.

cmd/restic/cmd_cache.go

@@ -11,6 +11,7 @@ import (
 	"github.com/restic/restic/internal/cache"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/fs"
+	"github.com/restic/restic/internal/ui"
 	"github.com/restic/restic/internal/ui/table"
 	"github.com/spf13/cobra"
 )
@@ -138,7 +139,7 @@ func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
 			if err != nil {
 				return err
 			}
-			size = fmt.Sprintf("%11s", formatBytes(uint64(bytes)))
+			size = fmt.Sprintf("%11s", ui.FormatBytes(uint64(bytes)))
 		}
 
 		name := entry.Name()

cmd/restic/cmd_cat.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 
 	"github.com/spf13/cobra"
@@ -24,7 +25,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runCat(globalOptions, args)
+		return runCat(cmd.Context(), globalOptions, args)
 	},
 }
 
@@ -32,40 +33,32 @@ func init() {
 	cmdRoot.AddCommand(cmdCat)
 }
 
-func runCat(gopts GlobalOptions, args []string) error {
+func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 	if len(args) < 1 || (args[0] != "masterkey" && args[0] != "config" && len(args) != 2) {
 		return errors.Fatal("type or ID not specified")
 	}
 
-	repo, err := OpenRepository(gopts)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
 
 	if !gopts.NoLock {
-		lock, err := lockRepo(gopts.ctx, repo)
+		var lock *restic.Lock
+		lock, ctx, err = lockRepo(ctx, repo)
+		defer unlockRepo(lock)
 		if err != nil {
 			return err
 		}
-
-		defer unlockRepo(lock)
 	}
 
 	tpe := args[0]
 
 	var id restic.ID
-	if tpe != "masterkey" && tpe != "config" {
+	if tpe != "masterkey" && tpe != "config" && tpe != "snapshot" {
 		id, err = restic.ParseID(args[1])
 		if err != nil {
-			if tpe != "snapshot" {
-				return errors.Fatalf("unable to parse ID: %v\n", err)
-			}
-
-			// find snapshot id with prefix
-			id, err = restic.FindSnapshot(gopts.ctx, repo.Backend(), args[1])
-			if err != nil {
-				return errors.Fatalf("could not find snapshot: %v\n", err)
-			}
+			return errors.Fatalf("unable to parse ID: %v\n", err)
 		}
 	}
 
@@ -79,7 +72,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 		Println(string(buf))
 		return nil
 	case "index":
-		buf, err := repo.LoadUnpacked(gopts.ctx, restic.IndexFile, id, nil)
+		buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id, nil)
 		if err != nil {
 			return err
 		}
@@ -87,9 +80,9 @@ func runCat(gopts GlobalOptions, args []string) error {
 		Println(string(buf))
 		return nil
 	case "snapshot":
-		sn, err := restic.LoadSnapshot(gopts.ctx, repo, id)
+		sn, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1])
 		if err != nil {
-			return err
+			return errors.Fatalf("could not find snapshot: %v\n", err)
 		}
 
 		buf, err := json.MarshalIndent(sn, "", "  ")
@@ -100,19 +93,12 @@ func runCat(gopts GlobalOptions, args []string) error {
 		Println(string(buf))
 		return nil
 	case "key":
-		h := restic.Handle{Type: restic.KeyFile, Name: id.String()}
-		buf, err := backend.LoadAll(gopts.ctx, nil, repo.Backend(), h)
+		key, err := repository.LoadKey(ctx, repo, id)
 		if err != nil {
 			return err
 		}
 
-		key := &repository.Key{}
-		err = json.Unmarshal(buf, key)
-		if err != nil {
-			return err
-		}
-
-		buf, err = json.MarshalIndent(&key, "", "  ")
+		buf, err := json.MarshalIndent(&key, "", "  ")
 		if err != nil {
 			return err
 		}
@@ -128,7 +114,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 		Println(string(buf))
 		return nil
 	case "lock":
-		lock, err := restic.LoadLock(gopts.ctx, repo, id)
+		lock, err := restic.LoadLock(ctx, repo, id)
 		if err != nil {
 			return err
 		}
@@ -143,7 +129,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 
 	case "pack":
 		h := restic.Handle{Type: restic.PackFile, Name: id.String()}
-		buf, err := backend.LoadAll(gopts.ctx, nil, repo.Backend(), h)
+		buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h)
 		if err != nil {
 			return err
 		}
@@ -157,7 +143,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 		return err
 
 	case "blob":
-		err = repo.LoadIndex(gopts.ctx)
+		err = repo.LoadIndex(ctx)
 		if err != nil {
 			return err
 		}
@@ -168,7 +154,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 				continue
 			}
 
-			buf, err := repo.LoadBlob(gopts.ctx, t, id, nil)
+			buf, err := repo.LoadBlob(ctx, t, id, nil)
 			if err != nil {
 				return err
 			}

cmd/restic/cmd_check.go

@@ -1,8 +1,9 @@
 package main
 
 import (
-	"io/ioutil"
+	"context"
 	"math/rand"
+	"os"
 	"strconv"
 	"strings"
 	"sync"
@@ -34,7 +35,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runCheck(checkOptions, globalOptions, args)
+		return runCheck(cmd.Context(), checkOptions, globalOptions, args)
 	},
 	PreRunE: func(cmd *cobra.Command, args []string) error {
 		return checkFlags(checkOptions)
@@ -170,7 +171,7 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func())
 	}
 
 	// use a cache in a temporary directory
-	tempdir, err := ioutil.TempDir(cachedir, "restic-check-cache-")
+	tempdir, err := os.MkdirTemp(cachedir, "restic-check-cache-")
 	if err != nil {
 		// if an error occurs, don't use any cache
 		Warnf("unable to create temporary directory for cache during check, disabling cache: %v\n", err)
@@ -191,25 +192,26 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func())
 	return cleanup
 }
 
-func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
+func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args []string) error {
 	if len(args) != 0 {
 		return errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags")
 	}
 
 	cleanup := prepareCheckCache(opts, &gopts)
-	AddCleanupHandler(func() error {
+	AddCleanupHandler(func(code int) (int, error) {
 		cleanup()
-		return nil
+		return code, nil
 	})
 
-	repo, err := OpenRepository(gopts)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
 
 	if !gopts.NoLock {
 		Verbosef("create exclusive lock for repository\n")
-		lock, err := lockRepoExclusive(gopts.ctx, repo)
+		var lock *restic.Lock
+		lock, ctx, err = lockRepoExclusive(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -217,13 +219,13 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	chkr := checker.New(repo, opts.CheckUnused)
-	err = chkr.LoadSnapshots(gopts.ctx)
+	err = chkr.LoadSnapshots(ctx)
 	if err != nil {
 		return err
 	}
 
 	Verbosef("load indexes\n")
-	hints, errs := chkr.LoadIndex(gopts.ctx)
+	hints, errs := chkr.LoadIndex(ctx)
 
 	errorsFound := false
 	suggestIndexRebuild := false
@@ -243,7 +245,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	if suggestIndexRebuild {
-		Printf("This is non-critical, you can run `restic rebuild-index' to correct this\n")
+		Printf("Duplicate packs/old indexes are non-critical, you can run `restic rebuild-index' to correct this.\n")
 	}
 	if mixedFound {
 		Printf("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")
@@ -260,13 +262,13 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 	errChan := make(chan error)
 
 	Verbosef("check all packs\n")
-	go chkr.Packs(gopts.ctx, errChan)
+	go chkr.Packs(ctx, errChan)
 
 	for err := range errChan {
 		if checker.IsOrphanedPack(err) {
 			orphanedPacks++
 			Verbosef("%v\n", err)
-		} else if _, ok := err.(*checker.ErrLegacyLayout); ok {
+		} else if err == checker.ErrLegacyLayout {
 			Verbosef("repository still uses the S3 legacy layout\nPlease run `restic migrate s3legacy` to correct this.\n")
 		} else {
 			errorsFound = true
@@ -287,13 +289,17 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 		defer wg.Done()
 		bar := newProgressMax(!gopts.Quiet, 0, "snapshots")
 		defer bar.Done()
-		chkr.Structure(gopts.ctx, bar, errChan)
+		chkr.Structure(ctx, bar, errChan)
 	}()
 
 	for err := range errChan {
 		errorsFound = true
 		if e, ok := err.(*checker.TreeError); ok {
-			Warnf("error for tree %v:\n", e.ID.Str())
+			var clean string
+			if stdoutCanUpdateStatus() {
+				clean = clearLine(0)
+			}
+			Warnf(clean+"error for tree %v:\n", e.ID.Str())
 			for _, treeErr := range e.Errors {
 				Warnf("  %v\n", treeErr)
 			}
@@ -308,7 +314,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 	wg.Wait()
 
 	if opts.CheckUnused {
-		for _, id := range chkr.UnusedBlobs(gopts.ctx) {
+		for _, id := range chkr.UnusedBlobs(ctx) {
 			Verbosef("unused blob %v\n", id)
 			errorsFound = true
 		}
@@ -320,7 +326,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 		p := newProgressMax(!gopts.Quiet, packCount, "packs")
 		errChan := make(chan error)
 
-		go chkr.ReadPacks(gopts.ctx, packs, p, errChan)
+		go chkr.ReadPacks(ctx, packs, p, errChan)
 
 		for err := range errChan {
 			errorsFound = true

cmd/restic/cmd_copy.go

@@ -32,16 +32,14 @@ This can be mitigated by the "--copy-chunker-params" option when initializing a
 new destination repository using the "init" command.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runCopy(copyOptions, globalOptions, args)
+		return runCopy(cmd.Context(), copyOptions, globalOptions, args)
 	},
 }
 
 // CopyOptions bundles all options for the copy command.
 type CopyOptions struct {
 	secondaryRepoOptions
-	Hosts []string
-	Tags  restic.TagLists
-	Paths []string
+	snapshotFilterOptions
 }
 
 var copyOptions CopyOptions
@@ -51,12 +49,10 @@ func init() {
 
 	f := cmdCopy.Flags()
 	initSecondaryRepoOptions(f, &copyOptions.secondaryRepoOptions, "destination", "to copy snapshots from")
-	f.StringArrayVarP(&copyOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)")
-	f.Var(&copyOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot ID is given")
-	f.StringArrayVar(&copyOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
+	initMultiSnapshotFilterOptions(f, &copyOptions.snapshotFilterOptions, true)
 }
 
-func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
+func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []string) error {
 	secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "destination")
 	if err != nil {
 		return err
@@ -66,28 +62,26 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 		gopts, secondaryGopts = secondaryGopts, gopts
 	}
 
-	ctx, cancel := context.WithCancel(gopts.ctx)
-	defer cancel()
-
-	srcRepo, err := OpenRepository(gopts)
+	srcRepo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
 
-	dstRepo, err := OpenRepository(secondaryGopts)
+	dstRepo, err := OpenRepository(ctx, secondaryGopts)
 	if err != nil {
 		return err
 	}
 
 	if !gopts.NoLock {
-		srcLock, err := lockRepo(ctx, srcRepo)
+		var srcLock *restic.Lock
+		srcLock, ctx, err = lockRepo(ctx, srcRepo)
 		defer unlockRepo(srcLock)
 		if err != nil {
 			return err
 		}
 	}
 
-	dstLock, err := lockRepo(ctx, dstRepo)
+	dstLock, ctx, err := lockRepo(ctx, dstRepo)
 	defer unlockRepo(dstLock)
 	if err != nil {
 		return err
@@ -126,7 +120,6 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 	visitedTrees := restic.NewIDSet()
 
 	for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) {
-		Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
 
 		// check whether the destination has a snapshot with the same persistent ID which has similar snapshot fields
 		srcOriginal := *sn.ID()
@@ -137,7 +130,8 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 			isCopy := false
 			for _, originalSn := range originalSns {
 				if similarSnapshots(originalSn, sn) {
-					Verbosef("skipping source snapshot %s, was already copied to snapshot %s\n", sn.ID().Str(), originalSn.ID().Str())
+					Verboseff("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
+					Verboseff("skipping source snapshot %s, was already copied to snapshot %s\n", sn.ID().Str(), originalSn.ID().Str())
 					isCopy = true
 					break
 				}
@@ -146,6 +140,7 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 				continue
 			}
 		}
+		Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
 		Verbosef("  copy started, this may take a while...\n")
 		if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree, gopts.Quiet); err != nil {
 			return err

cmd/restic/cmd_debug.go

@@ -13,6 +13,7 @@ import (
 	"os"
 	"runtime"
 	"sort"
+	"sync"
 	"time"
 
 	"github.com/klauspost/compress/zstd"
@@ -22,6 +23,7 @@ import (
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/index"
 	"github.com/restic/restic/internal/pack"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
@@ -46,7 +48,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runDebugDump(globalOptions, args)
+		return runDebugDump(cmd.Context(), globalOptions, args)
 	},
 }
 
@@ -104,10 +106,9 @@ type Blob struct {
 
 func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
 
-	return repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
-		h := restic.Handle{Type: restic.PackFile, Name: id.String()}
-
-		blobs, _, err := pack.List(repo.Key(), backend.ReaderAt(ctx, repo.Backend(), h), size)
+	var m sync.Mutex
+	return restic.ParallelList(ctx, repo.Backend(), restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
+		blobs, _, err := repo.ListPack(ctx, id, size)
 		if err != nil {
 			Warnf("error for pack %v: %v\n", id.Str(), err)
 			return nil
@@ -126,12 +127,14 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
 			}
 		}
 
+		m.Lock()
+		defer m.Unlock()
 		return prettyPrintJSON(wr, p)
 	})
 }
 
 func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error {
-	return repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
+	return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
 		Printf("index_id: %v\n", id)
 		if err != nil {
 			return err
@@ -141,18 +144,19 @@ func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) erro
 	})
 }
 
-func runDebugDump(gopts GlobalOptions, args []string) error {
+func runDebugDump(ctx context.Context, gopts GlobalOptions, args []string) error {
 	if len(args) != 1 {
 		return errors.Fatal("type not specified")
 	}
 
-	repo, err := OpenRepository(gopts)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
 
 	if !gopts.NoLock {
-		lock, err := lockRepo(gopts.ctx, repo)
+		var lock *restic.Lock
+		lock, ctx, err = lockRepo(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -163,20 +167,20 @@ func runDebugDump(gopts GlobalOptions, args []string) error {
 
 	switch tpe {
 	case "indexes":
-		return dumpIndexes(gopts.ctx, repo, gopts.stdout)
+		return dumpIndexes(ctx, repo, gopts.stdout)
 	case "snapshots":
-		return debugPrintSnapshots(gopts.ctx, repo, gopts.stdout)
+		return debugPrintSnapshots(ctx, repo, gopts.stdout)
 	case "packs":
-		return printPacks(gopts.ctx, repo, gopts.stdout)
+		return printPacks(ctx, repo, gopts.stdout)
 	case "all":
 		Printf("snapshots:\n")
-		err := debugPrintSnapshots(gopts.ctx, repo, gopts.stdout)
+		err := debugPrintSnapshots(ctx, repo, gopts.stdout)
 		if err != nil {
 			return err
 		}
 
 		Printf("\nindexes:\n")
-		err = dumpIndexes(gopts.ctx, repo, gopts.stdout)
+		err = dumpIndexes(ctx, repo, gopts.stdout)
 		if err != nil {
 			return err
 		}
@@ -192,7 +196,7 @@ var cmdDebugExamine = &cobra.Command{
 	Short:             "Examine a pack file",
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runDebugExamine(globalOptions, args)
+		return runDebugExamine(cmd.Context(), globalOptions, args)
 	},
 }
 
@@ -311,97 +315,104 @@ func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte {
 	return out
 }
 
-func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list []restic.Blob) error {
+func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, list []restic.Blob) error {
 	dec, err := zstd.NewReader(nil)
 	if err != nil {
 		panic(err)
 	}
 	be := repo.Backend()
 	h := restic.Handle{
-		Name: pack.String(),
+		Name: packID.String(),
 		Type: restic.PackFile,
 	}
-	for _, blob := range list {
-		Printf("      loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length)
-		buf := make([]byte, blob.Length)
-		err := be.Load(ctx, h, int(blob.Length), int64(blob.Offset), func(rd io.Reader) error {
-			n, err := io.ReadFull(rd, buf)
+
+	wg, ctx := errgroup.WithContext(ctx)
+
+	if reuploadBlobs {
+		repo.StartPackUploader(ctx, wg)
+	}
+
+	wg.Go(func() error {
+		for _, blob := range list {
+			Printf("      loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length)
+			buf := make([]byte, blob.Length)
+			err := be.Load(ctx, h, int(blob.Length), int64(blob.Offset), func(rd io.Reader) error {
+				n, err := io.ReadFull(rd, buf)
+				if err != nil {
+					return fmt.Errorf("read error after %d bytes: %v", n, err)
+				}
+				return nil
+			})
 			if err != nil {
-				return fmt.Errorf("read error after %d bytes: %v", n, err)
+				Warnf("error read: %v\n", err)
+				continue
 			}
-			return nil
-		})
-		if err != nil {
-			Warnf("error read: %v\n", err)
-			continue
-		}
 
-		key := repo.Key()
+			key := repo.Key()
 
-		nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
-		plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
-		outputPrefix := ""
-		filePrefix := ""
-		if err != nil {
-			Warnf("error decrypting blob: %v\n", err)
-			if tryRepair || repairByte {
-				plaintext = tryRepairWithBitflip(ctx, key, buf, repairByte)
+			nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
+			plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
+			outputPrefix := ""
+			filePrefix := ""
+			if err != nil {
+				Warnf("error decrypting blob: %v\n", err)
+				if tryRepair || repairByte {
+					plaintext = tryRepairWithBitflip(ctx, key, buf, repairByte)
+				}
+				if plaintext != nil {
+					outputPrefix = "repaired "
+					filePrefix = "repaired-"
+				} else {
+					plaintext = decryptUnsigned(ctx, key, buf)
+					err = storePlainBlob(blob.ID, "damaged-", plaintext)
+					if err != nil {
+						return err
+					}
+					continue
+				}
 			}
-			if plaintext != nil {
-				outputPrefix = "repaired "
-				filePrefix = "repaired-"
+
+			if blob.IsCompressed() {
+				decompressed, err := dec.DecodeAll(plaintext, nil)
+				if err != nil {
+					Printf("         failed to decompress blob %v\n", blob.ID)
+				}
+				if decompressed != nil {
+					plaintext = decompressed
+				}
+			}
+
+			id := restic.Hash(plaintext)
+			var prefix string
+			if !id.Equal(blob.ID) {
+				Printf("         successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v\n", outputPrefix, len(plaintext), id, blob.ID)
+				prefix = "wrong-hash-"
 			} else {
-				plaintext = decryptUnsigned(ctx, key, buf)
-				err = storePlainBlob(blob.ID, "damaged-", plaintext)
+				Printf("         successfully %vdecrypted blob (length %v), hash is %v, ID matches\n", outputPrefix, len(plaintext), id)
+				prefix = "correct-"
+			}
+			if extractPack {
+				err = storePlainBlob(id, filePrefix+prefix, plaintext)
 				if err != nil {
 					return err
 				}
-				continue
+			}
+			if reuploadBlobs {
+				_, _, _, err := repo.SaveBlob(ctx, blob.Type, plaintext, id, true)
+				if err != nil {
+					return err
+				}
+				Printf("         uploaded %v %v\n", blob.Type, id)
 			}
 		}
 
-		if blob.IsCompressed() {
-			decompressed, err := dec.DecodeAll(plaintext, nil)
-			if err != nil {
-				Printf("         failed to decompress blob %v\n", blob.ID)
-			}
-			if decompressed != nil {
-				plaintext = decompressed
-			}
-		}
-
-		id := restic.Hash(plaintext)
-		var prefix string
-		if !id.Equal(blob.ID) {
-			Printf("         successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v\n", outputPrefix, len(plaintext), id, blob.ID)
-			prefix = "wrong-hash-"
-		} else {
-			Printf("         successfully %vdecrypted blob (length %v), hash is %v, ID matches\n", outputPrefix, len(plaintext), id)
-			prefix = "correct-"
-		}
-		if extractPack {
-			err = storePlainBlob(id, filePrefix+prefix, plaintext)
-			if err != nil {
-				return err
-			}
-		}
 		if reuploadBlobs {
-			_, _, _, err := repo.SaveBlob(ctx, blob.Type, plaintext, id, true)
-			if err != nil {
-				return err
-			}
-			Printf("         uploaded %v %v\n", blob.Type, id)
+			return repo.Flush(ctx)
 		}
-	}
+		return nil
+	})
 
-	if reuploadBlobs {
-		err := repo.Flush(ctx)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
+	return wg.Wait()
 }
 
 func storePlainBlob(id restic.ID, prefix string, plain []byte) error {
@@ -426,8 +437,8 @@ func storePlainBlob(id restic.ID, prefix string, plain []byte) error {
 	return nil
 }
 
-func runDebugExamine(gopts GlobalOptions, args []string) error {
-	repo, err := OpenRepository(gopts)
+func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) error {
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
@@ -436,10 +447,7 @@ func runDebugExamine(gopts GlobalOptions, args []string) error {
 	for _, name := range args {
 		id, err := restic.ParseID(name)
 		if err != nil {
-			name, err = restic.Find(gopts.ctx, repo.Backend(), restic.PackFile, name)
-			if err == nil {
-				id, err = restic.ParseID(name)
-			}
+			id, err = restic.Find(ctx, repo.Backend(), restic.PackFile, name)
 			if err != nil {
 				Warnf("error: %v\n", err)
 				continue
@@ -453,20 +461,21 @@ func runDebugExamine(gopts GlobalOptions, args []string) error {
 	}
 
 	if !gopts.NoLock {
-		lock, err := lockRepo(gopts.ctx, repo)
+		var lock *restic.Lock
+		lock, ctx, err = lockRepo(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
 		}
 	}
 
-	err = repo.LoadIndex(gopts.ctx)
+	err = repo.LoadIndex(ctx)
 	if err != nil {
 		return err
 	}
 
 	for _, id := range ids {
-		err := examinePack(gopts.ctx, repo, id)
+		err := examinePack(ctx, repo, id)
 		if err != nil {
 			Warnf("error: %v\n", err)
 		}
@@ -525,7 +534,7 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro
 	Printf("  ========================================\n")
 	Printf("  inspect the pack itself\n")
 
-	blobs, _, err := pack.List(repo.Key(), backend.ReaderAt(ctx, repo.Backend(), h), fi.Size)
+	blobs, _, err := repo.ListPack(ctx, id, fi.Size)
 	if err != nil {
 		return fmt.Errorf("pack %v: %v", id.Str(), err)
 	}
@@ -556,7 +565,7 @@ func checkPackSize(blobs []restic.Blob, fileSize int64) {
 	size += uint64(pack.CalculateHeaderSize(blobs))
 
 	if uint64(fileSize) != size {
-		Printf("      file sizes do not match: computed %v from index, file size is %v\n", size, fileSize)
+		Printf("      file sizes do not match: computed %v, file size is %v\n", size, fileSize)
 	} else {
 		Printf("      file sizes match\n")
 	}

cmd/restic/cmd_diff.go

@@ -11,6 +11,7 @@ import (
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui"
 	"github.com/spf13/cobra"
 )
 
@@ -35,7 +36,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runDiff(diffOptions, globalOptions, args)
+		return runDiff(cmd.Context(), diffOptions, globalOptions, args)
 	},
 }
 
@@ -54,11 +55,11 @@ func init() {
 }
 
 func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, error) {
-	id, err := restic.FindSnapshot(ctx, be, desc)
+	sn, err := restic.FindSnapshot(ctx, be, repo, desc)
 	if err != nil {
 		return nil, errors.Fatal(err.Error())
 	}
-	return restic.LoadSnapshot(ctx, repo, id)
+	return sn, err
 }
 
 // Comparer collects all things needed to compare two snapshots.
@@ -321,21 +322,19 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
 	return nil
 }
 
-func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
+func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []string) error {
 	if len(args) != 2 {
 		return errors.Fatalf("specify two snapshot IDs")
 	}
 
-	ctx, cancel := context.WithCancel(gopts.ctx)
-	defer cancel()
-
-	repo, err := OpenRepository(gopts)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
 
 	if !gopts.NoLock {
-		lock, err := lockRepo(ctx, repo)
+		var lock *restic.Lock
+		lock, ctx, err = lockRepo(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -427,8 +426,8 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 		Printf("Others:      %5d new, %5d removed\n", stats.Added.Others, stats.Removed.Others)
 		Printf("Data Blobs:  %5d new, %5d removed\n", stats.Added.DataBlobs, stats.Removed.DataBlobs)
 		Printf("Tree Blobs:  %5d new, %5d removed\n", stats.Added.TreeBlobs, stats.Removed.TreeBlobs)
-		Printf("  Added:   %-5s\n", formatBytes(uint64(stats.Added.Bytes)))
-		Printf("  Removed: %-5s\n", formatBytes(uint64(stats.Removed.Bytes)))
+		Printf("  Added:   %-5s\n", ui.FormatBytes(uint64(stats.Added.Bytes)))
+		Printf("  Removed: %-5s\n", ui.FormatBytes(uint64(stats.Removed.Bytes)))
 	}
 
 	return nil

cmd/restic/cmd_dump.go

@@ -34,15 +34,13 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runDump(dumpOptions, globalOptions, args)
+		return runDump(cmd.Context(), dumpOptions, globalOptions, args)
 	},
 }
 
 // DumpOptions collects all options for the dump command.
 type DumpOptions struct {
-	Hosts   []string
-	Paths   []string
-	Tags    restic.TagLists
+	snapshotFilterOptions
 	Archive string
 }
 
@@ -52,9 +50,7 @@ func init() {
 	cmdRoot.AddCommand(cmdDump)
 
 	flags := cmdDump.Flags()
-	flags.StringArrayVarP(&dumpOptions.Hosts, "host", "H", nil, `only consider snapshots for this host when the snapshot ID is "latest" (can be specified multiple times)`)
-	flags.Var(&dumpOptions.Tags, "tag", "only consider snapshots which include this `taglist` for snapshot ID \"latest\"")
-	flags.StringArrayVar(&dumpOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
+	initSingleSnapshotFilterOptions(flags, &dumpOptions.snapshotFilterOptions)
 	flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"")
 }
 
@@ -111,9 +107,7 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
 	return fmt.Errorf("path %q not found in snapshot", item)
 }
 
-func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
-	ctx := gopts.ctx
-
+func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []string) error {
 	if len(args) != 2 {
 		return errors.Fatal("no file and no snapshot ID specified")
 	}
@@ -131,36 +125,23 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 
 	splittedPath := splitPath(path.Clean(pathToPrint))
 
-	repo, err := OpenRepository(gopts)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
 
 	if !gopts.NoLock {
-		lock, err := lockRepo(ctx, repo)
+		var lock *restic.Lock
+		lock, ctx, err = lockRepo(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
 		}
 	}
 
-	var id restic.ID
-
-	if snapshotIDString == "latest" {
-		id, err = restic.FindLatestSnapshot(ctx, repo.Backend(), repo, opts.Paths, opts.Tags, opts.Hosts, nil)
-		if err != nil {
-			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Hosts:%v", err, opts.Paths, opts.Hosts)
-		}
-	} else {
-		id, err = restic.FindSnapshot(ctx, repo.Backend(), snapshotIDString)
-		if err != nil {
-			Exitf(1, "invalid id %q: %v", snapshotIDString, err)
-		}
-	}
-
-	sn, err := restic.LoadSnapshot(gopts.ctx, repo, id)
+	sn, err := restic.FindFilteredSnapshot(ctx, repo.Backend(), repo, opts.Paths, opts.Tags, opts.Hosts, nil, snapshotIDString)
 	if err != nil {
-		Exitf(2, "loading snapshot %q failed: %v", snapshotIDString, err)
+		return errors.Fatalf("failed to find snapshot: %v", err)
 	}
 
 	err = repo.LoadIndex(ctx)
@@ -170,13 +151,13 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 
 	tree, err := restic.LoadTree(ctx, repo, *sn.Tree)
 	if err != nil {
-		Exitf(2, "loading tree for snapshot %q failed: %v", snapshotIDString, err)
+		return errors.Fatalf("loading tree for snapshot %q failed: %v", snapshotIDString, err)
 	}
 
 	d := dump.New(opts.Archive, repo, os.Stdout)
 	err = printFromTree(ctx, tree, repo, "/", splittedPath, d)
 	if err != nil {
-		Exitf(2, "cannot dump file: %v", err)
+		return errors.Fatalf("cannot dump file: %v", err)
 	}
 
 	return nil

cmd/restic/cmd_find.go

@@ -38,7 +38,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runFind(findOptions, globalOptions, args)
+		return runFind(cmd.Context(), findOptions, globalOptions, args)
 	},
 }
 
@@ -51,9 +51,7 @@ type FindOptions struct {
 	PackID, ShowPackID bool
 	CaseInsensitive    bool
 	ListLong           bool
-	Hosts              []string
-	Paths              []string
-	Tags               restic.TagLists
+	snapshotFilterOptions
 }
 
 var findOptions FindOptions
@@ -72,9 +70,7 @@ func init() {
 	f.BoolVarP(&findOptions.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern")
 	f.BoolVarP(&findOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
 
-	f.StringArrayVarP(&findOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)")
-	f.Var(&findOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot-ID is given")
-	f.StringArrayVar(&findOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
+	initMultiSnapshotFilterOptions(f, &findOptions.snapshotFilterOptions, true)
 }
 
 type findPattern struct {
@@ -471,7 +467,7 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
 
 	// remember which packs were found in the index
 	indexPackIDs := make(map[string]struct{})
-	for pb := range f.repo.Index().Each(wctx) {
+	f.repo.Index().Each(wctx, func(pb restic.PackedBlob) {
 		idStr := pb.PackID.String()
 		// keep entry in packIDs as Each() returns individual index entries
 		matchingID := false
@@ -489,7 +485,7 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
 			f.blobIDs[pb.ID.String()] = struct{}{}
 			indexPackIDs[idStr] = struct{}{}
 		}
-	}
+	})
 
 	for id := range indexPackIDs {
 		delete(packIDs, id)
@@ -538,7 +534,7 @@ func (f *Finder) findObjectsPacks(ctx context.Context) {
 	}
 }
 
-func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
+func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []string) error {
 	if len(args) == 0 {
 		return errors.Fatal("wrong number of arguments")
 	}
@@ -572,31 +568,29 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 		return errors.Fatal("cannot have several ID types")
 	}
 
-	repo, err := OpenRepository(gopts)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
 
 	if !gopts.NoLock {
-		lock, err := lockRepo(gopts.ctx, repo)
+		var lock *restic.Lock
+		lock, ctx, err = lockRepo(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
 		}
 	}
 
-	snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile)
+	snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
 	if err != nil {
 		return err
 	}
 
-	if err = repo.LoadIndex(gopts.ctx); err != nil {
+	if err = repo.LoadIndex(ctx); err != nil {
 		return err
 	}
 
-	ctx, cancel := context.WithCancel(gopts.ctx)
-	defer cancel()
-
 	f := &Finder{
 		repo:        repo,
 		pat:         pat,

cmd/restic/cmd_forget.go

@@ -32,7 +32,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runForget(forgetOptions, globalOptions, args)
+		return runForget(cmd.Context(), forgetOptions, globalOptions, args)
 	},
 }
 
@@ -52,9 +52,7 @@ type ForgetOptions struct {
 	WithinYearly  restic.Duration
 	KeepTags      restic.TagLists
 
-	Hosts   []string
-	Tags    restic.TagLists
-	Paths   []string
+	snapshotFilterOptions
 	Compact bool
 
 	// Grouping
@@ -81,9 +79,9 @@ func init() {
 	f.VarP(&forgetOptions.WithinWeekly, "keep-within-weekly", "", "keep weekly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
 	f.VarP(&forgetOptions.WithinMonthly, "keep-within-monthly", "", "keep monthly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
 	f.VarP(&forgetOptions.WithinYearly, "keep-within-yearly", "", "keep yearly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
-
 	f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)")
-	f.StringArrayVar(&forgetOptions.Hosts, "host", nil, "only consider snapshots with the given `host` (can be specified multiple times)")
+
+	initMultiSnapshotFilterOptions(f, &forgetOptions.snapshotFilterOptions, false)
 	f.StringArrayVar(&forgetOptions.Hosts, "hostname", nil, "only consider snapshots with the given `hostname` (can be specified multiple times)")
 	err := f.MarkDeprecated("hostname", "use --host")
 	if err != nil {
@@ -91,9 +89,6 @@ func init() {
 		panic(err)
 	}
 
-	f.Var(&forgetOptions.Tags, "tag", "only consider snapshots which include this `taglist` in the format `tag[,tag,...]` (can be specified multiple times)")
-
-	f.StringArrayVar(&forgetOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` (can be specified multiple times)")
 	f.BoolVarP(&forgetOptions.Compact, "compact", "c", false, "use compact output format")
 
 	f.StringVarP(&forgetOptions.GroupBy, "group-by", "g", "host,paths", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
@@ -104,13 +99,13 @@ func init() {
 	addPruneOptions(cmdForget)
 }
 
-func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
+func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, args []string) error {
 	err := verifyPruneOptions(&pruneOptions)
 	if err != nil {
 		return err
 	}
 
-	repo, err := OpenRepository(gopts)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
@@ -120,16 +115,14 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	if !opts.DryRun || !gopts.NoLock {
-		lock, err := lockRepoExclusive(gopts.ctx, repo)
+		var lock *restic.Lock
+		lock, ctx, err = lockRepoExclusive(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
 		}
 	}
 
-	ctx, cancel := context.WithCancel(gopts.ctx)
-	defer cancel()
-
 	var snapshots restic.Snapshots
 	removeSnIDs := restic.NewIDSet()
 
@@ -224,7 +217,7 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 
 	if len(removeSnIDs) > 0 {
 		if !opts.DryRun {
-			err := DeleteFilesChecked(gopts, repo, removeSnIDs, restic.SnapshotFile)
+			err := DeleteFilesChecked(ctx, gopts, repo, removeSnIDs, restic.SnapshotFile)
 			if err != nil {
 				return err
 			}
@@ -244,10 +237,14 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 
 	if len(removeSnIDs) > 0 && opts.Prune {
 		if !gopts.JSON {
-			Verbosef("%d snapshots have been removed, running prune\n", len(removeSnIDs))
+			if opts.DryRun {
+				Verbosef("%d snapshots would be removed, running prune dry run\n", len(removeSnIDs))
+			} else {
+				Verbosef("%d snapshots have been removed, running prune\n", len(removeSnIDs))
+			}
 		}
 		pruneOptions.DryRun = opts.DryRun
-		return runPruneWithRepo(pruneOptions, gopts, repo, removeSnIDs)
+		return runPruneWithRepo(ctx, pruneOptions, gopts, repo, removeSnIDs)
 	}
 
 	return nil

cmd/restic/cmd_generate.go

@@ -10,7 +10,7 @@ import (
 
 var cmdGenerate = &cobra.Command{
 	Use:   "generate [flags]",
-	Short: "Generate manual pages and auto-completion files (bash, fish, zsh)",
+	Short: "Generate manual pages and auto-completion files (bash, fish, zsh, powershell)",
 	Long: `
 The "generate" command writes automatically generated files (like the man pages
 and the auto-completion files for bash, fish and zsh).
@@ -25,10 +25,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 }
 
 type generateOptions struct {
-	ManDir             string
-	BashCompletionFile string
-	FishCompletionFile string
-	ZSHCompletionFile  string
+	ManDir                   string
+	BashCompletionFile       string
+	FishCompletionFile       string
+	ZSHCompletionFile        string
+	PowerShellCompletionFile string
 }
 
 var genOpts generateOptions
@@ -40,6 +41,7 @@ func init() {
 	fs.StringVar(&genOpts.BashCompletionFile, "bash-completion", "", "write bash completion `file`")
 	fs.StringVar(&genOpts.FishCompletionFile, "fish-completion", "", "write fish completion `file`")
 	fs.StringVar(&genOpts.ZSHCompletionFile, "zsh-completion", "", "write zsh completion `file`")
+	fs.StringVar(&genOpts.PowerShellCompletionFile, "powershell-completion", "", "write powershell completion `file`")
 }
 
 func writeManpages(dir string) error {
@@ -75,6 +77,11 @@ func writeZSHCompletion(file string) error {
 	return cmdRoot.GenZshCompletionFile(file)
 }
 
+func writePowerShellCompletion(file string) error {
+	Verbosef("writing powershell completion file to %v\n", file)
+	return cmdRoot.GenPowerShellCompletionFile(file)
+}
+
 func runGenerate(cmd *cobra.Command, args []string) error {
 	if genOpts.ManDir != "" {
 		err := writeManpages(genOpts.ManDir)
@@ -104,6 +111,13 @@ func runGenerate(cmd *cobra.Command, args []string) error {
 		}
 	}
 
+	if genOpts.PowerShellCompletionFile != "" {
+		err := writePowerShellCompletion(genOpts.PowerShellCompletionFile)
+		if err != nil {
+			return err
+		}
+	}
+
 	var empty generateOptions
 	if genOpts == empty {
 		return errors.Fatal("nothing to do, please specify at least one output file/dir")

cmd/restic/cmd_init.go

@@ -1,6 +1,8 @@
 package main
 
 import (
+	"context"
+	"encoding/json"
 	"strconv"
 
 	"github.com/restic/chunker"
@@ -25,7 +27,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runInit(initOptions, globalOptions, args)
+		return runInit(cmd.Context(), initOptions, globalOptions, args)
 	},
 }
 
@@ -47,7 +49,7 @@ func init() {
 	f.StringVar(&initOptions.RepositoryVersion, "repository-version", "stable", "repository format version to use, allowed values are a format version, 'latest' and 'stable'")
 }
 
-func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
+func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []string) error {
 	var version uint
 	if opts.RepositoryVersion == "latest" || opts.RepositoryVersion == "" {
 		version = restic.MaxRepoVersion
@@ -64,7 +66,7 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
 		return errors.Fatalf("only repository versions between %v and %v are allowed", restic.MinRepoVersion, restic.MaxRepoVersion)
 	}
 
-	chunkerPolynomial, err := maybeReadChunkerPolynomial(opts, gopts)
+	chunkerPolynomial, err := maybeReadChunkerPolynomial(ctx, opts, gopts)
 	if err != nil {
 		return err
 	}
@@ -81,7 +83,7 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	be, err := create(repo, gopts.extended)
+	be, err := create(ctx, repo, gopts.extended)
 	if err != nil {
 		return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
 	}
@@ -94,28 +96,38 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	err = s.Init(gopts.ctx, version, gopts.password, chunkerPolynomial)
+	err = s.Init(ctx, version, gopts.password, chunkerPolynomial)
 	if err != nil {
 		return errors.Fatalf("create key in repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
 	}
 
-	Verbosef("created restic repository %v at %s\n", s.Config().ID[:10], location.StripPassword(gopts.Repo))
-	Verbosef("\n")
-	Verbosef("Please note that knowledge of your password is required to access\n")
-	Verbosef("the repository. Losing your password means that your data is\n")
-	Verbosef("irrecoverably lost.\n")
+	if !gopts.JSON {
+		Verbosef("created restic repository %v at %s\n", s.Config().ID[:10], location.StripPassword(gopts.Repo))
+		Verbosef("\n")
+		Verbosef("Please note that knowledge of your password is required to access\n")
+		Verbosef("the repository. Losing your password means that your data is\n")
+		Verbosef("irrecoverably lost.\n")
+
+	} else {
+		status := initSuccess{
+			MessageType: "initialized",
+			ID:          s.Config().ID,
+			Repository:  location.StripPassword(gopts.Repo),
+		}
+		return json.NewEncoder(gopts.stdout).Encode(status)
+	}
 
 	return nil
 }
 
-func maybeReadChunkerPolynomial(opts InitOptions, gopts GlobalOptions) (*chunker.Pol, error) {
+func maybeReadChunkerPolynomial(ctx context.Context, opts InitOptions, gopts GlobalOptions) (*chunker.Pol, error) {
 	if opts.CopyChunkerParameters {
 		otherGopts, _, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "secondary")
 		if err != nil {
 			return nil, err
 		}
 
-		otherRepo, err := OpenRepository(otherGopts)
+		otherRepo, err := OpenRepository(ctx, otherGopts)
 		if err != nil {
 			return nil, err
 		}
@@ -129,3 +141,9 @@ func maybeReadChunkerPolynomial(opts InitOptions, gopts GlobalOptions) (*chunker
 	}
 	return nil, nil
 }
+
+type initSuccess struct {
+	MessageType string `json:"message_type"` // "initialized"
+	ID          string `json:"id"`
+	Repository  string `json:"repository"`
+}

cmd/restic/cmd_key.go

@@ -3,9 +3,9 @@ package main
 import (
 	"context"
 	"encoding/json"
-	"io/ioutil"
 	"os"
 	"strings"
+	"sync"
 
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
@@ -28,7 +28,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runKey(globalOptions, args)
+		return runKey(cmd.Context(), globalOptions, args)
 	},
 }
 
@@ -56,23 +56,26 @@ func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions
 		Created  string `json:"created"`
 	}
 
+	var m sync.Mutex
 	var keys []keyInfo
 
-	err := s.List(ctx, restic.KeyFile, func(id restic.ID, size int64) error {
-		k, err := repository.LoadKey(ctx, s, id.String())
+	err := restic.ParallelList(ctx, s.Backend(), restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
+		k, err := repository.LoadKey(ctx, s, id)
 		if err != nil {
 			Warnf("LoadKey() failed: %v\n", err)
 			return nil
 		}
 
 		key := keyInfo{
-			Current:  id.String() == s.KeyName(),
+			Current:  id == s.KeyID(),
 			ID:       id.Str(),
 			UserName: k.Username,
 			HostName: k.Hostname,
 			Created:  k.Created.Local().Format(TimeFormat),
 		}
 
+		m.Lock()
+		defer m.Unlock()
 		keys = append(keys, key)
 		return nil
 	})
@@ -120,18 +123,18 @@ func getNewPassword(gopts GlobalOptions) (string, error) {
 		"enter password again: ")
 }
 
-func addKey(gopts GlobalOptions, repo *repository.Repository) error {
+func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOptions) error {
 	pw, err := getNewPassword(gopts)
 	if err != nil {
 		return err
 	}
 
-	id, err := repository.AddKey(gopts.ctx, repo, pw, keyUsername, keyHostname, repo.Key())
+	id, err := repository.AddKey(ctx, repo, pw, keyUsername, keyHostname, repo.Key())
 	if err != nil {
 		return errors.Fatalf("creating new key failed: %v\n", err)
 	}
 
-	err = switchToNewKeyAndRemoveIfBroken(gopts.ctx, repo, id, pw)
+	err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw)
 	if err != nil {
 		return err
 	}
@@ -141,40 +144,40 @@ func addKey(gopts GlobalOptions, repo *repository.Repository) error {
 	return nil
 }
 
-func deleteKey(ctx context.Context, repo *repository.Repository, name string) error {
-	if name == repo.KeyName() {
+func deleteKey(ctx context.Context, repo *repository.Repository, id restic.ID) error {
+	if id == repo.KeyID() {
 		return errors.Fatal("refusing to remove key currently used to access repository")
 	}
 
-	h := restic.Handle{Type: restic.KeyFile, Name: name}
+	h := restic.Handle{Type: restic.KeyFile, Name: id.String()}
 	err := repo.Backend().Remove(ctx, h)
 	if err != nil {
 		return err
 	}
 
-	Verbosef("removed key %v\n", name)
+	Verbosef("removed key %v\n", id)
 	return nil
 }
 
-func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
+func changePassword(ctx context.Context, repo *repository.Repository, gopts GlobalOptions) error {
 	pw, err := getNewPassword(gopts)
 	if err != nil {
 		return err
 	}
 
-	id, err := repository.AddKey(gopts.ctx, repo, pw, "", "", repo.Key())
+	id, err := repository.AddKey(ctx, repo, pw, "", "", repo.Key())
 	if err != nil {
 		return errors.Fatalf("creating new key failed: %v\n", err)
 	}
-	oldID := repo.KeyName()
+	oldID := repo.KeyID()
 
-	err = switchToNewKeyAndRemoveIfBroken(gopts.ctx, repo, id, pw)
+	err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw)
 	if err != nil {
 		return err
 	}
 
-	h := restic.Handle{Type: restic.KeyFile, Name: oldID}
-	err = repo.Backend().Remove(gopts.ctx, h)
+	h := restic.Handle{Type: restic.KeyFile, Name: oldID.String()}
+	err = repo.Backend().Remove(ctx, h)
 	if err != nil {
 		return err
 	}
@@ -187,32 +190,29 @@ func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
 func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repository, key *repository.Key, pw string) error {
 	// Verify new key to make sure it really works. A broken key can render the
 	// whole repository inaccessible
-	err := repo.SearchKey(ctx, pw, 0, key.Name())
+	err := repo.SearchKey(ctx, pw, 0, key.ID().String())
 	if err != nil {
 		// the key is invalid, try to remove it
-		h := restic.Handle{Type: restic.KeyFile, Name: key.Name()}
+		h := restic.Handle{Type: restic.KeyFile, Name: key.ID().String()}
 		_ = repo.Backend().Remove(ctx, h)
 		return errors.Fatalf("failed to access repository with new key: %v", err)
 	}
 	return nil
 }
 
-func runKey(gopts GlobalOptions, args []string) error {
+func runKey(ctx context.Context, gopts GlobalOptions, args []string) error {
 	if len(args) < 1 || (args[0] == "remove" && len(args) != 2) || (args[0] != "remove" && len(args) != 1) {
 		return errors.Fatal("wrong number of arguments")
 	}
 
-	ctx, cancel := context.WithCancel(gopts.ctx)
-	defer cancel()
-
-	repo, err := OpenRepository(gopts)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
 
 	switch args[0] {
 	case "list":
-		lock, err := lockRepo(ctx, repo)
+		lock, ctx, err := lockRepo(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -220,15 +220,15 @@ func runKey(gopts GlobalOptions, args []string) error {
 
 		return listKeys(ctx, repo, gopts)
 	case "add":
-		lock, err := lockRepo(ctx, repo)
+		lock, ctx, err := lockRepo(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
 		}
 
-		return addKey(gopts, repo)
+		return addKey(ctx, repo, gopts)
 	case "remove":
-		lock, err := lockRepoExclusive(ctx, repo)
+		lock, ctx, err := lockRepoExclusive(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -239,22 +239,22 @@ func runKey(gopts GlobalOptions, args []string) error {
 			return err
 		}
 
-		return deleteKey(gopts.ctx, repo, id)
+		return deleteKey(ctx, repo, id)
 	case "passwd":
-		lock, err := lockRepoExclusive(ctx, repo)
+		lock, ctx, err := lockRepoExclusive(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
 		}
 
-		return changePassword(gopts, repo)
+		return changePassword(ctx, repo, gopts)
 	}
 
 	return nil
 }
 
 func loadPasswordFromFile(pwdFile string) (string, error) {
-	s, err := ioutil.ReadFile(pwdFile)
+	s, err := os.ReadFile(pwdFile)
 	if os.IsNotExist(err) {
 		return "", errors.Fatalf("%s does not exist", pwdFile)
 	}

cmd/restic/cmd_list.go

@@ -1,8 +1,10 @@
 package main
 
 import (
+	"context"
+
 	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/index"
 	"github.com/restic/restic/internal/restic"
 
 	"github.com/spf13/cobra"
@@ -21,7 +23,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runList(cmd, globalOptions, args)
+		return runList(cmd.Context(), cmd, globalOptions, args)
 	},
 }
 
@@ -29,18 +31,19 @@ func init() {
 	cmdRoot.AddCommand(cmdList)
 }
 
-func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error {
+func runList(ctx context.Context, cmd *cobra.Command, opts GlobalOptions, args []string) error {
 	if len(args) != 1 {
 		return errors.Fatal("type not specified, usage: " + cmd.Use)
 	}
 
-	repo, err := OpenRepository(opts)
+	repo, err := OpenRepository(ctx, opts)
 	if err != nil {
 		return err
 	}
 
 	if !opts.NoLock && args[0] != "locks" {
-		lock, err := lockRepo(opts.ctx, repo)
+		var lock *restic.Lock
+		lock, ctx, err = lockRepo(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -60,20 +63,20 @@ func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error {
 	case "locks":
 		t = restic.LockFile
 	case "blobs":
-		return repository.ForAllIndexes(opts.ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
+		return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
 			if err != nil {
 				return err
 			}
-			for blobs := range idx.Each(opts.ctx) {
+			idx.Each(ctx, func(blobs restic.PackedBlob) {
 				Printf("%v %v\n", blobs.Type, blobs.ID)
-			}
+			})
 			return nil
 		})
 	default:
 		return errors.Fatal("invalid type")
 	}
 
-	return repo.List(opts.ctx, t, func(id restic.ID, size int64) error {
+	return repo.List(ctx, t, func(id restic.ID, size int64) error {
 		Printf("%s\n", id)
 		return nil
 	})

cmd/restic/cmd_ls.go

@@ -42,16 +42,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runLs(lsOptions, globalOptions, args)
+		return runLs(cmd.Context(), lsOptions, globalOptions, args)
 	},
 }
 
 // LsOptions collects all options for the ls command.
 type LsOptions struct {
-	ListLong  bool
-	Hosts     []string
-	Tags      restic.TagLists
-	Paths     []string
+	ListLong bool
+	snapshotFilterOptions
 	Recursive bool
 }
 
@@ -61,10 +59,8 @@ func init() {
 	cmdRoot.AddCommand(cmdLs)
 
 	flags := cmdLs.Flags()
+	initSingleSnapshotFilterOptions(flags, &lsOptions.snapshotFilterOptions)
 	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
-	flags.StringArrayVarP(&lsOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when snapshot ID \"latest\" is given (can be specified multiple times)")
-	flags.Var(&lsOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when snapshot ID \"latest\" is given (can be specified multiple times)")
-	flags.StringArrayVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when snapshot ID \"latest\" is given (can be specified multiple times)")
 	flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories")
 }
 
@@ -115,7 +111,7 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
 	return enc.Encode(n)
 }
 
-func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
+func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error {
 	if len(args) == 0 {
 		return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'")
 	}
@@ -165,23 +161,20 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 		return false
 	}
 
-	repo, err := OpenRepository(gopts)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
 
-	snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile)
+	snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
 	if err != nil {
 		return err
 	}
 
-	if err = repo.LoadIndex(gopts.ctx); err != nil {
+	if err = repo.LoadIndex(ctx); err != nil {
 		return err
 	}
 
-	ctx, cancel := context.WithCancel(gopts.ctx)
-	defer cancel()
-
 	var (
 		printSnapshot func(sn *restic.Snapshot)
 		printNode     func(path string, node *restic.Node)
@@ -217,45 +210,48 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, args[:1]) {
-		printSnapshot(sn)
+	sn, err := restic.FindFilteredSnapshot(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, nil, args[0])
+	if err != nil {
+		return err
+	}
 
-		err := walker.Walk(ctx, repo, *sn.Tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
-			if err != nil {
-				return false, err
-			}
-			if node == nil {
-				return false, nil
-			}
-
-			if withinDir(nodepath) {
-				// if we're within a dir, print the node
-				printNode(nodepath, node)
-
-				// if recursive listing is requested, signal the walker that it
-				// should continue walking recursively
-				if opts.Recursive {
-					return false, nil
-				}
-			}
-
-			// if there's an upcoming match deeper in the tree (but we're not
-			// there yet), signal the walker to descend into any subdirs
-			if approachingMatchingTree(nodepath) {
-				return false, nil
-			}
-
-			// otherwise, signal the walker to not walk recursively into any
-			// subdirs
-			if node.Type == "dir" {
-				return false, walker.ErrSkipNode
-			}
-			return false, nil
-		})
+	printSnapshot(sn)
 
+	err = walker.Walk(ctx, repo, *sn.Tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
 		if err != nil {
-			return err
+			return false, err
 		}
+		if node == nil {
+			return false, nil
+		}
+
+		if withinDir(nodepath) {
+			// if we're within a dir, print the node
+			printNode(nodepath, node)
+
+			// if recursive listing is requested, signal the walker that it
+			// should continue walking recursively
+			if opts.Recursive {
+				return false, nil
+			}
+		}
+
+		// if there's an upcoming match deeper in the tree (but we're not
+		// there yet), signal the walker to descend into any subdirs
+		if approachingMatchingTree(nodepath) {
+			return false, nil
+		}
+
+		// otherwise, signal the walker to not walk recursively into any
+		// subdirs
+		if node.Type == "dir" {
+			return false, walker.ErrSkipNode
+		}
+		return false, nil
+	})
+
+	if err != nil {
+		return err
 	}
 
 	return nil

cmd/restic/cmd_migrate.go

@@ -1,6 +1,8 @@
 package main
 
 import (
+	"context"
+
 	"github.com/restic/restic/internal/migrations"
 	"github.com/restic/restic/internal/restic"
 
@@ -22,7 +24,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runMigrate(migrateOptions, globalOptions, args)
+		return runMigrate(cmd.Context(), migrateOptions, globalOptions, args)
 	},
 }
 
@@ -39,13 +41,12 @@ func init() {
 	f.BoolVarP(&migrateOptions.Force, "force", "f", false, `apply a migration a second time`)
 }
 
-func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository) error {
-	ctx := gopts.ctx
+func checkMigrations(ctx context.Context, repo restic.Repository) error {
 	Printf("available migrations:\n")
 	found := false
 
 	for _, m := range migrations.All {
-		ok, err := m.Check(ctx, repo)
+		ok, _, err := m.Check(ctx, repo)
 		if err != nil {
 			return err
 		}
@@ -57,27 +58,28 @@ func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos
 	}
 
 	if !found {
-		Printf("no migrations found")
+		Printf("no migrations found\n")
 	}
 
 	return nil
 }
 
-func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository, args []string) error {
-	ctx := gopts.ctx
-
+func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, repo restic.Repository, args []string) error {
 	var firsterr error
 	for _, name := range args {
 		for _, m := range migrations.All {
 			if m.Name() == name {
-				ok, err := m.Check(ctx, repo)
+				ok, reason, err := m.Check(ctx, repo)
 				if err != nil {
 					return err
 				}
 
 				if !ok {
 					if !opts.Force {
-						Warnf("migration %v cannot be applied: check failed\nIf you want to apply this migration anyway, re-run with option --force\n", m.Name())
+						if reason == "" {
+							reason = "check failed"
+						}
+						Warnf("migration %v cannot be applied: %v\nIf you want to apply this migration anyway, re-run with option --force\n", m.Name(), reason)
 						continue
 					}
 
@@ -91,7 +93,7 @@ func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos
 					checkGopts := gopts
 					// the repository is already locked
 					checkGopts.NoLock = true
-					err = runCheck(checkOptions, checkGopts, []string{})
+					err = runCheck(ctx, checkOptions, checkGopts, []string{})
 					if err != nil {
 						return err
 					}
@@ -114,21 +116,21 @@ func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos
 	return firsterr
 }
 
-func runMigrate(opts MigrateOptions, gopts GlobalOptions, args []string) error {
-	repo, err := OpenRepository(gopts)
+func runMigrate(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, args []string) error {
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
 
-	lock, err := lockRepoExclusive(gopts.ctx, repo)
+	lock, ctx, err := lockRepoExclusive(ctx, repo)
 	defer unlockRepo(lock)
 	if err != nil {
 		return err
 	}
 
 	if len(args) == 0 {
-		return checkMigrations(opts, gopts, repo)
+		return checkMigrations(ctx, repo)
 	}
 
-	return applyMigrations(opts, gopts, repo, args)
+	return applyMigrations(ctx, opts, gopts, repo, args)
 }

cmd/restic/cmd_mount.go

@@ -4,6 +4,7 @@
 package main
 
 import (
+	"context"
 	"os"
 	"strings"
 	"time"
@@ -17,8 +18,8 @@ import (
 	resticfs "github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/fuse"
 
-	systemFuse "bazil.org/fuse"
-	"bazil.org/fuse/fs"
+	systemFuse "github.com/anacrolix/fuse"
+	"github.com/anacrolix/fuse/fs"
 )
 
 var cmdMount = &cobra.Command{
@@ -67,7 +68,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runMount(mountOptions, globalOptions, args)
+		return runMount(cmd.Context(), mountOptions, globalOptions, args)
 	},
 }
 
@@ -76,11 +77,9 @@ type MountOptions struct {
 	OwnerRoot            bool
 	AllowOther           bool
 	NoDefaultPermissions bool
-	Hosts                []string
-	Tags                 restic.TagLists
-	Paths                []string
-	TimeTemplate         string
-	PathTemplates        []string
+	snapshotFilterOptions
+	TimeTemplate  string
+	PathTemplates []string
 }
 
 var mountOptions MountOptions
@@ -93,9 +92,7 @@ func init() {
 	mountFlags.BoolVar(&mountOptions.AllowOther, "allow-other", false, "allow other users to access the data in the mounted directory")
 	mountFlags.BoolVar(&mountOptions.NoDefaultPermissions, "no-default-permissions", false, "for 'allow-other', ignore Unix permissions and allow users to read all snapshot files")
 
-	mountFlags.StringArrayVarP(&mountOptions.Hosts, "host", "H", nil, `only consider snapshots for this host (can be specified multiple times)`)
-	mountFlags.Var(&mountOptions.Tags, "tag", "only consider snapshots which include this `taglist`")
-	mountFlags.StringArrayVar(&mountOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`")
+	initMultiSnapshotFilterOptions(mountFlags, &mountOptions.snapshotFilterOptions, true)
 
 	mountFlags.StringArrayVar(&mountOptions.PathTemplates, "path-template", nil, "set `template` for path names (can be specified multiple times)")
 	mountFlags.StringVar(&mountOptions.TimeTemplate, "snapshot-template", time.RFC3339, "set `template` to use for snapshot dirs")
@@ -103,7 +100,7 @@ func init() {
 	_ = mountFlags.MarkDeprecated("snapshot-template", "use --time-template")
 }
 
-func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
+func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args []string) error {
 	if opts.TimeTemplate == "" {
 		return errors.Fatal("time template string cannot be empty")
 	}
@@ -119,20 +116,21 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
 	debug.Log("start mount")
 	defer debug.Log("finish mount")
 
-	repo, err := OpenRepository(gopts)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
 
 	if !gopts.NoLock {
-		lock, err := lockRepo(gopts.ctx, repo)
+		var lock *restic.Lock
+		lock, ctx, err = lockRepo(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
 		}
 	}
 
-	err = repo.LoadIndex(gopts.ctx)
+	err = repo.LoadIndex(ctx)
 	if err != nil {
 		return err
 	}
@@ -158,13 +156,17 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	AddCleanupHandler(func() error {
+	AddCleanupHandler(func(code int) (int, error) {
 		debug.Log("running umount cleanup handler for mount at %v", mountpoint)
 		err := umount(mountpoint)
 		if err != nil {
 			Warnf("unable to umount (maybe already umounted or still in use?): %v\n", err)
 		}
-		return nil
+		// replace error code of sigint
+		if code == 130 {
+			code = 0
+		}
+		return code, nil
 	})
 
 	c, err := systemFuse.Mount(mountpoint, mountOptions...)

cmd/restic/cmd_prune.go

@@ -9,9 +9,11 @@ import (
 
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/index"
 	"github.com/restic/restic/internal/pack"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui"
 
 	"github.com/spf13/cobra"
 )
@@ -34,7 +36,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runPrune(pruneOptions, globalOptions)
+		return runPrune(cmd.Context(), pruneOptions, globalOptions)
 	},
 }
 
@@ -134,7 +136,7 @@ func verifyPruneOptions(opts *PruneOptions) error {
 	return nil
 }
 
-func runPrune(opts PruneOptions, gopts GlobalOptions) error {
+func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error {
 	err := verifyPruneOptions(&opts)
 	if err != nil {
 		return err
@@ -144,7 +146,7 @@ func runPrune(opts PruneOptions, gopts GlobalOptions) error {
 		return errors.Fatal("disabled compression and `--repack-uncompressed` are mutually exclusive")
 	}
 
-	repo, err := OpenRepository(gopts)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
@@ -165,16 +167,16 @@ func runPrune(opts PruneOptions, gopts GlobalOptions) error {
 		opts.unsafeRecovery = true
 	}
 
-	lock, err := lockRepoExclusive(gopts.ctx, repo)
+	lock, ctx, err := lockRepoExclusive(ctx, repo)
 	defer unlockRepo(lock)
 	if err != nil {
 		return err
 	}
 
-	return runPruneWithRepo(opts, gopts, repo, restic.NewIDSet())
+	return runPruneWithRepo(ctx, opts, gopts, repo, restic.NewIDSet())
 }
 
-func runPruneWithRepo(opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet) error {
+func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet) error {
 	// we do not need index updates while pruning!
 	repo.DisableAutoIndexUpdate()
 
@@ -184,22 +186,26 @@ func runPruneWithRepo(opts PruneOptions, gopts GlobalOptions, repo *repository.R
 
 	Verbosef("loading indexes...\n")
 	// loading the index before the snapshots is ok, as we use an exclusive lock here
-	err := repo.LoadIndex(gopts.ctx)
+	err := repo.LoadIndex(ctx)
 	if err != nil {
 		return err
 	}
 
-	plan, stats, err := planPrune(opts, gopts, repo, ignoreSnapshots)
+	plan, stats, err := planPrune(ctx, opts, repo, ignoreSnapshots, gopts.Quiet)
 	if err != nil {
 		return err
 	}
 
-	err = printPruneStats(gopts, stats)
+	if opts.DryRun {
+		Verbosef("\nWould have made the following changes:")
+	}
+
+	err = printPruneStats(stats)
 	if err != nil {
 		return err
 	}
 
-	return doPrune(opts, gopts, repo, plan)
+	return doPrune(ctx, opts, gopts, repo, plan)
 }
 
 type pruneStats struct {
@@ -212,13 +218,14 @@ type pruneStats struct {
 		repackrm  uint
 	}
 	size struct {
-		used      uint64
-		duplicate uint64
-		unused    uint64
-		remove    uint64
-		repack    uint64
-		repackrm  uint64
-		unref     uint64
+		used         uint64
+		duplicate    uint64
+		unused       uint64
+		remove       uint64
+		repack       uint64
+		repackrm     uint64
+		unref        uint64
+		uncompressed uint64
 	}
 	packs struct {
 		used       uint
@@ -232,11 +239,11 @@ type pruneStats struct {
 }
 
 type prunePlan struct {
-	removePacksFirst restic.IDSet   // packs to remove first (unreferenced packs)
-	repackPacks      restic.IDSet   // packs to repack
-	keepBlobs        restic.BlobSet // blobs to keep during repacking
-	removePacks      restic.IDSet   // packs to remove
-	ignorePacks      restic.IDSet   // packs to ignore when rebuilding the index
+	removePacksFirst restic.IDSet          // packs to remove first (unreferenced packs)
+	repackPacks      restic.IDSet          // packs to repack
+	keepBlobs        restic.CountedBlobSet // blobs to keep during repacking
+	removePacks      restic.IDSet          // packs to remove
+	ignorePacks      restic.IDSet          // packs to ignore when rebuilding the index
 }
 
 type packInfo struct {
@@ -251,15 +258,15 @@ type packInfo struct {
 type packInfoWithID struct {
 	ID restic.ID
 	packInfo
+	mustCompress bool
 }
 
 // planPrune selects which files to rewrite and which to delete and which blobs to keep.
 // Also some summary statistics are returned.
-func planPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, ignoreSnapshots restic.IDSet) (prunePlan, pruneStats, error) {
-	ctx := gopts.ctx
+func planPrune(ctx context.Context, opts PruneOptions, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (prunePlan, pruneStats, error) {
 	var stats pruneStats
 
-	usedBlobs, err := getUsedBlobs(gopts, repo, ignoreSnapshots)
+	usedBlobs, err := getUsedBlobs(ctx, repo, ignoreSnapshots, quiet)
 	if err != nil {
 		return prunePlan{}, stats, err
 	}
@@ -271,19 +278,25 @@ func planPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, i
 	}
 
 	Verbosef("collecting packs for deletion and repacking\n")
-	plan, err := decidePackAction(ctx, opts, gopts, repo, indexPack, &stats)
+	plan, err := decidePackAction(ctx, opts, repo, indexPack, &stats, quiet)
 	if err != nil {
 		return prunePlan{}, stats, err
 	}
 
 	if len(plan.repackPacks) != 0 {
+		blobCount := keepBlobs.Len()
 		// when repacking, we do not want to keep blobs which are
 		// already contained in kept packs, so delete them from keepBlobs
-		for blob := range repo.Index().Each(ctx) {
+		repo.Index().Each(ctx, func(blob restic.PackedBlob) {
 			if plan.removePacks.Has(blob.PackID) || plan.repackPacks.Has(blob.PackID) {
-				continue
+				return
 			}
 			keepBlobs.Delete(blob.BlobHandle)
+		})
+
+		if keepBlobs.Len() < blobCount/2 {
+			// replace with copy to shrink map to necessary size if there's a chance to benefit
+			keepBlobs = keepBlobs.Copy()
 		}
 	} else {
 		// keepBlobs is only needed if packs are repacked
@@ -294,46 +307,40 @@ func planPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, i
 	return plan, stats, nil
 }
 
-func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs restic.BlobSet, stats *pruneStats) (restic.BlobSet, map[restic.ID]packInfo, error) {
-	keepBlobs := restic.NewBlobSet()
-	duplicateBlobs := make(map[restic.BlobHandle]uint8)
-
+func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs restic.CountedBlobSet, stats *pruneStats) (restic.CountedBlobSet, map[restic.ID]packInfo, error) {
 	// iterate over all blobs in index to find out which blobs are duplicates
-	for blob := range idx.Each(ctx) {
+	// The counter in usedBlobs describes how many instances of the blob exist in the repository index
+	// Thus 0 == blob is missing, 1 == blob exists once, >= 2 == duplicates exist
+	idx.Each(ctx, func(blob restic.PackedBlob) {
 		bh := blob.BlobHandle
-		size := uint64(blob.Length)
-		switch {
-		case usedBlobs.Has(bh): // used blob, move to keepBlobs
-			usedBlobs.Delete(bh)
-			keepBlobs.Insert(bh)
-			stats.size.used += size
-			stats.blobs.used++
-		case keepBlobs.Has(bh): // duplicate blob
-			count, ok := duplicateBlobs[bh]
-			if !ok {
-				count = 2 // this one is already the second blob!
-			} else if count < math.MaxUint8 {
+		count, ok := usedBlobs[bh]
+		if ok {
+			if count < math.MaxUint8 {
 				// don't overflow, but saturate count at 255
 				// this can lead to a non-optimal pack selection, but won't cause
 				// problems otherwise
 				count++
 			}
-			duplicateBlobs[bh] = count
-			stats.size.duplicate += size
-			stats.blobs.duplicate++
-		default:
-			stats.size.unused += size
-			stats.blobs.unused++
+
+			usedBlobs[bh] = count
+		}
+	})
+
+	// Check if all used blobs have been found in index
+	missingBlobs := restic.NewBlobSet()
+	for bh, count := range usedBlobs {
+		if count == 0 {
+			// blob does not exist in any pack files
+			missingBlobs.Insert(bh)
 		}
 	}
 
-	// Check if all used blobs have been found in index
-	if len(usedBlobs) != 0 {
+	if len(missingBlobs) != 0 {
 		Warnf("%v not found in the index\n\n"+
 			"Integrity check failed: Data seems to be missing.\n"+
 			"Will not start prune to prevent (additional) data loss!\n"+
 			"Please report this error (along with the output of the 'prune' run) at\n"+
-			"https://github.com/restic/restic/issues/new/choose\n", usedBlobs)
+			"https://github.com/restic/restic/issues/new/choose\n", missingBlobs)
 		return nil, nil, errorIndexIncomplete
 	}
 
@@ -345,8 +352,9 @@ func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs re
 		indexPack[pid] = packInfo{tpe: restic.NumBlobTypes, usedSize: uint64(hdrSize)}
 	}
 
+	hasDuplicates := false
 	// iterate over all blobs in index to generate packInfo
-	for blob := range idx.Each(ctx) {
+	idx.Each(ctx, func(blob restic.PackedBlob) {
 		ip := indexPack[blob.PackID]
 
 		// Set blob type if not yet set
@@ -361,64 +369,95 @@ func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs re
 
 		bh := blob.BlobHandle
 		size := uint64(blob.Length)
-		_, isDuplicate := duplicateBlobs[bh]
+		dupCount := usedBlobs[bh]
 		switch {
-		case isDuplicate: // duplicate blobs will be handled later
-		case keepBlobs.Has(bh): // used blob, not duplicate
+		case dupCount >= 2:
+			hasDuplicates = true
+			// mark as unused for now, we will later on select one copy
+			ip.unusedSize += size
+			ip.unusedBlobs++
+
+			// count as duplicate, will later on change one copy to be counted as used
+			stats.size.duplicate += size
+			stats.blobs.duplicate++
+		case dupCount == 1: // used blob, not duplicate
 			ip.usedSize += size
 			ip.usedBlobs++
+
+			stats.size.used += size
+			stats.blobs.used++
 		default: // unused blob
 			ip.unusedSize += size
 			ip.unusedBlobs++
+
+			stats.size.unused += size
+			stats.blobs.unused++
 		}
 		if !blob.IsCompressed() {
 			ip.uncompressed = true
 		}
 		// update indexPack
 		indexPack[blob.PackID] = ip
-	}
+	})
 
 	// if duplicate blobs exist, those will be set to either "used" or "unused":
 	// - mark only one occurence of duplicate blobs as used
 	// - if there are already some used blobs in a pack, possibly mark duplicates in this pack as "used"
 	// - if there are no used blobs in a pack, possibly mark duplicates as "unused"
-	if len(duplicateBlobs) > 0 {
+	if hasDuplicates {
 		// iterate again over all blobs in index (this is pretty cheap, all in-mem)
-		for blob := range idx.Each(ctx) {
+		idx.Each(ctx, func(blob restic.PackedBlob) {
 			bh := blob.BlobHandle
-			count, isDuplicate := duplicateBlobs[bh]
-			if !isDuplicate {
-				continue
+			count, ok := usedBlobs[bh]
+			// skip non-duplicate, aka. normal blobs
+			// count == 0 is used to mark that this was a duplicate blob with only a single occurence remaining
+			if !ok || count == 1 {
+				return
 			}
 
 			ip := indexPack[blob.PackID]
 			size := uint64(blob.Length)
 			switch {
-			case count == 0:
-				// used duplicate exists ->  mark as unused
-				ip.unusedSize += size
-				ip.unusedBlobs++
-			case ip.usedBlobs > 0, count == 1:
-				// other used blobs in pack or "last" occurency ->  mark as used
+			case ip.usedBlobs > 0, count == 0:
+				// other used blobs in pack or "last" occurence ->  transition to used
 				ip.usedSize += size
 				ip.usedBlobs++
-				// let other occurences be marked as unused
-				duplicateBlobs[bh] = 0
+				ip.unusedSize -= size
+				ip.unusedBlobs--
+				// same for the global statistics
+				stats.size.used += size
+				stats.blobs.used++
+				stats.size.duplicate -= size
+				stats.blobs.duplicate--
+				// let other occurences remain marked as unused
+				usedBlobs[bh] = 1
 			default:
-				// mark as unused and decrease counter
-				ip.unusedSize += size
-				ip.unusedBlobs++
-				duplicateBlobs[bh] = count - 1
+				// remain unused and decrease counter
+				count--
+				if count == 1 {
+					// setting count to 1 would lead to forgetting that this blob had duplicates
+					// thus use the special value zero. This will select the last instance of the blob for keeping.
+					count = 0
+				}
+				usedBlobs[bh] = count
 			}
 			// update indexPack
 			indexPack[blob.PackID] = ip
+		})
+	}
+
+	// Sanity check. If no duplicates exist, all blobs have value 1. After handling
+	// duplicates, this also applies to duplicates.
+	for _, count := range usedBlobs {
+		if count != 1 {
+			panic("internal error during blob selection")
 		}
 	}
 
-	return keepBlobs, indexPack, nil
+	return usedBlobs, indexPack, nil
 }
 
-func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo restic.Repository, indexPack map[restic.ID]packInfo, stats *pruneStats) (prunePlan, error) {
+func decidePackAction(ctx context.Context, opts PruneOptions, repo restic.Repository, indexPack map[restic.ID]packInfo, stats *pruneStats, quiet bool) (prunePlan, error) {
 	removePacksFirst := restic.NewIDSet()
 	removePacks := restic.NewIDSet()
 	repackPacks := restic.NewIDSet()
@@ -434,7 +473,7 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
 	}
 
 	// loop over all packs and decide what to do
-	bar := newProgressMax(!gopts.Quiet, uint64(len(indexPack)), "packs processed")
+	bar := newProgressMax(quiet, uint64(len(indexPack)), "packs processed")
 	err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
 		p, ok := indexPack[id]
 		if !ok {
@@ -464,14 +503,15 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
 			stats.packs.partlyUsed++
 		}
 
+		if p.uncompressed {
+			stats.size.uncompressed += p.unusedSize + p.usedSize
+		}
 		mustCompress := false
 		if repoVersion >= 2 {
 			// repo v2: always repack tree blobs if uncompressed
 			// compress data blobs if requested
 			mustCompress = (p.tpe == restic.TreeBlob || opts.RepackUncompressed) && p.uncompressed
 		}
-		// use a flag that pack must be compressed
-		p.uncompressed = mustCompress
 
 		// decide what to do
 		switch {
@@ -490,12 +530,12 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
 				// All blobs in pack are used and not mixed => keep pack!
 				stats.packs.keep++
 			} else {
-				repackSmallCandidates = append(repackSmallCandidates, packInfoWithID{ID: id, packInfo: p})
+				repackSmallCandidates = append(repackSmallCandidates, packInfoWithID{ID: id, packInfo: p, mustCompress: mustCompress})
 			}
 
 		default:
 			// all other packs are candidates for repacking
-			repackCandidates = append(repackCandidates, packInfoWithID{ID: id, packInfo: p})
+			repackCandidates = append(repackCandidates, packInfoWithID{ID: id, packInfo: p, mustCompress: mustCompress})
 		}
 
 		delete(indexPack, id)
@@ -569,6 +609,9 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
 		stats.size.repack += p.unusedSize + p.usedSize
 		stats.blobs.repackrm += p.unusedBlobs
 		stats.size.repackrm += p.unusedSize
+		if p.uncompressed {
+			stats.size.uncompressed -= p.unusedSize + p.usedSize
+		}
 	}
 
 	// calculate limit for number of unused bytes in the repo after repacking
@@ -583,7 +626,7 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
 		case reachedRepackSize:
 			stats.packs.keep++
 
-		case p.tpe != restic.DataBlob, p.uncompressed:
+		case p.tpe != restic.DataBlob, p.mustCompress:
 			// repacking non-data packs / uncompressed-trees is only limited by repackSize
 			repack(p.ID, p.packInfo)
 
@@ -600,6 +643,11 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
 	stats.packs.repack = uint(len(repackPacks))
 	stats.packs.remove = uint(len(removePacks))
 
+	if repo.Config().Version < 2 {
+		// compression not supported for repository format version 1
+		stats.size.uncompressed = 0
+	}
+
 	return prunePlan{removePacksFirst: removePacksFirst,
 		removePacks: removePacks,
 		repackPacks: repackPacks,
@@ -608,30 +656,33 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
 }
 
 // printPruneStats prints out the statistics
-func printPruneStats(gopts GlobalOptions, stats pruneStats) error {
-	Verboseff("\nused:         %10d blobs / %s\n", stats.blobs.used, formatBytes(stats.size.used))
+func printPruneStats(stats pruneStats) error {
+	Verboseff("\nused:         %10d blobs / %s\n", stats.blobs.used, ui.FormatBytes(stats.size.used))
 	if stats.blobs.duplicate > 0 {
-		Verboseff("duplicates:   %10d blobs / %s\n", stats.blobs.duplicate, formatBytes(stats.size.duplicate))
+		Verboseff("duplicates:   %10d blobs / %s\n", stats.blobs.duplicate, ui.FormatBytes(stats.size.duplicate))
 	}
-	Verboseff("unused:       %10d blobs / %s\n", stats.blobs.unused, formatBytes(stats.size.unused))
+	Verboseff("unused:       %10d blobs / %s\n", stats.blobs.unused, ui.FormatBytes(stats.size.unused))
 	if stats.size.unref > 0 {
-		Verboseff("unreferenced:                    %s\n", formatBytes(stats.size.unref))
+		Verboseff("unreferenced:                    %s\n", ui.FormatBytes(stats.size.unref))
 	}
 	totalBlobs := stats.blobs.used + stats.blobs.unused + stats.blobs.duplicate
 	totalSize := stats.size.used + stats.size.duplicate + stats.size.unused + stats.size.unref
 	unusedSize := stats.size.duplicate + stats.size.unused
-	Verboseff("total:        %10d blobs / %s\n", totalBlobs, formatBytes(totalSize))
-	Verboseff("unused size: %s of total size\n", formatPercent(unusedSize, totalSize))
+	Verboseff("total:        %10d blobs / %s\n", totalBlobs, ui.FormatBytes(totalSize))
+	Verboseff("unused size: %s of total size\n", ui.FormatPercent(unusedSize, totalSize))
 
-	Verbosef("\nto repack:    %10d blobs / %s\n", stats.blobs.repack, formatBytes(stats.size.repack))
-	Verbosef("this removes: %10d blobs / %s\n", stats.blobs.repackrm, formatBytes(stats.size.repackrm))
-	Verbosef("to delete:    %10d blobs / %s\n", stats.blobs.remove, formatBytes(stats.size.remove+stats.size.unref))
+	Verbosef("\nto repack:    %10d blobs / %s\n", stats.blobs.repack, ui.FormatBytes(stats.size.repack))
+	Verbosef("this removes: %10d blobs / %s\n", stats.blobs.repackrm, ui.FormatBytes(stats.size.repackrm))
+	Verbosef("to delete:    %10d blobs / %s\n", stats.blobs.remove, ui.FormatBytes(stats.size.remove+stats.size.unref))
 	totalPruneSize := stats.size.remove + stats.size.repackrm + stats.size.unref
-	Verbosef("total prune:  %10d blobs / %s\n", stats.blobs.remove+stats.blobs.repackrm, formatBytes(totalPruneSize))
-	Verbosef("remaining:    %10d blobs / %s\n", totalBlobs-(stats.blobs.remove+stats.blobs.repackrm), formatBytes(totalSize-totalPruneSize))
+	Verbosef("total prune:  %10d blobs / %s\n", stats.blobs.remove+stats.blobs.repackrm, ui.FormatBytes(totalPruneSize))
+	if stats.size.uncompressed > 0 {
+		Verbosef("not yet compressed:              %s\n", ui.FormatBytes(stats.size.uncompressed))
+	}
+	Verbosef("remaining:    %10d blobs / %s\n", totalBlobs-(stats.blobs.remove+stats.blobs.repackrm), ui.FormatBytes(totalSize-totalPruneSize))
 	unusedAfter := unusedSize - stats.size.remove - stats.size.repackrm
 	Verbosef("unused size after prune: %s (%s of remaining size)\n",
-		formatBytes(unusedAfter), formatPercent(unusedAfter, totalSize-totalPruneSize))
+		ui.FormatBytes(unusedAfter), ui.FormatPercent(unusedAfter, totalSize-totalPruneSize))
 	Verbosef("\n")
 	Verboseff("totally used packs: %10d\n", stats.packs.used)
 	Verboseff("partly used packs:  %10d\n", stats.packs.partlyUsed)
@@ -652,11 +703,10 @@ func printPruneStats(gopts GlobalOptions, stats pruneStats) error {
 // - rebuild the index while ignoring all files that will be deleted
 // - delete the files
 // plan.removePacks and plan.ignorePacks are modified in this function.
-func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, plan prunePlan) (err error) {
-	ctx := gopts.ctx
-
+func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo restic.Repository, plan prunePlan) (err error) {
 	if opts.DryRun {
 		if !gopts.JSON && gopts.verbosity >= 2 {
+			Printf("Repeated prune dry-runs can report slightly different amounts of data to keep or repack. This is expected behavior.\n\n")
 			if len(plan.removePacksFirst) > 0 {
 				Printf("Would have removed the following unreferenced packs:\n%v\n\n", plan.removePacksFirst)
 			}
@@ -670,7 +720,7 @@ func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, pla
 	// unreferenced packs can be safely deleted first
 	if len(plan.removePacksFirst) != 0 {
 		Verbosef("deleting unreferenced packs\n")
-		DeleteFiles(gopts, repo, plan.removePacksFirst, restic.PackFile)
+		DeleteFiles(ctx, gopts, repo, plan.removePacksFirst, restic.PackFile)
 	}
 
 	if len(plan.repackPacks) != 0 {
@@ -692,6 +742,9 @@ func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, pla
 				"https://github.com/restic/restic/issues/new/choose\n", plan.keepBlobs)
 			return errors.Fatal("internal error: blobs were not repacked")
 		}
+
+		// allow GC of the blob set
+		plan.keepBlobs = nil
 	}
 
 	if len(plan.ignorePacks) == 0 {
@@ -702,13 +755,13 @@ func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, pla
 
 	if opts.unsafeRecovery {
 		Verbosef("deleting index files\n")
-		indexFiles := repo.Index().(*repository.MasterIndex).IDs()
-		err = DeleteFilesChecked(gopts, repo, indexFiles, restic.IndexFile)
+		indexFiles := repo.Index().(*index.MasterIndex).IDs()
+		err = DeleteFilesChecked(ctx, gopts, repo, indexFiles, restic.IndexFile)
 		if err != nil {
 			return errors.Fatalf("%s", err)
 		}
 	} else if len(plan.ignorePacks) != 0 {
-		err = rebuildIndexFiles(gopts, repo, plan.ignorePacks, nil)
+		err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil)
 		if err != nil {
 			return errors.Fatalf("%s", err)
 		}
@@ -716,11 +769,11 @@ func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, pla
 
 	if len(plan.removePacks) != 0 {
 		Verbosef("removing %d old packs\n", len(plan.removePacks))
-		DeleteFiles(gopts, repo, plan.removePacks, restic.PackFile)
+		DeleteFiles(ctx, gopts, repo, plan.removePacks, restic.PackFile)
 	}
 
 	if opts.unsafeRecovery {
-		_, err = writeIndexFiles(gopts, repo, plan.ignorePacks, nil)
+		_, err = writeIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil)
 		if err != nil {
 			return errors.Fatalf("%s", err)
 		}
@@ -730,31 +783,29 @@ func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, pla
 	return nil
 }
 
-func writeIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) (restic.IDSet, error) {
+func writeIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) (restic.IDSet, error) {
 	Verbosef("rebuilding index\n")
 
 	bar := newProgressMax(!gopts.Quiet, 0, "packs processed")
-	obsoleteIndexes, err := repo.Index().Save(gopts.ctx, repo, removePacks, extraObsolete, bar)
+	obsoleteIndexes, err := repo.Index().Save(ctx, repo, removePacks, extraObsolete, bar)
 	bar.Done()
 	return obsoleteIndexes, err
 }
 
-func rebuildIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error {
-	obsoleteIndexes, err := writeIndexFiles(gopts, repo, removePacks, extraObsolete)
+func rebuildIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error {
+	obsoleteIndexes, err := writeIndexFiles(ctx, gopts, repo, removePacks, extraObsolete)
 	if err != nil {
 		return err
 	}
 
 	Verbosef("deleting obsolete index files\n")
-	return DeleteFilesChecked(gopts, repo, obsoleteIndexes, restic.IndexFile)
+	return DeleteFilesChecked(ctx, gopts, repo, obsoleteIndexes, restic.IndexFile)
 }
 
-func getUsedBlobs(gopts GlobalOptions, repo restic.Repository, ignoreSnapshots restic.IDSet) (usedBlobs restic.BlobSet, err error) {
-	ctx := gopts.ctx
-
+func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (usedBlobs restic.CountedBlobSet, err error) {
 	var snapshotTrees restic.IDs
 	Verbosef("loading all snapshots...\n")
-	err = restic.ForAllSnapshots(gopts.ctx, repo.Backend(), repo, ignoreSnapshots,
+	err = restic.ForAllSnapshots(ctx, repo.Backend(), repo, ignoreSnapshots,
 		func(id restic.ID, sn *restic.Snapshot, err error) error {
 			if err != nil {
 				debug.Log("failed to load snapshot %v (error %v)", id, err)
@@ -770,9 +821,9 @@ func getUsedBlobs(gopts GlobalOptions, repo restic.Repository, ignoreSnapshots r
 
 	Verbosef("finding data that is still in use for %d snapshots\n", len(snapshotTrees))
 
-	usedBlobs = restic.NewBlobSet()
+	usedBlobs = restic.NewCountedBlobSet()
 
-	bar := newProgressMax(!gopts.Quiet, uint64(len(snapshotTrees)), "snapshots")
+	bar := newProgressMax(!quiet, uint64(len(snapshotTrees)), "snapshots")
 	defer bar.Done()
 
 	err = restic.FindUsedBlobs(ctx, repo, snapshotTrees, usedBlobs, bar)

cmd/restic/cmd_rebuild_index.go

@@ -1,6 +1,9 @@
 package main
 
 import (
+	"context"
+
+	"github.com/restic/restic/internal/index"
 	"github.com/restic/restic/internal/pack"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
@@ -22,7 +25,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runRebuildIndex(rebuildIndexOptions, globalOptions)
+		return runRebuildIndex(cmd.Context(), rebuildIndexOptions, globalOptions)
 	},
 }
 
@@ -40,24 +43,22 @@ func init() {
 
 }
 
-func runRebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions) error {
-	repo, err := OpenRepository(gopts)
+func runRebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOptions) error {
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
 
-	lock, err := lockRepoExclusive(gopts.ctx, repo)
+	lock, ctx, err := lockRepoExclusive(ctx, repo)
 	defer unlockRepo(lock)
 	if err != nil {
 		return err
 	}
 
-	return rebuildIndex(opts, gopts, repo, restic.NewIDSet())
+	return rebuildIndex(ctx, opts, gopts, repo, restic.NewIDSet())
 }
 
-func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error {
-	ctx := gopts.ctx
-
+func rebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error {
 	var obsoleteIndexes restic.IDs
 	packSizeFromList := make(map[restic.ID]int64)
 	packSizeFromIndex := make(map[restic.ID]int64)
@@ -74,8 +75,8 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor
 		}
 	} else {
 		Verbosef("loading indexes...\n")
-		mi := repository.NewMasterIndex()
-		err := repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
+		mi := index.NewMasterIndex()
+		err := index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
 			if err != nil {
 				Warnf("removing invalid index %v: %v\n", id, err)
 				obsoleteIndexes = append(obsoleteIndexes, id)
@@ -141,7 +142,7 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor
 		}
 	}
 
-	err = rebuildIndexFiles(gopts, repo, removePacks, obsoleteIndexes)
+	err = rebuildIndexFiles(ctx, gopts, repo, removePacks, obsoleteIndexes)
 	if err != nil {
 		return err
 	}

cmd/restic/cmd_recover.go

@@ -27,7 +27,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runRecover(globalOptions)
+		return runRecover(cmd.Context(), globalOptions)
 	},
 }
 
@@ -35,30 +35,30 @@ func init() {
 	cmdRoot.AddCommand(cmdRecover)
 }
 
-func runRecover(gopts GlobalOptions) error {
+func runRecover(ctx context.Context, gopts GlobalOptions) error {
 	hostname, err := os.Hostname()
 	if err != nil {
 		return err
 	}
 
-	repo, err := OpenRepository(gopts)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
 
-	lock, err := lockRepo(gopts.ctx, repo)
+	lock, ctx, err := lockRepo(ctx, repo)
 	defer unlockRepo(lock)
 	if err != nil {
 		return err
 	}
 
-	snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile)
+	snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
 	if err != nil {
 		return err
 	}
 
 	Verbosef("load index files\n")
-	if err = repo.LoadIndex(gopts.ctx); err != nil {
+	if err = repo.LoadIndex(ctx); err != nil {
 		return err
 	}
 
@@ -66,16 +66,16 @@ func runRecover(gopts GlobalOptions) error {
 	// tree. If it is not referenced, we have a root tree.
 	trees := make(map[restic.ID]bool)
 
-	for blob := range repo.Index().Each(gopts.ctx) {
+	repo.Index().Each(ctx, func(blob restic.PackedBlob) {
 		if blob.Type == restic.TreeBlob {
 			trees[blob.Blob.ID] = false
 		}
-	}
+	})
 
 	Verbosef("load %d trees\n", len(trees))
 	bar := newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
 	for id := range trees {
-		tree, err := restic.LoadTree(gopts.ctx, repo, id)
+		tree, err := restic.LoadTree(ctx, repo, id)
 		if err != nil {
 			Warnf("unable to load tree %v: %v\n", id.Str(), err)
 			continue
@@ -91,7 +91,7 @@ func runRecover(gopts GlobalOptions) error {
 	bar.Done()
 
 	Verbosef("load snapshots\n")
-	err = restic.ForAllSnapshots(gopts.ctx, snapshotLister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error {
+	err = restic.ForAllSnapshots(ctx, snapshotLister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error {
 		trees[*sn.Tree] = true
 		return nil
 	})
@@ -132,18 +132,18 @@ func runRecover(gopts GlobalOptions) error {
 		}
 	}
 
-	wg, ctx := errgroup.WithContext(gopts.ctx)
-	repo.StartPackUploader(ctx, wg)
+	wg, wgCtx := errgroup.WithContext(ctx)
+	repo.StartPackUploader(wgCtx, wg)
 
 	var treeID restic.ID
 	wg.Go(func() error {
 		var err error
-		treeID, err = restic.SaveTree(ctx, repo, tree)
+		treeID, err = restic.SaveTree(wgCtx, repo, tree)
 		if err != nil {
 			return errors.Fatalf("unable to save new tree to the repository: %v", err)
 		}
 
-		err = repo.Flush(ctx)
+		err = repo.Flush(wgCtx)
 		if err != nil {
 			return errors.Fatalf("unable to save blobs to the repository: %v", err)
 		}
@@ -154,7 +154,7 @@ func runRecover(gopts GlobalOptions) error {
 		return err
 	}
 
-	return createSnapshot(gopts.ctx, "/recover", hostname, []string{"recovered"}, repo, &treeID)
+	return createSnapshot(ctx, "/recover", hostname, []string{"recovered"}, repo, &treeID)
 
 }
 

cmd/restic/cmd_restore.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"strings"
 	"time"
 
@@ -30,7 +31,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runRestore(restoreOptions, globalOptions, args)
+		return runRestore(cmd.Context(), restoreOptions, globalOptions, args)
 	},
 }
 
@@ -41,10 +42,9 @@ type RestoreOptions struct {
 	Include            []string
 	InsensitiveInclude []string
 	Target             string
-	Hosts              []string
-	Paths              []string
-	Tags               restic.TagLists
-	Verify             bool
+	snapshotFilterOptions
+	Sparse bool
+	Verify bool
 }
 
 var restoreOptions RestoreOptions
@@ -59,36 +59,34 @@ func init() {
 	flags.StringArrayVar(&restoreOptions.InsensitiveInclude, "iinclude", nil, "same as `--include` but ignores the casing of filenames")
 	flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to")
 
-	flags.StringArrayVarP(&restoreOptions.Hosts, "host", "H", nil, `only consider snapshots for this host when the snapshot ID is "latest" (can be specified multiple times)`)
-	flags.Var(&restoreOptions.Tags, "tag", "only consider snapshots which include this `taglist` for snapshot ID \"latest\"")
-	flags.StringArrayVar(&restoreOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
+	initSingleSnapshotFilterOptions(flags, &restoreOptions.snapshotFilterOptions)
+	flags.BoolVar(&restoreOptions.Sparse, "sparse", false, "restore files as sparse")
 	flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content")
 }
 
-func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
-	ctx := gopts.ctx
+func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, args []string) error {
 	hasExcludes := len(opts.Exclude) > 0 || len(opts.InsensitiveExclude) > 0
 	hasIncludes := len(opts.Include) > 0 || len(opts.InsensitiveInclude) > 0
 
 	// Validate provided patterns
 	if len(opts.Exclude) > 0 {
-		if valid, invalidPatterns := filter.ValidatePatterns(opts.Exclude); !valid {
-			return errors.Fatalf("--exclude: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
+		if err := filter.ValidatePatterns(opts.Exclude); err != nil {
+			return errors.Fatalf("--exclude: %s", err)
 		}
 	}
 	if len(opts.InsensitiveExclude) > 0 {
-		if valid, invalidPatterns := filter.ValidatePatterns(opts.InsensitiveExclude); !valid {
-			return errors.Fatalf("--iexclude: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
+		if err := filter.ValidatePatterns(opts.InsensitiveExclude); err != nil {
+			return errors.Fatalf("--iexclude: %s", err)
 		}
 	}
 	if len(opts.Include) > 0 {
-		if valid, invalidPatterns := filter.ValidatePatterns(opts.Include); !valid {
-			return errors.Fatalf("--include: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
+		if err := filter.ValidatePatterns(opts.Include); err != nil {
+			return errors.Fatalf("--include: %s", err)
 		}
 	}
 	if len(opts.InsensitiveInclude) > 0 {
-		if valid, invalidPatterns := filter.ValidatePatterns(opts.InsensitiveInclude); !valid {
-			return errors.Fatalf("--iinclude: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
+		if err := filter.ValidatePatterns(opts.InsensitiveInclude); err != nil {
+			return errors.Fatalf("--iinclude: %s", err)
 		}
 	}
 
@@ -119,31 +117,23 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 
 	debug.Log("restore %v to %v", snapshotIDString, opts.Target)
 
-	repo, err := OpenRepository(gopts)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
 
 	if !gopts.NoLock {
-		lock, err := lockRepo(ctx, repo)
+		var lock *restic.Lock
+		lock, ctx, err = lockRepo(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
 		}
 	}
 
-	var id restic.ID
-
-	if snapshotIDString == "latest" {
-		id, err = restic.FindLatestSnapshot(ctx, repo.Backend(), repo, opts.Paths, opts.Tags, opts.Hosts, nil)
-		if err != nil {
-			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Hosts:%v", err, opts.Paths, opts.Hosts)
-		}
-	} else {
-		id, err = restic.FindSnapshot(ctx, repo.Backend(), snapshotIDString)
-		if err != nil {
-			Exitf(1, "invalid id %q: %v", snapshotIDString, err)
-		}
+	sn, err := restic.FindFilteredSnapshot(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, nil, snapshotIDString)
+	if err != nil {
+		return errors.Fatalf("failed to find snapshot: %v", err)
 	}
 
 	err = repo.LoadIndex(ctx)
@@ -151,10 +141,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	res, err := restorer.NewRestorer(ctx, repo, id)
-	if err != nil {
-		Exitf(2, "creating restorer failed: %v\n", err)
-	}
+	res := restorer.NewRestorer(ctx, repo, sn, opts.Sparse)
 
 	totalErrors := 0
 	res.Error = func(location string, err error) error {

cmd/restic/cmd_rewrite.go

@@ -0,0 +1,216 @@
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/spf13/cobra"
+	"golang.org/x/sync/errgroup"
+
+	"github.com/restic/restic/internal/backend"
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/walker"
+)
+
+var cmdRewrite = &cobra.Command{
+	Use:   "rewrite [flags] [snapshotID ...]",
+	Short: "Rewrite snapshots to exclude unwanted files",
+	Long: `
+The "rewrite" command excludes files from existing snapshots. It creates new
+snapshots containing the same data as the original ones, but without the files
+you specify to exclude. All metadata (time, host, tags) will be preserved.
+
+The snapshots to rewrite are specified using the --host, --tag and --path options,
+or by providing a list of snapshot IDs. Please note that specifying neither any of
+these options nor a snapshot ID will cause the command to rewrite all snapshots.
+
+The special tag 'rewrite' will be added to the new snapshots to distinguish
+them from the original ones, unless --forget is used. If the --forget option is
+used, the original snapshots will instead be directly removed from the repository.
+
+Please note that the --forget option only removes the snapshots and not the actual
+data stored in the repository. In order to delete the no longer referenced data,
+use the "prune" command.
+
+EXIT STATUS
+===========
+
+Exit status is 0 if the command was successful, and non-zero if there was any error.
+`,
+	DisableAutoGenTag: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runRewrite(cmd.Context(), rewriteOptions, globalOptions, args)
+	},
+}
+
+// RewriteOptions collects all options for the rewrite command.
+type RewriteOptions struct {
+	Forget bool
+	DryRun bool
+
+	snapshotFilterOptions
+	excludePatternOptions
+}
+
+var rewriteOptions RewriteOptions
+
+func init() {
+	cmdRoot.AddCommand(cmdRewrite)
+
+	f := cmdRewrite.Flags()
+	f.BoolVarP(&rewriteOptions.Forget, "forget", "", false, "remove original snapshots after creating new ones")
+	f.BoolVarP(&rewriteOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done")
+
+	initMultiSnapshotFilterOptions(f, &rewriteOptions.snapshotFilterOptions, true)
+	initExcludePatternOptions(f, &rewriteOptions.excludePatternOptions)
+}
+
+func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, opts RewriteOptions) (bool, error) {
+	if sn.Tree == nil {
+		return false, errors.Errorf("snapshot %v has nil tree", sn.ID().Str())
+	}
+
+	rejectByNameFuncs, err := opts.excludePatternOptions.CollectPatterns()
+	if err != nil {
+		return false, err
+	}
+
+	selectByName := func(nodepath string) bool {
+		for _, reject := range rejectByNameFuncs {
+			if reject(nodepath) {
+				return false
+			}
+		}
+		return true
+	}
+
+	wg, wgCtx := errgroup.WithContext(ctx)
+	repo.StartPackUploader(wgCtx, wg)
+
+	var filteredTree restic.ID
+	wg.Go(func() error {
+		filteredTree, err = walker.FilterTree(wgCtx, repo, "/", *sn.Tree, &walker.TreeFilterVisitor{
+			SelectByName: selectByName,
+			PrintExclude: func(path string) { Verbosef(fmt.Sprintf("excluding %s\n", path)) },
+		})
+		if err != nil {
+			return err
+		}
+
+		return repo.Flush(wgCtx)
+	})
+	err = wg.Wait()
+	if err != nil {
+		return false, err
+	}
+
+	if filteredTree == *sn.Tree {
+		debug.Log("Snapshot %v not modified", sn)
+		return false, nil
+	}
+
+	debug.Log("Snapshot %v modified", sn)
+	if opts.DryRun {
+		Verbosef("would save new snapshot\n")
+
+		if opts.Forget {
+			Verbosef("would remove old snapshot\n")
+		}
+
+		return true, nil
+	}
+
+	// Always set the original snapshot id as this essentially a new snapshot.
+	sn.Original = sn.ID()
+	*sn.Tree = filteredTree
+
+	if !opts.Forget {
+		sn.AddTags([]string{"rewrite"})
+	}
+
+	// Save the new snapshot.
+	id, err := restic.SaveSnapshot(ctx, repo, sn)
+	if err != nil {
+		return false, err
+	}
+
+	if opts.Forget {
+		h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
+		if err = repo.Backend().Remove(ctx, h); err != nil {
+			return false, err
+		}
+		debug.Log("removed old snapshot %v", sn.ID())
+		Verbosef("removed old snapshot %v\n", sn.ID().Str())
+	}
+	Verbosef("saved new snapshot %v\n", id.Str())
+	return true, nil
+}
+
+func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error {
+	if opts.excludePatternOptions.Empty() {
+		return errors.Fatal("Nothing to do: no excludes provided")
+	}
+
+	repo, err := OpenRepository(ctx, gopts)
+	if err != nil {
+		return err
+	}
+
+	if !opts.DryRun {
+		var lock *restic.Lock
+		var err error
+		if opts.Forget {
+			Verbosef("create exclusive lock for repository\n")
+			lock, ctx, err = lockRepoExclusive(ctx, repo)
+		} else {
+			lock, ctx, err = lockRepo(ctx, repo)
+		}
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	} else {
+		repo.SetDryRun()
+	}
+
+	snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
+	if err != nil {
+		return err
+	}
+
+	if err = repo.LoadIndex(ctx); err != nil {
+		return err
+	}
+
+	changedCount := 0
+	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, args) {
+		Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
+		changed, err := rewriteSnapshot(ctx, repo, sn, opts)
+		if err != nil {
+			return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err)
+		}
+		if changed {
+			changedCount++
+		}
+	}
+
+	Verbosef("\n")
+	if changedCount == 0 {
+		if !opts.DryRun {
+			Verbosef("no snapshots were modified\n")
+		} else {
+			Verbosef("no snapshots would be modified\n")
+		}
+	} else {
+		if !opts.DryRun {
+			Verbosef("modified %v snapshots\n", changedCount)
+		} else {
+			Verbosef("would modify %v snapshots\n", changedCount)
+		}
+	}
+
+	return nil
+}

cmd/restic/cmd_self_update.go

@@ -1,8 +1,9 @@
-// xbuild selfupdate
+//go:build selfupdate
 
 package main
 
 import (
+	"context"
 	"os"
 	"path/filepath"
 
@@ -27,7 +28,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runSelfUpdate(selfUpdateOptions, globalOptions, args)
+		return runSelfUpdate(cmd.Context(), selfUpdateOptions, globalOptions, args)
 	},
 }
 
@@ -45,7 +46,7 @@ func init() {
 	flags.StringVar(&selfUpdateOptions.Output, "output", "", "Save the downloaded file as `filename` (default: running binary itself)")
 }
 
-func runSelfUpdate(opts SelfUpdateOptions, gopts GlobalOptions, args []string) error {
+func runSelfUpdate(ctx context.Context, opts SelfUpdateOptions, gopts GlobalOptions, args []string) error {
 	if opts.Output == "" {
 		file, err := os.Executable()
 		if err != nil {
@@ -73,7 +74,7 @@ func runSelfUpdate(opts SelfUpdateOptions, gopts GlobalOptions, args []string) e
 
 	Verbosef("writing restic to %v\n", opts.Output)
 
-	v, err := selfupdate.DownloadLatestStableRelease(gopts.ctx, opts.Output, version, Verbosef)
+	v, err := selfupdate.DownloadLatestStableRelease(ctx, opts.Output, version, Verbosef)
 	if err != nil {
 		return errors.Fatalf("unable to update restic: %v", err)
 	}

cmd/restic/cmd_snapshots.go

@@ -26,15 +26,13 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runSnapshots(snapshotOptions, globalOptions, args)
+		return runSnapshots(cmd.Context(), snapshotOptions, globalOptions, args)
 	},
 }
 
 // SnapshotOptions bundles all options for the snapshots command.
 type SnapshotOptions struct {
-	Hosts   []string
-	Tags    restic.TagLists
-	Paths   []string
+	snapshotFilterOptions
 	Compact bool
 	Last    bool // This option should be removed in favour of Latest.
 	Latest  int
@@ -47,9 +45,7 @@ func init() {
 	cmdRoot.AddCommand(cmdSnapshots)
 
 	f := cmdSnapshots.Flags()
-	f.StringArrayVarP(&snapshotOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host` (can be specified multiple times)")
-	f.Var(&snapshotOptions.Tags, "tag", "only consider snapshots which include this `taglist` in the format `tag[,tag,...]` (can be specified multiple times)")
-	f.StringArrayVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)")
+	initMultiSnapshotFilterOptions(f, &snapshotOptions.snapshotFilterOptions, true)
 	f.BoolVarP(&snapshotOptions.Compact, "compact", "c", false, "use compact output format")
 	f.BoolVar(&snapshotOptions.Last, "last", false, "only show the last snapshot for each host and path")
 	err := f.MarkDeprecated("last", "use --latest 1")
@@ -61,23 +57,21 @@ func init() {
 	f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "`group` snapshots by host, paths and/or tags, separated by comma")
 }
 
-func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error {
-	repo, err := OpenRepository(gopts)
+func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions, args []string) error {
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
 
 	if !gopts.NoLock {
-		lock, err := lockRepo(gopts.ctx, repo)
+		var lock *restic.Lock
+		lock, ctx, err = lockRepo(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
 		}
 	}
 
-	ctx, cancel := context.WithCancel(gopts.ctx)
-	defer cancel()
-
 	var snapshots restic.Snapshots
 	for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) {
 		snapshots = append(snapshots, sn)

cmd/restic/cmd_stats.go

@@ -7,7 +7,9 @@ import (
 	"path/filepath"
 
 	"github.com/restic/restic/internal/backend"
+	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui"
 	"github.com/restic/restic/internal/walker"
 
 	"github.com/minio/sha256-simd"
@@ -47,7 +49,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runStats(globalOptions, args)
+		return runStats(cmd.Context(), globalOptions, args)
 	},
 }
 
@@ -56,10 +58,7 @@ type StatsOptions struct {
 	// the mode of counting to perform (see consts for available modes)
 	countMode string
 
-	// filter snapshots by, if given by user
-	Hosts []string
-	Tags  restic.TagLists
-	Paths []string
+	snapshotFilterOptions
 }
 
 var statsOptions StatsOptions
@@ -68,34 +67,30 @@ func init() {
 	cmdRoot.AddCommand(cmdStats)
 	f := cmdStats.Flags()
 	f.StringVar(&statsOptions.countMode, "mode", countModeRestoreSize, "counting mode: restore-size (default), files-by-contents, blobs-per-file or raw-data")
-	f.StringArrayVarP(&statsOptions.Hosts, "host", "H", nil, "only consider snapshots with the given `host` (can be specified multiple times)")
-	f.Var(&statsOptions.Tags, "tag", "only consider snapshots which include this `taglist` in the format `tag[,tag,...]` (can be specified multiple times)")
-	f.StringArrayVar(&statsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` (can be specified multiple times)")
+	initMultiSnapshotFilterOptions(f, &statsOptions.snapshotFilterOptions, true)
 }
 
-func runStats(gopts GlobalOptions, args []string) error {
+func runStats(ctx context.Context, gopts GlobalOptions, args []string) error {
 	err := verifyStatsInput(gopts, args)
 	if err != nil {
 		return err
 	}
 
-	ctx, cancel := context.WithCancel(gopts.ctx)
-	defer cancel()
-
-	repo, err := OpenRepository(gopts)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
 
 	if !gopts.NoLock {
-		lock, err := lockRepo(ctx, repo)
+		var lock *restic.Lock
+		lock, ctx, err = lockRepo(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
 		}
 	}
 
-	snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile)
+	snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
 	if err != nil {
 		return err
 	}
@@ -135,8 +130,22 @@ func runStats(gopts GlobalOptions, args []string) error {
 				return fmt.Errorf("blob %v not found", blobHandle)
 			}
 			stats.TotalSize += uint64(pbs[0].Length)
+			if repo.Config().Version >= 2 {
+				stats.TotalUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].DataLength())))
+				if pbs[0].IsCompressed() {
+					stats.TotalCompressedBlobsSize += uint64(pbs[0].Length)
+					stats.TotalCompressedBlobsUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].DataLength())))
+				}
+			}
 			stats.TotalBlobCount++
 		}
+		if stats.TotalCompressedBlobsSize > 0 {
+			stats.CompressionRatio = float64(stats.TotalCompressedBlobsUncompressedSize) / float64(stats.TotalCompressedBlobsSize)
+		}
+		if stats.TotalUncompressedSize > 0 {
+			stats.CompressionProgress = float64(stats.TotalCompressedBlobsUncompressedSize) / float64(stats.TotalUncompressedSize) * 100
+			stats.CompressionSpaceSaving = (1 - float64(stats.TotalSize)/float64(stats.TotalUncompressedSize)) * 100
+		}
 	}
 
 	if gopts.JSON {
@@ -148,15 +157,26 @@ func runStats(gopts GlobalOptions, args []string) error {
 	}
 
 	Printf("Stats in %s mode:\n", statsOptions.countMode)
-	Printf("Snapshots processed:   %d\n", stats.SnapshotsCount)
-
+	Printf("     Snapshots processed:  %d\n", stats.SnapshotsCount)
 	if stats.TotalBlobCount > 0 {
-		Printf("   Total Blob Count:   %d\n", stats.TotalBlobCount)
+		Printf("        Total Blob Count:  %d\n", stats.TotalBlobCount)
 	}
 	if stats.TotalFileCount > 0 {
-		Printf("   Total File Count:   %d\n", stats.TotalFileCount)
+		Printf("        Total File Count:  %d\n", stats.TotalFileCount)
+	}
+	if stats.TotalUncompressedSize > 0 {
+		Printf(" Total Uncompressed Size:  %-5s\n", ui.FormatBytes(stats.TotalUncompressedSize))
+	}
+	Printf("              Total Size:  %-5s\n", ui.FormatBytes(stats.TotalSize))
+	if stats.CompressionProgress > 0 {
+		Printf("    Compression Progress:  %.2f%%\n", stats.CompressionProgress)
+	}
+	if stats.CompressionRatio > 0 {
+		Printf("       Compression Ratio:  %.2fx\n", stats.CompressionRatio)
+	}
+	if stats.CompressionSpaceSaving > 0 {
+		Printf("Compression Space Saving:  %.2f%%\n", stats.CompressionSpaceSaving)
 	}
-	Printf("         Total Size:   %-5s\n", formatBytes(stats.TotalSize))
 
 	return nil
 }
@@ -282,9 +302,15 @@ func verifyStatsInput(gopts GlobalOptions, args []string) error {
 // to collect information about it, as well as state needed
 // for a successful and efficient walk.
 type statsContainer struct {
-	TotalSize      uint64 `json:"total_size"`
-	TotalFileCount uint64 `json:"total_file_count"`
-	TotalBlobCount uint64 `json:"total_blob_count,omitempty"`
+	TotalSize                            uint64  `json:"total_size"`
+	TotalUncompressedSize                uint64  `json:"total_uncompressed_size,omitempty"`
+	TotalCompressedBlobsSize             uint64  `json:"-"`
+	TotalCompressedBlobsUncompressedSize uint64  `json:"-"`
+	CompressionRatio                     float64 `json:"compression_ratio,omitempty"`
+	CompressionProgress                  float64 `json:"compression_progress,omitempty"`
+	CompressionSpaceSaving               float64 `json:"compression_space_saving,omitempty"`
+	TotalFileCount                       uint64  `json:"total_file_count,omitempty"`
+	TotalBlobCount                       uint64  `json:"total_blob_count,omitempty"`
 	// holds count of all considered snapshots
 	SnapshotsCount int `json:"snapshots_count"`
 

cmd/restic/cmd_tag.go

@@ -29,15 +29,13 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runTag(tagOptions, globalOptions, args)
+		return runTag(cmd.Context(), tagOptions, globalOptions, args)
 	},
 }
 
 // TagOptions bundles all options for the 'tag' command.
 type TagOptions struct {
-	Hosts      []string
-	Paths      []string
-	Tags       restic.TagLists
+	snapshotFilterOptions
 	SetTags    restic.TagLists
 	AddTags    restic.TagLists
 	RemoveTags restic.TagLists
@@ -52,10 +50,7 @@ func init() {
 	tagFlags.Var(&tagOptions.SetTags, "set", "`tags` which will replace the existing tags in the format `tag[,tag,...]` (can be given multiple times)")
 	tagFlags.Var(&tagOptions.AddTags, "add", "`tags` which will be added to the existing tags in the format `tag[,tag,...]` (can be given multiple times)")
 	tagFlags.Var(&tagOptions.RemoveTags, "remove", "`tags` which will be removed from the existing tags in the format `tag[,tag,...]` (can be given multiple times)")
-
-	tagFlags.StringArrayVarP(&tagOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)")
-	tagFlags.Var(&tagOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot-ID is given")
-	tagFlags.StringArrayVar(&tagOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
+	initMultiSnapshotFilterOptions(tagFlags, &tagOptions.snapshotFilterOptions, true)
 }
 
 func changeTags(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, setTags, addTags, removeTags []string) (bool, error) {
@@ -100,7 +95,7 @@ func changeTags(ctx context.Context, repo *repository.Repository, sn *restic.Sna
 	return changed, nil
 }
 
-func runTag(opts TagOptions, gopts GlobalOptions, args []string) error {
+func runTag(ctx context.Context, opts TagOptions, gopts GlobalOptions, args []string) error {
 	if len(opts.SetTags) == 0 && len(opts.AddTags) == 0 && len(opts.RemoveTags) == 0 {
 		return errors.Fatal("nothing to do!")
 	}
@@ -108,14 +103,15 @@ func runTag(opts TagOptions, gopts GlobalOptions, args []string) error {
 		return errors.Fatal("--set and --add/--remove cannot be given at the same time")
 	}
 
-	repo, err := OpenRepository(gopts)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
 
 	if !gopts.NoLock {
 		Verbosef("create exclusive lock for repository\n")
-		lock, err := lockRepoExclusive(gopts.ctx, repo)
+		var lock *restic.Lock
+		lock, ctx, err = lockRepoExclusive(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -123,8 +119,6 @@ func runTag(opts TagOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	changeCnt := 0
-	ctx, cancel := context.WithCancel(gopts.ctx)
-	defer cancel()
 	for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) {
 		changed, err := changeTags(ctx, repo, sn, opts.SetTags.Flatten(), opts.AddTags.Flatten(), opts.RemoveTags.Flatten())
 		if err != nil {

cmd/restic/cmd_unlock.go

@@ -1,6 +1,8 @@
 package main
 
 import (
+	"context"
+
 	"github.com/restic/restic/internal/restic"
 	"github.com/spf13/cobra"
 )
@@ -18,7 +20,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runUnlock(unlockOptions, globalOptions)
+		return runUnlock(cmd.Context(), unlockOptions, globalOptions)
 	},
 }
 
@@ -35,8 +37,8 @@ func init() {
 	unlockCmd.Flags().BoolVar(&unlockOptions.RemoveAll, "remove-all", false, "remove all locks, even non-stale ones")
 }
 
-func runUnlock(opts UnlockOptions, gopts GlobalOptions) error {
-	repo, err := OpenRepository(gopts)
+func runUnlock(ctx context.Context, opts UnlockOptions, gopts GlobalOptions) error {
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
@@ -46,11 +48,13 @@ func runUnlock(opts UnlockOptions, gopts GlobalOptions) error {
 		fn = restic.RemoveAllLocks
 	}
 
-	err = fn(gopts.ctx, repo)
+	processed, err := fn(ctx, repo)
 	if err != nil {
 		return err
 	}
 
-	Verbosef("successfully removed locks\n")
+	if processed > 0 {
+		Verbosef("successfully removed %d locks\n", processed)
+	}
 	return nil
 }

cmd/restic/delete.go

@@ -1,6 +1,8 @@
 package main
 
 import (
+	"context"
+
 	"golang.org/x/sync/errgroup"
 
 	"github.com/restic/restic/internal/restic"
@@ -8,22 +10,22 @@ import (
 
 // DeleteFiles deletes the given fileList of fileType in parallel
 // it will print a warning if there is an error, but continue deleting the remaining files
-func DeleteFiles(gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) {
-	_ = deleteFiles(gopts, true, repo, fileList, fileType)
+func DeleteFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) {
+	_ = deleteFiles(ctx, gopts, true, repo, fileList, fileType)
 }
 
 // DeleteFilesChecked deletes the given fileList of fileType in parallel
 // if an error occurs, it will cancel and return this error
-func DeleteFilesChecked(gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
-	return deleteFiles(gopts, false, repo, fileList, fileType)
+func DeleteFilesChecked(ctx context.Context, gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
+	return deleteFiles(ctx, gopts, false, repo, fileList, fileType)
 }
 
 // deleteFiles deletes the given fileList of fileType in parallel
 // if ignoreError=true, it will print a warning if there was an error, else it will abort.
-func deleteFiles(gopts GlobalOptions, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
+func deleteFiles(ctx context.Context, gopts GlobalOptions, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
 	totalCount := len(fileList)
 	fileChan := make(chan restic.ID)
-	wg, ctx := errgroup.WithContext(gopts.ctx)
+	wg, ctx := errgroup.WithContext(ctx)
 	wg.Go(func() error {
 		defer close(fileChan)
 		for id := range fileList {

cmd/restic/exclude.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bufio"
 	"bytes"
 	"fmt"
 	"io"
@@ -15,6 +16,8 @@ import (
 	"github.com/restic/restic/internal/filter"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/textfile"
+	"github.com/spf13/pflag"
 )
 
 type rejectionCache struct {
@@ -410,3 +413,115 @@ func parseSizeStr(sizeStr string) (int64, error) {
 	}
 	return value * unit, nil
 }
+
+// readExcludePatternsFromFiles reads all exclude files and returns the list of
+// exclude patterns. For each line, leading and trailing white space is removed
+// and comment lines are ignored. For each remaining pattern, environment
+// variables are resolved. For adding a literal dollar sign ($), write $$ to
+// the file.
+func readExcludePatternsFromFiles(excludeFiles []string) ([]string, error) {
+	getenvOrDollar := func(s string) string {
+		if s == "$" {
+			return "$"
+		}
+		return os.Getenv(s)
+	}
+
+	var excludes []string
+	for _, filename := range excludeFiles {
+		err := func() (err error) {
+			data, err := textfile.Read(filename)
+			if err != nil {
+				return err
+			}
+
+			scanner := bufio.NewScanner(bytes.NewReader(data))
+			for scanner.Scan() {
+				line := strings.TrimSpace(scanner.Text())
+
+				// ignore empty lines
+				if line == "" {
+					continue
+				}
+
+				// strip comments
+				if strings.HasPrefix(line, "#") {
+					continue
+				}
+
+				line = os.Expand(line, getenvOrDollar)
+				excludes = append(excludes, line)
+			}
+			return scanner.Err()
+		}()
+		if err != nil {
+			return nil, err
+		}
+	}
+	return excludes, nil
+}
+
+type excludePatternOptions struct {
+	Excludes                []string
+	InsensitiveExcludes     []string
+	ExcludeFiles            []string
+	InsensitiveExcludeFiles []string
+}
+
+func initExcludePatternOptions(f *pflag.FlagSet, opts *excludePatternOptions) {
+	f.StringArrayVarP(&opts.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
+	f.StringArrayVar(&opts.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
+	f.StringArrayVar(&opts.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
+	f.StringArrayVar(&opts.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns")
+}
+
+func (opts *excludePatternOptions) Empty() bool {
+	return len(opts.Excludes) == 0 && len(opts.InsensitiveExcludes) == 0 && len(opts.ExcludeFiles) == 0 && len(opts.InsensitiveExcludeFiles) == 0
+}
+
+func (opts excludePatternOptions) CollectPatterns() ([]RejectByNameFunc, error) {
+	var fs []RejectByNameFunc
+	// add patterns from file
+	if len(opts.ExcludeFiles) > 0 {
+		excludePatterns, err := readExcludePatternsFromFiles(opts.ExcludeFiles)
+		if err != nil {
+			return nil, err
+		}
+
+		if err := filter.ValidatePatterns(excludePatterns); err != nil {
+			return nil, errors.Fatalf("--exclude-file: %s", err)
+		}
+
+		opts.Excludes = append(opts.Excludes, excludePatterns...)
+	}
+
+	if len(opts.InsensitiveExcludeFiles) > 0 {
+		excludes, err := readExcludePatternsFromFiles(opts.InsensitiveExcludeFiles)
+		if err != nil {
+			return nil, err
+		}
+
+		if err := filter.ValidatePatterns(excludes); err != nil {
+			return nil, errors.Fatalf("--iexclude-file: %s", err)
+		}
+
+		opts.InsensitiveExcludes = append(opts.InsensitiveExcludes, excludes...)
+	}
+
+	if len(opts.InsensitiveExcludes) > 0 {
+		if err := filter.ValidatePatterns(opts.InsensitiveExcludes); err != nil {
+			return nil, errors.Fatalf("--iexclude: %s", err)
+		}
+
+		fs = append(fs, rejectByInsensitivePattern(opts.InsensitiveExcludes))
+	}
+
+	if len(opts.Excludes) > 0 {
+		if err := filter.ValidatePatterns(opts.Excludes); err != nil {
+			return nil, errors.Fatalf("--exclude: %s", err)
+		}
+
+		fs = append(fs, rejectByPattern(opts.Excludes))
+	}
+	return fs, nil
+}

cmd/restic/exclude_test.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
@@ -85,17 +84,16 @@ func TestIsExcludedByFile(t *testing.T) {
 	}
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
-			tempDir, cleanup := test.TempDir(t)
-			defer cleanup()
+			tempDir := test.TempDir(t)
 
 			foo := filepath.Join(tempDir, "foo")
-			err := ioutil.WriteFile(foo, []byte("foo"), 0666)
+			err := os.WriteFile(foo, []byte("foo"), 0666)
 			if err != nil {
 				t.Fatalf("could not write file: %v", err)
 			}
 			if tc.tagFile != "" {
 				tagFile := filepath.Join(tempDir, tc.tagFile)
-				err = ioutil.WriteFile(tagFile, []byte(tc.content), 0666)
+				err = os.WriteFile(tagFile, []byte(tc.content), 0666)
 				if err != nil {
 					t.Fatalf("could not write tagfile: %v", err)
 				}
@@ -116,8 +114,7 @@ func TestIsExcludedByFile(t *testing.T) {
 // cancel each other out. It was initially written to demonstrate a bug in
 // rejectIfPresent.
 func TestMultipleIsExcludedByFile(t *testing.T) {
-	tempDir, cleanup := test.TempDir(t)
-	defer cleanup()
+	tempDir := test.TempDir(t)
 
 	// Create some files in a temporary directory.
 	// Files in UPPERCASE will be used as exclusion triggers later on.
@@ -150,7 +147,7 @@ func TestMultipleIsExcludedByFile(t *testing.T) {
 		// create directories first, then the file
 		p := filepath.Join(tempDir, filepath.FromSlash(f.path))
 		errs = append(errs, os.MkdirAll(filepath.Dir(p), 0700))
-		errs = append(errs, ioutil.WriteFile(p, []byte(f.path), 0600))
+		errs = append(errs, os.WriteFile(p, []byte(f.path), 0600))
 	}
 	test.OKs(t, errs) // see if anything went wrong during the creation
 
@@ -241,8 +238,7 @@ func TestParseInvalidSizeStr(t *testing.T) {
 // TestIsExcludedByFileSize is for testing the instance of
 // --exclude-larger-than parameters
 func TestIsExcludedByFileSize(t *testing.T) {
-	tempDir, cleanup := test.TempDir(t)
-	defer cleanup()
+	tempDir := test.TempDir(t)
 
 	// Max size of file is set to be 1k
 	maxSizeStr := "1k"

cmd/restic/find.go

@@ -5,77 +5,60 @@ import (
 
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/restic"
+	"github.com/spf13/pflag"
 )
 
+type snapshotFilterOptions struct {
+	Hosts []string
+	Tags  restic.TagLists
+	Paths []string
+}
+
+// initMultiSnapshotFilterOptions is used for commands that work on multiple snapshots
+// MUST be combined with restic.FindFilteredSnapshots or FindFilteredSnapshots
+func initMultiSnapshotFilterOptions(flags *pflag.FlagSet, options *snapshotFilterOptions, addHostShorthand bool) {
+	hostShorthand := "H"
+	if !addHostShorthand {
+		hostShorthand = ""
+	}
+	flags.StringArrayVarP(&options.Hosts, "host", hostShorthand, nil, "only consider snapshots for this `host` (can be specified multiple times)")
+	flags.Var(&options.Tags, "tag", "only consider snapshots including `tag[,tag,...]` (can be specified multiple times)")
+	flags.StringArrayVar(&options.Paths, "path", nil, "only consider snapshots including this (absolute) `path` (can be specified multiple times)")
+}
+
+// initSingleSnapshotFilterOptions is used for commands that work on a single snapshot
+// MUST be combined with restic.FindFilteredSnapshot
+func initSingleSnapshotFilterOptions(flags *pflag.FlagSet, options *snapshotFilterOptions) {
+	flags.StringArrayVarP(&options.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when snapshot ID \"latest\" is given (can be specified multiple times)")
+	flags.Var(&options.Tags, "tag", "only consider snapshots including `tag[,tag,...]`, when snapshot ID \"latest\" is given (can be specified multiple times)")
+	flags.StringArrayVar(&options.Paths, "path", nil, "only consider snapshots including this (absolute) `path`, when snapshot ID \"latest\" is given (can be specified multiple times)")
+}
+
 // FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots.
 func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic.LoaderUnpacked, hosts []string, tags []restic.TagList, paths []string, snapshotIDs []string) <-chan *restic.Snapshot {
 	out := make(chan *restic.Snapshot)
 	go func() {
 		defer close(out)
-		if len(snapshotIDs) != 0 {
-			// memorize snapshots list to prevent repeated backend listings
-			be, err := backend.MemorizeList(ctx, be, restic.SnapshotFile)
-			if err != nil {
-				Warnf("could not load snapshots: %v\n", err)
-				return
-			}
-
-			var (
-				id         restic.ID
-				usedFilter bool
-			)
-			ids := make(restic.IDs, 0, len(snapshotIDs))
-			// Process all snapshot IDs given as arguments.
-			for _, s := range snapshotIDs {
-				if s == "latest" {
-					usedFilter = true
-					id, err = restic.FindLatestSnapshot(ctx, be, loader, paths, tags, hosts, nil)
-					if err != nil {
-						Warnf("Ignoring %q, no snapshot matched given filter (Paths:%v Tags:%v Hosts:%v)\n", s, paths, tags, hosts)
-						continue
-					}
-				} else {
-					id, err = restic.FindSnapshot(ctx, be, s)
-					if err != nil {
-						Warnf("Ignoring %q: %v\n", s, err)
-						continue
-					}
-				}
-				ids = append(ids, id)
-			}
-
-			// Give the user some indication their filters are not used.
-			if !usedFilter && (len(hosts) != 0 || len(tags) != 0 || len(paths) != 0) {
-				Warnf("Ignoring filters as there are explicit snapshot ids given\n")
-			}
-
-			for _, id := range ids.Uniq() {
-				sn, err := restic.LoadSnapshot(ctx, loader, id)
-				if err != nil {
-					Warnf("Ignoring %q, could not load snapshot: %v\n", id, err)
-					continue
-				}
-				select {
-				case <-ctx.Done():
-					return
-				case out <- sn:
-				}
-			}
-			return
-		}
-
-		snapshots, err := restic.FindFilteredSnapshots(ctx, be, loader, hosts, tags, paths)
+		be, err := backend.MemorizeList(ctx, be, restic.SnapshotFile)
 		if err != nil {
 			Warnf("could not load snapshots: %v\n", err)
 			return
 		}
 
-		for _, sn := range snapshots {
-			select {
-			case <-ctx.Done():
-				return
-			case out <- sn:
+		err = restic.FindFilteredSnapshots(ctx, be, loader, hosts, tags, paths, snapshotIDs, func(id string, sn *restic.Snapshot, err error) error {
+			if err != nil {
+				Warnf("Ignoring %q: %v\n", id, err)
+			} else {
+				select {
+				case <-ctx.Done():
+					return ctx.Err()
+				case out <- sn:
+				}
 			}
+			return nil
+		})
+		if err != nil {
+			Warnf("could not load snapshots: %v\n", err)
 		}
 	}()
 	return out

cmd/restic/flags_test.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"io/ioutil"
+	"io"
 	"testing"
 )
 
@@ -10,7 +10,7 @@ import (
 func TestFlags(t *testing.T) {
 	for _, cmd := range cmdRoot.Commands() {
 		t.Run(cmd.Name(), func(t *testing.T) {
-			cmd.Flags().SetOutput(ioutil.Discard)
+			cmd.Flags().SetOutput(io.Discard)
 			err := cmd.ParseFlags([]string{"--help"})
 			if err.Error() == "pflag: help requested" {
 				err = nil

cmd/restic/format.go

@@ -3,59 +3,10 @@ package main
 import (
 	"fmt"
 	"os"
-	"time"
 
 	"github.com/restic/restic/internal/restic"
 )
 
-func formatBytes(c uint64) string {
-	b := float64(c)
-
-	switch {
-	case c > 1<<40:
-		return fmt.Sprintf("%.3f TiB", b/(1<<40))
-	case c > 1<<30:
-		return fmt.Sprintf("%.3f GiB", b/(1<<30))
-	case c > 1<<20:
-		return fmt.Sprintf("%.3f MiB", b/(1<<20))
-	case c > 1<<10:
-		return fmt.Sprintf("%.3f KiB", b/(1<<10))
-	default:
-		return fmt.Sprintf("%d B", c)
-	}
-}
-
-func formatSeconds(sec uint64) string {
-	hours := sec / 3600
-	sec -= hours * 3600
-	min := sec / 60
-	sec -= min * 60
-	if hours > 0 {
-		return fmt.Sprintf("%d:%02d:%02d", hours, min, sec)
-	}
-
-	return fmt.Sprintf("%d:%02d", min, sec)
-}
-
-func formatPercent(numerator uint64, denominator uint64) string {
-	if denominator == 0 {
-		return ""
-	}
-
-	percent := 100.0 * float64(numerator) / float64(denominator)
-
-	if percent > 100 {
-		percent = 100
-	}
-
-	return fmt.Sprintf("%3.2f%%", percent)
-}
-
-func formatDuration(d time.Duration) string {
-	sec := uint64(d / time.Second)
-	return formatSeconds(sec)
-}
-
 func formatNode(path string, n *restic.Node, long bool) string {
 	if !long {
 		return path

cmd/restic/global.go

@@ -22,6 +22,7 @@ import (
 	"github.com/restic/restic/internal/backend/location"
 	"github.com/restic/restic/internal/backend/rclone"
 	"github.com/restic/restic/internal/backend/rest"
+	"github.com/restic/restic/internal/backend/retry"
 	"github.com/restic/restic/internal/backend/s3"
 	"github.com/restic/restic/internal/backend/sftp"
 	"github.com/restic/restic/internal/backend/swift"
@@ -41,7 +42,7 @@ import (
 	"golang.org/x/term"
 )
 
-var version = "0.14.0"
+var version = "0.15.0"
 
 // TimeFormat is the format used for all timestamps printed by restic.
 const TimeFormat = "2006-01-02 15:04:05"
@@ -68,7 +69,6 @@ type GlobalOptions struct {
 	backend.TransportOptions
 	limiter.Limits
 
-	ctx      context.Context
 	password string
 	stdout   io.Writer
 	stderr   io.Writer
@@ -93,28 +93,26 @@ var globalOptions = GlobalOptions{
 }
 
 var isReadingPassword bool
+var internalGlobalCtx context.Context
 
 func init() {
 	var cancel context.CancelFunc
-	globalOptions.ctx, cancel = context.WithCancel(context.Background())
-	AddCleanupHandler(func() error {
+	internalGlobalCtx, cancel = context.WithCancel(context.Background())
+	AddCleanupHandler(func(code int) (int, error) {
 		// Must be called before the unlock cleanup handler to ensure that the latter is
 		// not blocked due to limited number of backend connections, see #1434
 		cancel()
-		return nil
+		return code, nil
 	})
 
-	// parse target pack size from env, on error the default value will be used
-	targetPackSize, _ := strconv.ParseUint(os.Getenv("RESTIC_PACK_SIZE"), 10, 32)
-
 	f := cmdRoot.PersistentFlags()
-	f.StringVarP(&globalOptions.Repo, "repo", "r", os.Getenv("RESTIC_REPOSITORY"), "`repository` to backup to or restore from (default: $RESTIC_REPOSITORY)")
-	f.StringVarP(&globalOptions.RepositoryFile, "repository-file", "", os.Getenv("RESTIC_REPOSITORY_FILE"), "`file` to read the repository location from (default: $RESTIC_REPOSITORY_FILE)")
-	f.StringVarP(&globalOptions.PasswordFile, "password-file", "p", os.Getenv("RESTIC_PASSWORD_FILE"), "`file` to read the repository password from (default: $RESTIC_PASSWORD_FILE)")
-	f.StringVarP(&globalOptions.KeyHint, "key-hint", "", os.Getenv("RESTIC_KEY_HINT"), "`key` ID of key to try decrypting first (default: $RESTIC_KEY_HINT)")
-	f.StringVarP(&globalOptions.PasswordCommand, "password-command", "", os.Getenv("RESTIC_PASSWORD_COMMAND"), "shell `command` to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND)")
+	f.StringVarP(&globalOptions.Repo, "repo", "r", "", "`repository` to backup to or restore from (default: $RESTIC_REPOSITORY)")
+	f.StringVarP(&globalOptions.RepositoryFile, "repository-file", "", "", "`file` to read the repository location from (default: $RESTIC_REPOSITORY_FILE)")
+	f.StringVarP(&globalOptions.PasswordFile, "password-file", "p", "", "`file` to read the repository password from (default: $RESTIC_PASSWORD_FILE)")
+	f.StringVarP(&globalOptions.KeyHint, "key-hint", "", "", "`key` ID of key to try decrypting first (default: $RESTIC_KEY_HINT)")
+	f.StringVarP(&globalOptions.PasswordCommand, "password-command", "", "", "shell `command` to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND)")
 	f.BoolVarP(&globalOptions.Quiet, "quiet", "q", false, "do not output comprehensive progress report")
-	f.CountVarP(&globalOptions.Verbose, "verbose", "v", "be verbose (specify multiple times or a level using --verbose=`n`, max level/times is 3)")
+	f.CountVarP(&globalOptions.Verbose, "verbose", "v", "be verbose (specify multiple times or a level using --verbose=`n`, max level/times is 2)")
 	f.BoolVar(&globalOptions.NoLock, "no-lock", false, "do not lock the repository, this allows some operations on read-only repositories")
 	f.BoolVarP(&globalOptions.JSON, "json", "", false, "set output mode to JSON for commands that support it")
 	f.StringVar(&globalOptions.CacheDir, "cache-dir", "", "set the cache `directory`. (default: use system default cache directory)")
@@ -124,18 +122,26 @@ func init() {
 	f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)")
 	f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories")
 	f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max)")
-	f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum rate in KiB/s. (default: unlimited)")
-	f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum rate in KiB/s. (default: unlimited)")
-	f.UintVar(&globalOptions.PackSize, "pack-size", uint(targetPackSize), "set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)")
+	f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum `rate` in KiB/s. (default: unlimited)")
+	f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum `rate` in KiB/s. (default: unlimited)")
+	f.UintVar(&globalOptions.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)")
 	f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)")
 	// Use our "generate" command instead of the cobra provided "completion" command
 	cmdRoot.CompletionOptions.DisableDefaultCmd = true
 
+	globalOptions.Repo = os.Getenv("RESTIC_REPOSITORY")
+	globalOptions.RepositoryFile = os.Getenv("RESTIC_REPOSITORY_FILE")
+	globalOptions.PasswordFile = os.Getenv("RESTIC_PASSWORD_FILE")
+	globalOptions.KeyHint = os.Getenv("RESTIC_KEY_HINT")
+	globalOptions.PasswordCommand = os.Getenv("RESTIC_PASSWORD_COMMAND")
 	comp := os.Getenv("RESTIC_COMPRESSION")
 	if comp != "" {
 		// ignore error as there's no good way to handle it
 		_ = globalOptions.Compression.Set(comp)
 	}
+	// parse target pack size from env, on error the default value will be used
+	targetPackSize, _ := strconv.ParseUint(os.Getenv("RESTIC_PACK_SIZE"), 10, 32)
+	globalOptions.PackSize = uint(targetPackSize)
 
 	restoreTerminal()
 }
@@ -194,20 +200,20 @@ func restoreTerminal() {
 		return
 	}
 
-	AddCleanupHandler(func() error {
+	AddCleanupHandler(func(code int) (int, error) {
 		// Restoring the terminal configuration while restic runs in the
 		// background, causes restic to get stopped on unix systems with
 		// a SIGTTOU signal. Thus only restore the terminal settings if
 		// they might have been modified, which is the case while reading
 		// a password.
 		if !isReadingPassword {
-			return nil
+			return code, nil
 		}
 		err := checkErrno(term.Restore(fd, state))
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "unable to restore terminal state: %v\n", err)
 		}
-		return err
+		return code, err
 	})
 }
 
@@ -275,17 +281,6 @@ func Warnf(format string, args ...interface{}) {
 	}
 }
 
-// Exitf uses Warnf to write the message and then terminates the process with
-// the given exit code.
-func Exitf(exitcode int, format string, args ...interface{}) {
-	if !(strings.HasSuffix(format, "\n")) {
-		format += "\n"
-	}
-
-	Warnf(format, args...)
-	Exit(exitcode)
-}
-
 // resolvePassword determines the password to be used for opening the repository.
 func resolvePassword(opts GlobalOptions, envStr string) (string, error) {
 	if opts.PasswordFile != "" && opts.PasswordCommand != "" {
@@ -423,20 +418,24 @@ func ReadRepo(opts GlobalOptions) (string, error) {
 const maxKeys = 20
 
 // OpenRepository reads the password and opens the repository.
-func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
+func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Repository, error) {
 	repo, err := ReadRepo(opts)
 	if err != nil {
 		return nil, err
 	}
 
-	be, err := open(repo, opts, opts.extended)
+	be, err := open(ctx, repo, opts, opts.extended)
 	if err != nil {
 		return nil, err
 	}
 
-	be = backend.NewRetryBackend(be, 10, func(msg string, err error, d time.Duration) {
+	report := func(msg string, err error, d time.Duration) {
 		Warnf("%v returned error, retrying after %v: %v\n", msg, d, err)
-	})
+	}
+	success := func(msg string, retries int) {
+		Warnf("%v operation successful after %d retries\n", msg, retries)
+	}
+	be = retry.New(be, 10, report, success)
 
 	// wrap backend if a test specified a hook
 	if opts.backendTestHook != nil {
@@ -469,7 +468,7 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
 			continue
 		}
 
-		err = s.SearchKey(opts.ctx, opts.password, maxKeys, opts.KeyHint)
+		err = s.SearchKey(ctx, opts.password, maxKeys, opts.KeyHint)
 		if err != nil && passwordTriesLeft > 1 {
 			opts.password = ""
 			fmt.Fprintf(os.Stderr, "%s. Try again\n", err)
@@ -488,7 +487,11 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
 			id = id[:8]
 		}
 		if !opts.JSON {
-			Verbosef("repository %v opened (repository version %v) successfully, password is correct\n", id, s.Config().Version)
+			extra := ""
+			if s.Config().Version >= 2 {
+				extra = ", compression level " + opts.Compression.String()
+			}
+			Verbosef("repository %v opened (version %v%s)\n", id, s.Config().Version, extra)
 		}
 	}
 
@@ -686,7 +689,7 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
 }
 
 // Open the backend specified by a location config.
-func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) {
+func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) {
 	debug.Log("parsing location %v", location.StripPassword(s))
 	loc, err := location.Parse(s)
 	if err != nil {
@@ -711,19 +714,19 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
 
 	switch loc.Scheme {
 	case "local":
-		be, err = local.Open(globalOptions.ctx, cfg.(local.Config))
+		be, err = local.Open(ctx, cfg.(local.Config))
 	case "sftp":
-		be, err = sftp.Open(globalOptions.ctx, cfg.(sftp.Config))
+		be, err = sftp.Open(ctx, cfg.(sftp.Config))
 	case "s3":
-		be, err = s3.Open(globalOptions.ctx, cfg.(s3.Config), rt)
+		be, err = s3.Open(ctx, cfg.(s3.Config), rt)
 	case "gs":
 		be, err = gs.Open(cfg.(gs.Config), rt)
 	case "azure":
-		be, err = azure.Open(cfg.(azure.Config), rt)
+		be, err = azure.Open(ctx, cfg.(azure.Config), rt)
 	case "swift":
-		be, err = swift.Open(globalOptions.ctx, cfg.(swift.Config), rt)
+		be, err = swift.Open(ctx, cfg.(swift.Config), rt)
 	case "b2":
-		be, err = b2.Open(globalOptions.ctx, cfg.(b2.Config), rt)
+		be, err = b2.Open(ctx, cfg.(b2.Config), rt)
 	case "rest":
 		be, err = rest.Open(cfg.(rest.Config), rt)
 	case "rclone":
@@ -751,7 +754,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
 	}
 
 	// check if config is there
-	fi, err := be.Stat(globalOptions.ctx, restic.Handle{Type: restic.ConfigFile})
+	fi, err := be.Stat(ctx, restic.Handle{Type: restic.ConfigFile})
 	if err != nil {
 		return nil, errors.Fatalf("unable to open config file: %v\nIs there a repository at the following location?\n%v", err, location.StripPassword(s))
 	}
@@ -764,7 +767,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
 }
 
 // Create the backend specified by URI.
-func create(s string, opts options.Options) (restic.Backend, error) {
+func create(ctx context.Context, s string, opts options.Options) (restic.Backend, error) {
 	debug.Log("parsing location %v", s)
 	loc, err := location.Parse(s)
 	if err != nil {
@@ -783,23 +786,23 @@ func create(s string, opts options.Options) (restic.Backend, error) {
 
 	switch loc.Scheme {
 	case "local":
-		return local.Create(globalOptions.ctx, cfg.(local.Config))
+		return local.Create(ctx, cfg.(local.Config))
 	case "sftp":
-		return sftp.Create(globalOptions.ctx, cfg.(sftp.Config))
+		return sftp.Create(ctx, cfg.(sftp.Config))
 	case "s3":
-		return s3.Create(globalOptions.ctx, cfg.(s3.Config), rt)
+		return s3.Create(ctx, cfg.(s3.Config), rt)
 	case "gs":
 		return gs.Create(cfg.(gs.Config), rt)
 	case "azure":
-		return azure.Create(cfg.(azure.Config), rt)
+		return azure.Create(ctx, cfg.(azure.Config), rt)
 	case "swift":
-		return swift.Open(globalOptions.ctx, cfg.(swift.Config), rt)
+		return swift.Open(ctx, cfg.(swift.Config), rt)
 	case "b2":
-		return b2.Create(globalOptions.ctx, cfg.(b2.Config), rt)
+		return b2.Create(ctx, cfg.(b2.Config), rt)
 	case "rest":
-		return rest.Create(globalOptions.ctx, cfg.(rest.Config), rt)
+		return rest.Create(ctx, cfg.(rest.Config), rt)
 	case "rclone":
-		return rclone.Create(globalOptions.ctx, cfg.(rclone.Config))
+		return rclone.Create(ctx, cfg.(rclone.Config))
 	}
 
 	debug.Log("invalid repository scheme: %v", s)

cmd/restic/global_debug.go

@@ -84,9 +84,9 @@ func runDebug() error {
 	}
 
 	if prof != nil {
-		AddCleanupHandler(func() error {
+		AddCleanupHandler(func(code int) (int, error) {
 			prof.Stop()
-			return nil
+			return code, nil
 		})
 	}
 

cmd/restic/global_test.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"bytes"
-	"io/ioutil"
+	"os"
 	"path/filepath"
 	"testing"
 
@@ -31,8 +31,7 @@ func Test_PrintFunctionsRespectsGlobalStdout(t *testing.T) {
 }
 
 func TestReadRepo(t *testing.T) {
-	tempDir, cleanup := test.TempDir(t)
-	defer cleanup()
+	tempDir := test.TempDir(t)
 
 	// test --repo option
 	var opts GlobalOptions
@@ -43,7 +42,7 @@ func TestReadRepo(t *testing.T) {
 
 	// test --repository-file option
 	foo := filepath.Join(tempDir, "foo")
-	err = ioutil.WriteFile(foo, []byte(tempDir+"\n"), 0666)
+	err = os.WriteFile(foo, []byte(tempDir+"\n"), 0666)
 	rtest.OK(t, err)
 
 	var opts2 GlobalOptions

cmd/restic/integration_filter_pattern_test.go

@@ -1,14 +1,7 @@
-//go:build go1.16
-// +build go1.16
-
-// Before Go 1.16 filepath.Match returned early on a failed match,
-// and thus did not report any later syntax error in the pattern.
-// https://go.dev/doc/go1.16#path/filepath
-
 package main
 
 import (
-	"io/ioutil"
+	"os"
 	"path/filepath"
 	"testing"
 
@@ -24,14 +17,14 @@ func TestBackupFailsWhenUsingInvalidPatterns(t *testing.T) {
 	var err error
 
 	// Test --exclude
-	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts)
+	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
 
 	rtest.Equals(t, `Fatal: --exclude: invalid pattern(s) provided:
 *[._]log[.-][0-9]
 !*[._]log[.-][0-9]`, err.Error())
 
 	// Test --iexclude
-	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts)
+	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
 
 	rtest.Equals(t, `Fatal: --iexclude: invalid pattern(s) provided:
 *[._]log[.-][0-9]
@@ -46,7 +39,7 @@ func TestBackupFailsWhenUsingInvalidPatternsFromFile(t *testing.T) {
 
 	// Create an exclude file with some invalid patterns
 	excludeFile := env.base + "/excludefile"
-	fileErr := ioutil.WriteFile(excludeFile, []byte("*.go\n*[._]log[.-][0-9]\n!*[._]log[.-][0-9]"), 0644)
+	fileErr := os.WriteFile(excludeFile, []byte("*.go\n*[._]log[.-][0-9]\n!*[._]log[.-][0-9]"), 0644)
 	if fileErr != nil {
 		t.Fatalf("Could not write exclude file: %v", fileErr)
 	}
@@ -54,14 +47,14 @@ func TestBackupFailsWhenUsingInvalidPatternsFromFile(t *testing.T) {
 	var err error
 
 	// Test --exclude-file:
-	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludeFiles: []string{excludeFile}}, env.gopts)
+	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{ExcludeFiles: []string{excludeFile}}}, env.gopts)
 
 	rtest.Equals(t, `Fatal: --exclude-file: invalid pattern(s) provided:
 *[._]log[.-][0-9]
 !*[._]log[.-][0-9]`, err.Error())
 
 	// Test --iexclude-file
-	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{InsensitiveExcludeFiles: []string{excludeFile}}, env.gopts)
+	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludeFiles: []string{excludeFile}}}, env.gopts)
 
 	rtest.Equals(t, `Fatal: --iexclude-file: invalid pattern(s) provided:
 *[._]log[.-][0-9]

cmd/restic/integration_fuse_test.go

@@ -4,9 +4,11 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
+	"sync"
 	"testing"
 	"time"
 
@@ -53,11 +55,12 @@ func waitForMount(t testing.TB, dir string) {
 	t.Errorf("subdir %q of dir %s never appeared", mountTestSubdir, dir)
 }
 
-func testRunMount(t testing.TB, gopts GlobalOptions, dir string) {
+func testRunMount(t testing.TB, gopts GlobalOptions, dir string, wg *sync.WaitGroup) {
+	defer wg.Done()
 	opts := MountOptions{
 		TimeTemplate: time.RFC3339,
 	}
-	rtest.OK(t, runMount(opts, gopts, []string{dir}))
+	rtest.OK(t, runMount(context.TODO(), opts, gopts, []string{dir}))
 }
 
 func testRunUmount(t testing.TB, gopts GlobalOptions, dir string) {
@@ -86,8 +89,11 @@ func listSnapshots(t testing.TB, dir string) []string {
 func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Repository, mountpoint, repodir string, snapshotIDs restic.IDs, expectedSnapshotsInFuseDir int) {
 	t.Logf("checking for %d snapshots: %v", len(snapshotIDs), snapshotIDs)
 
-	go testRunMount(t, global, mountpoint)
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go testRunMount(t, global, mountpoint, &wg)
 	waitForMount(t, mountpoint)
+	defer wg.Wait()
 	defer testRunUmount(t, global, mountpoint)
 
 	if !snapshotsDirExists(t, mountpoint) {
@@ -119,7 +125,7 @@ func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Reposit
 	}
 
 	for _, id := range snapshotIDs {
-		snapshot, err := restic.LoadSnapshot(global.ctx, repo, id)
+		snapshot, err := restic.LoadSnapshot(context.TODO(), repo, id)
 		rtest.OK(t, err)
 
 		ts := snapshot.Time.Format(time.RFC3339)
@@ -160,7 +166,7 @@ func TestMount(t *testing.T) {
 
 	testRunInit(t, env.gopts)
 
-	repo, err := OpenRepository(env.gopts)
+	repo, err := OpenRepository(context.TODO(), env.gopts)
 	rtest.OK(t, err)
 
 	checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, []restic.ID{}, 0)
@@ -205,7 +211,7 @@ func TestMountSameTimestamps(t *testing.T) {
 
 	rtest.SetupTarTestFixture(t, env.base, filepath.Join("testdata", "repo-same-timestamps.tar.gz"))
 
-	repo, err := OpenRepository(env.gopts)
+	repo, err := OpenRepository(context.TODO(), env.gopts)
 	rtest.OK(t, err)
 
 	ids := []restic.ID{

cmd/restic/integration_helpers_test.go

@@ -2,14 +2,13 @@ package main
 
 import (
 	"bytes"
-	"context"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
 	"testing"
 
+	"github.com/restic/restic/internal/backend/retry"
 	"github.com/restic/restic/internal/options"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
@@ -172,8 +171,9 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) {
 
 	repository.TestUseLowSecurityKDFParameters(t)
 	restic.TestDisableCheckPolynomial(t)
+	retry.TestFastRetries(t)
 
-	tempdir, err := ioutil.TempDir(rtest.TestTempDir, "restic-test-")
+	tempdir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-")
 	rtest.OK(t, err)
 
 	env = &testEnvironment{
@@ -193,7 +193,6 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) {
 		Repo:     env.repo,
 		Quiet:    true,
 		CacheDir: env.cache,
-		ctx:      context.Background(),
 		password: rtest.TestPassword,
 		stdout:   os.Stdout,
 		stderr:   os.Stderr,

cmd/restic/integration_helpers_unix_test.go

@@ -6,7 +6,6 @@ package main
 import (
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"syscall"
@@ -57,7 +56,7 @@ func nlink(info os.FileInfo) uint64 {
 func createFileSetPerHardlink(dir string) map[uint64][]string {
 	var stat syscall.Stat_t
 	linkTests := make(map[uint64][]string)
-	files, err := ioutil.ReadDir(dir)
+	files, err := os.ReadDir(dir)
 	if err != nil {
 		return nil
 	}

cmd/restic/integration_helpers_windows_test.go

@@ -6,7 +6,6 @@ package main
 import (
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 )
 
@@ -39,7 +38,7 @@ func inode(info os.FileInfo) uint64 {
 
 func createFileSetPerHardlink(dir string) map[uint64][]string {
 	linkTests := make(map[uint64][]string)
-	files, err := ioutil.ReadDir(dir)
+	files, err := os.ReadDir(dir)
 	if err != nil {
 		return nil
 	}

cmd/restic/integration_rewrite_test.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+
+	"github.com/restic/restic/internal/restic"
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool) {
+	opts := RewriteOptions{
+		excludePatternOptions: excludePatternOptions{
+			Excludes: excludes,
+		},
+		Forget: forget,
+	}
+
+	rtest.OK(t, runRewrite(context.TODO(), opts, gopts, nil))
+}
+
+func createBasicRewriteRepo(t testing.TB, env *testEnvironment) restic.ID {
+	testSetupBackupData(t, env)
+
+	// create backup
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{}, env.gopts)
+	snapshotIDs := testRunList(t, "snapshots", env.gopts)
+	rtest.Assert(t, len(snapshotIDs) == 1, "expected one snapshot, got %v", snapshotIDs)
+	testRunCheck(t, env.gopts)
+
+	return snapshotIDs[0]
+}
+
+func TestRewrite(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+	createBasicRewriteRepo(t, env)
+
+	// exclude some data
+	testRunRewriteExclude(t, env.gopts, []string{"3"}, false)
+	snapshotIDs := testRunList(t, "snapshots", env.gopts)
+	rtest.Assert(t, len(snapshotIDs) == 2, "expected two snapshots, got %v", snapshotIDs)
+	testRunCheck(t, env.gopts)
+}
+
+func TestRewriteUnchanged(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+	snapshotID := createBasicRewriteRepo(t, env)
+
+	// use an exclude that will not exclude anything
+	testRunRewriteExclude(t, env.gopts, []string{"3dflkhjgdflhkjetrlkhjgfdlhkj"}, false)
+	newSnapshotIDs := testRunList(t, "snapshots", env.gopts)
+	rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs)
+	rtest.Assert(t, snapshotID == newSnapshotIDs[0], "snapshot id changed unexpectedly")
+	testRunCheck(t, env.gopts)
+}
+
+func TestRewriteReplace(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+	snapshotID := createBasicRewriteRepo(t, env)
+
+	// exclude some data
+	testRunRewriteExclude(t, env.gopts, []string{"3"}, true)
+	newSnapshotIDs := testRunList(t, "snapshots", env.gopts)
+	rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs)
+	rtest.Assert(t, snapshotID != newSnapshotIDs[0], "snapshot id should have changed")
+	// check forbids unused blobs, thus remove them first
+	testRunPrune(t, env.gopts, PruneOptions{MaxUnused: "0"})
+	testRunCheck(t, env.gopts)
+}

cmd/restic/integration_test.go

@@ -8,7 +8,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	mrand "math/rand"
 	"os"
 	"path/filepath"
@@ -23,6 +22,7 @@ import (
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/filter"
 	"github.com/restic/restic/internal/fs"
+	"github.com/restic/restic/internal/index"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
@@ -52,26 +52,26 @@ func testRunInit(t testing.TB, opts GlobalOptions) {
 	restic.TestDisableCheckPolynomial(t)
 	restic.TestSetLockTimeout(t, 0)
 
-	rtest.OK(t, runInit(InitOptions{}, opts, nil))
+	rtest.OK(t, runInit(context.TODO(), InitOptions{}, opts, nil))
 	t.Logf("repository initialized at %v", opts.Repo)
 }
 
 func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts BackupOptions, gopts GlobalOptions) error {
-	ctx, cancel := context.WithCancel(gopts.ctx)
+	ctx, cancel := context.WithCancel(context.TODO())
 	defer cancel()
 
 	var wg errgroup.Group
 	term := termstatus.New(gopts.stdout, gopts.stderr, gopts.Quiet)
 	wg.Go(func() error { term.Run(ctx); return nil })
 
-	gopts.stdout = ioutil.Discard
+	gopts.stdout = io.Discard
 	t.Logf("backing up %v in %v", target, dir)
 	if dir != "" {
 		cleanup := rtest.Chdir(t, dir)
 		defer cleanup()
 	}
 
-	backupErr := runBackup(opts, gopts, term, target)
+	backupErr := runBackup(ctx, opts, gopts, term, target)
 
 	cancel()
 
@@ -95,7 +95,7 @@ func testRunList(t testing.TB, tpe string, opts GlobalOptions) restic.IDs {
 		globalOptions.stdout = os.Stdout
 	}()
 
-	rtest.OK(t, runList(cmdList, opts, []string{tpe}))
+	rtest.OK(t, runList(context.TODO(), cmdList, opts, []string{tpe}))
 	return parseIDsFromReader(t, buf)
 }
 
@@ -106,11 +106,13 @@ func testRunRestore(t testing.TB, opts GlobalOptions, dir string, snapshotID res
 func testRunRestoreLatest(t testing.TB, gopts GlobalOptions, dir string, paths []string, hosts []string) {
 	opts := RestoreOptions{
 		Target: dir,
-		Hosts:  hosts,
-		Paths:  paths,
+		snapshotFilterOptions: snapshotFilterOptions{
+			Hosts: hosts,
+			Paths: paths,
+		},
 	}
 
-	rtest.OK(t, runRestore(opts, gopts, []string{"latest"}))
+	rtest.OK(t, runRestore(context.TODO(), opts, gopts, []string{"latest"}))
 }
 
 func testRunRestoreExcludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, excludes []string) {
@@ -119,7 +121,7 @@ func testRunRestoreExcludes(t testing.TB, gopts GlobalOptions, dir string, snaps
 		Exclude: excludes,
 	}
 
-	rtest.OK(t, runRestore(opts, gopts, []string{snapshotID.String()}))
+	rtest.OK(t, runRestore(context.TODO(), opts, gopts, []string{snapshotID.String()}))
 }
 
 func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, includes []string) {
@@ -128,11 +130,11 @@ func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snaps
 		Include: includes,
 	}
 
-	rtest.OK(t, runRestore(opts, gopts, []string{snapshotID.String()}))
+	rtest.OK(t, runRestore(context.TODO(), opts, gopts, []string{snapshotID.String()}))
 }
 
 func testRunRestoreAssumeFailure(t testing.TB, snapshotID string, opts RestoreOptions, gopts GlobalOptions) error {
-	err := runRestore(opts, gopts, []string{snapshotID})
+	err := runRestore(context.TODO(), opts, gopts, []string{snapshotID})
 
 	return err
 }
@@ -142,7 +144,7 @@ func testRunCheck(t testing.TB, gopts GlobalOptions) {
 		ReadData:    true,
 		CheckUnused: true,
 	}
-	rtest.OK(t, runCheck(opts, gopts, nil))
+	rtest.OK(t, runCheck(context.TODO(), opts, gopts, nil))
 }
 
 func testRunCheckOutput(gopts GlobalOptions) (string, error) {
@@ -157,7 +159,7 @@ func testRunCheckOutput(gopts GlobalOptions) (string, error) {
 		ReadData: true,
 	}
 
-	err := runCheck(opts, gopts, nil)
+	err := runCheck(context.TODO(), opts, gopts, nil)
 	return buf.String(), err
 }
 
@@ -175,17 +177,17 @@ func testRunDiffOutput(gopts GlobalOptions, firstSnapshotID string, secondSnapsh
 	opts := DiffOptions{
 		ShowMetadata: false,
 	}
-	err := runDiff(opts, gopts, []string{firstSnapshotID, secondSnapshotID})
+	err := runDiff(context.TODO(), opts, gopts, []string{firstSnapshotID, secondSnapshotID})
 	return buf.String(), err
 }
 
 func testRunRebuildIndex(t testing.TB, gopts GlobalOptions) {
-	globalOptions.stdout = ioutil.Discard
+	globalOptions.stdout = io.Discard
 	defer func() {
 		globalOptions.stdout = os.Stdout
 	}()
 
-	rtest.OK(t, runRebuildIndex(RebuildIndexOptions{}, gopts))
+	rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{}, gopts))
 }
 
 func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
@@ -200,7 +202,7 @@ func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
 
 	opts := LsOptions{}
 
-	rtest.OK(t, runLs(opts, gopts, []string{snapshotID}))
+	rtest.OK(t, runLs(context.TODO(), opts, gopts, []string{snapshotID}))
 
 	return strings.Split(buf.String(), "\n")
 }
@@ -216,7 +218,7 @@ func testRunFind(t testing.TB, wantJSON bool, gopts GlobalOptions, pattern strin
 
 	opts := FindOptions{}
 
-	rtest.OK(t, runFind(opts, gopts, []string{pattern}))
+	rtest.OK(t, runFind(context.TODO(), opts, gopts, []string{pattern}))
 
 	return buf.Bytes()
 }
@@ -232,7 +234,7 @@ func testRunSnapshots(t testing.TB, gopts GlobalOptions) (newest *Snapshot, snap
 
 	opts := SnapshotOptions{}
 
-	rtest.OK(t, runSnapshots(opts, globalOptions, []string{}))
+	rtest.OK(t, runSnapshots(context.TODO(), opts, globalOptions, []string{}))
 
 	snapshots := []Snapshot{}
 	rtest.OK(t, json.Unmarshal(buf.Bytes(), &snapshots))
@@ -249,7 +251,7 @@ func testRunSnapshots(t testing.TB, gopts GlobalOptions) (newest *Snapshot, snap
 
 func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
 	opts := ForgetOptions{}
-	rtest.OK(t, runForget(opts, gopts, args))
+	rtest.OK(t, runForget(context.TODO(), opts, gopts, args))
 }
 
 func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) {
@@ -267,7 +269,7 @@ func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) {
 		Last:   1,
 	}
 
-	rtest.OK(t, runForget(opts, gopts, args))
+	rtest.OK(t, runForget(context.TODO(), opts, gopts, args))
 
 	var forgets []*ForgetGroup
 	rtest.OK(t, json.Unmarshal(buf.Bytes(), &forgets))
@@ -286,7 +288,7 @@ func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) {
 	defer func() {
 		gopts.backendTestHook = oldHook
 	}()
-	rtest.OK(t, runPrune(opts, gopts))
+	rtest.OK(t, runPrune(context.TODO(), opts, gopts))
 }
 
 func testSetupBackupData(t testing.TB, env *testEnvironment) string {
@@ -416,7 +418,7 @@ func TestBackupNonExistingFile(t *testing.T) {
 	defer cleanup()
 
 	testSetupBackupData(t, env)
-	globalOptions.stderr = ioutil.Discard
+	globalOptions.stderr = io.Discard
 	defer func() {
 		globalOptions.stderr = os.Stderr
 	}()
@@ -435,25 +437,25 @@ func TestBackupNonExistingFile(t *testing.T) {
 }
 
 func removePacksExcept(gopts GlobalOptions, t *testing.T, keep restic.IDSet, removeTreePacks bool) {
-	r, err := OpenRepository(gopts)
+	r, err := OpenRepository(context.TODO(), gopts)
 	rtest.OK(t, err)
 
 	// Get all tree packs
-	rtest.OK(t, r.LoadIndex(gopts.ctx))
+	rtest.OK(t, r.LoadIndex(context.TODO()))
 
 	treePacks := restic.NewIDSet()
-	for pb := range r.Index().Each(context.TODO()) {
+	r.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
 		if pb.Type == restic.TreeBlob {
 			treePacks.Insert(pb.PackID)
 		}
-	}
+	})
 
 	// remove all packs containing data blobs
-	rtest.OK(t, r.List(gopts.ctx, restic.PackFile, func(id restic.ID, size int64) error {
+	rtest.OK(t, r.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
 		if treePacks.Has(id) != removeTreePacks || keep.Has(id) {
 			return nil
 		}
-		return r.Backend().Remove(gopts.ctx, restic.Handle{Type: restic.PackFile, Name: id.String()})
+		return r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()})
 	}))
 }
 
@@ -477,7 +479,7 @@ func TestBackupSelfHealing(t *testing.T) {
 
 	testRunRebuildIndex(t, env.gopts)
 	// now the repo is also missing the data blob in the index; check should report this
-	rtest.Assert(t, runCheck(CheckOptions{}, env.gopts, nil) != nil,
+	rtest.Assert(t, runCheck(context.TODO(), CheckOptions{}, env.gopts, nil) != nil,
 		"check should have reported an error")
 
 	// second backup should report an error but "heal" this situation
@@ -500,26 +502,26 @@ func TestBackupTreeLoadError(t *testing.T) {
 	// Backup a subdirectory first, such that we can remove the tree pack for the subdirectory
 	testRunBackup(t, env.testdata, []string{"test"}, opts, env.gopts)
 
-	r, err := OpenRepository(env.gopts)
+	r, err := OpenRepository(context.TODO(), env.gopts)
 	rtest.OK(t, err)
-	rtest.OK(t, r.LoadIndex(env.gopts.ctx))
+	rtest.OK(t, r.LoadIndex(context.TODO()))
 	treePacks := restic.NewIDSet()
-	for pb := range r.Index().Each(context.TODO()) {
+	r.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
 		if pb.Type == restic.TreeBlob {
 			treePacks.Insert(pb.PackID)
 		}
-	}
+	})
 
 	testRunBackup(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
 	testRunCheck(t, env.gopts)
 
 	// delete the subdirectory pack first
 	for id := range treePacks {
-		rtest.OK(t, r.Backend().Remove(env.gopts.ctx, restic.Handle{Type: restic.PackFile, Name: id.String()}))
+		rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()}))
 	}
 	testRunRebuildIndex(t, env.gopts)
 	// now the repo is missing the tree blob in the index; check should report this
-	rtest.Assert(t, runCheck(CheckOptions{}, env.gopts, nil) != nil, "check should have reported an error")
+	rtest.Assert(t, runCheck(context.TODO(), CheckOptions{}, env.gopts, nil) != nil, "check should have reported an error")
 	// second backup should report an error but "heal" this situation
 	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
 	rtest.Assert(t, err != nil, "backup should have reported an error for the subdirectory")
@@ -529,7 +531,7 @@ func TestBackupTreeLoadError(t *testing.T) {
 	removePacksExcept(env.gopts, t, restic.NewIDSet(), true)
 	testRunRebuildIndex(t, env.gopts)
 	// now the repo is also missing the data blob in the index; check should report this
-	rtest.Assert(t, runCheck(CheckOptions{}, env.gopts, nil) != nil, "check should have reported an error")
+	rtest.Assert(t, runCheck(context.TODO(), CheckOptions{}, env.gopts, nil) != nil, "check should have reported an error")
 	// second backup should report an error but "heal" this situation
 	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
 	rtest.Assert(t, err != nil, "backup should have reported an error")
@@ -638,7 +640,7 @@ func TestBackupErrors(t *testing.T) {
 	}()
 	opts := BackupOptions{}
 	gopts := env.gopts
-	gopts.stderr = ioutil.Discard
+	gopts.stderr = io.Discard
 	err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, gopts)
 	rtest.Assert(t, err != nil, "Assumed failure, but no error occurred.")
 	rtest.Assert(t, err == ErrInvalidSourceData, "Wrong error returned")
@@ -759,7 +761,7 @@ func testRunCopy(t testing.TB, srcGopts GlobalOptions, dstGopts GlobalOptions) {
 		},
 	}
 
-	rtest.OK(t, runCopy(copyOpts, gopts, nil))
+	rtest.OK(t, runCopy(context.TODO(), copyOpts, gopts, nil))
 }
 
 func TestCopy(t *testing.T) {
@@ -901,15 +903,15 @@ func TestInitCopyChunkerParams(t *testing.T) {
 			password: env2.gopts.password,
 		},
 	}
-	rtest.Assert(t, runInit(initOpts, env.gopts, nil) != nil, "expected invalid init options to fail")
+	rtest.Assert(t, runInit(context.TODO(), initOpts, env.gopts, nil) != nil, "expected invalid init options to fail")
 
 	initOpts.CopyChunkerParameters = true
-	rtest.OK(t, runInit(initOpts, env.gopts, nil))
+	rtest.OK(t, runInit(context.TODO(), initOpts, env.gopts, nil))
 
-	repo, err := OpenRepository(env.gopts)
+	repo, err := OpenRepository(context.TODO(), env.gopts)
 	rtest.OK(t, err)
 
-	otherRepo, err := OpenRepository(env2.gopts)
+	otherRepo, err := OpenRepository(context.TODO(), env2.gopts)
 	rtest.OK(t, err)
 
 	rtest.Assert(t, repo.Config().ChunkerPolynomial == otherRepo.Config().ChunkerPolynomial,
@@ -918,7 +920,7 @@ func TestInitCopyChunkerParams(t *testing.T) {
 }
 
 func testRunTag(t testing.TB, opts TagOptions, gopts GlobalOptions) {
-	rtest.OK(t, runTag(opts, gopts, []string{}))
+	rtest.OK(t, runTag(context.TODO(), opts, gopts, []string{}))
 }
 
 func TestTag(t *testing.T) {
@@ -1010,7 +1012,7 @@ func testRunKeyListOtherIDs(t testing.TB, gopts GlobalOptions) []string {
 		globalOptions.stdout = os.Stdout
 	}()
 
-	rtest.OK(t, runKey(gopts, []string{"list"}))
+	rtest.OK(t, runKey(context.TODO(), gopts, []string{"list"}))
 
 	scanner := bufio.NewScanner(buf)
 	exp := regexp.MustCompile(`^ ([a-f0-9]+) `)
@@ -1031,7 +1033,7 @@ func testRunKeyAddNewKey(t testing.TB, newPassword string, gopts GlobalOptions)
 		testKeyNewPassword = ""
 	}()
 
-	rtest.OK(t, runKey(gopts, []string{"add"}))
+	rtest.OK(t, runKey(context.TODO(), gopts, []string{"add"}))
 }
 
 func testRunKeyAddNewKeyUserHost(t testing.TB, gopts GlobalOptions) {
@@ -1045,11 +1047,11 @@ func testRunKeyAddNewKeyUserHost(t testing.TB, gopts GlobalOptions) {
 	rtest.OK(t, cmdKey.Flags().Parse([]string{"--user=john", "--host=example.com"}))
 
 	t.Log("adding key for john@example.com")
-	rtest.OK(t, runKey(gopts, []string{"add"}))
+	rtest.OK(t, runKey(context.TODO(), gopts, []string{"add"}))
 
-	repo, err := OpenRepository(gopts)
+	repo, err := OpenRepository(context.TODO(), gopts)
 	rtest.OK(t, err)
-	key, err := repository.SearchKey(gopts.ctx, repo, testKeyNewPassword, 2, "")
+	key, err := repository.SearchKey(context.TODO(), repo, testKeyNewPassword, 2, "")
 	rtest.OK(t, err)
 
 	rtest.Equals(t, "john", key.Username)
@@ -1062,13 +1064,13 @@ func testRunKeyPasswd(t testing.TB, newPassword string, gopts GlobalOptions) {
 		testKeyNewPassword = ""
 	}()
 
-	rtest.OK(t, runKey(gopts, []string{"passwd"}))
+	rtest.OK(t, runKey(context.TODO(), gopts, []string{"passwd"}))
 }
 
 func testRunKeyRemove(t testing.TB, gopts GlobalOptions, IDs []string) {
 	t.Logf("remove %d keys: %q\n", len(IDs), IDs)
 	for _, id := range IDs {
-		rtest.OK(t, runKey(gopts, []string{"remove", id}))
+		rtest.OK(t, runKey(context.TODO(), gopts, []string{"remove", id}))
 	}
 }
 
@@ -1098,7 +1100,7 @@ func TestKeyAddRemove(t *testing.T) {
 
 	env.gopts.password = passwordList[len(passwordList)-1]
 	t.Logf("testing access with last password %q\n", env.gopts.password)
-	rtest.OK(t, runKey(env.gopts, []string{"list"}))
+	rtest.OK(t, runKey(context.TODO(), env.gopts, []string{"list"}))
 	testRunCheck(t, env.gopts)
 
 	testRunKeyAddNewKeyUserHost(t, env.gopts)
@@ -1126,16 +1128,16 @@ func TestKeyProblems(t *testing.T) {
 		testKeyNewPassword = ""
 	}()
 
-	err := runKey(env.gopts, []string{"passwd"})
+	err := runKey(context.TODO(), env.gopts, []string{"passwd"})
 	t.Log(err)
 	rtest.Assert(t, err != nil, "expected passwd change to fail")
 
-	err = runKey(env.gopts, []string{"add"})
+	err = runKey(context.TODO(), env.gopts, []string{"add"})
 	t.Log(err)
 	rtest.Assert(t, err != nil, "expected key adding to fail")
 
 	t.Logf("testing access with initial password %q\n", env.gopts.password)
-	rtest.OK(t, runKey(env.gopts, []string{"list"}))
+	rtest.OK(t, runKey(context.TODO(), env.gopts, []string{"list"}))
 	testRunCheck(t, env.gopts)
 }
 
@@ -1195,7 +1197,7 @@ func TestRestoreFilter(t *testing.T) {
 			if ok, _ := filter.Match(pat, filepath.Base(testFile.name)); !ok {
 				rtest.OK(t, err)
 			} else {
-				rtest.Assert(t, os.IsNotExist(errors.Cause(err)),
+				rtest.Assert(t, os.IsNotExist(err),
 					"expected %v to not exist in restore step %v, but it exists, err %v", testFile.name, i+1, err)
 			}
 		}
@@ -1240,7 +1242,7 @@ func TestRestoreLatest(t *testing.T) {
 	opts := BackupOptions{}
 
 	// chdir manually here so we can get the current directory. This is not the
-	// same as the temp dir returned by ioutil.TempDir() on darwin.
+	// same as the temp dir returned by os.MkdirTemp() on darwin.
 	back := rtest.Chdir(t, filepath.Dir(env.testdata))
 	defer back()
 
@@ -1281,15 +1283,15 @@ func TestRestoreLatest(t *testing.T) {
 
 	testRunRestoreLatest(t, env.gopts, filepath.Join(env.base, "restore1"), []string{filepath.Dir(p1)}, nil)
 	rtest.OK(t, testFileSize(p1rAbs, int64(102)))
-	if _, err := os.Stat(p2rAbs); os.IsNotExist(errors.Cause(err)) {
-		rtest.Assert(t, os.IsNotExist(errors.Cause(err)),
+	if _, err := os.Stat(p2rAbs); os.IsNotExist(err) {
+		rtest.Assert(t, os.IsNotExist(err),
 			"expected %v to not exist in restore, but it exists, err %v", p2rAbs, err)
 	}
 
 	testRunRestoreLatest(t, env.gopts, filepath.Join(env.base, "restore2"), []string{filepath.Dir(p2)}, nil)
 	rtest.OK(t, testFileSize(p2rAbs, int64(103)))
-	if _, err := os.Stat(p1rAbs); os.IsNotExist(errors.Cause(err)) {
-		rtest.Assert(t, os.IsNotExist(errors.Cause(err)),
+	if _, err := os.Stat(p1rAbs); os.IsNotExist(err) {
+		rtest.Assert(t, os.IsNotExist(err),
 			"expected %v to not exist in restore, but it exists, err %v", p1rAbs, err)
 	}
 }
@@ -1305,7 +1307,7 @@ func TestRestoreWithPermissionFailure(t *testing.T) {
 	rtest.Assert(t, len(snapshots) > 0,
 		"no snapshots found in repo (%v)", datafile)
 
-	globalOptions.stderr = ioutil.Discard
+	globalOptions.stderr = io.Discard
 	defer func() {
 		globalOptions.stderr = os.Stderr
 	}()
@@ -1475,11 +1477,11 @@ func TestRebuildIndex(t *testing.T) {
 }
 
 func TestRebuildIndexAlwaysFull(t *testing.T) {
-	indexFull := repository.IndexFull
+	indexFull := index.IndexFull
 	defer func() {
-		repository.IndexFull = indexFull
+		index.IndexFull = indexFull
 	}()
-	repository.IndexFull = func(*repository.Index, bool) bool { return true }
+	index.IndexFull = func(*index.Index, bool) bool { return true }
 	testRebuildIndex(t, nil)
 }
 
@@ -1539,7 +1541,7 @@ func TestRebuildIndexFailsOnAppendOnly(t *testing.T) {
 	datafile := filepath.Join("..", "..", "internal", "checker", "testdata", "duplicate-packs-in-index-test-repo.tar.gz")
 	rtest.SetupTarTestFixture(t, env.base, datafile)
 
-	globalOptions.stdout = ioutil.Discard
+	globalOptions.stdout = io.Discard
 	defer func() {
 		globalOptions.stdout = os.Stdout
 	}()
@@ -1547,7 +1549,7 @@ func TestRebuildIndexFailsOnAppendOnly(t *testing.T) {
 	env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
 		return &appendOnlyBackend{r}, nil
 	}
-	err := runRebuildIndex(RebuildIndexOptions{}, env.gopts)
+	err := runRebuildIndex(context.TODO(), RebuildIndexOptions{}, env.gopts)
 	if err == nil {
 		t.Error("expected rebuildIndex to fail")
 	}
@@ -1643,18 +1645,18 @@ func testPrune(t *testing.T, pruneOpts PruneOptions, checkOpts CheckOptions) {
 	testRunForgetJSON(t, env.gopts)
 	testRunForget(t, env.gopts, firstSnapshot[0].String())
 	testRunPrune(t, env.gopts, pruneOpts)
-	rtest.OK(t, runCheck(checkOpts, env.gopts, nil))
+	rtest.OK(t, runCheck(context.TODO(), checkOpts, env.gopts, nil))
 }
 
 var pruneDefaultOptions = PruneOptions{MaxUnused: "5%"}
 
 func listPacks(gopts GlobalOptions, t *testing.T) restic.IDSet {
-	r, err := OpenRepository(gopts)
+	r, err := OpenRepository(context.TODO(), gopts)
 	rtest.OK(t, err)
 
 	packs := restic.NewIDSet()
 
-	rtest.OK(t, r.List(gopts.ctx, restic.PackFile, func(id restic.ID, size int64) error {
+	rtest.OK(t, r.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
 		packs.Insert(id)
 		return nil
 	}))
@@ -1695,7 +1697,7 @@ func TestPruneWithDamagedRepository(t *testing.T) {
 		env.gopts.backendTestHook = oldHook
 	}()
 	// prune should fail
-	rtest.Assert(t, runPrune(pruneDefaultOptions, env.gopts) == errorPacksMissing,
+	rtest.Assert(t, runPrune(context.TODO(), pruneDefaultOptions, env.gopts) == errorPacksMissing,
 		"prune should have reported index not complete error")
 }
 
@@ -1767,7 +1769,7 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o
 	if checkOK {
 		testRunCheck(t, env.gopts)
 	} else {
-		rtest.Assert(t, runCheck(optionsCheck, env.gopts, nil) != nil,
+		rtest.Assert(t, runCheck(context.TODO(), optionsCheck, env.gopts, nil) != nil,
 			"check should have reported an error")
 	}
 
@@ -1775,7 +1777,7 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o
 		testRunPrune(t, env.gopts, optionsPrune)
 		testRunCheck(t, env.gopts)
 	} else {
-		rtest.Assert(t, runPrune(optionsPrune, env.gopts) != nil,
+		rtest.Assert(t, runPrune(context.TODO(), optionsPrune, env.gopts) != nil,
 			"prune should have reported an error")
 	}
 }
@@ -1846,10 +1848,10 @@ func TestListOnce(t *testing.T) {
 	testRunForgetJSON(t, env.gopts)
 	testRunForget(t, env.gopts, firstSnapshot[0].String())
 	testRunPrune(t, env.gopts, pruneOpts)
-	rtest.OK(t, runCheck(checkOpts, env.gopts, nil))
+	rtest.OK(t, runCheck(context.TODO(), checkOpts, env.gopts, nil))
 
-	rtest.OK(t, runRebuildIndex(RebuildIndexOptions{}, env.gopts))
-	rtest.OK(t, runRebuildIndex(RebuildIndexOptions{ReadAllPacks: true}, env.gopts))
+	rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{}, env.gopts))
+	rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{ReadAllPacks: true}, env.gopts))
 }
 
 func TestHardLink(t *testing.T) {
@@ -1859,7 +1861,7 @@ func TestHardLink(t *testing.T) {
 
 	datafile := filepath.Join("testdata", "test.hl.tar.gz")
 	fd, err := os.Open(datafile)
-	if os.IsNotExist(errors.Cause(err)) {
+	if os.IsNotExist(err) {
 		t.Skipf("unable to find data file %q, skipping", datafile)
 		return
 	}
@@ -2202,7 +2204,7 @@ func TestFindListOnce(t *testing.T) {
 	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "3")}, opts, env.gopts)
 	thirdSnapshot := restic.NewIDSet(testRunList(t, "snapshots", env.gopts)...)
 
-	repo, err := OpenRepository(env.gopts)
+	repo, err := OpenRepository(context.TODO(), env.gopts)
 	rtest.OK(t, err)
 
 	snapshotIDs := restic.NewIDSet()

cmd/restic/lock.go

@@ -2,32 +2,36 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"sync"
 	"time"
 
 	"github.com/restic/restic/internal/debug"
-	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 )
 
+type lockContext struct {
+	cancel    context.CancelFunc
+	refreshWG sync.WaitGroup
+}
+
 var globalLocks struct {
-	locks         []*restic.Lock
-	cancelRefresh chan struct{}
-	refreshWG     sync.WaitGroup
+	locks map[*restic.Lock]*lockContext
 	sync.Mutex
 	sync.Once
 }
 
-func lockRepo(ctx context.Context, repo *repository.Repository) (*restic.Lock, error) {
+func lockRepo(ctx context.Context, repo restic.Repository) (*restic.Lock, context.Context, error) {
 	return lockRepository(ctx, repo, false)
 }
 
-func lockRepoExclusive(ctx context.Context, repo *repository.Repository) (*restic.Lock, error) {
+func lockRepoExclusive(ctx context.Context, repo restic.Repository) (*restic.Lock, context.Context, error) {
 	return lockRepository(ctx, repo, true)
 }
 
-func lockRepository(ctx context.Context, repo *repository.Repository, exclusive bool) (*restic.Lock, error) {
+// lockRepository wraps the ctx such that it is cancelled when the repository is unlocked
+// cancelling the original context also stops the lock refresh
+func lockRepository(ctx context.Context, repo restic.Repository, exclusive bool) (*restic.Lock, context.Context, error) {
 	// make sure that a repository is unlocked properly and after cancel() was
 	// called by the cleanup handler in global.go
 	globalLocks.Do(func() {
@@ -41,53 +45,114 @@ func lockRepository(ctx context.Context, repo *repository.Repository, exclusive
 
 	lock, err := lockFn(ctx, repo)
 	if err != nil {
-		return nil, errors.WithMessage(err, "unable to create lock in backend")
+		return nil, ctx, fmt.Errorf("unable to create lock in backend: %w", err)
 	}
 	debug.Log("create lock %p (exclusive %v)", lock, exclusive)
 
-	globalLocks.Lock()
-	if globalLocks.cancelRefresh == nil {
-		debug.Log("start goroutine for lock refresh")
-		globalLocks.cancelRefresh = make(chan struct{})
-		globalLocks.refreshWG = sync.WaitGroup{}
-		globalLocks.refreshWG.Add(1)
-		go refreshLocks(&globalLocks.refreshWG, globalLocks.cancelRefresh)
+	ctx, cancel := context.WithCancel(ctx)
+	lockInfo := &lockContext{
+		cancel: cancel,
 	}
+	lockInfo.refreshWG.Add(2)
+	refreshChan := make(chan struct{})
 
-	globalLocks.locks = append(globalLocks.locks, lock)
+	globalLocks.Lock()
+	globalLocks.locks[lock] = lockInfo
+	go refreshLocks(ctx, lock, lockInfo, refreshChan)
+	go monitorLockRefresh(ctx, lock, lockInfo, refreshChan)
 	globalLocks.Unlock()
 
-	return lock, err
+	return lock, ctx, err
 }
 
 var refreshInterval = 5 * time.Minute
 
-func refreshLocks(wg *sync.WaitGroup, done <-chan struct{}) {
-	debug.Log("start")
-	defer func() {
-		wg.Done()
-		globalLocks.Lock()
-		globalLocks.cancelRefresh = nil
-		globalLocks.Unlock()
-	}()
+// consider a lock refresh failed a bit before the lock actually becomes stale
+// the difference allows to compensate for a small time drift between clients.
+var refreshabilityTimeout = restic.StaleLockTimeout - refreshInterval*3/2
 
+func refreshLocks(ctx context.Context, lock *restic.Lock, lockInfo *lockContext, refreshed chan<- struct{}) {
+	debug.Log("start")
 	ticker := time.NewTicker(refreshInterval)
+	lastRefresh := lock.Time
+
+	defer func() {
+		ticker.Stop()
+		// ensure that the context was cancelled before removing the lock
+		lockInfo.cancel()
+
+		// remove the lock from the repo
+		debug.Log("unlocking repository with lock %v", lock)
+		if err := lock.Unlock(); err != nil {
+			debug.Log("error while unlocking: %v", err)
+			Warnf("error while unlocking: %v", err)
+		}
+
+		lockInfo.refreshWG.Done()
+	}()
 
 	for {
 		select {
-		case <-done:
+		case <-ctx.Done():
 			debug.Log("terminate")
 			return
 		case <-ticker.C:
+			if time.Since(lastRefresh) > refreshabilityTimeout {
+				// the lock is too old, wait until the expiry monitor cancels the context
+				continue
+			}
+
 			debug.Log("refreshing locks")
-			globalLocks.Lock()
-			for _, lock := range globalLocks.locks {
-				err := lock.Refresh(context.TODO())
-				if err != nil {
-					Warnf("unable to refresh lock: %v\n", err)
+			err := lock.Refresh(context.TODO())
+			if err != nil {
+				Warnf("unable to refresh lock: %v\n", err)
+			} else {
+				lastRefresh = lock.Time
+				// inform monitor gorountine about successful refresh
+				select {
+				case <-ctx.Done():
+				case refreshed <- struct{}{}:
 				}
 			}
-			globalLocks.Unlock()
+		}
+	}
+}
+
+func monitorLockRefresh(ctx context.Context, lock *restic.Lock, lockInfo *lockContext, refreshed <-chan struct{}) {
+	// time.Now() might use a monotonic timer which is paused during standby
+	// convert to unix time to ensure we compare real time values
+	lastRefresh := time.Now().UnixNano()
+	pollDuration := 1 * time.Second
+	if refreshInterval < pollDuration {
+		// require for TestLockFailedRefresh
+		pollDuration = refreshInterval / 5
+	}
+	// timers are paused during standby, which is a problem as the refresh timeout
+	// _must_ expire if the host was too long in standby. Thus fall back to periodic checks
+	// https://github.com/golang/go/issues/35012
+	timer := time.NewTimer(pollDuration)
+	defer func() {
+		timer.Stop()
+		lockInfo.cancel()
+		lockInfo.refreshWG.Done()
+	}()
+
+	for {
+		select {
+		case <-ctx.Done():
+			debug.Log("terminate expiry monitoring")
+			return
+		case <-refreshed:
+			lastRefresh = time.Now().UnixNano()
+		case <-timer.C:
+			if time.Now().UnixNano()-lastRefresh < refreshabilityTimeout.Nanoseconds() {
+				// restart timer
+				timer.Reset(pollDuration)
+				continue
+			}
+
+			Warnf("Fatal: failed to refresh lock in time\n")
+			return
 		}
 	}
 }
@@ -98,40 +163,35 @@ func unlockRepo(lock *restic.Lock) {
 	}
 
 	globalLocks.Lock()
-	defer globalLocks.Unlock()
+	lockInfo, exists := globalLocks.locks[lock]
+	delete(globalLocks.locks, lock)
+	globalLocks.Unlock()
 
-	for i := 0; i < len(globalLocks.locks); i++ {
-		if lock == globalLocks.locks[i] {
-			// remove the lock from the repo
-			debug.Log("unlocking repository with lock %v", lock)
-			if err := lock.Unlock(); err != nil {
-				debug.Log("error while unlocking: %v", err)
-				Warnf("error while unlocking: %v", err)
-				return
-			}
-
-			// remove the lock from the list of locks
-			globalLocks.locks = append(globalLocks.locks[:i], globalLocks.locks[i+1:]...)
-			return
-		}
+	if !exists {
+		debug.Log("unable to find lock %v in the global list of locks, ignoring", lock)
+		return
 	}
-
-	debug.Log("unable to find lock %v in the global list of locks, ignoring", lock)
+	lockInfo.cancel()
+	lockInfo.refreshWG.Wait()
 }
 
-func unlockAll() error {
+func unlockAll(code int) (int, error) {
 	globalLocks.Lock()
-	defer globalLocks.Unlock()
-
+	locks := globalLocks.locks
 	debug.Log("unlocking %d locks", len(globalLocks.locks))
-	for _, lock := range globalLocks.locks {
-		if err := lock.Unlock(); err != nil {
-			debug.Log("error while unlocking: %v", err)
-			return err
-		}
-		debug.Log("successfully removed lock")
+	for _, lockInfo := range globalLocks.locks {
+		lockInfo.cancel()
 	}
-	globalLocks.locks = globalLocks.locks[:0]
+	globalLocks.locks = make(map[*restic.Lock]*lockContext)
+	globalLocks.Unlock()
 
-	return nil
+	for _, lockInfo := range locks {
+		lockInfo.refreshWG.Wait()
+	}
+
+	return code, nil
+}
+
+func init() {
+	globalLocks.locks = make(map[*restic.Lock]*lockContext)
 }

cmd/restic/lock_test.go

@@ -0,0 +1,170 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func openTestRepo(t *testing.T, wrapper backendWrapper) (*repository.Repository, func(), *testEnvironment) {
+	env, cleanup := withTestEnvironment(t)
+	if wrapper != nil {
+		env.gopts.backendTestHook = wrapper
+	}
+	testRunInit(t, env.gopts)
+
+	repo, err := OpenRepository(context.TODO(), env.gopts)
+	rtest.OK(t, err)
+	return repo, cleanup, env
+}
+
+func checkedLockRepo(ctx context.Context, t *testing.T, repo restic.Repository) (*restic.Lock, context.Context) {
+	lock, wrappedCtx, err := lockRepo(ctx, repo)
+	rtest.OK(t, err)
+	rtest.OK(t, wrappedCtx.Err())
+	if lock.Stale() {
+		t.Fatal("lock returned stale lock")
+	}
+	return lock, wrappedCtx
+}
+
+func TestLock(t *testing.T) {
+	repo, cleanup, _ := openTestRepo(t, nil)
+	defer cleanup()
+
+	lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo)
+	unlockRepo(lock)
+	if wrappedCtx.Err() == nil {
+		t.Fatal("unlock did not cancel context")
+	}
+}
+
+func TestLockCancel(t *testing.T) {
+	repo, cleanup, _ := openTestRepo(t, nil)
+	defer cleanup()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	lock, wrappedCtx := checkedLockRepo(ctx, t, repo)
+	cancel()
+	if wrappedCtx.Err() == nil {
+		t.Fatal("canceled parent context did not cancel context")
+	}
+
+	// unlockRepo should not crash
+	unlockRepo(lock)
+}
+
+func TestLockUnlockAll(t *testing.T) {
+	repo, cleanup, _ := openTestRepo(t, nil)
+	defer cleanup()
+
+	lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo)
+	_, err := unlockAll(0)
+	rtest.OK(t, err)
+	if wrappedCtx.Err() == nil {
+		t.Fatal("canceled parent context did not cancel context")
+	}
+
+	// unlockRepo should not crash
+	unlockRepo(lock)
+}
+
+func TestLockConflict(t *testing.T) {
+	repo, cleanup, env := openTestRepo(t, nil)
+	defer cleanup()
+	repo2, err := OpenRepository(context.TODO(), env.gopts)
+	rtest.OK(t, err)
+
+	lock, _, err := lockRepoExclusive(context.Background(), repo)
+	rtest.OK(t, err)
+	defer unlockRepo(lock)
+	_, _, err = lockRepo(context.Background(), repo2)
+	if err == nil {
+		t.Fatal("second lock should have failed")
+	}
+}
+
+type writeOnceBackend struct {
+	restic.Backend
+	written bool
+}
+
+func (b *writeOnceBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
+	if b.written {
+		return fmt.Errorf("fail after first write")
+	}
+	b.written = true
+	return b.Backend.Save(ctx, h, rd)
+}
+
+func TestLockFailedRefresh(t *testing.T) {
+	repo, cleanup, _ := openTestRepo(t, func(r restic.Backend) (restic.Backend, error) {
+		return &writeOnceBackend{Backend: r}, nil
+	})
+	defer cleanup()
+
+	// reduce locking intervals to be suitable for testing
+	ri, rt := refreshInterval, refreshabilityTimeout
+	refreshInterval = 20 * time.Millisecond
+	refreshabilityTimeout = 100 * time.Millisecond
+	defer func() {
+		refreshInterval, refreshabilityTimeout = ri, rt
+	}()
+
+	lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo)
+
+	select {
+	case <-wrappedCtx.Done():
+		// expected lock refresh failure
+	case <-time.After(time.Second):
+		t.Fatal("failed lock refresh did not cause context cancellation")
+	}
+	// unlockRepo should not crash
+	unlockRepo(lock)
+}
+
+type loggingBackend struct {
+	restic.Backend
+	t *testing.T
+}
+
+func (b *loggingBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
+	b.t.Logf("save %v @ %v", h, time.Now())
+	return b.Backend.Save(ctx, h, rd)
+}
+
+func TestLockSuccessfulRefresh(t *testing.T) {
+	repo, cleanup, _ := openTestRepo(t, func(r restic.Backend) (restic.Backend, error) {
+		return &loggingBackend{
+			Backend: r,
+			t:       t,
+		}, nil
+	})
+	defer cleanup()
+
+	t.Logf("test for successful lock refresh %v", time.Now())
+	// reduce locking intervals to be suitable for testing
+	ri, rt := refreshInterval, refreshabilityTimeout
+	refreshInterval = 40 * time.Millisecond
+	refreshabilityTimeout = 200 * time.Millisecond
+	defer func() {
+		refreshInterval, refreshabilityTimeout = ri, rt
+	}()
+
+	lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo)
+
+	select {
+	case <-wrappedCtx.Done():
+		t.Fatal("lock refresh failed")
+	case <-time.After(2 * refreshabilityTimeout):
+		// expected lock refresh to work
+	}
+	// unlockRepo should not crash
+	unlockRepo(lock)
+}

cmd/restic/main.go

@@ -85,17 +85,15 @@ func needsPassword(cmd string) bool {
 
 var logBuffer = bytes.NewBuffer(nil)
 
-func init() {
+func main() {
 	// install custom global logger into a buffer, if an error occurs
 	// we can show the logs
 	log.SetOutput(logBuffer)
-}
 
-func main() {
 	debug.Log("main %#v", os.Args)
 	debug.Log("restic %s compiled with %v on %v/%v",
 		version, runtime.Version(), runtime.GOOS, runtime.GOARCH)
-	err := cmdRoot.Execute()
+	err := cmdRoot.ExecuteContext(internalGlobalCtx)
 
 	switch {
 	case restic.IsAlreadyLocked(err):

cmd/restic/progress.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/restic/restic/internal/ui"
 	"github.com/restic/restic/internal/ui/progress"
 	"github.com/restic/restic/internal/ui/termstatus"
 )
@@ -39,10 +40,11 @@ func newProgressMax(show bool, max uint64, description string) *progress.Counter
 	return progress.New(interval, max, func(v uint64, max uint64, d time.Duration, final bool) {
 		var status string
 		if max == 0 {
-			status = fmt.Sprintf("[%s]          %d %s", formatDuration(d), v, description)
+			status = fmt.Sprintf("[%s]          %d %s",
+				ui.FormatDuration(d), v, description)
 		} else {
 			status = fmt.Sprintf("[%s] %s  %d / %d %s",
-				formatDuration(d), formatPercent(v, max), v, max, description)
+				ui.FormatDuration(d), ui.FormatPercent(v, max), v, max, description)
 		}
 
 		printProgress(status, canUpdateStatus)

cmd/restic/secondary_repo.go

@@ -24,11 +24,11 @@ type secondaryRepoOptions struct {
 }
 
 func initSecondaryRepoOptions(f *pflag.FlagSet, opts *secondaryRepoOptions, repoPrefix string, repoUsage string) {
-	f.StringVarP(&opts.LegacyRepo, "repo2", "", os.Getenv("RESTIC_REPOSITORY2"), repoPrefix+" `repository` "+repoUsage+" (default: $RESTIC_REPOSITORY2)")
-	f.StringVarP(&opts.LegacyRepositoryFile, "repository-file2", "", os.Getenv("RESTIC_REPOSITORY_FILE2"), "`file` from which to read the "+repoPrefix+" repository location "+repoUsage+" (default: $RESTIC_REPOSITORY_FILE2)")
-	f.StringVarP(&opts.LegacyPasswordFile, "password-file2", "", os.Getenv("RESTIC_PASSWORD_FILE2"), "`file` to read the "+repoPrefix+" repository password from (default: $RESTIC_PASSWORD_FILE2)")
-	f.StringVarP(&opts.LegacyKeyHint, "key-hint2", "", os.Getenv("RESTIC_KEY_HINT2"), "key ID of key to try decrypting the "+repoPrefix+" repository first (default: $RESTIC_KEY_HINT2)")
-	f.StringVarP(&opts.LegacyPasswordCommand, "password-command2", "", os.Getenv("RESTIC_PASSWORD_COMMAND2"), "shell `command` to obtain the "+repoPrefix+" repository password from (default: $RESTIC_PASSWORD_COMMAND2)")
+	f.StringVarP(&opts.LegacyRepo, "repo2", "", "", repoPrefix+" `repository` "+repoUsage+" (default: $RESTIC_REPOSITORY2)")
+	f.StringVarP(&opts.LegacyRepositoryFile, "repository-file2", "", "", "`file` from which to read the "+repoPrefix+" repository location "+repoUsage+" (default: $RESTIC_REPOSITORY_FILE2)")
+	f.StringVarP(&opts.LegacyPasswordFile, "password-file2", "", "", "`file` to read the "+repoPrefix+" repository password from (default: $RESTIC_PASSWORD_FILE2)")
+	f.StringVarP(&opts.LegacyKeyHint, "key-hint2", "", "", "key ID of key to try decrypting the "+repoPrefix+" repository first (default: $RESTIC_KEY_HINT2)")
+	f.StringVarP(&opts.LegacyPasswordCommand, "password-command2", "", "", "shell `command` to obtain the "+repoPrefix+" repository password from (default: $RESTIC_PASSWORD_COMMAND2)")
 
 	// hide repo2 options
 	_ = f.MarkDeprecated("repo2", "use --repo or --from-repo instead")
@@ -37,11 +37,23 @@ func initSecondaryRepoOptions(f *pflag.FlagSet, opts *secondaryRepoOptions, repo
 	_ = f.MarkHidden("key-hint2")
 	_ = f.MarkHidden("password-command2")
 
-	f.StringVarP(&opts.Repo, "from-repo", "", os.Getenv("RESTIC_FROM_REPOSITORY"), "source `repository` "+repoUsage+" (default: $RESTIC_FROM_REPOSITORY)")
-	f.StringVarP(&opts.RepositoryFile, "from-repository-file", "", os.Getenv("RESTIC_FROM_REPOSITORY_FILE"), "`file` from which to read the source repository location "+repoUsage+" (default: $RESTIC_FROM_REPOSITORY_FILE)")
-	f.StringVarP(&opts.PasswordFile, "from-password-file", "", os.Getenv("RESTIC_FROM_PASSWORD_FILE"), "`file` to read the source repository password from (default: $RESTIC_FROM_PASSWORD_FILE)")
-	f.StringVarP(&opts.KeyHint, "from-key-hint", "", os.Getenv("RESTIC_FROM_KEY_HINT"), "key ID of key to try decrypting the source repository first (default: $RESTIC_FROM_KEY_HINT)")
-	f.StringVarP(&opts.PasswordCommand, "from-password-command", "", os.Getenv("RESTIC_FROM_PASSWORD_COMMAND"), "shell `command` to obtain the source repository password from (default: $RESTIC_FROM_PASSWORD_COMMAND)")
+	opts.LegacyRepo = os.Getenv("RESTIC_REPOSITORY2")
+	opts.LegacyRepositoryFile = os.Getenv("RESTIC_REPOSITORY_FILE2")
+	opts.LegacyPasswordFile = os.Getenv("RESTIC_PASSWORD_FILE2")
+	opts.LegacyKeyHint = os.Getenv("RESTIC_KEY_HINT2")
+	opts.LegacyPasswordCommand = os.Getenv("RESTIC_PASSWORD_COMMAND2")
+
+	f.StringVarP(&opts.Repo, "from-repo", "", "", "source `repository` "+repoUsage+" (default: $RESTIC_FROM_REPOSITORY)")
+	f.StringVarP(&opts.RepositoryFile, "from-repository-file", "", "", "`file` from which to read the source repository location "+repoUsage+" (default: $RESTIC_FROM_REPOSITORY_FILE)")
+	f.StringVarP(&opts.PasswordFile, "from-password-file", "", "", "`file` to read the source repository password from (default: $RESTIC_FROM_PASSWORD_FILE)")
+	f.StringVarP(&opts.KeyHint, "from-key-hint", "", "", "key ID of key to try decrypting the source repository first (default: $RESTIC_FROM_KEY_HINT)")
+	f.StringVarP(&opts.PasswordCommand, "from-password-command", "", "", "shell `command` to obtain the source repository password from (default: $RESTIC_FROM_PASSWORD_COMMAND)")
+
+	opts.Repo = os.Getenv("RESTIC_FROM_REPOSITORY")
+	opts.RepositoryFile = os.Getenv("RESTIC_FROM_REPOSITORY_FILE")
+	opts.PasswordFile = os.Getenv("RESTIC_FROM_PASSWORD_FILE")
+	opts.KeyHint = os.Getenv("RESTIC_FROM_KEY_HINT")
+	opts.PasswordCommand = os.Getenv("RESTIC_FROM_PASSWORD_COMMAND")
 }
 
 func fillSecondaryGlobalOpts(opts secondaryRepoOptions, gopts GlobalOptions, repoPrefix string) (GlobalOptions, bool, error) {

cmd/restic/secondary_repo_test.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"io/ioutil"
+	"os"
 	"path/filepath"
 	"testing"
 
@@ -160,14 +160,12 @@ func TestFillSecondaryGlobalOpts(t *testing.T) {
 	}
 
 	//Create temp dir to create password file.
-	dir, cleanup := rtest.TempDir(t)
-	defer cleanup()
-
-	cleanup = rtest.Chdir(t, dir)
+	dir := rtest.TempDir(t)
+	cleanup := rtest.Chdir(t, dir)
 	defer cleanup()
 
 	//Create temporary password file
-	err := ioutil.WriteFile(filepath.Join(dir, "passwordFileDst"), []byte("secretDst"), 0666)
+	err := os.WriteFile(filepath.Join(dir, "passwordFileDst"), []byte("secretDst"), 0666)
 	rtest.OK(t, err)
 
 	// Test all valid cases

doc/020_installation.rst

@@ -265,16 +265,11 @@ binary, you can get it with `docker pull` like this:
 
     $ docker pull restic/restic
 
-.. note::
-   | Another docker container which offers more configuration options is
-   | available as a contribution (Thank you!). You can find it at
-   | https://github.com/Lobaro/restic-backup-docker
-
 From Source
 ***********
 
 restic is written in the Go programming language and you need at least
-Go version 1.15. Building restic may also work with older versions of Go,
+Go version 1.18. Building restic may also work with older versions of Go,
 but that's not supported. See the `Getting
 started <https://golang.org/doc/install>`__ guide of the Go project for
 instructions how to install Go.
@@ -313,14 +308,14 @@ compiler. Building restic with gccgo may work, but is not supported.
 Autocompletion
 **************
 
-Restic can write out man pages and bash/fish/zsh compatible autocompletion scripts:
+Restic can write out man pages and bash/fish/zsh/powershell compatible autocompletion scripts:
 
 .. code-block:: console
 
     $ ./restic generate --help
 
     The "generate" command writes automatically generated files (like the man pages
-    and the auto-completion files for bash, fish and zsh).
+    and the auto-completion files for bash, fish, zsh and powershell).
 
     Usage:
       restic generate [flags] [command]
@@ -330,6 +325,7 @@ Restic can write out man pages and bash/fish/zsh compatible autocompletion scrip
           --fish-completion file   write fish completion file
       -h, --help                   help for generate
           --man directory          write man pages to directory
+          --powershell-completion  write powershell completion file
           --zsh-completion file    write zsh completion file
 
 Example for using sudo to write a bash completion script directly to the system-wide location:

doc/030_preparing_a_new_repo.rst

@@ -58,9 +58,9 @@ versions.
 +--------------------+-------------------------+---------------------+------------------+
 | Repository version | Required restic version | Major new features  | Comment          |
 +====================+=========================+=====================+==================+
-| ``1``              | Any                     |                     | Current default  |
+| ``1``              | Any                     |                     |                  |
 +--------------------+-------------------------+---------------------+------------------+
-| ``2``              | 0.14.0 or newer         | Compression support |                  |
+| ``2``              | 0.14.0 or newer         | Compression support | Current default  |
 +--------------------+-------------------------+---------------------+------------------+
 
 
@@ -86,10 +86,11 @@ command and enter the same password twice:
 
 .. warning::
 
-   On Linux, storing the backup repository on a CIFS (SMB) share is not
-   recommended due to compatibility issues. Either use another backend
-   or set the environment variable `GODEBUG` to `asyncpreemptoff=1`.
-   Refer to GitHub issue `#2659 <https://github.com/restic/restic/issues/2659>`_ for further explanations.
+   On Linux, storing the backup repository on a CIFS (SMB) share or backing up
+   data from a CIFS share is not recommended due to compatibility issues in
+   older Linux kernels. Either use another backend or set the environment
+   variable `GODEBUG` to `asyncpreemptoff=1`. Refer to GitHub issue
+   `#2659 <https://github.com/restic/restic/issues/2659>`_ for further explanations.
 
 SFTP
 ****
@@ -221,6 +222,8 @@ REST server uses exactly the same directory structure as local backend,
 so you should be able to access it both locally and via HTTP, even
 simultaneously.
 
+.. _Amazon S3:
+
 Amazon S3
 *********
 
@@ -301,7 +304,7 @@ credentials of your Minio Server.
 .. code-block:: console
 
     $ export AWS_ACCESS_KEY_ID=<YOUR-MINIO-ACCESS-KEY-ID>
-    $ export AWS_SECRET_ACCESS_KEY= <YOUR-MINIO-SECRET-ACCESS-KEY>
+    $ export AWS_SECRET_ACCESS_KEY=<YOUR-MINIO-SECRET-ACCESS-KEY>
 
 Now you can easily initialize restic to use Minio server as a backend with
 this command.
@@ -464,6 +467,19 @@ The policy of the new container created by restic can be changed using environme
 Backblaze B2
 ************
 
+.. warning::
+
+   Due to issues with error handling in the current B2 library that restic uses,
+   the recommended way to utilize Backblaze B2 is by using its S3-compatible API.
+   
+   Follow the documentation to `generate S3-compatible access keys`_ and then
+   setup restic as described at :ref:`Amazon S3`. This is expected to work better
+   than using the Backblaze B2 backend directly.
+
+   Different from the B2 backend, restic's S3 backend will only hide no longer
+   necessary files. Thus, make sure to setup lifecycle rules to eventually
+   delete hidden files.
+
 Restic can backup data to any Backblaze B2 bucket. You need to first setup the
 following environment variables with the credentials you can find in the
 dashboard on the "Buckets" page when signed into your B2 account:
@@ -502,11 +518,13 @@ The number of concurrent connections to the B2 service can be set with the ``-o
 b2.connections=10`` switch. By default, at most five parallel connections are
 established.
 
+.. _generate S3-compatible access keys: https://help.backblaze.com/hc/en-us/articles/360047425453-Getting-Started-with-the-S3-Compatible-API
+
 Microsoft Azure Blob Storage
 ****************************
 
 You can also store backups on Microsoft Azure Blob Storage. Export the Azure
-account name and key as follows:
+Blob Storage account name and key as follows:
 
 .. code-block:: console
 
@@ -618,6 +636,13 @@ initiate a new repository in the path ``bar`` in the remote ``foo``:
 
 Restic takes care of starting and stopping rclone.
 
+.. note:: If you get an error message saying "cannot implicitly run relative
+          executable rclone found in current directory", this means that an
+          rclone executable was found in the current directory. For security
+          reasons restic will not run this implicitly, instead you have to
+          use the ``-o rclone.program=./rclone`` extended option to override
+          this security check and explicitly tell restic to use the executable.
+
 As a more concrete example, suppose you have configured a remote named
 ``b2prod`` for Backblaze B2 with rclone, with a bucket called ``yggdrasil``.
 You can then use rclone to list files in the bucket like this:

doc/040_backup.rst

@@ -204,6 +204,7 @@ Combined with ``--verbose``, you can see a list of changes:
     modified  /archive.tar.gz, saved in 0.140s (25.542 MiB added)
     Would be added to the repository: 25.551 MiB
 
+.. _backup-excluding-files:
 Excluding Files
 ***************
 
@@ -299,7 +300,7 @@ directory, then selectively add back some of them.
 
 ::
 
-    $HOME/**/*
+    $HOME/*
     !$HOME/Documents
     !$HOME/code
     !$HOME/.emacs.d
@@ -555,6 +556,7 @@ environment variables. The following lists these environment variables:
     RESTIC_COMPRESSION                  Compression mode (only available for repository format version 2)
     RESTIC_PROGRESS_FPS                 Frames per second by which the progress bar is updated
     RESTIC_PACK_SIZE                    Target size for pack files
+    RESTIC_READ_CONCURRENCY             Concurrency for file reads
 
     TMPDIR                              Location for temporary files