Add version for 0.16.0

Tweak changelog entries

.dockerignore

@@ -1,12 +0,0 @@
-# Folders
-.git/
-.github/
-changelog/
-doc/
-docker/
-helpers/
-
-# Files
-.gitignore
-.golangci.yml
-*.md

.github/dependabot.yml

@@ -0,0 +1,13 @@
+version: 2
+updates:
+  # Dependencies listed in go.mod
+  - package-ecosystem: "gomod"
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "monthly"
+
+  # Dependencies listed in .github/workflows/*.yml
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"

.github/workflows/docker.yml

@@ -0,0 +1,59 @@
+
+name: Create and publish a Docker image
+
+on:
+  push:
+    tags:
+      - 'v*'
+    branches:
+      - 'master'
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    if: github.repository == 'restic/restic'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          push: true
+          context: .
+          file: docker/Dockerfile.release
+          platforms: linux/386,linux/amd64,linux/arm,linux/arm64
+          pull: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/tests.yml

@@ -7,9 +7,13 @@ on:
 
   # run tests for all pull requests
   pull_request:
+  merge_group:
+
+permissions:
+  contents: read
 
 env:
-  latest_go: "1.19.x"
+  latest_go: "1.20.x"
   GO111MODULE: on
 
 jobs:
@@ -19,47 +23,36 @@ jobs:
         # list of jobs to run:
         include:
           - job_name: Windows
-            go: 1.19.x
+            go: 1.20.x
             os: windows-latest
-            install_verb: install
 
           - job_name: macOS
-            go: 1.19.x
+            go: 1.20.x
             os: macOS-latest
             test_fuse: false
-            install_verb: install
 
           - job_name: Linux
-            go: 1.19.x
+            go: 1.20.x
             os: ubuntu-latest
             test_cloud_backends: true
             test_fuse: true
             check_changelog: true
-            install_verb: install
+
+          - job_name: Linux (race)
+            go: 1.20.x
+            os: ubuntu-latest
+            test_fuse: true
+            test_opts: "-race"
+
+          - job_name: Linux
+            go: 1.19.x
+            os: ubuntu-latest
+            test_fuse: true
 
           - job_name: Linux
             go: 1.18.x
             os: ubuntu-latest
             test_fuse: true
-            install_verb: install
-
-          - job_name: Linux
-            go: 1.17.x
-            os: ubuntu-latest
-            test_fuse: true
-            install_verb: install
-
-          - job_name: Linux
-            go: 1.16.x
-            os: ubuntu-latest
-            test_fuse: true
-            install_verb: get
-
-          - job_name: Linux
-            go: 1.15.x
-            os: ubuntu-latest
-            test_fuse: true
-            install_verb: get
 
     name: ${{ matrix.job_name }} Go ${{ matrix.go }}
     runs-on: ${{ matrix.os }}
@@ -69,14 +62,14 @@ jobs:
 
     steps:
       - name: Set up Go ${{ matrix.go }}
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v4
         with:
           go-version: ${{ matrix.go }}
 
       - name: Get programs (Linux/macOS)
         run: |
           echo "build Go tools"
-          go ${{ matrix.install_verb }} github.com/restic/rest-server/cmd/rest-server@latest
+          go install github.com/restic/rest-server/cmd/rest-server@latest
 
           echo "install minio server"
           mkdir $HOME/bin
@@ -98,7 +91,7 @@ jobs:
           chmod 755 $HOME/bin/rclone
           rm -rf rclone*
 
-          # add $HOME/bin to path ($GOBIN was already added to the path by setup-go@v2)
+          # add $HOME/bin to path ($GOBIN was already added to the path by setup-go@v3)
           echo $HOME/bin >> $GITHUB_PATH
         if: matrix.os == 'ubuntu-latest' || matrix.os == 'macOS-latest'
 
@@ -108,7 +101,7 @@ jobs:
           $ProgressPreference = 'SilentlyContinue'
 
           echo "build Go tools"
-          go ${{ matrix.install_verb }} github.com/restic/rest-server/...
+          go install github.com/restic/rest-server/...
 
           echo "install minio server"
           mkdir $Env:USERPROFILE/bin
@@ -120,7 +113,7 @@ jobs:
           unzip rclone.zip
           copy rclone*/rclone.exe $Env:USERPROFILE/bin
 
-          # add $USERPROFILE/bin to path ($GOBIN was already added to the path by setup-go@v2)
+          # add $USERPROFILE/bin to path ($GOBIN was already added to the path by setup-go@v3)
           echo $Env:USERPROFILE\bin >> $Env:GITHUB_PATH
 
           echo "install tar"
@@ -142,17 +135,25 @@ jobs:
         if: matrix.os == 'windows-latest'
 
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Build with build.go
         run: |
           go run build.go
 
+      - name: Minimal test
+        run: |
+          ./restic init
+          ./restic backup .
+        env:
+          RESTIC_REPOSITORY: ../testrepo
+          RESTIC_PASSWORD: password
+
       - name: Run local Tests
         env:
           RESTIC_TEST_FUSE: ${{ matrix.test_fuse }}
         run: |
-          go test -cover ./...
+          go test -cover ${{matrix.test_opts}} ./...
 
       - name: Test cloud backends
         env:
@@ -193,7 +194,9 @@ jobs:
 
         # only run cloud backend tests for pull requests from and pushes to our
         # own repo, otherwise the secrets are not available
-        if: (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && matrix.test_cloud_backends
+        # Skip for Dependabot pull requests as these are run without secrets
+        # https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#responding-to-events
+        if: ((github.repository == 'restic/restic' && github.event_name == 'push') || github.event.pull_request.head.repo.full_name == github.repository) && (github.actor != 'dependabot[bot]') && matrix.test_cloud_backends
 
       - name: Check changelog files with calens
         run: |
@@ -207,70 +210,53 @@ jobs:
   cross_compile:
     strategy:
 
-      # ATTENTION: the list of architectures must be in sync with helpers/build-release-binaries/main.go!
       matrix:
-        # run cross-compile in two batches parallel so the overall tests run faster
-        targets:
-          - "linux/386 linux/amd64 linux/arm linux/arm64 linux/ppc64le linux/mips linux/mipsle linux/mips64 linux/mips64le linux/s390x \
-            openbsd/386 openbsd/amd64"
-
-          - "freebsd/386 freebsd/amd64 freebsd/arm \
-            aix/ppc64 \
-            darwin/amd64 darwin/arm64 \
-            netbsd/386 netbsd/amd64 \
-            windows/386 windows/amd64 \
-            solaris/amd64"
+        # run cross-compile in three batches parallel so the overall tests run faster
+        subset:
+          - "0/3"
+          - "1/3"
+          - "2/3"
 
     env:
       GOPROXY: https://proxy.golang.org
 
     runs-on: ubuntu-latest
 
-    name: Cross Compile for ${{ matrix.targets }}
+    name: Cross Compile for subset ${{ matrix.subset }}
 
     steps:
       - name: Set up Go ${{ env.latest_go }}
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v4
         with:
           go-version: ${{ env.latest_go }}
 
-      - name: Install gox
-        run: |
-          go install github.com/mitchellh/gox@latest
-
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
-      - name: Cross-compile with gox for ${{ matrix.targets }}
-        env:
-          GOFLAGS: "-trimpath"
-          GOX_ARCHS: "${{ matrix.targets }}"
+      - name: Cross-compile for subset ${{ matrix.subset }}
         run: |
-          mkdir build-output
-          gox -parallel 2 -verbose -osarch "$GOX_ARCHS" -output "build-output/{{.Dir}}_{{.OS}}_{{.Arch}}" ./cmd/restic
-          gox -parallel 2 -verbose -osarch "$GOX_ARCHS" -tags debug -output "build-output/{{.Dir}}_{{.OS}}_{{.Arch}}_debug" ./cmd/restic
+          mkdir build-output build-output-debug
+          go run ./helpers/build-release-binaries/main.go -o build-output -s . --platform-subset ${{ matrix.subset }}
+          go run ./helpers/build-release-binaries/main.go -o build-output-debug -s . --platform-subset ${{ matrix.subset }} --tags debug
 
   lint:
     name: lint
     runs-on: ubuntu-latest
     steps:
       - name: Set up Go ${{ env.latest_go }}
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v4
         with:
           go-version: ${{ env.latest_go }}
 
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v2
+        uses: golangci/golangci-lint-action@v3
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.48
-          # Optional: show only new issues if it's a pull request. The default value is `false`.
-          only-new-issues: true
+          version: v1.52.2
           args: --verbose --timeout 5m
-          skip-go-installation: true
 
         # only run golangci-lint for pull requests, otherwise ALL hints get
         # reported. We need to slowly address all issues until we can enable
@@ -283,16 +269,31 @@ jobs:
           go mod tidy
           git diff --exit-code go.mod go.sum
 
+  analyze:
+    name: Analyze results
+    needs: [test, cross_compile, lint]
+    if: always()
+
+    permissions: # no need to access code
+      contents: none
+
+    runs-on: ubuntu-latest
+    steps:
+      - name: Decide whether the needed jobs succeeded or failed
+        uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe
+        with:
+          jobs: ${{ toJSON(needs) }}
+
   docker:
     name: docker
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v4
         with:
           # list of Docker images to use as base name for tags
           images: |
@@ -308,14 +309,14 @@ jobs:
             type=sha
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v4
         with:
           push: false
           context: .

.gitignore

@@ -1,3 +1,4 @@
 /restic
+/restic.exe
 /.vagrant
 /.vscode

.golangci.yml

@@ -10,13 +10,10 @@ linters:
     # make sure all errors returned by functions are handled
     - errcheck
 
-    # find unused code
-    - deadcode
-
     # show how code can be simplified
     - gosimple
 
-    # # make sure code is formatted
+    # make sure code is formatted
     - gofmt
 
     # examine code and report suspicious constructs, such as Printf calls whose
@@ -35,12 +32,6 @@ linters:
     # find unused variables, functions, structs, types, etc.
     - unused
 
-    # find unused struct fields
-    - structcheck
-
-    # find unused global variables
-    - varcheck
-
     # parse and typecheck code
     - typecheck
 
@@ -55,3 +46,8 @@ issues:
     - exported (function|method|var|type|const) .* should have comment or be unexported
     # revive: ignore constants in all caps
     - don't use ALL_CAPS in Go names; use CamelCase
+    # revive: lots of packages don't have such a comment
+    - "package-comments: should have a package comment"
+    # staticcheck: there's no easy way to replace these packages
+    - "SA1019: \"golang.org/x/crypto/poly1305\" is deprecated"
+    - "SA1019: \"golang.org/x/crypto/openpgp\" is deprecated"

CONTRIBUTING.md

@@ -58,6 +58,19 @@ Please be aware that the debug log file will contain potentially sensitive
 things like file and directory names, so please either redact it before
 uploading it somewhere or post only the parts that are really relevant.
 
+If restic gets stuck, please also include a stacktrace in the description.
+On non-Windows systems, you can send a SIGQUIT signal to restic or press
+`Ctrl-\` to achieve the same result. This causes restic to print a stacktrace
+and then exit immediatelly. This will not damage your repository, however,
+it might be necessary to manually clean up stale lock files using
+`restic unlock`.
+
+On Windows, please set the environment variable `RESTIC_DEBUG_STACKTRACE_SIGINT`
+to `true` and press `Ctrl-C` to create a stacktrace.
+
+If you think restic uses too much memory or a too large cache directory, then
+please include the output of `restic stats --mode debug`.
+
 
 Development Environment
 =======================
@@ -76,12 +89,42 @@ Then use the `go` tool to build restic:
 
     $ go build ./cmd/restic
     $ ./restic version
-    restic 0.10.0-dev (compiled manually) compiled with go1.15.2 on linux/amd64
+    restic 0.14.0-dev (compiled manually) compiled with go1.19 on linux/amd64
+
+To create a debug build use:
+
+    $ go build -tags debug ./cmd/restic
 
 You can run all tests with the following command:
 
     $ go test ./...
 
+
+Performance and Memory Usage Issues
+===================================
+
+Debug builds of restic support the `--block-profile`, `--cpu-profile`,
+`--mem-profile`, and `--trace-profile` options which collect performance data
+that later on can be analyzed using the go tools:
+
+    $ restic --cpu-profile . [...]
+    $ go tool pprof -http localhost:12345 cpu.pprof
+
+To analyze a trace profile use `go tool trace -http=localhost:12345 trace.out`.
+
+As the memory usage of restic changes over time, it may be useful to capture a
+snapshot of the current heap. This is possible using then `--listen-profile`
+option. Then while restic runs you can query and afterwards analyze the heap statistics.
+
+    $ restic --listen-profile localhost:12345 [...]
+    $ curl http://localhost:12345/debug/pprof/heap -o heap.pprof
+    $ go tool pprof -http localhost:12345 heap.pprof
+
+Further useful tools are setting the environment variable `GODEBUG=gctrace=1`,
+which provides information about garbage collector runs. For a graphical variant
+combine this with gcvis.
+
+
 Providing Patches
 =================
 

VERSION

@@ -1 +1 @@
-0.14.0
+0.16.0

build.go

@@ -3,8 +3,8 @@
 // This program aims to make building Go programs for end users easier by just
 // calling it with `go run`, without having to setup a GOPATH.
 //
-// This program needs Go >= 1.12. It'll use Go modules for compilation. It
-// builds the package configured as Main in the Config struct.
+// This program checks for a minimum Go version. It will use Go modules for
+// compilation. It builds the package configured as Main in the Config struct.
 
 // BSD 2-Clause License
 //
@@ -43,7 +43,6 @@ package main
 import (
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -59,7 +58,7 @@ var config = Config{
 	Main:             "./cmd/restic",                           // package name for the main package
 	DefaultBuildTags: []string{"selfupdate"},                   // specify build tags which are always used
 	Tests:            []string{"./..."},                        // tests to run
-	MinVersion:       GoVersion{Major: 1, Minor: 14, Patch: 0}, // minimum Go version supported
+	MinVersion:       GoVersion{Major: 1, Minor: 18, Patch: 0}, // minimum Go version supported
 }
 
 // Config configures the build.
@@ -179,7 +178,7 @@ func test(cwd string, env map[string]string, args ...string) error {
 // getVersion returns the version string from the file VERSION in the current
 // directory.
 func getVersionFromFile() string {
-	buf, err := ioutil.ReadFile("VERSION")
+	buf, err := os.ReadFile("VERSION")
 	if err != nil {
 		verbosePrintf("error reading file VERSION: %v\n", err)
 		return ""
@@ -319,12 +318,8 @@ func (v GoVersion) String() string {
 }
 
 func main() {
-	if !goVersion.AtLeast(GoVersion{1, 12, 0}) {
-		die("Go version (%v) is too old, restic requires Go >= 1.12\n", goVersion)
-	}
-
 	if !goVersion.AtLeast(config.MinVersion) {
-		fmt.Fprintf(os.Stderr, "%s detected, this program requires at least %s\n", goVersion, config.MinVersion)
+		fmt.Fprintf(os.Stderr, "Detected version %s is too old, restic requires at least %s\n", goVersion, config.MinVersion)
 		os.Exit(1)
 	}
 
@@ -385,6 +380,12 @@ func main() {
 		}
 	}
 
+	solarisMinVersion := GoVersion{Major: 1, Minor: 20, Patch: 0}
+	if env["GOARCH"] == "solaris" && !goVersion.AtLeast(solarisMinVersion) {
+		fmt.Fprintf(os.Stderr, "Detected version %s is too old, restic requires at least %s for Solaris\n", goVersion, solarisMinVersion)
+		os.Exit(1)
+	}
+
 	verbosePrintf("detected Go version %v\n", goVersion)
 
 	preserveSymbols := false

changelog/0.15.0_2023-01-12/issue-14

@@ -0,0 +1,8 @@
+Enhancement: Implement `rewrite` command
+
+Restic now has a `rewrite` command which allows to rewrite existing snapshots
+to remove unwanted files.
+
+https://github.com/restic/restic/issues/14
+https://github.com/restic/restic/pull/2731
+https://github.com/restic/restic/pull/4079

changelog/0.15.0_2023-01-12/issue-1734

@@ -0,0 +1,15 @@
+Enhancement: Inform about successful retries after errors
+
+When a recoverable error is encountered, restic shows a warning message saying
+that it's retrying, e.g.:
+
+`Save(<data/956b9ced99>) returned error, retrying after 357.131936ms: ...`
+
+This message can be confusing in that it never clearly states whether the retry
+is successful or not. This has now been fixed such that restic follows up with
+a message confirming a successful retry, e.g.:
+
+`Save(<data/956b9ced99>) operation successful after 1 retries`
+
+https://github.com/restic/restic/issues/1734
+https://github.com/restic/restic/pull/2661

changelog/0.15.0_2023-01-12/issue-1866

@@ -0,0 +1,12 @@
+Enhancement: Improve handling of directories with duplicate entries
+
+If for some reason a directory contains a duplicate entry, the `backup` command
+would previously fail with a `node "path/to/file" already present` or `nodes
+are not ordered got "path/to/file", last "path/to/file"` error.
+
+The error handling has been improved to only report a warning in this case. Make
+sure to check that the filesystem in question is not damaged if you see this!
+
+https://github.com/restic/restic/issues/1866
+https://github.com/restic/restic/issues/3937
+https://github.com/restic/restic/pull/3880

changelog/0.15.0_2023-01-12/issue-2015

@@ -0,0 +1,10 @@
+Bugfix: Make `mount` return exit code 0 after receiving Ctrl-C / SIGINT
+
+To stop the `mount` command, a user has to press Ctrl-C or send a SIGINT
+signal to restic. This used to cause restic to exit with a non-zero exit code.
+
+The exit code has now been changed to zero as the above is the expected way
+to stop the `mount` command and should therefore be considered successful.
+
+https://github.com/restic/restic/issues/2015
+https://github.com/restic/restic/pull/3894

changelog/0.15.0_2023-01-12/issue-2134

@@ -0,0 +1,19 @@
+Enhancement: Support B2 API keys restricted to hiding but not deleting files
+
+When the B2 backend does not have the necessary permissions to permanently
+delete files, it now automatically falls back to hiding files. This allows
+using restic with an application key which is not allowed to delete files.
+This can prevent an attacker from deleting backups with such an API key.
+
+To use this feature create an application key without the `deleteFiles`
+capability. It is recommended to restrict the key to just one bucket.
+For example using the `b2` command line tool:
+
+`b2 create-key --bucket <bucketName> <keyName> listBuckets,readFiles,writeFiles,listFiles`
+
+Alternatively, you can use the S3 backend to access B2, as described
+in the documentation. In this mode, files are also only hidden instead
+of being deleted permanently.
+
+https://github.com/restic/restic/issues/2134
+https://github.com/restic/restic/pull/2398

changelog/0.15.0_2023-01-12/issue-2152

@@ -0,0 +1,11 @@
+Enhancement: Make `init` open only one connection for the SFTP backend
+
+The `init` command using the SFTP backend used to connect twice to the
+repository. This could be inconvenient if the user must enter a password,
+or cause `init` to fail if the server does not correctly close the first SFTP
+connection.
+
+This has now been fixed by reusing the first/initial SFTP connection opened.
+
+https://github.com/restic/restic/issues/2152
+https://github.com/restic/restic/pull/3882

changelog/0.15.0_2023-01-12/issue-2533

@@ -0,0 +1,13 @@
+Enhancement: Handle cache corruption on disk and in downloads
+
+In rare situations, like for example after a system crash, the data stored
+in the cache might be corrupted. This could cause restic to fail and required
+manually deleting the cache.
+
+Restic now automatically removes broken data from the cache, allowing it
+to recover from such a situation without user intervention. In addition,
+restic retries downloads which return corrupt data in order to also handle
+temporary download problems.
+
+https://github.com/restic/restic/issues/2533
+https://github.com/restic/restic/pull/3521

changelog/0.15.0_2023-01-12/issue-2591

@@ -0,0 +1,17 @@
+Bugfix: Don't read password from stdin for `backup --stdin`
+
+The `backup` command when used with `--stdin` previously tried to read first
+the password, then the data to be backed up from standard input. This meant
+it would often confuse part of the data for the password.
+
+From now on, it will instead exit with the message `Fatal: cannot read both
+password and data from stdin` unless the password is passed in some other
+way (such as `--restic-password-file`, `RESTIC_PASSWORD`, etc).
+
+To enter the password interactively a password command has to be used. For
+example on Linux, `mysqldump somedatabase | restic backup --stdin
+--password-command='sh -c "systemd-ask-password < /dev/tty"'` securely reads
+the password from the terminal.
+
+https://github.com/restic/restic/issues/2591
+https://github.com/restic/restic/pull/4011

changelog/0.15.0_2023-01-12/issue-2699

@@ -0,0 +1,9 @@
+Enhancement: Support restoring symbolic links on Windows
+
+The `restore` command now supports restoring symbolic links on Windows. Because
+of Windows specific restrictions this is only possible when running restic with
+the `SeCreateSymbolicLinkPrivilege` privilege or as an administrator.
+
+https://github.com/restic/restic/issues/1078
+https://github.com/restic/restic/issues/2699
+https://github.com/restic/restic/pull/2875

changelog/0.15.0_2023-01-12/issue-2715

@@ -0,0 +1,20 @@
+Enhancement: Stricter repository lock handling
+
+Previously, restic commands kept running even if they failed to refresh their
+locks in time. This could be a problem e.g. in case the client system running
+a backup entered the standby power mode while the backup was still in progress
+(which would prevent the client from refreshing its lock), and after a short
+delay another host successfully runs `unlock` and `prune` on the repository,
+which would remove all data added by the in-progress backup. If the backup
+client later continues its backup, even though its lock had expired in the
+meantime, this would lead to an incomplete snapshot.
+
+To address this, lock handling is now much stricter. Commands requiring a lock
+are canceled if the lock is not refreshed successfully in time. In addition,
+if a lock file is not readable restic will not allow starting a command. It may
+be necessary to remove invalid lock files manually or use `unlock --remove-all`.
+Please make sure that no other restic processes are running concurrently before
+doing this, however.
+
+https://github.com/restic/restic/issues/2715
+https://github.com/restic/restic/pull/3569

changelog/0.15.0_2023-01-12/issue-2724

@@ -0,0 +1,9 @@
+Change: Include full snapshot ID in JSON output of `backup`
+
+We have changed the JSON output of the backup command to include the full
+snapshot ID instead of just a shortened version, as the latter can be ambiguous
+in some rare cases. To derive the short ID, please truncate the full ID down to
+eight characters.
+
+https://github.com/restic/restic/issues/2724
+https://github.com/restic/restic/pull/3993

changelog/0.15.0_2023-01-12/issue-3029

@@ -0,0 +1,8 @@
+Enhancement: Add support for `credential_process` to S3 backend
+
+Restic now uses a newer library for the S3 backend, which adds support for the
+`credential_process` option in the AWS credential configuration.
+
+https://github.com/restic/restic/issues/3029
+https://github.com/restic/restic/issues/4034
+https://github.com/restic/restic/pull/4025

changelog/0.15.0_2023-01-12/issue-3096

@@ -0,0 +1,8 @@
+Enhancement: Make `mount` command support macOS using macFUSE 4.x
+
+Restic now uses a different FUSE library for mounting snapshots and making them
+available as a FUSE filesystem using the `mount` command. This adds support for
+macFUSE 4.x which can be used to make this work on recent macOS versions.
+
+https://github.com/restic/restic/issues/3096
+https://github.com/restic/restic/pull/4024

changelog/0.15.0_2023-01-12/issue-3124

@@ -0,0 +1,7 @@
+Enhancement: Support JSON output for the `init` command
+
+The `init` command used to ignore the `--json` option, but now outputs a JSON
+message if the repository was created successfully.
+
+https://github.com/restic/restic/issues/3124
+https://github.com/restic/restic/pull/3132

changelog/0.15.0_2023-01-12/issue-3161

@@ -0,0 +1,14 @@
+Bugfix: Delete files on Backblaze B2 more reliably
+
+Restic used to only delete the latest version of files stored in B2. In most
+cases this worked well as there was only a single version of the file. However,
+due to retries while uploading it is possible for multiple file versions to be
+stored at B2. This could lead to various problems for files that should have
+been deleted but still existed.
+
+The implementation has now been changed to delete all versions of files, which
+doubles the amount of Class B transactions necessary to delete files, but
+assures that no file versions are left behind.
+
+https://github.com/restic/restic/issues/3161
+https://github.com/restic/restic/pull/3885

changelog/0.15.0_2023-01-12/issue-3336

@@ -0,0 +1,12 @@
+Bugfix: Make SFTP backend report no space left on device
+
+Backing up to an SFTP backend would spew repeated SSH_FX_FAILURE messages when
+the remote disk was full. Restic now reports "sftp: no space left on device"
+and exits immediately when it detects this condition.
+
+A fix for this issue was implemented in restic 0.12.1, but unfortunately the
+fix itself contained a bug that prevented it from taking effect.
+
+https://github.com/restic/restic/issues/3336
+https://github.com/restic/restic/pull/3345
+https://github.com/restic/restic/pull/4075

changelog/0.15.0_2023-01-12/issue-3567

@@ -0,0 +1,10 @@
+Bugfix: Improve handling of interrupted syscalls in `mount` command
+
+Accessing restic's FUSE mount could result in "input/output" errors when using
+programs in which syscalls can be interrupted. This is for example the case for
+Go programs. This has now been fixed by improved error handling of interrupted
+syscalls.
+
+https://github.com/restic/restic/issues/3567
+https://github.com/restic/restic/issues/3694
+https://github.com/restic/restic/pull/3875

changelog/0.15.0_2023-01-12/issue-3897

@@ -0,0 +1,7 @@
+Bugfix: Fix stuck `copy` command when `-o <backend>.connections=1`
+
+When running the `copy` command with `-o <backend>.connections=1` the
+command would be infinitely stuck. This has now been fixed.
+
+https://github.com/restic/restic/issues/3897
+https://github.com/restic/restic/pull/3898

changelog/0.15.0_2023-01-12/issue-3918

@@ -0,0 +1,9 @@
+Bugfix: Correct prune statistics for partially compressed repositories
+
+In a partially compressed repository, one data blob can exist both in an
+uncompressed and a compressed version. This caused the `prune` statistics to
+become inaccurate and e.g. report a too high value for the unused size, such
+as "unused size after prune: 16777215.991 TiB". This has now been fixed.
+
+https://github.com/restic/restic/issues/3918
+https://github.com/restic/restic/pull/3980

changelog/0.15.0_2023-01-12/issue-3929

@@ -0,0 +1,11 @@
+Change: Make `unlock` display message only when locks were actually removed
+
+The `unlock` command used to print the "successfully removed locks" message
+whenever it was run, regardless of lock files having being removed or not.
+
+This has now been changed such that it only prints the message if any lock
+files were actually removed. In addition, it also reports the number of
+removed lock files.
+
+https://github.com/restic/restic/issues/3929
+https://github.com/restic/restic/pull/3935

changelog/0.15.0_2023-01-12/issue-3932

@@ -0,0 +1,15 @@
+Enhancement: Improve handling of ErrDot errors in rclone and sftp backends
+
+Since Go 1.19, restic can no longer implicitly run relative executables which
+are found in the current directory (e.g. `rclone` if found in `.`). This is a
+security feature of Go to prevent against running unintended and possibly
+harmful executables.
+
+The error message for this was just "cannot run executable found relative to
+current directory". This has now been improved to yield a more specific error
+message, informing the user how to explicitly allow running the executable
+using the `-o rclone.program` and `-o sftp.command` extended options with `./`.
+
+https://github.com/restic/restic/issues/3932
+https://pkg.go.dev/os/exec#hdr-Executables_in_the_current_directory
+https://go.dev/blog/path-security

changelog/0.15.0_2023-01-12/issue-4003

@@ -0,0 +1,8 @@
+Bugfix: Make `backup` no longer hang on Solaris when seeing a FIFO file
+
+The `backup` command used to hang on Solaris whenever it encountered a FIFO
+file (named pipe), due to a bug in the handling of extended attributes. This
+bug has now been fixed.
+
+https://github.com/restic/restic/issues/4003
+https://github.com/restic/restic/pull/4053

changelog/0.15.0_2023-01-12/issue-4016

@@ -0,0 +1,8 @@
+Bugfix: Support ExFAT-formatted local backends on macOS Ventura
+
+ExFAT-formatted disks could not be used as local backends starting from macOS
+Ventura. Restic commands would fail with an "inappropriate ioctl for device"
+error. This has now been fixed.
+
+https://github.com/restic/restic/issues/4016
+https://github.com/restic/restic/pull/4021

changelog/0.15.0_2023-01-12/issue-4033

@@ -0,0 +1,11 @@
+Change: Don't print skipped snapshots by default in `copy` command
+
+The `copy` command used to print each snapshot that was skipped because it
+already existed in the target repository. The amount of this output could
+practically bury the list of snapshots that were actually copied.
+
+From now on, the skipped snapshots are by default not printed at all, but
+this can be re-enabled by increasing the verbosity level of the command.
+
+https://github.com/restic/restic/issues/4033
+https://github.com/restic/restic/pull/4066

changelog/0.15.0_2023-01-12/issue-4085

@@ -0,0 +1,10 @@
+Bugfix: Make `init` ignore "Access Denied" errors when creating S3 buckets
+
+In restic 0.9.0 through 0.13.0, the `init` command ignored some permission
+errors from S3 backends when trying to check for bucket existence, so that
+manually created buckets with custom permissions could be used for backups.
+
+This feature became broken in 0.14.0, but has now been restored again.
+
+https://github.com/restic/restic/issues/4085
+https://github.com/restic/restic/pull/4086

changelog/0.15.0_2023-01-12/issue-4103

@@ -0,0 +1,10 @@
+Bugfix: Don't generate negative UIDs and GIDs in tar files from `dump`
+
+When using a 32-bit build of restic, the `dump` command could in some cases
+create tar files containing negative UIDs and GIDs, which cannot be read by
+GNU tar. This corner case especially applies to backups from stdin on Windows.
+
+This is now fixed such that `dump` creates valid tar files in these cases too.
+
+https://github.com/restic/restic/issues/4103
+https://github.com/restic/restic/pull/4104

changelog/0.15.0_2023-01-12/issue-79

@@ -0,0 +1,17 @@
+Enhancement: Restore files with long runs of zeros as sparse files
+
+When using `restore --sparse`, the restorer may now write files containing long
+runs of zeros as sparse files (also called files with holes), where the zeros
+are not actually written to disk.
+
+How much space is saved by writing sparse files depends on the operating
+system, file system and the distribution of zeros in the file.
+
+During backup restic still reads the whole file including sparse regions, but
+with optimized processing speed of sparse regions.
+
+https://github.com/restic/restic/issues/79
+https://github.com/restic/restic/issues/3903
+https://github.com/restic/restic/pull/2601
+https://github.com/restic/restic/pull/3854
+https://forum.restic.net/t/sparse-file-support/1264

changelog/0.15.0_2023-01-12/pull-2750

@@ -0,0 +1,7 @@
+Enhancement: Make backup file read concurrency configurable
+
+The `backup` command now supports a `--read-concurrency` option which allows
+tuning restic for very fast storage like NVMe disks by controlling the number
+of concurrent file reads during the backup process.
+
+https://github.com/restic/restic/pull/2750

changelog/0.15.0_2023-01-12/pull-3780

@@ -0,0 +1,8 @@
+Bugfix: Make `restore` replace existing symlinks
+
+When restoring a symlink, restic used to report an error if the target path
+already existed. This has now been fixed such that the potentially existing
+target path is first removed before the symlink is restored.
+
+https://github.com/restic/restic/issues/2578
+https://github.com/restic/restic/pull/3780

changelog/0.15.0_2023-01-12/pull-3899

@@ -0,0 +1,6 @@
+Enhancement: Optimize prune memory usage
+
+The `prune` command needs large amounts of memory in order to determine what to
+keep and what to remove. This is now optimized to use up to 30% less memory.
+
+https://github.com/restic/restic/pull/3899

changelog/0.15.0_2023-01-12/pull-3905

@@ -0,0 +1,6 @@
+Enhancement: Improve speed of parent snapshot detection in `backup` command
+
+Backing up a large number of files using `--files-from-verbatim` or `--files-from-raw`
+options could require a long time to find the parent snapshot. This has been improved.
+
+https://github.com/restic/restic/pull/3905

changelog/0.15.0_2023-01-12/pull-3915

@@ -0,0 +1,12 @@
+Enhancement: Add compression statistics to the `stats` command
+
+When executed with `--mode raw-data` on a repository that supports compression,
+the `stats` command now calculates and displays, for the selected repository or
+snapshots: the uncompressed size of the data; the compression progress
+(percentage of data that has been compressed); the compression ratio of the
+compressed data; the total space saving.
+
+It also takes into account both the compressed and uncompressed data if the
+repository is only partially compressed.
+
+https://github.com/restic/restic/pull/3915

changelog/0.15.0_2023-01-12/pull-3925

@@ -0,0 +1,6 @@
+Enhancement: Provide command completion for PowerShell
+
+Restic already provided generation of completion files for bash, fish and zsh.
+Now powershell is supported, too.
+
+https://github.com/restic/restic/pull/3925/files

changelog/0.15.0_2023-01-12/pull-3931

@@ -0,0 +1,10 @@
+Enhancement: Allow `backup` file tree scanner to be disabled
+
+The `backup` command walks the file tree in a separate scanner process to find
+the total size and file/directory count, and uses this to provide an ETA. This
+can slow down backups, especially of network filesystems.
+
+The command now has a new option `--no-scan` which can be used to disable this
+scanning in order to speed up backups when needed.
+
+https://github.com/restic/restic/pull/3931

changelog/0.15.0_2023-01-12/pull-3943

@@ -0,0 +1,9 @@
+Enhancement: Ignore additional/unknown files in repository
+
+If a restic repository had additional files in it (not created by restic),
+commands like `find` and `restore` could become confused and fail with an
+`multiple IDs with prefix "12345678" found` error. These commands now
+ignore such additional files.
+
+https://github.com/restic/restic/pull/3943
+https://forum.restic.net/t/which-protocol-should-i-choose-for-remote-linux-backups/5446/17

changelog/0.15.0_2023-01-12/pull-3951

@@ -0,0 +1,7 @@
+Bugfix: Make `ls` return exit code 1 if snapshot cannot be loaded
+
+The `ls` command used to show a warning and return exit code 0 when failing
+to load a snapshot. This has now been fixed such that it instead returns exit
+code 1 (still showing a warning).
+
+https://github.com/restic/restic/pull/3951

changelog/0.15.0_2023-01-12/pull-3955

@@ -0,0 +1,9 @@
+Enhancement: Improve `backup` performance for small files
+
+When backing up small files restic was slower than it could be. In particular
+this affected backups using maximum compression.
+
+This has been fixed by reworking the internal parallelism of the backup
+command, making it back up small files around two times faster.
+
+https://github.com/restic/restic/pull/3955

changelog/0.15.0_2023-01-12/pull-4041

@@ -0,0 +1,7 @@
+Change: Update dependencies and require Go 1.18 or newer
+
+Most dependencies have been updated. Since some libraries require newer language
+features, support for Go 1.15-1.17 has been dropped, which means that restic now
+requires at least Go 1.18 to build.
+
+https://github.com/restic/restic/pull/4041

changelog/0.15.0_2023-01-12/pull-4100

@@ -0,0 +1,11 @@
+Bugfix: Make `self-update` enabled by default only in release builds
+
+The `self-update` command was previously included by default in all builds of
+restic as opposed to only in official release builds, even if the `selfupdate`
+tag was not explicitly enabled when building.
+
+This has now been corrected, and the `self-update` command is only available
+if restic was built with `-tags selfupdate` (as done for official release
+builds by `build.go`).
+
+https://github.com/restic/restic/pull/4100

changelog/0.15.1_2023-01-30/issue-3750

@@ -0,0 +1,10 @@
+Bugfix: Remove `b2_download_file_by_name: 404` warning from B2 backend
+
+In some cases the B2 backend could print `b2_download_file_by_name: 404: :
+b2.b2err` warnings. These are only debug messages and can be safely ignored.
+
+Restic now uses an updated library for accessing B2, which removes the warning.
+
+https://github.com/restic/restic/issues/3750
+https://github.com/restic/restic/issues/4144
+https://github.com/restic/restic/pull/4146

changelog/0.15.1_2023-01-30/issue-4147

@@ -0,0 +1,7 @@
+Bugfix: Make `prune --quiet` not print progress bar
+
+A regression in restic 0.15.0 caused `prune --quiet` to show a progress bar
+while deciding how to process each pack files. This has now been fixed.
+
+https://github.com/restic/restic/issues/4147
+https://github.com/restic/restic/pull/4153

changelog/0.15.1_2023-01-30/pull-4152

@@ -0,0 +1,19 @@
+Enhancement: Ignore empty lock files
+
+With restic 0.15.0 the checks for stale locks became much stricter than before.
+In particular, empty or unreadable locks were no longer silently ignored. This
+made restic to complain with `Load(<lock/1234567812>, 0, 0) returned error,
+retrying after 552.330144ms: load(<lock/1234567812>): invalid data returned`
+and fail in the end.
+
+The error message is now clarified and the implementation changed to ignore
+empty lock files which are sometimes created as the result of a failed uploads
+on some backends.
+
+Please note that unreadable lock files still have to cleaned up manually. To do
+so, you can run `restic unlock --remove-all` which removes all existing lock
+files. But first make sure that no other restic process is currently using the
+repository.
+
+https://github.com/restic/restic/issues/4143
+https://github.com/restic/restic/pull/4152

changelog/0.15.1_2023-01-30/pull-4163

@@ -0,0 +1,13 @@
+Bugfix: Make `self-update --output` work with new filename on Windows
+
+Since restic 0.14.0 the `self-update` command did not work when a custom output
+filename was specified via the `--output` option. This has now been fixed.
+
+As a workaround, either use an older restic version to run the self-update or
+create an empty file with the output filename before updating e.g. using CMD:
+
+`type nul > new-file.exe`
+`restic self-update --output new-file.exe`
+
+https://github.com/restic/restic/pull/4163
+https://forum.restic.net/t/self-update-windows-started-failing-after-release-of-0-15/5836

changelog/0.15.1_2023-01-30/pull-4167

@@ -0,0 +1,6 @@
+Bugfix: Add missing ETA in `backup` progress bar
+
+A regression in restic 0.15.0 caused the ETA to be missing from the progress
+bar displayed by the `backup` command. This has now been fixed.
+
+https://github.com/restic/restic/pull/4167

changelog/0.15.2_2023-04-24/issue-2260

@@ -0,0 +1,12 @@
+Bugfix: Sanitize filenames printed by `backup` during processing
+
+The `backup` command would previously not sanitize the filenames it printed
+during processing, potentially causing newlines or terminal control characters
+to mangle the status output or even change the state of a terminal.
+
+Filenames are now checked and quoted if they contain non-printable or
+non-Unicode characters.
+
+https://github.com/restic/restic/issues/2260
+https://github.com/restic/restic/issues/4191
+https://github.com/restic/restic/pull/4192

changelog/0.15.2_2023-04-24/issue-4211

@@ -0,0 +1,8 @@
+Bugfix: Make `dump` interpret `--host` and `--path` correctly
+
+A regression in restic 0.15.0 caused `dump` to confuse its `--host=<host>` and
+`--path=<path>` options: it looked for snapshots with paths called `<host>`
+from hosts called `<path>`. It now treats the options as intended.
+
+https://github.com/restic/restic/issues/4211
+https://github.com/restic/restic/pull/4212

changelog/0.15.2_2023-04-24/issue-4239

@@ -0,0 +1,11 @@
+Bugfix: Correct number of blocks reported in mount point
+
+Restic mount points reported an incorrect number of 512-byte (POSIX standard)
+blocks for files and links due to a rounding bug. In particular, empty files
+were reported as taking one block instead of zero.
+
+The rounding is now fixed: the number of blocks reported is the file size
+(or link target size) divided by 512 and rounded up to a whole number.
+
+https://github.com/restic/restic/issues/4239
+https://github.com/restic/restic/pull/4240

changelog/0.15.2_2023-04-24/issue-4253

@@ -0,0 +1,18 @@
+Bugfix: Minimize risk of spurious filesystem loops with `mount`
+
+When a backup contains a directory that has the same name as its parent, say
+`a/b/b`, and the GNU `find` command was run on this backup in a restic mount,
+`find` would refuse to traverse the lowest `b` directory, instead printing
+`File system loop detected`. This was due to the way the restic mount command
+generates inode numbers for directories in the mount point.
+
+The rule for generating these inode numbers was changed in 0.15.0. It has
+now been changed again to avoid this issue. A perfect rule does not exist,
+but the probability of this behavior occurring is now extremely small.
+
+When it does occur, the mount point is not broken, and scripts that traverse
+the mount point should work as long as they don't rely on inode numbers for
+detecting filesystem loops.
+
+https://github.com/restic/restic/issues/4253
+https://github.com/restic/restic/pull/4255

changelog/0.15.2_2023-04-24/issue-4275

@@ -0,0 +1,4 @@
+Security: Update golang.org/x/net to address CVE-2022-41723
+
+https://github.com/restic/restic/issues/4275
+https://github.com/restic/restic/pull/4213

changelog/0.15.2_2023-04-24/pull-4180

@@ -0,0 +1,6 @@
+Enhancement: Add release binaries for riscv64 architecture on Linux
+
+Builds for the `riscv64` architecture on Linux are now included in the
+release binaries.
+
+https://github.com/restic/restic/pull/4180

changelog/0.15.2_2023-04-24/pull-4219

@@ -0,0 +1,5 @@
+Enhancement: Upgrade Minio to version 7.0.49
+
+The upgraded version now allows use of the `ap-southeast-4` region (Melbourne).
+
+https://github.com/restic/restic/pull/4219

changelog/0.16.0_2023-07-31/issue-1495

@@ -0,0 +1,7 @@
+Enhancement: Sort snapshots by timestamp in `restic find`
+
+The `find` command used to print snapshots in an arbitrary order. Restic now
+prints snapshots sorted by timestamp.
+
+https://github.com/restic/restic/issues/1495
+https://github.com/restic/restic/pull/4409

changelog/0.16.0_2023-07-31/issue-1759

@@ -0,0 +1,21 @@
+Enhancement: Add `repair index` and `repair snapshots` commands
+
+The `rebuild-index` command has been renamed to `repair index`. The old name
+will still work, but is deprecated.
+
+When a snapshot was damaged, the only option up to now was to completely forget
+the snapshot, even if only some unimportant files in it were damaged and other
+files were still fine.
+
+Restic now has a `repair snapshots` command, which can salvage any non-damaged
+files and parts of files in the snapshots by removing damaged directories and
+missing file contents. Please note that the damaged data may still be lost
+and see the "Troubleshooting" section in the documentation for more details.
+
+https://github.com/restic/restic/issues/1759
+https://github.com/restic/restic/issues/1714
+https://github.com/restic/restic/issues/1798
+https://github.com/restic/restic/issues/2334
+https://github.com/restic/restic/pull/2876
+https://forum.restic.net/t/corrupted-repo-how-to-repair/799
+https://forum.restic.net/t/recovery-options-for-damaged-repositories/1571

changelog/0.16.0_2023-07-31/issue-1926

@@ -0,0 +1,8 @@
+Enhancement: Allow certificate paths to be passed through environment variables
+
+Restic will now read paths to certificates from the environment variables
+`RESTIC_CACERT` or `RESTIC_TLS_CLIENT_CERT` if `--cacert` or `--tls-client-cert`
+are not specified. 
+
+https://github.com/restic/restic/issues/1926  
+https://github.com/restic/restic/pull/4384

changelog/0.16.0_2023-07-31/issue-2359

@@ -0,0 +1,11 @@
+Enhancement: Provide multi-platform Docker images
+
+The official Docker images are now built for the architectures linux/386,
+linux/amd64, linux/arm and linux/arm64.
+
+As an alternative to the Docker Hub, the Docker images are also
+available on ghcr.io, the GitHub Container Registry.
+
+https://github.com/restic/restic/issues/2359
+https://github.com/restic/restic/issues/4269
+https://github.com/restic/restic/pull/4364

changelog/0.16.0_2023-07-31/issue-2468

@@ -0,0 +1,10 @@
+Enhancement: Add support for non-global Azure clouds
+
+The `azure` backend previously only supported storages using the global domain
+`core.windows.net`. This meant that backups to other domains such as Azure
+China (`core.chinacloudapi.cn`) or Azure Germany (`core.cloudapi.de`) were
+not supported. Restic now allows overriding the global domain using the 
+environment variable `AZURE_ENDPOINT_SUFFIX`.
+
+https://github.com/restic/restic/issues/2468
+https://github.com/restic/restic/pull/4387

changelog/0.16.0_2023-07-31/issue-2565

@@ -0,0 +1,10 @@
+Bugfix: Support "unlimited" in `forget --keep-*` options
+
+Restic would previously forget snapshots that should have been kept when a
+negative value was passed to the `--keep-*` options. Negative values are now
+forbidden. To keep all snapshots, the special value `unlimited` is now
+supported. For example, `--keep-monthly unlimited` will keep all monthly
+snapshots.
+
+https://github.com/restic/restic/issues/2565
+https://github.com/restic/restic/pull/4234

changelog/0.16.0_2023-07-31/issue-3311

@@ -0,0 +1,12 @@
+Bugfix: Support non-UTF8 paths as symlink target
+
+Earlier restic versions did not correctly `backup` and `restore` symlinks that
+contain a non-UTF8 target. Note that this only affected systems that still use
+a non-Unicode encoding for filesystem paths.
+
+The repository format is now extended to add support for such symlinks. Please
+note that snapshots must have been created with at least restic version 0.16.0
+for `restore` to correctly handle non-UTF8 symlink targets when restoring them.
+
+https://github.com/restic/restic/issues/3311
+https://github.com/restic/restic/pull/3802

changelog/0.16.0_2023-07-31/issue-3328

@@ -0,0 +1,9 @@
+Enhancement: Reduce memory usage by up to 25%
+
+The in-memory index has been optimized to be more garbage collection friendly.
+Restic now defaults to `GOGC=50` to run the Go garbage collector more
+frequently.
+
+https://github.com/restic/restic/issues/3328
+https://github.com/restic/restic/pull/4352
+https://github.com/restic/restic/pull/4353

changelog/0.16.0_2023-07-31/issue-3397

@@ -0,0 +1,11 @@
+Enhancement: Improve accuracy of ETA displayed during backup
+
+Restic's `backup` command displayed an ETA that did not adapt when the rate of
+progress made during the backup changed during the course of the backup.
+
+Restic now uses recent progress when computing the ETA. It is important to
+realize that the estimate may still be wrong, because restic cannot predict
+the future, but the hope is that the ETA will be more accurate in most cases.
+
+https://github.com/restic/restic/issues/3397
+https://github.com/restic/restic/pull/3563

changelog/0.16.0_2023-07-31/issue-3624

@@ -0,0 +1,9 @@
+Enhancement: Keep oldest snapshot when there are not enough snapshots
+
+The `forget` command now additionally preserves the oldest snapshot if fewer
+snapshots than allowed by the `--keep-*` parameters would otherwise be kept.
+This maximizes the amount of history kept within the specified limits.
+
+https://github.com/restic/restic/issues/3624
+https://github.com/restic/restic/pull/4366
+https://forum.restic.net/t/keeping-yearly-snapshots-policy-when-backup-began-during-the-year/4670/2

changelog/0.16.0_2023-07-31/issue-3698

@@ -0,0 +1,8 @@
+Enhancement: Add support for Managed / Workload Identity to `azure` backend
+
+Restic now additionally supports authenticating to Azure using Workload
+Identity or Managed Identity credentials, which are automatically injected in
+several environments such as a managed Kubernetes cluster.
+
+https://github.com/restic/restic/issues/3698
+https://github.com/restic/restic/pull/4029

changelog/0.16.0_2023-07-31/issue-3871

@@ -0,0 +1,22 @@
+Enhancement: Support `<snapshot>:<subfolder>` syntax to select subfolders
+
+Commands like `diff` or `restore` always worked with the full snapshot. This
+did not allow comparing only a specific subfolder or only restoring that folder
+(`restore --include subfolder` filters the restored files, but still creates the
+directories included in `subfolder`).
+
+The commands `diff`, `dump`, `ls` and `restore` now support the
+`<snapshot>:<subfolder>` syntax, where `snapshot` is the ID of a snapshot (or
+the string `latest`) and `subfolder` is a path within the snapshot. The
+commands will then only work with the specified path of the snapshot. The
+`subfolder` must be a path to a folder as returned by `ls`. Two examples:
+
+`restic restore -t target latest:/some/path`
+`restic diff 12345678:/some/path 90abcef:/some/path`
+
+For debugging purposes, the `cat` command now supports `cat tree
+<snapshot>:<subfolder>` to return the directory metadata for the given
+subfolder.
+
+https://github.com/restic/restic/issues/3871
+https://github.com/restic/restic/pull/4334

changelog/0.16.0_2023-07-31/issue-3941

@@ -0,0 +1,17 @@
+Enhancement: Support `--group-by` for backup parent selection
+
+Previously, the `backup` command by default selected the parent snapshot based
+on the hostname and the backup targets. When the backup path list changed, the
+`backup` command was unable to determine a suitable parent snapshot and had to
+read all files again.
+
+The new `--group-by` option for the `backup` command allows filtering snapshots
+for the parent selection by `host`, `paths` and `tags`. It defaults to
+`host,paths` which selects the latest snapshot with hostname and paths matching
+those of the backup run. This matches the behavior of prior restic versions.
+
+The new `--group-by` option should be set to the same value as passed to
+`forget --group-by`.
+
+https://github.com/restic/restic/issues/3941
+https://github.com/restic/restic/pull/4081

changelog/0.16.0_2023-07-31/issue-4130

@@ -0,0 +1,9 @@
+Enhancement: Cancel current command if cache becomes unusable
+
+If the cache directory was removed or ran out of space while restic was
+running, this would previously cause further caching attempts to fail and
+thereby drastically slow down the command execution. Now, the currently running
+command is instead canceled.
+
+https://github.com/restic/restic/issues/4130
+https://github.com/restic/restic/pull/4166

changelog/0.16.0_2023-07-31/issue-4159

@@ -0,0 +1,12 @@
+Enhancement: Add `--human-readable` option to `ls` and `find` commands
+
+Previously, when using the `-l` option with the `ls` and `find` commands, the
+displayed size was always in bytes, without an option for a more human readable
+format such as MiB or GiB.
+
+The new `--human-readable` option will convert longer size values into more 
+human friendly values with an appropriate suffix depending on the output size.
+For example, a size of `14680064` will be shown as `14.000 MiB`.
+
+https://github.com/restic/restic/issues/4159
+https://github.com/restic/restic/pull/4351

changelog/0.16.0_2023-07-31/issue-4188

@@ -0,0 +1,8 @@
+Enhancement: Include restic version in snapshot metadata
+
+The restic version used to backup a snapshot is now included in its metadata
+and shown when inspecting a snapshot using `restic cat snapshot <snapshotID>`
+or `restic snapshots --json`.
+
+https://github.com/restic/restic/issues/4188
+https://github.com/restic/restic/pull/4378

changelog/0.16.0_2023-07-31/issue-4199

@@ -0,0 +1,9 @@
+Bugfix: Avoid lock refresh issues on slow network connections
+
+On network connections with a low upload speed, backups and other operations
+could fail with the error message `Fatal: failed to refresh lock in time`.
+
+This has now been fixed by reworking the lock refresh handling.
+
+https://github.com/restic/restic/issues/4199
+https://github.com/restic/restic/pull/4304

changelog/0.16.0_2023-07-31/issue-426

@@ -0,0 +1,14 @@
+Enhancement: Show progress bar during restore
+
+The `restore` command now shows a progress report while restoring files.
+
+Example: `[0:42]  5.76%  23 files 12.98 MiB, total 3456 files 23.54 GiB`
+
+JSON output is now also supported.
+
+https://github.com/restic/restic/issues/426
+https://github.com/restic/restic/issues/3413
+https://github.com/restic/restic/issues/3627
+https://github.com/restic/restic/pull/3991
+https://github.com/restic/restic/pull/4314
+https://forum.restic.net/t/progress-bar-for-restore/5210

changelog/0.16.0_2023-07-31/issue-4274

@@ -0,0 +1,11 @@
+Bugfix: Improve lock refresh handling after standby
+
+If the restic process was stopped or the host running restic entered standby
+during a long running operation such as a backup, this previously resulted in
+the operation failing with `Fatal: failed to refresh lock in time`.
+
+This has now been fixed such that restic first checks whether it is safe to
+continue the current operation and only throws an error if not.
+
+https://github.com/restic/restic/issues/4274
+https://github.com/restic/restic/pull/4374

changelog/0.16.0_2023-07-31/issue-4375

@@ -0,0 +1,8 @@
+Enhancement: Add support for extended attributes on symlinks
+
+Restic now supports extended attributes on symlinks when backing up,
+restoring, or FUSE-mounting snapshots. This includes, for example, the
+`security.selinux` xattr on Linux distributions that use SELinux.
+
+https://github.com/restic/restic/issues/4375
+https://github.com/restic/restic/pull/4379

changelog/0.16.0_2023-07-31/issue-719

@@ -0,0 +1,8 @@
+Enhancement: Add `--retry-lock` option
+
+This option allows specifying a duration for which restic will wait if the
+repository is already locked.
+
+https://github.com/restic/restic/issues/719
+https://github.com/restic/restic/pull/2214
+https://github.com/restic/restic/pull/4107

changelog/0.16.0_2023-07-31/pull-3261

@@ -0,0 +1,8 @@
+Enhancement: Reduce file fragmentation for local backend
+
+Before this change, local backend files could become fragmented.
+Now restic will try to preallocate space for pack files to avoid
+their fragmentation.
+
+https://github.com/restic/restic/issues/2679
+https://github.com/restic/restic/pull/3261

changelog/0.16.0_2023-07-31/pull-4176

@@ -0,0 +1,7 @@
+Change: Fix JSON message type of `scan_finished` for the `backup` command
+
+Restic incorrectly set the `message_type` of the `scan_finished` message to
+`status` instead of `verbose_status`. This has now been corrected so that
+the messages report the correct type.
+
+https://github.com/restic/restic/pull/4176

changelog/0.16.0_2023-07-31/pull-4201

@@ -0,0 +1,7 @@
+Change: Require Go 1.20 for Solaris builds
+
+Building restic on Solaris now requires Go 1.20, as the library used to access
+Azure uses the mmap syscall, which is only available on Solaris starting from
+Go 1.20. All other platforms however continue to build with Go 1.18.
+
+https://github.com/restic/restic/pull/4201

changelog/0.16.0_2023-07-31/pull-4220

@@ -0,0 +1,6 @@
+Enhancement: Add `jq` binary to Docker image
+
+The Docker image now contains `jq`, which can be useful to process JSON data
+output by restic.
+
+https://github.com/restic/restic/pull/4220

changelog/0.16.0_2023-07-31/pull-4226

@@ -0,0 +1,7 @@
+Enhancement: Allow specifying region of new buckets in the `gs` backend
+
+Previously, buckets used by the Google Cloud Storage backend would always get
+created in the "us" region. It is now possible to specify the region where a
+bucket should be created by using the `-o gs.region=us` option.
+
+https://github.com/restic/restic/pull/4226

changelog/0.16.0_2023-07-31/pull-4318

@@ -0,0 +1,8 @@
+Bugfix: Correctly clean up status bar output of the `backup` command
+
+Due to a regression in restic 0.15.2, the status bar of the `backup` command
+could leave some output behind. This happened if filenames were printed that
+are wider than the current terminal width. This has now been fixed.
+
+https://github.com/restic/restic/issues/4319
+https://github.com/restic/restic/pull/4318

changelog/0.16.0_2023-07-31/pull-4333

@@ -0,0 +1,3 @@
+Bugfix: `generate` and `init` no longer silently ignore unexpected arguments
+
+https://github.com/restic/restic/pull/4333

changelog/0.16.0_2023-07-31/pull-4400

@@ -0,0 +1,8 @@
+Bugfix: Ignore missing folders in `rest` backend
+
+If a repository accessed via the REST backend was missing folders, then restic
+would fail with an error while trying to list the data in the repository. This
+has been now fixed.
+
+https://github.com/restic/restic/pull/4400
+https://github.com/restic/rest-server/issues/235

changelog/TEMPLATE

@@ -1,16 +1,17 @@
 # The first line must start with Bugfix:, Enhancement: or Change:,
-# including the colon. Use present tense. Remove lines starting with '#'
-# from this template.
+# including the colon. Use present tense and the imperative mood. Remove
+# lines starting with '#' from this template.
 Enhancement: Allow custom bar in the foo command
 
 # Describe the problem in the past tense, the new behavior in the present
 # tense. Mention the affected commands, backends, operating systems, etc.
 # Focus on user-facing behavior, not the implementation.
+# Use "Restic now ..." instead of "We have changed ...".
 
 Restic foo always used the system-wide bar when deciding how to frob an
-item in the baz backend. It now permits selecting the bar with --bar or
-the environment variable RESTIC_BAR. The system-wide bar is still the
-default.
+item in the `baz` backend. It now permits selecting the bar with `--bar`
+or the environment variable `RESTIC_BAR`. The system-wide bar is still
+the default.
 
 # The last section is a list of issue, PR and forum URLs.
 # The first issue ID determines the filename for the changelog entry:

cmd/restic/cleanup.go

@@ -11,7 +11,7 @@ import (
 
 var cleanupHandlers struct {
 	sync.Mutex
-	list []func() error
+	list []func(code int) (int, error)
 	done bool
 	ch   chan os.Signal
 }
@@ -25,7 +25,7 @@ func init() {
 // AddCleanupHandler adds the function f to the list of cleanup handlers so
 // that it is executed when all the cleanup handlers are run, e.g. when SIGINT
 // is received.
-func AddCleanupHandler(f func() error) {
+func AddCleanupHandler(f func(code int) (int, error)) {
 	cleanupHandlers.Lock()
 	defer cleanupHandlers.Unlock()
 
@@ -36,22 +36,24 @@ func AddCleanupHandler(f func() error) {
 }
 
 // RunCleanupHandlers runs all registered cleanup handlers
-func RunCleanupHandlers() {
+func RunCleanupHandlers(code int) int {
 	cleanupHandlers.Lock()
 	defer cleanupHandlers.Unlock()
 
 	if cleanupHandlers.done {
-		return
+		return code
 	}
 	cleanupHandlers.done = true
 
 	for _, f := range cleanupHandlers.list {
-		err := f()
+		var err error
+		code, err = f(code)
 		if err != nil {
 			Warnf("error in cleanup handler: %v\n", err)
 		}
 	}
 	cleanupHandlers.list = nil
+	return code
 }
 
 // CleanupHandler handles the SIGINT signals.
@@ -60,6 +62,12 @@ func CleanupHandler(c <-chan os.Signal) {
 		debug.Log("signal %v received, cleaning up", s)
 		Warnf("%ssignal %v received, cleaning up\n", clearLine(0), s)
 
+		if val, _ := os.LookupEnv("RESTIC_DEBUG_STACKTRACE_SIGINT"); val != "" {
+			_, _ = os.Stderr.WriteString("\n--- STACKTRACE START ---\n\n")
+			_, _ = os.Stderr.WriteString(debug.DumpStacktrace())
+			_, _ = os.Stderr.WriteString("\n--- STACKTRACE END ---\n")
+		}
+
 		code := 0
 
 		if s == syscall.SIGINT {
@@ -75,6 +83,7 @@ func CleanupHandler(c <-chan os.Signal) {
 // Exit runs the cleanup handlers and then terminates the process with the
 // given exit code.
 func Exit(code int) {
-	RunCleanupHandlers()
+	code = RunCleanupHandlers(code)
+	debug.Log("exiting with status code %d", code)
 	os.Exit(code)
 }

cmd/restic/cmd_backup.go

@@ -6,11 +6,11 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -21,11 +21,11 @@ import (
 	"github.com/restic/restic/internal/archiver"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/filter"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/textfile"
+	"github.com/restic/restic/internal/ui"
 	"github.com/restic/restic/internal/ui/backup"
 	"github.com/restic/restic/internal/ui/termstatus"
 )
@@ -56,8 +56,9 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
 	},
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
+		ctx := cmd.Context()
 		var wg sync.WaitGroup
-		cancelCtx, cancel := context.WithCancel(globalOptions.ctx)
+		cancelCtx, cancel := context.WithCancel(ctx)
 		defer func() {
 			// shutdown termstatus
 			cancel()
@@ -71,35 +72,44 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
 			term.Run(cancelCtx)
 		}()
 
-		return runBackup(backupOptions, globalOptions, term, args)
+		// use the terminal for stdout/stderr
+		prevStdout, prevStderr := globalOptions.stdout, globalOptions.stderr
+		defer func() {
+			globalOptions.stdout, globalOptions.stderr = prevStdout, prevStderr
+		}()
+		stdioWrapper := ui.NewStdioWrapper(term)
+		globalOptions.stdout, globalOptions.stderr = stdioWrapper.Stdout(), stdioWrapper.Stderr()
+
+		return runBackup(ctx, backupOptions, globalOptions, term, args)
 	},
 }
 
 // BackupOptions bundles all options for the backup command.
 type BackupOptions struct {
-	Parent                  string
-	Force                   bool
-	Excludes                []string
-	InsensitiveExcludes     []string
-	ExcludeFiles            []string
-	InsensitiveExcludeFiles []string
-	ExcludeOtherFS          bool
-	ExcludeIfPresent        []string
-	ExcludeCaches           bool
-	ExcludeLargerThan       string
-	Stdin                   bool
-	StdinFilename           string
-	Tags                    restic.TagLists
-	Host                    string
-	FilesFrom               []string
-	FilesFromVerbatim       []string
-	FilesFromRaw            []string
-	TimeStamp               string
-	WithAtime               bool
-	IgnoreInode             bool
-	IgnoreCtime             bool
-	UseFsSnapshot           bool
-	DryRun                  bool
+	excludePatternOptions
+
+	Parent            string
+	GroupBy           restic.SnapshotGroupByOptions
+	Force             bool
+	ExcludeOtherFS    bool
+	ExcludeIfPresent  []string
+	ExcludeCaches     bool
+	ExcludeLargerThan string
+	Stdin             bool
+	StdinFilename     string
+	Tags              restic.TagLists
+	Host              string
+	FilesFrom         []string
+	FilesFromVerbatim []string
+	FilesFromRaw      []string
+	TimeStamp         string
+	WithAtime         bool
+	IgnoreInode       bool
+	IgnoreCtime       bool
+	UseFsSnapshot     bool
+	DryRun            bool
+	ReadConcurrency   uint
+	NoScan            bool
 }
 
 var backupOptions BackupOptions
@@ -111,12 +121,13 @@ func init() {
 	cmdRoot.AddCommand(cmdBackup)
 
 	f := cmdBackup.Flags()
-	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: last snapshot in the repository that has the same target files/directories, and is not newer than the snapshot time)")
+	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: latest snapshot in the group determined by --group-by and not newer than the timestamp determined by --time)")
+	backupOptions.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true}
+	f.VarP(&backupOptions.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
 	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
-	f.StringArrayVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
-	f.StringArrayVar(&backupOptions.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
-	f.StringArrayVar(&backupOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
-	f.StringArrayVar(&backupOptions.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns")
+
+	initExcludePatternOptions(f, &backupOptions.excludePatternOptions)
+
 	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems, don't cross filesystem boundaries and subvolumes")
 	f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes `filename[:header]`, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)")
 	f.BoolVar(&backupOptions.ExcludeCaches, "exclude-caches", false, `excludes cache directories that are marked with a CACHEDIR.TAG file. See https://bford.info/cachedir/ for the Cache Directory Tagging Standard`)
@@ -124,7 +135,7 @@ func init() {
 	f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
 	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "`filename` to use when reading from stdin")
 	f.Var(&backupOptions.Tags, "tag", "add `tags` for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times)")
-
+	f.UintVar(&backupOptions.ReadConcurrency, "read-concurrency", 0, "read `n` files concurrently (default: $RESTIC_READ_CONCURRENCY or 2)")
 	f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag")
 	f.StringVar(&backupOptions.Host, "hostname", "", "set the `hostname` for the snapshot manually")
 	err := f.MarkDeprecated("hostname", "use --host")
@@ -132,7 +143,6 @@ func init() {
 		// MarkDeprecated only returns an error when the flag could not be found
 		panic(err)
 	}
-
 	f.StringArrayVar(&backupOptions.FilesFrom, "files-from", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
 	f.StringArrayVar(&backupOptions.FilesFromVerbatim, "files-from-verbatim", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
 	f.StringArrayVar(&backupOptions.FilesFromRaw, "files-from-raw", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
@@ -141,9 +151,14 @@ func init() {
 	f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number changes when checking for modified files")
 	f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files")
 	f.BoolVarP(&backupOptions.DryRun, "dry-run", "n", false, "do not upload or write any data, just show what would be done")
+	f.BoolVar(&backupOptions.NoScan, "no-scan", false, "do not run scanner to estimate size of backup")
 	if runtime.GOOS == "windows" {
 		f.BoolVar(&backupOptions.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)")
 	}
+
+	// parse read concurrency from env, on error the default value will be used
+	readConcurrency, _ := strconv.ParseUint(os.Getenv("RESTIC_READ_CONCURRENCY"), 10, 32)
+	backupOptions.ReadConcurrency = uint(readConcurrency)
 }
 
 // filterExisting returns a slice of all existing items, or an error if no
@@ -183,7 +198,7 @@ func readLines(filename string) ([]string, error) {
 	)
 
 	if filename == "-" {
-		data, err = ioutil.ReadAll(os.Stdin)
+		data, err = io.ReadAll(os.Stdin)
 	} else {
 		data, err = textfile.Read(filename)
 	}
@@ -260,6 +275,10 @@ func readFilenamesRaw(r io.Reader) (names []string, err error) {
 // Check returns an error when an invalid combination of options was set.
 func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
 	if gopts.password == "" {
+		if opts.Stdin {
+			return errors.Fatal("cannot read both password and data from stdin")
+		}
+
 		filesFrom := append(append(opts.FilesFrom, opts.FilesFromVerbatim...), opts.FilesFromRaw...)
 		for _, filename := range filesFrom {
 			if filename == "-" {
@@ -289,7 +308,7 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
 
 // collectRejectByNameFuncs returns a list of all functions which may reject data
 // from being saved in a snapshot based on path only
-func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, targets []string) (fs []RejectByNameFunc, err error) {
+func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (fs []RejectByNameFunc, err error) {
 	// exclude restic cache
 	if repo.Cache != nil {
 		f, err := rejectResticCache(repo)
@@ -300,48 +319,11 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, t
 		fs = append(fs, f)
 	}
 
-	// add patterns from file
-	if len(opts.ExcludeFiles) > 0 {
-		excludes, err := readExcludePatternsFromFiles(opts.ExcludeFiles)
-		if err != nil {
-			return nil, err
-		}
-
-		if valid, invalidPatterns := filter.ValidatePatterns(excludes); !valid {
-			return nil, errors.Fatalf("--exclude-file: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
-		}
-
-		opts.Excludes = append(opts.Excludes, excludes...)
-	}
-
-	if len(opts.InsensitiveExcludeFiles) > 0 {
-		excludes, err := readExcludePatternsFromFiles(opts.InsensitiveExcludeFiles)
-		if err != nil {
-			return nil, err
-		}
-
-		if valid, invalidPatterns := filter.ValidatePatterns(excludes); !valid {
-			return nil, errors.Fatalf("--iexclude-file: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
-		}
-
-		opts.InsensitiveExcludes = append(opts.InsensitiveExcludes, excludes...)
-	}
-
-	if len(opts.InsensitiveExcludes) > 0 {
-		if valid, invalidPatterns := filter.ValidatePatterns(opts.InsensitiveExcludes); !valid {
-			return nil, errors.Fatalf("--iexclude: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
-		}
-
-		fs = append(fs, rejectByInsensitivePattern(opts.InsensitiveExcludes))
-	}
-
-	if len(opts.Excludes) > 0 {
-		if valid, invalidPatterns := filter.ValidatePatterns(opts.Excludes); !valid {
-			return nil, errors.Fatalf("--exclude: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
-		}
-
-		fs = append(fs, rejectByPattern(opts.Excludes))
+	fsPatterns, err := opts.excludePatternOptions.CollectPatterns()
+	if err != nil {
+		return nil, err
 	}
+	fs = append(fs, fsPatterns...)
 
 	if opts.ExcludeCaches {
 		opts.ExcludeIfPresent = append(opts.ExcludeIfPresent, "CACHEDIR.TAG:Signature: 8a477f597d28d172789f06886806bc55")
@@ -361,7 +343,7 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, t
 
 // collectRejectFuncs returns a list of all functions which may reject data
 // from being saved in a snapshot based on path and file info
-func collectRejectFuncs(opts BackupOptions, repo *repository.Repository, targets []string) (fs []RejectFunc, err error) {
+func collectRejectFuncs(opts BackupOptions, targets []string) (fs []RejectFunc, err error) {
 	// allowed devices
 	if opts.ExcludeOtherFS && !opts.Stdin {
 		f, err := rejectByDevice(targets)
@@ -382,53 +364,6 @@ func collectRejectFuncs(opts BackupOptions, repo *repository.Repository, targets
 	return fs, nil
 }
 
-// readExcludePatternsFromFiles reads all exclude files and returns the list of
-// exclude patterns. For each line, leading and trailing white space is removed
-// and comment lines are ignored. For each remaining pattern, environment
-// variables are resolved. For adding a literal dollar sign ($), write $$ to
-// the file.
-func readExcludePatternsFromFiles(excludeFiles []string) ([]string, error) {
-	getenvOrDollar := func(s string) string {
-		if s == "$" {
-			return "$"
-		}
-		return os.Getenv(s)
-	}
-
-	var excludes []string
-	for _, filename := range excludeFiles {
-		err := func() (err error) {
-			data, err := textfile.Read(filename)
-			if err != nil {
-				return err
-			}
-
-			scanner := bufio.NewScanner(bytes.NewReader(data))
-			for scanner.Scan() {
-				line := strings.TrimSpace(scanner.Text())
-
-				// ignore empty lines
-				if line == "" {
-					continue
-				}
-
-				// strip comments
-				if strings.HasPrefix(line, "#") {
-					continue
-				}
-
-				line = os.Expand(line, getenvOrDollar)
-				excludes = append(excludes, line)
-			}
-			return scanner.Err()
-		}()
-		if err != nil {
-			return nil, err
-		}
-	}
-	return excludes, nil
-}
-
 // collectTargets returns a list of target files/dirs from several sources.
 func collectTargets(opts BackupOptions, args []string) (targets []string, err error) {
 	if opts.Stdin {
@@ -451,7 +386,7 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 			var expanded []string
 			expanded, err := filepath.Glob(line)
 			if err != nil {
-				return nil, errors.WithMessage(err, fmt.Sprintf("pattern: %s", line))
+				return nil, fmt.Errorf("pattern: %s: %w", line, err)
 			}
 			if len(expanded) == 0 {
 				Warnf("pattern %q does not match any files, skipping\n", line)
@@ -498,31 +433,35 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 
 // parent returns the ID of the parent snapshot. If there is none, nil is
 // returned.
-func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string, timeStampLimit time.Time) (parentID *restic.ID, err error) {
-	// Force using a parent
-	if !opts.Force && opts.Parent != "" {
-		id, err := restic.FindSnapshot(ctx, repo.Backend(), opts.Parent)
-		if err != nil {
-			return nil, errors.Fatalf("invalid id %q: %v", opts.Parent, err)
-		}
-
-		parentID = &id
+func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string, timeStampLimit time.Time) (*restic.Snapshot, error) {
+	if opts.Force {
+		return nil, nil
 	}
 
-	// Find last snapshot to set it as parent, if not already set
-	if !opts.Force && parentID == nil {
-		id, err := restic.FindLatestSnapshot(ctx, repo.Backend(), repo, targets, []restic.TagList{}, []string{opts.Host}, &timeStampLimit)
-		if err == nil {
-			parentID = &id
-		} else if err != restic.ErrNoSnapshotFound {
-			return nil, err
-		}
+	snName := opts.Parent
+	if snName == "" {
+		snName = "latest"
+	}
+	f := restic.SnapshotFilter{TimestampLimit: timeStampLimit}
+	if opts.GroupBy.Host {
+		f.Hosts = []string{opts.Host}
+	}
+	if opts.GroupBy.Path {
+		f.Paths = targets
+	}
+	if opts.GroupBy.Tag {
+		f.Tags = []restic.TagList{opts.Tags.Flatten()}
 	}
 
-	return parentID, nil
+	sn, _, err := f.FindLatest(ctx, repo.Backend(), repo, snName)
+	// Snapshot not found is ok if no explicit parent was set
+	if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) {
+		err = nil
+	}
+	return sn, err
 }
 
-func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
+func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
 	err := opts.Check(gopts, args)
 	if err != nil {
 		return err
@@ -545,7 +484,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		Verbosef("open repository\n")
 	}
 
-	repo, err := OpenRepository(gopts)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
@@ -556,58 +495,48 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	} else {
 		progressPrinter = backup.NewTextProgress(term, gopts.verbosity)
 	}
-	progressReporter := backup.NewProgress(progressPrinter)
+	progressReporter := backup.NewProgress(progressPrinter,
+		calculateProgressInterval(!gopts.Quiet, gopts.JSON))
+	defer progressReporter.Done()
 
 	if opts.DryRun {
 		repo.SetDryRun()
-		progressReporter.SetDryRun()
 	}
 
-	// use the terminal for stdout/stderr
-	prevStdout, prevStderr := gopts.stdout, gopts.stderr
-	defer func() {
-		gopts.stdout, gopts.stderr = prevStdout, prevStderr
-	}()
-	gopts.stdout, gopts.stderr = progressPrinter.Stdout(), progressPrinter.Stderr()
-
-	progressReporter.SetMinUpdatePause(calculateProgressInterval(!gopts.Quiet, gopts.JSON))
-
-	wg, wgCtx := errgroup.WithContext(gopts.ctx)
-	cancelCtx, cancel := context.WithCancel(wgCtx)
-	defer cancel()
-	wg.Go(func() error { return progressReporter.Run(cancelCtx) })
-
 	if !gopts.JSON {
 		progressPrinter.V("lock repository")
 	}
-	lock, err := lockRepo(gopts.ctx, repo)
-	defer unlockRepo(lock)
-	if err != nil {
-		return err
+	if !opts.DryRun {
+		var lock *restic.Lock
+		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
 	}
 
 	// rejectByNameFuncs collect functions that can reject items from the backup based on path only
-	rejectByNameFuncs, err := collectRejectByNameFuncs(opts, repo, targets)
+	rejectByNameFuncs, err := collectRejectByNameFuncs(opts, repo)
 	if err != nil {
 		return err
 	}
 
 	// rejectFuncs collect functions that can reject items from the backup based on path and file info
-	rejectFuncs, err := collectRejectFuncs(opts, repo, targets)
+	rejectFuncs, err := collectRejectFuncs(opts, targets)
 	if err != nil {
 		return err
 	}
 
-	var parentSnapshotID *restic.ID
+	var parentSnapshot *restic.Snapshot
 	if !opts.Stdin {
-		parentSnapshotID, err = findParentSnapshot(gopts.ctx, repo, opts, targets, timeStamp)
+		parentSnapshot, err = findParentSnapshot(ctx, repo, opts, targets, timeStamp)
 		if err != nil {
 			return err
 		}
 
 		if !gopts.JSON {
-			if parentSnapshotID != nil {
-				progressPrinter.P("using parent snapshot %v\n", parentSnapshotID.Str())
+			if parentSnapshot != nil {
+				progressPrinter.P("using parent snapshot %v\n", parentSnapshot.ID().Str())
 			} else {
 				progressPrinter.P("no parent snapshot found, will read all files\n")
 			}
@@ -617,7 +546,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	if !gopts.JSON {
 		progressPrinter.V("load index files")
 	}
-	err = repo.LoadIndex(gopts.ctx)
+	err = repo.LoadIndex(ctx)
 	if err != nil {
 		return err
 	}
@@ -674,18 +603,24 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		targets = []string{filename}
 	}
 
-	sc := archiver.NewScanner(targetFS)
-	sc.SelectByName = selectByNameFilter
-	sc.Select = selectFilter
-	sc.Error = progressReporter.ScannerError
-	sc.Result = progressReporter.ReportTotal
+	wg, wgCtx := errgroup.WithContext(ctx)
+	cancelCtx, cancel := context.WithCancel(wgCtx)
+	defer cancel()
 
-	if !gopts.JSON {
-		progressPrinter.V("start scan on %v", targets)
+	if !opts.NoScan {
+		sc := archiver.NewScanner(targetFS)
+		sc.SelectByName = selectByNameFilter
+		sc.Select = selectFilter
+		sc.Error = progressPrinter.ScannerError
+		sc.Result = progressReporter.ReportTotal
+
+		if !gopts.JSON {
+			progressPrinter.V("start scan on %v", targets)
+		}
+		wg.Go(func() error { return sc.Scan(cancelCtx, targets) })
 	}
-	wg.Go(func() error { return sc.Scan(cancelCtx, targets) })
 
-	arch := archiver.New(repo, targetFS, archiver.Options{})
+	arch := archiver.New(repo, targetFS, archiver.Options{ReadConcurrency: backupOptions.ReadConcurrency})
 	arch.SelectByName = selectByNameFilter
 	arch.Select = selectFilter
 	arch.WithAtime = opts.WithAtime
@@ -707,22 +642,19 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		arch.ChangeIgnoreFlags |= archiver.ChangeIgnoreCtime
 	}
 
-	if parentSnapshotID == nil {
-		parentSnapshotID = &restic.ID{}
-	}
-
 	snapshotOpts := archiver.SnapshotOptions{
 		Excludes:       opts.Excludes,
 		Tags:           opts.Tags.Flatten(),
 		Time:           timeStamp,
 		Hostname:       opts.Host,
-		ParentSnapshot: *parentSnapshotID,
+		ParentSnapshot: parentSnapshot,
+		ProgramVersion: "restic " + version,
 	}
 
 	if !gopts.JSON {
 		progressPrinter.V("start backup on %v", targets)
 	}
-	_, id, err := arch.Snapshot(gopts.ctx, targets, snapshotOpts)
+	_, id, err := arch.Snapshot(ctx, targets, snapshotOpts)
 
 	// cleanly shutdown all running goroutines
 	cancel()
@@ -736,7 +668,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}
 
 	// Report finished execution
-	progressReporter.Finish(id)
+	progressReporter.Finish(id, opts.DryRun)
 	if !gopts.JSON && !opts.DryRun {
 		progressPrinter.P("snapshot %s saved\n", id.Str())
 	}

cmd/restic/cmd_backup_integration_test.go

@@ -0,0 +1,569 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/restic/restic/internal/fs"
+	"github.com/restic/restic/internal/restic"
+	rtest "github.com/restic/restic/internal/test"
+	"github.com/restic/restic/internal/ui/termstatus"
+)
+
+func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts BackupOptions, gopts GlobalOptions) error {
+	return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
+		t.Logf("backing up %v in %v", target, dir)
+		if dir != "" {
+			cleanup := rtest.Chdir(t, dir)
+			defer cleanup()
+		}
+
+		opts.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true}
+		return runBackup(ctx, opts, gopts, term, target)
+	})
+}
+
+func testRunBackup(t testing.TB, dir string, target []string, opts BackupOptions, gopts GlobalOptions) {
+	err := testRunBackupAssumeFailure(t, dir, target, opts, gopts)
+	rtest.Assert(t, err == nil, "Error while backing up")
+}
+
+func TestBackup(t *testing.T) {
+	testBackup(t, false)
+}
+
+func TestBackupWithFilesystemSnapshots(t *testing.T) {
+	if runtime.GOOS == "windows" && fs.HasSufficientPrivilegesForVSS() == nil {
+		testBackup(t, true)
+	}
+}
+
+func testBackup(t *testing.T, useFsSnapshot bool) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{UseFsSnapshot: useFsSnapshot}
+
+	// first backup
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	testListSnapshots(t, env.gopts, 1)
+
+	testRunCheck(t, env.gopts)
+	stat1 := dirStats(env.repo)
+
+	// second backup, implicit incremental
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	snapshotIDs := testListSnapshots(t, env.gopts, 2)
+
+	stat2 := dirStats(env.repo)
+	if stat2.size > stat1.size+stat1.size/10 {
+		t.Error("repository size has grown by more than 10 percent")
+	}
+	t.Logf("repository grown by %d bytes", stat2.size-stat1.size)
+
+	testRunCheck(t, env.gopts)
+	// third backup, explicit incremental
+	opts.Parent = snapshotIDs[0].String()
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	snapshotIDs = testListSnapshots(t, env.gopts, 3)
+
+	stat3 := dirStats(env.repo)
+	if stat3.size > stat1.size+stat1.size/10 {
+		t.Error("repository size has grown by more than 10 percent")
+	}
+	t.Logf("repository grown by %d bytes", stat3.size-stat2.size)
+
+	// restore all backups and compare
+	for i, snapshotID := range snapshotIDs {
+		restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
+		t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir)
+		testRunRestore(t, env.gopts, restoredir, snapshotID)
+		diff := directoriesContentsDiff(env.testdata, filepath.Join(restoredir, "testdata"))
+		rtest.Assert(t, diff == "", "directories are not equal: %v", diff)
+	}
+
+	testRunCheck(t, env.gopts)
+}
+
+func TestBackupWithRelativePath(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{}
+
+	// first backup
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	firstSnapshotID := testListSnapshots(t, env.gopts, 1)[0]
+
+	// second backup, implicit incremental
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+
+	// that the correct parent snapshot was used
+	latestSn, _ := testRunSnapshots(t, env.gopts)
+	rtest.Assert(t, latestSn != nil, "missing latest snapshot")
+	rtest.Assert(t, latestSn.Parent != nil && latestSn.Parent.Equal(firstSnapshotID), "second snapshot selected unexpected parent %v instead of %v", latestSn.Parent, firstSnapshotID)
+}
+
+func TestBackupParentSelection(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{}
+
+	// first backup
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata/0/0"}, opts, env.gopts)
+	firstSnapshotID := testListSnapshots(t, env.gopts, 1)[0]
+
+	// second backup, sibling path
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata/0/tests"}, opts, env.gopts)
+	testListSnapshots(t, env.gopts, 2)
+
+	// third backup, incremental for the first backup
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata/0/0"}, opts, env.gopts)
+
+	// test that the correct parent snapshot was used
+	latestSn, _ := testRunSnapshots(t, env.gopts)
+	rtest.Assert(t, latestSn != nil, "missing latest snapshot")
+	rtest.Assert(t, latestSn.Parent != nil && latestSn.Parent.Equal(firstSnapshotID), "third snapshot selected unexpected parent %v instead of %v", latestSn.Parent, firstSnapshotID)
+}
+
+func TestDryRunBackup(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{}
+	dryOpts := BackupOptions{DryRun: true}
+
+	// dry run before first backup
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
+	snapshotIDs := testListSnapshots(t, env.gopts, 0)
+	packIDs := testRunList(t, "packs", env.gopts)
+	rtest.Assert(t, len(packIDs) == 0,
+		"expected no data, got %v", snapshotIDs)
+	indexIDs := testRunList(t, "index", env.gopts)
+	rtest.Assert(t, len(indexIDs) == 0,
+		"expected no index, got %v", snapshotIDs)
+
+	// first backup
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	snapshotIDs = testListSnapshots(t, env.gopts, 1)
+	packIDs = testRunList(t, "packs", env.gopts)
+	indexIDs = testRunList(t, "index", env.gopts)
+
+	// dry run between backups
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
+	snapshotIDsAfter := testListSnapshots(t, env.gopts, 1)
+	rtest.Equals(t, snapshotIDs, snapshotIDsAfter)
+	dataIDsAfter := testRunList(t, "packs", env.gopts)
+	rtest.Equals(t, packIDs, dataIDsAfter)
+	indexIDsAfter := testRunList(t, "index", env.gopts)
+	rtest.Equals(t, indexIDs, indexIDsAfter)
+
+	// second backup, implicit incremental
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	snapshotIDs = testListSnapshots(t, env.gopts, 2)
+	packIDs = testRunList(t, "packs", env.gopts)
+	indexIDs = testRunList(t, "index", env.gopts)
+
+	// another dry run
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
+	snapshotIDsAfter = testListSnapshots(t, env.gopts, 2)
+	rtest.Equals(t, snapshotIDs, snapshotIDsAfter)
+	dataIDsAfter = testRunList(t, "packs", env.gopts)
+	rtest.Equals(t, packIDs, dataIDsAfter)
+	indexIDsAfter = testRunList(t, "index", env.gopts)
+	rtest.Equals(t, indexIDs, indexIDsAfter)
+}
+
+func TestBackupNonExistingFile(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+
+	_ = withRestoreGlobalOptions(func() error {
+		globalOptions.stderr = io.Discard
+
+		p := filepath.Join(env.testdata, "0", "0", "9")
+		dirs := []string{
+			filepath.Join(p, "0"),
+			filepath.Join(p, "1"),
+			filepath.Join(p, "nonexisting"),
+			filepath.Join(p, "5"),
+		}
+
+		opts := BackupOptions{}
+
+		testRunBackup(t, "", dirs, opts, env.gopts)
+		return nil
+	})
+}
+
+func TestBackupSelfHealing(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testRunInit(t, env.gopts)
+
+	p := filepath.Join(env.testdata, "test/test")
+	rtest.OK(t, os.MkdirAll(filepath.Dir(p), 0755))
+	rtest.OK(t, appendRandomData(p, 5))
+
+	opts := BackupOptions{}
+
+	testRunBackup(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
+	testRunCheck(t, env.gopts)
+
+	// remove all data packs
+	removePacksExcept(env.gopts, t, restic.NewIDSet(), false)
+
+	testRunRebuildIndex(t, env.gopts)
+	// now the repo is also missing the data blob in the index; check should report this
+	testRunCheckMustFail(t, env.gopts)
+
+	// second backup should report an error but "heal" this situation
+	err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
+	rtest.Assert(t, err != nil,
+		"backup should have reported an error")
+	testRunCheck(t, env.gopts)
+}
+
+func TestBackupTreeLoadError(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testRunInit(t, env.gopts)
+	p := filepath.Join(env.testdata, "test/test")
+	rtest.OK(t, os.MkdirAll(filepath.Dir(p), 0755))
+	rtest.OK(t, appendRandomData(p, 5))
+
+	opts := BackupOptions{}
+	// Backup a subdirectory first, such that we can remove the tree pack for the subdirectory
+	testRunBackup(t, env.testdata, []string{"test"}, opts, env.gopts)
+
+	r, err := OpenRepository(context.TODO(), env.gopts)
+	rtest.OK(t, err)
+	rtest.OK(t, r.LoadIndex(context.TODO()))
+	treePacks := restic.NewIDSet()
+	r.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
+		if pb.Type == restic.TreeBlob {
+			treePacks.Insert(pb.PackID)
+		}
+	})
+
+	testRunBackup(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
+	testRunCheck(t, env.gopts)
+
+	// delete the subdirectory pack first
+	for id := range treePacks {
+		rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()}))
+	}
+	testRunRebuildIndex(t, env.gopts)
+	// now the repo is missing the tree blob in the index; check should report this
+	testRunCheckMustFail(t, env.gopts)
+	// second backup should report an error but "heal" this situation
+	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
+	rtest.Assert(t, err != nil, "backup should have reported an error for the subdirectory")
+	testRunCheck(t, env.gopts)
+
+	// remove all tree packs
+	removePacksExcept(env.gopts, t, restic.NewIDSet(), true)
+	testRunRebuildIndex(t, env.gopts)
+	// now the repo is also missing the data blob in the index; check should report this
+	testRunCheckMustFail(t, env.gopts)
+	// second backup should report an error but "heal" this situation
+	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
+	rtest.Assert(t, err != nil, "backup should have reported an error")
+	testRunCheck(t, env.gopts)
+}
+
+var backupExcludeFilenames = []string{
+	"testfile1",
+	"foo.tar.gz",
+	"private/secret/passwords.txt",
+	"work/source/test.c",
+}
+
+func TestBackupExclude(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testRunInit(t, env.gopts)
+
+	datadir := filepath.Join(env.base, "testdata")
+
+	for _, filename := range backupExcludeFilenames {
+		fp := filepath.Join(datadir, filename)
+		rtest.OK(t, os.MkdirAll(filepath.Dir(fp), 0755))
+
+		f, err := os.Create(fp)
+		rtest.OK(t, err)
+
+		fmt.Fprint(f, filename)
+		rtest.OK(t, f.Close())
+	}
+
+	snapshots := make(map[string]struct{})
+
+	opts := BackupOptions{}
+
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	snapshots, snapshotID := lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts))
+	files := testRunLs(t, env.gopts, snapshotID)
+	rtest.Assert(t, includes(files, "/testdata/foo.tar.gz"),
+		"expected file %q in first snapshot, but it's not included", "foo.tar.gz")
+
+	opts.Excludes = []string{"*.tar.gz"}
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	snapshots, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts))
+	files = testRunLs(t, env.gopts, snapshotID)
+	rtest.Assert(t, !includes(files, "/testdata/foo.tar.gz"),
+		"expected file %q not in first snapshot, but it's included", "foo.tar.gz")
+
+	opts.Excludes = []string{"*.tar.gz", "private/secret"}
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	_, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts))
+	files = testRunLs(t, env.gopts, snapshotID)
+	rtest.Assert(t, !includes(files, "/testdata/foo.tar.gz"),
+		"expected file %q not in first snapshot, but it's included", "foo.tar.gz")
+	rtest.Assert(t, !includes(files, "/testdata/private/secret/passwords.txt"),
+		"expected file %q not in first snapshot, but it's included", "passwords.txt")
+}
+
+func TestBackupErrors(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		return
+	}
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+
+	// Assume failure
+	inaccessibleFile := filepath.Join(env.testdata, "0", "0", "9", "0")
+	rtest.OK(t, os.Chmod(inaccessibleFile, 0000))
+	defer func() {
+		rtest.OK(t, os.Chmod(inaccessibleFile, 0644))
+	}()
+	opts := BackupOptions{}
+	err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	rtest.Assert(t, err != nil, "Assumed failure, but no error occurred.")
+	rtest.Assert(t, err == ErrInvalidSourceData, "Wrong error returned")
+	testListSnapshots(t, env.gopts, 1)
+}
+
+const (
+	incrementalFirstWrite  = 10 * 1042 * 1024
+	incrementalSecondWrite = 1 * 1042 * 1024
+	incrementalThirdWrite  = 1 * 1042 * 1024
+)
+
+func TestIncrementalBackup(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testRunInit(t, env.gopts)
+
+	datadir := filepath.Join(env.base, "testdata")
+	testfile := filepath.Join(datadir, "testfile")
+
+	rtest.OK(t, appendRandomData(testfile, incrementalFirstWrite))
+
+	opts := BackupOptions{}
+
+	testRunBackup(t, "", []string{datadir}, opts, env.gopts)
+	testRunCheck(t, env.gopts)
+	stat1 := dirStats(env.repo)
+
+	rtest.OK(t, appendRandomData(testfile, incrementalSecondWrite))
+
+	testRunBackup(t, "", []string{datadir}, opts, env.gopts)
+	testRunCheck(t, env.gopts)
+	stat2 := dirStats(env.repo)
+	if stat2.size-stat1.size > incrementalFirstWrite {
+		t.Errorf("repository size has grown by more than %d bytes", incrementalFirstWrite)
+	}
+	t.Logf("repository grown by %d bytes", stat2.size-stat1.size)
+
+	rtest.OK(t, appendRandomData(testfile, incrementalThirdWrite))
+
+	testRunBackup(t, "", []string{datadir}, opts, env.gopts)
+	testRunCheck(t, env.gopts)
+	stat3 := dirStats(env.repo)
+	if stat3.size-stat2.size > incrementalFirstWrite {
+		t.Errorf("repository size has grown by more than %d bytes", incrementalFirstWrite)
+	}
+	t.Logf("repository grown by %d bytes", stat3.size-stat2.size)
+}
+
+func TestBackupTags(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{}
+
+	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
+	testRunCheck(t, env.gopts)
+	newest, _ := testRunSnapshots(t, env.gopts)
+
+	if newest == nil {
+		t.Fatal("expected a backup, got nil")
+	}
+
+	rtest.Assert(t, len(newest.Tags) == 0,
+		"expected no tags, got %v", newest.Tags)
+	parent := newest
+
+	opts.Tags = restic.TagLists{[]string{"NL"}}
+	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
+	testRunCheck(t, env.gopts)
+	newest, _ = testRunSnapshots(t, env.gopts)
+
+	if newest == nil {
+		t.Fatal("expected a backup, got nil")
+	}
+
+	rtest.Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "NL",
+		"expected one NL tag, got %v", newest.Tags)
+	// Tagged backup should have untagged backup as parent.
+	rtest.Assert(t, parent.ID.Equal(*newest.Parent),
+		"expected parent to be %v, got %v", parent.ID, newest.Parent)
+}
+
+func TestBackupProgramVersion(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts)
+	newest, _ := testRunSnapshots(t, env.gopts)
+
+	if newest == nil {
+		t.Fatal("expected a backup, got nil")
+	}
+	resticVersion := "restic " + version
+	rtest.Assert(t, newest.ProgramVersion == resticVersion,
+		"expected %v, got %v", resticVersion, newest.ProgramVersion)
+}
+
+func TestQuietBackup(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{}
+
+	env.gopts.Quiet = false
+	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
+	testListSnapshots(t, env.gopts, 1)
+
+	testRunCheck(t, env.gopts)
+
+	env.gopts.Quiet = true
+	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
+	testListSnapshots(t, env.gopts, 2)
+
+	testRunCheck(t, env.gopts)
+}
+
+func TestHardLink(t *testing.T) {
+	// this test assumes a test set with a single directory containing hard linked files
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	datafile := filepath.Join("testdata", "test.hl.tar.gz")
+	fd, err := os.Open(datafile)
+	if os.IsNotExist(err) {
+		t.Skipf("unable to find data file %q, skipping", datafile)
+		return
+	}
+	rtest.OK(t, err)
+	rtest.OK(t, fd.Close())
+
+	testRunInit(t, env.gopts)
+
+	rtest.SetupTarTestFixture(t, env.testdata, datafile)
+
+	linkTests := createFileSetPerHardlink(env.testdata)
+
+	opts := BackupOptions{}
+
+	// first backup
+	testRunBackup(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
+	snapshotIDs := testListSnapshots(t, env.gopts, 1)
+
+	testRunCheck(t, env.gopts)
+
+	// restore all backups and compare
+	for i, snapshotID := range snapshotIDs {
+		restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
+		t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir)
+		testRunRestore(t, env.gopts, restoredir, snapshotID)
+		diff := directoriesContentsDiff(env.testdata, filepath.Join(restoredir, "testdata"))
+		rtest.Assert(t, diff == "", "directories are not equal %v", diff)
+
+		linkResults := createFileSetPerHardlink(filepath.Join(restoredir, "testdata"))
+		rtest.Assert(t, linksEqual(linkTests, linkResults),
+			"links are not equal")
+	}
+
+	testRunCheck(t, env.gopts)
+}
+
+func linksEqual(source, dest map[uint64][]string) bool {
+	for _, vs := range source {
+		found := false
+		for kd, vd := range dest {
+			if linkEqual(vs, vd) {
+				delete(dest, kd)
+				found = true
+				break
+			}
+		}
+		if !found {
+			return false
+		}
+	}
+
+	return len(dest) == 0
+}
+
+func linkEqual(source, dest []string) bool {
+	// equal if sliced are equal without considering order
+	if source == nil && dest == nil {
+		return true
+	}
+
+	if source == nil || dest == nil {
+		return false
+	}
+
+	if len(source) != len(dest) {
+		return false
+	}
+
+	for i := range source {
+		found := false
+		for j := range dest {
+			if source[i] == dest[j] {
+				found = true
+				break
+			}
+		}
+		if !found {
+			return false
+		}
+	}
+
+	return true
+}

cmd/restic/cmd_backup_test.go

@@ -14,8 +14,7 @@ import (
 )
 
 func TestCollectTargets(t *testing.T) {
-	dir, cleanup := rtest.TempDir(t)
-	defer cleanup()
+	dir := rtest.TempDir(t)
 
 	fooSpace := "foo "
 	barStar := "bar*"              // Must sort before the others, below.

cmd/restic/cmd_cache.go

@@ -11,6 +11,7 @@ import (
 	"github.com/restic/restic/internal/cache"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/fs"
+	"github.com/restic/restic/internal/ui"
 	"github.com/restic/restic/internal/ui/table"
 	"github.com/spf13/cobra"
 )
@@ -138,7 +139,7 @@ func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
 			if err != nil {
 				return err
 			}
-			size = fmt.Sprintf("%11s", formatBytes(uint64(bytes)))
+			size = fmt.Sprintf("%11s", ui.FormatBytes(uint64(bytes)))
 		}
 
 		name := entry.Name()
@@ -154,7 +155,7 @@ func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
 		})
 	}
 
-	_ = tab.Write(gopts.stdout)
+	_ = tab.Write(globalOptions.stdout)
 	Printf("%d cache dirs in %s\n", len(dirs), cachedir)
 
 	return nil

cmd/restic/cmd_cat.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 
 	"github.com/spf13/cobra"
@@ -12,7 +13,7 @@ import (
 )
 
 var cmdCat = &cobra.Command{
-	Use:   "cat [flags] [pack|blob|snapshot|index|key|masterkey|config|lock] ID",
+	Use:   "cat [flags] [masterkey|config|pack ID|blob ID|snapshot ID|index ID|key ID|lock ID|tree snapshot:subfolder]",
 	Short: "Print internal objects to stdout",
 	Long: `
 The "cat" command is used to print internal objects to stdout.
@@ -24,7 +25,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runCat(globalOptions, args)
+		return runCat(cmd.Context(), globalOptions, args)
 	},
 }
 
@@ -32,40 +33,32 @@ func init() {
 	cmdRoot.AddCommand(cmdCat)
 }
 
-func runCat(gopts GlobalOptions, args []string) error {
+func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 	if len(args) < 1 || (args[0] != "masterkey" && args[0] != "config" && len(args) != 2) {
 		return errors.Fatal("type or ID not specified")
 	}
 
-	repo, err := OpenRepository(gopts)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
 
 	if !gopts.NoLock {
-		lock, err := lockRepo(gopts.ctx, repo)
+		var lock *restic.Lock
+		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
+		defer unlockRepo(lock)
 		if err != nil {
 			return err
 		}
-
-		defer unlockRepo(lock)
 	}
 
 	tpe := args[0]
 
 	var id restic.ID
-	if tpe != "masterkey" && tpe != "config" {
+	if tpe != "masterkey" && tpe != "config" && tpe != "snapshot" && tpe != "tree" {
 		id, err = restic.ParseID(args[1])
 		if err != nil {
-			if tpe != "snapshot" {
-				return errors.Fatalf("unable to parse ID: %v\n", err)
-			}
-
-			// find snapshot id with prefix
-			id, err = restic.FindSnapshot(gopts.ctx, repo.Backend(), args[1])
-			if err != nil {
-				return errors.Fatalf("could not find snapshot: %v\n", err)
-			}
+			return errors.Fatalf("unable to parse ID: %v\n", err)
 		}
 	}
 
@@ -79,7 +72,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 		Println(string(buf))
 		return nil
 	case "index":
-		buf, err := repo.LoadUnpacked(gopts.ctx, restic.IndexFile, id, nil)
+		buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id)
 		if err != nil {
 			return err
 		}
@@ -87,9 +80,9 @@ func runCat(gopts GlobalOptions, args []string) error {
 		Println(string(buf))
 		return nil
 	case "snapshot":
-		sn, err := restic.LoadSnapshot(gopts.ctx, repo, id)
+		sn, _, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1])
 		if err != nil {
-			return err
+			return errors.Fatalf("could not find snapshot: %v\n", err)
 		}
 
 		buf, err := json.MarshalIndent(sn, "", "  ")
@@ -100,19 +93,12 @@ func runCat(gopts GlobalOptions, args []string) error {
 		Println(string(buf))
 		return nil
 	case "key":
-		h := restic.Handle{Type: restic.KeyFile, Name: id.String()}
-		buf, err := backend.LoadAll(gopts.ctx, nil, repo.Backend(), h)
+		key, err := repository.LoadKey(ctx, repo, id)
 		if err != nil {
 			return err
 		}
 
-		key := &repository.Key{}
-		err = json.Unmarshal(buf, key)
-		if err != nil {
-			return err
-		}
-
-		buf, err = json.MarshalIndent(&key, "", "  ")
+		buf, err := json.MarshalIndent(&key, "", "  ")
 		if err != nil {
 			return err
 		}
@@ -128,7 +114,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 		Println(string(buf))
 		return nil
 	case "lock":
-		lock, err := restic.LoadLock(gopts.ctx, repo, id)
+		lock, err := restic.LoadLock(ctx, repo, id)
 		if err != nil {
 			return err
 		}
@@ -143,7 +129,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 
 	case "pack":
 		h := restic.Handle{Type: restic.PackFile, Name: id.String()}
-		buf, err := backend.LoadAll(gopts.ctx, nil, repo.Backend(), h)
+		buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h)
 		if err != nil {
 			return err
 		}
@@ -157,7 +143,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 		return err
 
 	case "blob":
-		err = repo.LoadIndex(gopts.ctx)
+		err = repo.LoadIndex(ctx)
 		if err != nil {
 			return err
 		}
@@ -168,7 +154,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 				continue
 			}
 
-			buf, err := repo.LoadBlob(gopts.ctx, t, id, nil)
+			buf, err := repo.LoadBlob(ctx, t, id, nil)
 			if err != nil {
 				return err
 			}
@@ -179,6 +165,29 @@ func runCat(gopts GlobalOptions, args []string) error {
 
 		return errors.Fatal("blob not found")
 
+	case "tree":
+		sn, subfolder, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1])
+		if err != nil {
+			return errors.Fatalf("could not find snapshot: %v\n", err)
+		}
+
+		err = repo.LoadIndex(ctx)
+		if err != nil {
+			return err
+		}
+
+		sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
+		if err != nil {
+			return err
+		}
+
+		buf, err := repo.LoadBlob(ctx, restic.TreeBlob, *sn.Tree, nil)
+		if err != nil {
+			return err
+		}
+		_, err = globalOptions.stdout.Write(buf)
+		return err
+
 	default:
 		return errors.Fatal("invalid type")
 	}

cmd/restic/cmd_check.go

@@ -1,8 +1,9 @@
 package main
 
 import (
-	"io/ioutil"
+	"context"
 	"math/rand"
+	"os"
 	"strconv"
 	"strings"
 	"sync"
@@ -15,6 +16,7 @@ import (
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui"
 )
 
 var cmdCheck = &cobra.Command{
@@ -34,7 +36,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runCheck(checkOptions, globalOptions, args)
+		return runCheck(cmd.Context(), checkOptions, globalOptions, args)
 	},
 	PreRunE: func(cmd *cobra.Command, args []string) error {
 		return checkFlags(checkOptions)
@@ -64,7 +66,7 @@ func init() {
 		// MarkDeprecated only returns an error when the flag is not found
 		panic(err)
 	}
-	f.BoolVar(&checkOptions.WithCache, "with-cache", false, "use the cache")
+	f.BoolVar(&checkOptions.WithCache, "with-cache", false, "use existing cache, only read uncached data from repository")
 }
 
 func checkFlags(opts CheckOptions) error {
@@ -96,7 +98,7 @@ func checkFlags(opts CheckOptions) error {
 			}
 
 		} else {
-			fileSize, err := parseSizeStr(opts.ReadDataSubset)
+			fileSize, err := ui.ParseBytes(opts.ReadDataSubset)
 			if err != nil {
 				return argumentError
 			}
@@ -170,7 +172,7 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func())
 	}
 
 	// use a cache in a temporary directory
-	tempdir, err := ioutil.TempDir(cachedir, "restic-check-cache-")
+	tempdir, err := os.MkdirTemp(cachedir, "restic-check-cache-")
 	if err != nil {
 		// if an error occurs, don't use any cache
 		Warnf("unable to create temporary directory for cache during check, disabling cache: %v\n", err)
@@ -191,25 +193,26 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func())
 	return cleanup
 }
 
-func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
+func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args []string) error {
 	if len(args) != 0 {
 		return errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags")
 	}
 
 	cleanup := prepareCheckCache(opts, &gopts)
-	AddCleanupHandler(func() error {
+	AddCleanupHandler(func(code int) (int, error) {
 		cleanup()
-		return nil
+		return code, nil
 	})
 
-	repo, err := OpenRepository(gopts)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
 
 	if !gopts.NoLock {
 		Verbosef("create exclusive lock for repository\n")
-		lock, err := lockRepoExclusive(gopts.ctx, repo)
+		var lock *restic.Lock
+		lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -217,13 +220,13 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	chkr := checker.New(repo, opts.CheckUnused)
-	err = chkr.LoadSnapshots(gopts.ctx)
+	err = chkr.LoadSnapshots(ctx)
 	if err != nil {
 		return err
 	}
 
 	Verbosef("load indexes\n")
-	hints, errs := chkr.LoadIndex(gopts.ctx)
+	hints, errs := chkr.LoadIndex(ctx)
 
 	errorsFound := false
 	suggestIndexRebuild := false
@@ -243,7 +246,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	if suggestIndexRebuild {
-		Printf("This is non-critical, you can run `restic rebuild-index' to correct this\n")
+		Printf("Duplicate packs/old indexes are non-critical, you can run `restic repair index' to correct this.\n")
 	}
 	if mixedFound {
 		Printf("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")
@@ -260,13 +263,13 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 	errChan := make(chan error)
 
 	Verbosef("check all packs\n")
-	go chkr.Packs(gopts.ctx, errChan)
+	go chkr.Packs(ctx, errChan)
 
 	for err := range errChan {
 		if checker.IsOrphanedPack(err) {
 			orphanedPacks++
 			Verbosef("%v\n", err)
-		} else if _, ok := err.(*checker.ErrLegacyLayout); ok {
+		} else if err == checker.ErrLegacyLayout {
 			Verbosef("repository still uses the S3 legacy layout\nPlease run `restic migrate s3legacy` to correct this.\n")
 		} else {
 			errorsFound = true
@@ -287,13 +290,17 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 		defer wg.Done()
 		bar := newProgressMax(!gopts.Quiet, 0, "snapshots")
 		defer bar.Done()
-		chkr.Structure(gopts.ctx, bar, errChan)
+		chkr.Structure(ctx, bar, errChan)
 	}()
 
 	for err := range errChan {
 		errorsFound = true
 		if e, ok := err.(*checker.TreeError); ok {
-			Warnf("error for tree %v:\n", e.ID.Str())
+			var clean string
+			if stdoutCanUpdateStatus() {
+				clean = clearLine(0)
+			}
+			Warnf(clean+"error for tree %v:\n", e.ID.Str())
 			for _, treeErr := range e.Errors {
 				Warnf("  %v\n", treeErr)
 			}
@@ -308,7 +315,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 	wg.Wait()
 
 	if opts.CheckUnused {
-		for _, id := range chkr.UnusedBlobs(gopts.ctx) {
+		for _, id := range chkr.UnusedBlobs(ctx) {
 			Verbosef("unused blob %v\n", id)
 			errorsFound = true
 		}
@@ -320,7 +327,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 		p := newProgressMax(!gopts.Quiet, packCount, "packs")
 		errChan := make(chan error)
 
-		go chkr.ReadPacks(gopts.ctx, packs, p, errChan)
+		go chkr.ReadPacks(ctx, packs, p, errChan)
 
 		for err := range errChan {
 			errorsFound = true
@@ -357,7 +364,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 			if repoSize == 0 {
 				return errors.Fatal("Cannot read from a repository having size 0")
 			}
-			subsetSize, _ := parseSizeStr(opts.ReadDataSubset)
+			subsetSize, _ := ui.ParseBytes(opts.ReadDataSubset)
 			if subsetSize > repoSize {
 				subsetSize = repoSize
 			}

cmd/restic/cmd_check_integration_test.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"context"
+	"testing"
+
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func testRunCheck(t testing.TB, gopts GlobalOptions) {
+	t.Helper()
+	output, err := testRunCheckOutput(gopts, true)
+	if err != nil {
+		t.Error(output)
+		t.Fatalf("unexpected error: %+v", err)
+	}
+}
+
+func testRunCheckMustFail(t testing.TB, gopts GlobalOptions) {
+	t.Helper()
+	_, err := testRunCheckOutput(gopts, false)
+	rtest.Assert(t, err != nil, "expected non nil error after check of damaged repository")
+}
+
+func testRunCheckOutput(gopts GlobalOptions, checkUnused bool) (string, error) {
+	buf, err := withCaptureStdout(func() error {
+		opts := CheckOptions{
+			ReadData:    true,
+			CheckUnused: checkUnused,
+		}
+		return runCheck(context.TODO(), opts, gopts, nil)
+	})
+	return buf.String(), err
+}

cmd/restic/cmd_copy.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"golang.org/x/sync/errgroup"
@@ -32,16 +33,14 @@ This can be mitigated by the "--copy-chunker-params" option when initializing a
 new destination repository using the "init" command.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runCopy(copyOptions, globalOptions, args)
+		return runCopy(cmd.Context(), copyOptions, globalOptions, args)
 	},
 }
 
 // CopyOptions bundles all options for the copy command.
 type CopyOptions struct {
 	secondaryRepoOptions
-	Hosts []string
-	Tags  restic.TagLists
-	Paths []string
+	restic.SnapshotFilter
 }
 
 var copyOptions CopyOptions
@@ -51,12 +50,10 @@ func init() {
 
 	f := cmdCopy.Flags()
 	initSecondaryRepoOptions(f, &copyOptions.secondaryRepoOptions, "destination", "to copy snapshots from")
-	f.StringArrayVarP(&copyOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)")
-	f.Var(&copyOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot ID is given")
-	f.StringArrayVar(&copyOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
+	initMultiSnapshotFilter(f, &copyOptions.SnapshotFilter, true)
 }
 
-func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
+func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []string) error {
 	secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "destination")
 	if err != nil {
 		return err
@@ -66,28 +63,26 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 		gopts, secondaryGopts = secondaryGopts, gopts
 	}
 
-	ctx, cancel := context.WithCancel(gopts.ctx)
-	defer cancel()
-
-	srcRepo, err := OpenRepository(gopts)
+	srcRepo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
 
-	dstRepo, err := OpenRepository(secondaryGopts)
+	dstRepo, err := OpenRepository(ctx, secondaryGopts)
 	if err != nil {
 		return err
 	}
 
 	if !gopts.NoLock {
-		srcLock, err := lockRepo(ctx, srcRepo)
+		var srcLock *restic.Lock
+		srcLock, ctx, err = lockRepo(ctx, srcRepo, gopts.RetryLock, gopts.JSON)
 		defer unlockRepo(srcLock)
 		if err != nil {
 			return err
 		}
 	}
 
-	dstLock, err := lockRepo(ctx, dstRepo)
+	dstLock, ctx, err := lockRepo(ctx, dstRepo, gopts.RetryLock, gopts.JSON)
 	defer unlockRepo(dstLock)
 	if err != nil {
 		return err
@@ -114,7 +109,7 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	dstSnapshotByOriginal := make(map[restic.ID][]*restic.Snapshot)
-	for sn := range FindFilteredSnapshots(ctx, dstSnapshotLister, dstRepo, opts.Hosts, opts.Tags, opts.Paths, nil) {
+	for sn := range FindFilteredSnapshots(ctx, dstSnapshotLister, dstRepo, &opts.SnapshotFilter, nil) {
 		if sn.Original != nil && !sn.Original.IsNull() {
 			dstSnapshotByOriginal[*sn.Original] = append(dstSnapshotByOriginal[*sn.Original], sn)
 		}
@@ -125,9 +120,7 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 	// remember already processed trees across all snapshots
 	visitedTrees := restic.NewIDSet()
 
-	for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) {
-		Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
-
+	for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, &opts.SnapshotFilter, args) {
 		// check whether the destination has a snapshot with the same persistent ID which has similar snapshot fields
 		srcOriginal := *sn.ID()
 		if sn.Original != nil {
@@ -137,7 +130,8 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 			isCopy := false
 			for _, originalSn := range originalSns {
 				if similarSnapshots(originalSn, sn) {
-					Verbosef("skipping source snapshot %s, was already copied to snapshot %s\n", sn.ID().Str(), originalSn.ID().Str())
+					Verboseff("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
+					Verboseff("skipping source snapshot %s, was already copied to snapshot %s\n", sn.ID().Str(), originalSn.ID().Str())
 					isCopy = true
 					break
 				}
@@ -146,6 +140,7 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 				continue
 			}
 		}
+		Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
 		Verbosef("  copy started, this may take a while...\n")
 		if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree, gopts.Quiet); err != nil {
 			return err
@@ -242,5 +237,8 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep
 	bar := newProgressMax(!quiet, uint64(len(packList)), "packs copied")
 	_, err = repository.Repack(ctx, srcRepo, dstRepo, packList, copyBlobs, bar)
 	bar.Done()
-	return err
+	if err != nil {
+		return errors.Fatal(err.Error())
+	}
+	return nil
 }