Add version for 0.16.1

Don't load password when autocompleting CLI commands

.dockerignore

@@ -1,12 +0,0 @@
-# Folders
-.git/
-.github/
-changelog/
-doc/
-docker/
-helpers/
-
-# Files
-.gitignore
-.golangci.yml
-*.md

.github/ISSUE_TEMPLATE/Bug.md

@@ -32,23 +32,30 @@ Output of `restic version`
 --------------------------
 
 
-How did you run restic exactly?
--------------------------------
+
+What backend/service did you use to store the repository?
+---------------------------------------------------------
+
+
+
+Problem description / Steps to reproduce
+----------------------------------------
 
 <!--
 This section should include at least:
 
+ * A description of the problem you are having with restic.
+
  * The complete command line and any environment variables you used to
    configure restic's backend access. Make sure to replace sensitive values!
 
  * The output of the commands, what restic prints gives may give us much
    information to diagnose the problem!
+
+ * The more time you spend describing an easy way to reproduce the behavior (if
+   this is possible), the easier it is for the project developers to fix it!
 -->
 
-What backend/server/service did you use to store the repository?
-----------------------------------------------------------------
-
-
 
 Expected behavior
 -----------------
@@ -65,22 +72,12 @@ In this section, please try to concentrate on observations, so only describe
 what you observed directly.
 -->
 
-Steps to reproduce the behavior
--------------------------------
-
-<!--
-The more time you spend describing an easy way to reproduce the behavior (if
-this is possible), the easier it is for the project developers to fix it!
--->
-
 Do you have any idea what may have caused this?
 -----------------------------------------------
 
-
-
-Do you have an idea how to solve the issue?
--------------------------------------------
-
+<!--
+Did something noteworthy happen on your system, Internet connection, backend services, etc?
+-->
 
 
 Did restic help you today? Did it make you happy in any way?

.github/workflows/docker.yml

@@ -0,0 +1,66 @@
+
+name: Create and publish a Docker image
+
+on:
+  push:
+    tags:
+      - 'v*'
+    branches:
+      - 'master'
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    if: github.repository == 'restic/restic'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@b4bedf8053341df3b5a9f9e0f2cf4e79e27360c6
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226
+
+      - name: Ensure consistent binaries
+        run: |
+          echo "removing git directory for consistency with release binaries"
+          rm -rf .git
+        # remove VCS information from release builds, keep VCS for nightly builds on master
+        if: github.ref != 'refs/heads/master'
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          push: true
+          context: .
+          file: docker/Dockerfile.release
+          platforms: linux/386,linux/amd64,linux/arm,linux/arm64
+          pull: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/tests.yml

@@ -7,9 +7,13 @@ on:
 
   # run tests for all pull requests
   pull_request:
+  merge_group:
+
+permissions:
+  contents: read
 
 env:
-  latest_go: "1.19.x"
+  latest_go: "1.21.x"
   GO111MODULE: on
 
 jobs:
@@ -19,29 +23,34 @@ jobs:
         # list of jobs to run:
         include:
           - job_name: Windows
-            go: 1.19.x
+            go: 1.21.x
             os: windows-latest
 
           - job_name: macOS
-            go: 1.19.x
+            go: 1.21.x
             os: macOS-latest
             test_fuse: false
 
           - job_name: Linux
-            go: 1.19.x
+            go: 1.21.x
             os: ubuntu-latest
             test_cloud_backends: true
             test_fuse: true
             check_changelog: true
 
           - job_name: Linux (race)
-            go: 1.19.x
+            go: 1.21.x
             os: ubuntu-latest
             test_fuse: true
             test_opts: "-race"
 
           - job_name: Linux
-            go: 1.18.x
+            go: 1.20.x
+            os: ubuntu-latest
+            test_fuse: true
+
+          - job_name: Linux
+            go: 1.19.x
             os: ubuntu-latest
             test_fuse: true
 
@@ -53,7 +62,7 @@ jobs:
 
     steps:
       - name: Set up Go ${{ matrix.go }}
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: ${{ matrix.go }}
 
@@ -126,12 +135,20 @@ jobs:
         if: matrix.os == 'windows-latest'
 
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Build with build.go
         run: |
           go run build.go
 
+      - name: Minimal test
+        run: |
+          ./restic init
+          ./restic backup .
+        env:
+          RESTIC_REPOSITORY: ../testrepo
+          RESTIC_PASSWORD: password
+
       - name: Run local Tests
         env:
           RESTIC_TEST_FUSE: ${{ matrix.test_fuse }}
@@ -179,7 +196,7 @@ jobs:
         # own repo, otherwise the secrets are not available
         # Skip for Dependabot pull requests as these are run without secrets
         # https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#responding-to-events
-        if: (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && (github.actor != 'dependabot[bot]') && matrix.test_cloud_backends
+        if: ((github.repository == 'restic/restic' && github.event_name == 'push') || github.event.pull_request.head.repo.full_name == github.repository) && (github.actor != 'dependabot[bot]') && matrix.test_cloud_backends
 
       - name: Check changelog files with calens
         run: |
@@ -193,69 +210,52 @@ jobs:
   cross_compile:
     strategy:
 
-      # ATTENTION: the list of architectures must be in sync with helpers/build-release-binaries/main.go!
       matrix:
         # run cross-compile in three batches parallel so the overall tests run faster
-        targets:
-          - "linux/386 linux/amd64 linux/arm linux/arm64 linux/ppc64le linux/mips linux/mipsle linux/mips64 linux/mips64le linux/s390x"
-
-          - "openbsd/386 openbsd/amd64 \
-            freebsd/386 freebsd/amd64 freebsd/arm \
-            aix/ppc64 \
-            darwin/amd64 darwin/arm64"
-
-          - "netbsd/386 netbsd/amd64 \
-            windows/386 windows/amd64 \
-            solaris/amd64"
+        subset:
+          - "0/3"
+          - "1/3"
+          - "2/3"
 
     env:
       GOPROXY: https://proxy.golang.org
 
     runs-on: ubuntu-latest
 
-    name: Cross Compile for ${{ matrix.targets }}
+    name: Cross Compile for subset ${{ matrix.subset }}
 
     steps:
       - name: Set up Go ${{ env.latest_go }}
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: ${{ env.latest_go }}
 
-      - name: Install gox
-        run: |
-          go install github.com/mitchellh/gox@latest
-
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Cross-compile with gox for ${{ matrix.targets }}
-        env:
-          GOFLAGS: "-trimpath"
-          GOX_ARCHS: "${{ matrix.targets }}"
+      - name: Cross-compile for subset ${{ matrix.subset }}
         run: |
-          mkdir build-output
-          gox -parallel 2 -verbose -osarch "$GOX_ARCHS" -output "build-output/{{.Dir}}_{{.OS}}_{{.Arch}}" ./cmd/restic
-          gox -parallel 2 -verbose -osarch "$GOX_ARCHS" -tags debug -output "build-output/{{.Dir}}_{{.OS}}_{{.Arch}}_debug" ./cmd/restic
+          mkdir build-output build-output-debug
+          go run ./helpers/build-release-binaries/main.go -o build-output -s . --platform-subset ${{ matrix.subset }}
+          go run ./helpers/build-release-binaries/main.go -o build-output-debug -s . --platform-subset ${{ matrix.subset }} --tags debug
 
   lint:
     name: lint
     runs-on: ubuntu-latest
     steps:
       - name: Set up Go ${{ env.latest_go }}
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: ${{ env.latest_go }}
 
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.49
-          # Optional: show only new issues if it's a pull request. The default value is `false`.
-          only-new-issues: true
+          version: v1.52.2
           args: --verbose --timeout 5m
 
         # only run golangci-lint for pull requests, otherwise ALL hints get
@@ -269,12 +269,27 @@ jobs:
           go mod tidy
           git diff --exit-code go.mod go.sum
 
+  analyze:
+    name: Analyze results
+    needs: [test, cross_compile, lint]
+    if: always()
+
+    permissions: # no need to access code
+      contents: none
+
+    runs-on: ubuntu-latest
+    steps:
+      - name: Decide whether the needed jobs succeeded or failed
+        uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe
+        with:
+          jobs: ${{ toJSON(needs) }}
+
   docker:
     name: docker
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Docker meta
         id: meta
@@ -294,14 +309,14 @@ jobs:
             type=sha
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           push: false
           context: .

.gitignore

@@ -1,3 +1,4 @@
 /restic
+/restic.exe
 /.vagrant
 /.vscode

.golangci.yml

@@ -10,13 +10,10 @@ linters:
     # make sure all errors returned by functions are handled
     - errcheck
 
-    # find unused code
-    - deadcode
-
     # show how code can be simplified
     - gosimple
 
-    # # make sure code is formatted
+    # make sure code is formatted
     - gofmt
 
     # examine code and report suspicious constructs, such as Printf calls whose
@@ -35,12 +32,6 @@ linters:
     # find unused variables, functions, structs, types, etc.
     - unused
 
-    # find unused struct fields
-    - structcheck
-
-    # find unused global variables
-    - varcheck
-
     # parse and typecheck code
     - typecheck
 
@@ -57,3 +48,6 @@ issues:
     - don't use ALL_CAPS in Go names; use CamelCase
     # revive: lots of packages don't have such a comment
     - "package-comments: should have a package comment"
+    # staticcheck: there's no easy way to replace these packages
+    - "SA1019: \"golang.org/x/crypto/poly1305\" is deprecated"
+    - "SA1019: \"golang.org/x/crypto/openpgp\" is deprecated"

CHANGELOG.md

@@ -1,3 +1,635 @@
+Changelog for restic 0.16.1 (2023-10-24)
+=======================================
+
+The following sections list the changes in restic 0.16.1 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #4513: Make `key list` command honor `--no-lock`
+ * Fix #4516: Do not try to load password on command line autocomplete
+ * Fix #4523: Update zstd library to fix possible data corruption at max. compression
+ * Chg #4532: Update dependencies and require Go 1.19 or newer
+ * Enh #229: Show progress bar while loading the index
+ * Enh #4128: Automatically set `GOMAXPROCS` in resource-constrained containers
+ * Enh #4480: Allow setting REST password and username via environment variables
+ * Enh #4511: Include inode numbers in JSON output for `find` and `ls` commands
+ * Enh #4519: Add config option to set SFTP command arguments
+
+Details
+-------
+
+ * Bugfix #4513: Make `key list` command honor `--no-lock`
+
+   The `key list` command now supports the `--no-lock` options. This allows determining which
+   keys a repo can be accessed by without the need for having write access (e.g., read-only sftp
+   access, filesystem snapshot).
+
+   https://github.com/restic/restic/issues/4513
+   https://github.com/restic/restic/pull/4514
+
+ * Bugfix #4516: Do not try to load password on command line autocomplete
+
+   The command line autocompletion previously tried to load the repository password. This could
+   cause the autocompletion not to work. Now, this step gets skipped.
+
+   https://github.com/restic/restic/issues/4516
+   https://github.com/restic/restic/pull/4526
+
+ * Bugfix #4523: Update zstd library to fix possible data corruption at max. compression
+
+   In restic 0.16.0, backups where the compression level was set to `max` (using `--compression
+   max`) could in rare and very specific circumstances result in data corruption due to a bug in the
+   library used for compressing data.
+
+   Restic now uses the latest version of the library used to compress data, which includes a fix for
+   this issue. Please note that the `auto` compression level (which restic uses by default) was
+   never affected, and even if you used `max` compression, chances of being affected by this issue
+   were very small.
+
+   To check a repository for any corruption, run `restic check --read-data`. This will download
+   and verify the whole repository and can be used at any time to completely verify the integrity of
+   a repository. If the `check` command detects anomalies, follow the suggested steps.
+
+   To simplify any needed repository repair and minimize data loss, there is also a new and
+   experimental `repair packs` command that salvages all valid data from the affected pack files
+   (see `restic help repair packs` for more information).
+
+   https://github.com/restic/restic/issues/4523
+   https://github.com/restic/restic/pull/4530
+
+ * Change #4532: Update dependencies and require Go 1.19 or newer
+
+   We have updated all dependencies. Since some libraries require newer Go standard library
+   features, support for Go 1.18 has been dropped, which means that restic now requires at least Go
+   1.19 to build.
+
+   https://github.com/restic/restic/pull/4532
+   https://github.com/restic/restic/pull/4533
+
+ * Enhancement #229: Show progress bar while loading the index
+
+   Restic did not provide any feedback while loading index files. Now, there is a progress bar that
+   shows the index loading progress.
+
+   https://github.com/restic/restic/issues/229
+   https://github.com/restic/restic/pull/4419
+
+ * Enhancement #4128: Automatically set `GOMAXPROCS` in resource-constrained containers
+
+   When running restic in a Linux container with CPU-usage limits, restic now automatically
+   adjusts `GOMAXPROCS`. This helps to reduce the memory consumption on hosts with many CPU
+   cores.
+
+   https://github.com/restic/restic/issues/4128
+   https://github.com/restic/restic/pull/4485
+   https://github.com/restic/restic/pull/4531
+
+ * Enhancement #4480: Allow setting REST password and username via environment variables
+
+   Previously, it was only possible to specify the REST-server username and password in the
+   repository URL, or by using the `--repository-file` option. This meant it was not possible to
+   use authentication in contexts where the repository URL is stored in publicly accessible way.
+
+   Restic now allows setting the username and password using the `RESTIC_REST_USERNAME` and
+   `RESTIC_REST_PASSWORD` variables.
+
+   https://github.com/restic/restic/pull/4480
+
+ * Enhancement #4511: Include inode numbers in JSON output for `find` and `ls` commands
+
+   Restic used to omit the inode numbers in the JSON messages emitted for nodes by the `ls` command
+   as well as for matches by the `find` command. It now includes those values whenever they are
+   available.
+
+   https://github.com/restic/restic/pull/4511
+
+ * Enhancement #4519: Add config option to set SFTP command arguments
+
+   When using the `sftp` backend, scenarios where a custom identity file was needed for the SSH
+   connection, required the full command to be specified: `-o sftp.command='ssh
+   user@host:port -i /ssh/my_private_key -s sftp'`
+
+   Now, the `-o sftp.args=...` option can be passed to restic to specify custom arguments for the
+   SSH command executed by the SFTP backend. This simplifies the above example to `-o
+   sftp.args='-i /ssh/my_private_key'`.
+
+   https://github.com/restic/restic/issues/4241
+   https://github.com/restic/restic/pull/4519
+
+
+Changelog for restic 0.16.0 (2023-07-31)
+=======================================
+
+The following sections list the changes in restic 0.16.0 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #2565: Support "unlimited" in `forget --keep-*` options
+ * Fix #3311: Support non-UTF8 paths as symlink target
+ * Fix #4199: Avoid lock refresh issues on slow network connections
+ * Fix #4274: Improve lock refresh handling after standby
+ * Fix #4319: Correctly clean up status bar output of the `backup` command
+ * Fix #4333: `generate` and `init` no longer silently ignore unexpected arguments
+ * Fix #4400: Ignore missing folders in `rest` backend
+ * Chg #4176: Fix JSON message type of `scan_finished` for the `backup` command
+ * Chg #4201: Require Go 1.20 for Solaris builds
+ * Enh #426: Show progress bar during restore
+ * Enh #719: Add `--retry-lock` option
+ * Enh #1495: Sort snapshots by timestamp in `restic find`
+ * Enh #1759: Add `repair index` and `repair snapshots` commands
+ * Enh #1926: Allow certificate paths to be passed through environment variables
+ * Enh #2359: Provide multi-platform Docker images
+ * Enh #2468: Add support for non-global Azure clouds
+ * Enh #2679: Reduce file fragmentation for local backend
+ * Enh #3328: Reduce memory usage by up to 25%
+ * Enh #3397: Improve accuracy of ETA displayed during backup
+ * Enh #3624: Keep oldest snapshot when there are not enough snapshots
+ * Enh #3698: Add support for Managed / Workload Identity to `azure` backend
+ * Enh #3871: Support `<snapshot>:<subfolder>` syntax to select subfolders
+ * Enh #3941: Support `--group-by` for backup parent selection
+ * Enh #4130: Cancel current command if cache becomes unusable
+ * Enh #4159: Add `--human-readable` option to `ls` and `find` commands
+ * Enh #4188: Include restic version in snapshot metadata
+ * Enh #4220: Add `jq` binary to Docker image
+ * Enh #4226: Allow specifying region of new buckets in the `gs` backend
+ * Enh #4375: Add support for extended attributes on symlinks
+
+Details
+-------
+
+ * Bugfix #2565: Support "unlimited" in `forget --keep-*` options
+
+   Restic would previously forget snapshots that should have been kept when a negative value was
+   passed to the `--keep-*` options. Negative values are now forbidden. To keep all snapshots,
+   the special value `unlimited` is now supported. For example, `--keep-monthly unlimited`
+   will keep all monthly snapshots.
+
+   https://github.com/restic/restic/issues/2565
+   https://github.com/restic/restic/pull/4234
+
+ * Bugfix #3311: Support non-UTF8 paths as symlink target
+
+   Earlier restic versions did not correctly `backup` and `restore` symlinks that contain a
+   non-UTF8 target. Note that this only affected systems that still use a non-Unicode encoding
+   for filesystem paths.
+
+   The repository format is now extended to add support for such symlinks. Please note that
+   snapshots must have been created with at least restic version 0.16.0 for `restore` to
+   correctly handle non-UTF8 symlink targets when restoring them.
+
+   https://github.com/restic/restic/issues/3311
+   https://github.com/restic/restic/pull/3802
+
+ * Bugfix #4199: Avoid lock refresh issues on slow network connections
+
+   On network connections with a low upload speed, backups and other operations could fail with
+   the error message `Fatal: failed to refresh lock in time`.
+
+   This has now been fixed by reworking the lock refresh handling.
+
+   https://github.com/restic/restic/issues/4199
+   https://github.com/restic/restic/pull/4304
+
+ * Bugfix #4274: Improve lock refresh handling after standby
+
+   If the restic process was stopped or the host running restic entered standby during a long
+   running operation such as a backup, this previously resulted in the operation failing with
+   `Fatal: failed to refresh lock in time`.
+
+   This has now been fixed such that restic first checks whether it is safe to continue the current
+   operation and only throws an error if not.
+
+   https://github.com/restic/restic/issues/4274
+   https://github.com/restic/restic/pull/4374
+
+ * Bugfix #4319: Correctly clean up status bar output of the `backup` command
+
+   Due to a regression in restic 0.15.2, the status bar of the `backup` command could leave some
+   output behind. This happened if filenames were printed that are wider than the current
+   terminal width. This has now been fixed.
+
+   https://github.com/restic/restic/issues/4319
+   https://github.com/restic/restic/pull/4318
+
+ * Bugfix #4333: `generate` and `init` no longer silently ignore unexpected arguments
+
+   https://github.com/restic/restic/pull/4333
+
+ * Bugfix #4400: Ignore missing folders in `rest` backend
+
+   If a repository accessed via the REST backend was missing folders, then restic would fail with
+   an error while trying to list the data in the repository. This has been now fixed.
+
+   https://github.com/restic/rest-server/issues/235
+   https://github.com/restic/restic/pull/4400
+
+ * Change #4176: Fix JSON message type of `scan_finished` for the `backup` command
+
+   Restic incorrectly set the `message_type` of the `scan_finished` message to `status`
+   instead of `verbose_status`. This has now been corrected so that the messages report the
+   correct type.
+
+   https://github.com/restic/restic/pull/4176
+
+ * Change #4201: Require Go 1.20 for Solaris builds
+
+   Building restic on Solaris now requires Go 1.20, as the library used to access Azure uses the
+   mmap syscall, which is only available on Solaris starting from Go 1.20. All other platforms
+   however continue to build with Go 1.18.
+
+   https://github.com/restic/restic/pull/4201
+
+ * Enhancement #426: Show progress bar during restore
+
+   The `restore` command now shows a progress report while restoring files.
+
+   Example: `[0:42] 5.76% 23 files 12.98 MiB, total 3456 files 23.54 GiB`
+
+   JSON output is now also supported.
+
+   https://github.com/restic/restic/issues/426
+   https://github.com/restic/restic/issues/3413
+   https://github.com/restic/restic/issues/3627
+   https://github.com/restic/restic/pull/3991
+   https://github.com/restic/restic/pull/4314
+   https://forum.restic.net/t/progress-bar-for-restore/5210
+
+ * Enhancement #719: Add `--retry-lock` option
+
+   This option allows specifying a duration for which restic will wait if the repository is
+   already locked.
+
+   https://github.com/restic/restic/issues/719
+   https://github.com/restic/restic/pull/2214
+   https://github.com/restic/restic/pull/4107
+
+ * Enhancement #1495: Sort snapshots by timestamp in `restic find`
+
+   The `find` command used to print snapshots in an arbitrary order. Restic now prints snapshots
+   sorted by timestamp.
+
+   https://github.com/restic/restic/issues/1495
+   https://github.com/restic/restic/pull/4409
+
+ * Enhancement #1759: Add `repair index` and `repair snapshots` commands
+
+   The `rebuild-index` command has been renamed to `repair index`. The old name will still work,
+   but is deprecated.
+
+   When a snapshot was damaged, the only option up to now was to completely forget the snapshot,
+   even if only some unimportant files in it were damaged and other files were still fine.
+
+   Restic now has a `repair snapshots` command, which can salvage any non-damaged files and parts
+   of files in the snapshots by removing damaged directories and missing file contents. Please
+   note that the damaged data may still be lost and see the "Troubleshooting" section in the
+   documentation for more details.
+
+   https://github.com/restic/restic/issues/1759
+   https://github.com/restic/restic/issues/1714
+   https://github.com/restic/restic/issues/1798
+   https://github.com/restic/restic/issues/2334
+   https://github.com/restic/restic/pull/2876
+   https://forum.restic.net/t/corrupted-repo-how-to-repair/799
+   https://forum.restic.net/t/recovery-options-for-damaged-repositories/1571
+
+ * Enhancement #1926: Allow certificate paths to be passed through environment variables
+
+   Restic will now read paths to certificates from the environment variables `RESTIC_CACERT` or
+   `RESTIC_TLS_CLIENT_CERT` if `--cacert` or `--tls-client-cert` are not specified.
+
+   https://github.com/restic/restic/issues/1926
+   https://github.com/restic/restic/pull/4384
+
+ * Enhancement #2359: Provide multi-platform Docker images
+
+   The official Docker images are now built for the architectures linux/386, linux/amd64,
+   linux/arm and linux/arm64.
+
+   As an alternative to the Docker Hub, the Docker images are also available on ghcr.io, the GitHub
+   Container Registry.
+
+   https://github.com/restic/restic/issues/2359
+   https://github.com/restic/restic/issues/4269
+   https://github.com/restic/restic/pull/4364
+
+ * Enhancement #2468: Add support for non-global Azure clouds
+
+   The `azure` backend previously only supported storages using the global domain
+   `core.windows.net`. This meant that backups to other domains such as Azure China
+   (`core.chinacloudapi.cn`) or Azure Germany (`core.cloudapi.de`) were not supported.
+   Restic now allows overriding the global domain using the environment variable
+   `AZURE_ENDPOINT_SUFFIX`.
+
+   https://github.com/restic/restic/issues/2468
+   https://github.com/restic/restic/pull/4387
+
+ * Enhancement #2679: Reduce file fragmentation for local backend
+
+   Before this change, local backend files could become fragmented. Now restic will try to
+   preallocate space for pack files to avoid their fragmentation.
+
+   https://github.com/restic/restic/issues/2679
+   https://github.com/restic/restic/pull/3261
+
+ * Enhancement #3328: Reduce memory usage by up to 25%
+
+   The in-memory index has been optimized to be more garbage collection friendly. Restic now
+   defaults to `GOGC=50` to run the Go garbage collector more frequently.
+
+   https://github.com/restic/restic/issues/3328
+   https://github.com/restic/restic/pull/4352
+   https://github.com/restic/restic/pull/4353
+
+ * Enhancement #3397: Improve accuracy of ETA displayed during backup
+
+   Restic's `backup` command displayed an ETA that did not adapt when the rate of progress made
+   during the backup changed during the course of the backup.
+
+   Restic now uses recent progress when computing the ETA. It is important to realize that the
+   estimate may still be wrong, because restic cannot predict the future, but the hope is that the
+   ETA will be more accurate in most cases.
+
+   https://github.com/restic/restic/issues/3397
+   https://github.com/restic/restic/pull/3563
+
+ * Enhancement #3624: Keep oldest snapshot when there are not enough snapshots
+
+   The `forget` command now additionally preserves the oldest snapshot if fewer snapshots than
+   allowed by the `--keep-*` parameters would otherwise be kept. This maximizes the amount of
+   history kept within the specified limits.
+
+   https://github.com/restic/restic/issues/3624
+   https://github.com/restic/restic/pull/4366
+   https://forum.restic.net/t/keeping-yearly-snapshots-policy-when-backup-began-during-the-year/4670/2
+
+ * Enhancement #3698: Add support for Managed / Workload Identity to `azure` backend
+
+   Restic now additionally supports authenticating to Azure using Workload Identity or Managed
+   Identity credentials, which are automatically injected in several environments such as a
+   managed Kubernetes cluster.
+
+   https://github.com/restic/restic/issues/3698
+   https://github.com/restic/restic/pull/4029
+
+ * Enhancement #3871: Support `<snapshot>:<subfolder>` syntax to select subfolders
+
+   Commands like `diff` or `restore` always worked with the full snapshot. This did not allow
+   comparing only a specific subfolder or only restoring that folder (`restore --include
+   subfolder` filters the restored files, but still creates the directories included in
+   `subfolder`).
+
+   The commands `diff`, `dump`, `ls` and `restore` now support the `<snapshot>:<subfolder>`
+   syntax, where `snapshot` is the ID of a snapshot (or the string `latest`) and `subfolder` is a
+   path within the snapshot. The commands will then only work with the specified path of the
+   snapshot. The `subfolder` must be a path to a folder as returned by `ls`. Two examples:
+
+   `restic restore -t target latest:/some/path` `restic diff 12345678:/some/path
+   90abcef:/some/path`
+
+   For debugging purposes, the `cat` command now supports `cat tree <snapshot>:<subfolder>` to
+   return the directory metadata for the given subfolder.
+
+   https://github.com/restic/restic/issues/3871
+   https://github.com/restic/restic/pull/4334
+
+ * Enhancement #3941: Support `--group-by` for backup parent selection
+
+   Previously, the `backup` command by default selected the parent snapshot based on the
+   hostname and the backup targets. When the backup path list changed, the `backup` command was
+   unable to determine a suitable parent snapshot and had to read all files again.
+
+   The new `--group-by` option for the `backup` command allows filtering snapshots for the
+   parent selection by `host`, `paths` and `tags`. It defaults to `host,paths` which selects the
+   latest snapshot with hostname and paths matching those of the backup run. This matches the
+   behavior of prior restic versions.
+
+   The new `--group-by` option should be set to the same value as passed to `forget --group-by`.
+
+   https://github.com/restic/restic/issues/3941
+   https://github.com/restic/restic/pull/4081
+
+ * Enhancement #4130: Cancel current command if cache becomes unusable
+
+   If the cache directory was removed or ran out of space while restic was running, this would
+   previously cause further caching attempts to fail and thereby drastically slow down the
+   command execution. Now, the currently running command is instead canceled.
+
+   https://github.com/restic/restic/issues/4130
+   https://github.com/restic/restic/pull/4166
+
+ * Enhancement #4159: Add `--human-readable` option to `ls` and `find` commands
+
+   Previously, when using the `-l` option with the `ls` and `find` commands, the displayed size
+   was always in bytes, without an option for a more human readable format such as MiB or GiB.
+
+   The new `--human-readable` option will convert longer size values into more human friendly
+   values with an appropriate suffix depending on the output size. For example, a size of
+   `14680064` will be shown as `14.000 MiB`.
+
+   https://github.com/restic/restic/issues/4159
+   https://github.com/restic/restic/pull/4351
+
+ * Enhancement #4188: Include restic version in snapshot metadata
+
+   The restic version used to backup a snapshot is now included in its metadata and shown when
+   inspecting a snapshot using `restic cat snapshot <snapshotID>` or `restic snapshots
+   --json`.
+
+   https://github.com/restic/restic/issues/4188
+   https://github.com/restic/restic/pull/4378
+
+ * Enhancement #4220: Add `jq` binary to Docker image
+
+   The Docker image now contains `jq`, which can be useful to process JSON data output by restic.
+
+   https://github.com/restic/restic/pull/4220
+
+ * Enhancement #4226: Allow specifying region of new buckets in the `gs` backend
+
+   Previously, buckets used by the Google Cloud Storage backend would always get created in the
+   "us" region. It is now possible to specify the region where a bucket should be created by using
+   the `-o gs.region=us` option.
+
+   https://github.com/restic/restic/pull/4226
+
+ * Enhancement #4375: Add support for extended attributes on symlinks
+
+   Restic now supports extended attributes on symlinks when backing up, restoring, or
+   FUSE-mounting snapshots. This includes, for example, the `security.selinux` xattr on Linux
+   distributions that use SELinux.
+
+   https://github.com/restic/restic/issues/4375
+   https://github.com/restic/restic/pull/4379
+
+
+Changelog for restic 0.15.2 (2023-04-24)
+=======================================
+
+The following sections list the changes in restic 0.15.2 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Sec #4275: Update golang.org/x/net to address CVE-2022-41723
+ * Fix #2260: Sanitize filenames printed by `backup` during processing
+ * Fix #4211: Make `dump` interpret `--host` and `--path` correctly
+ * Fix #4239: Correct number of blocks reported in mount point
+ * Fix #4253: Minimize risk of spurious filesystem loops with `mount`
+ * Enh #4180: Add release binaries for riscv64 architecture on Linux
+ * Enh #4219: Upgrade Minio to version 7.0.49
+
+Details
+-------
+
+ * Security #4275: Update golang.org/x/net to address CVE-2022-41723
+
+   https://github.com/restic/restic/issues/4275
+   https://github.com/restic/restic/pull/4213
+
+ * Bugfix #2260: Sanitize filenames printed by `backup` during processing
+
+   The `backup` command would previously not sanitize the filenames it printed during
+   processing, potentially causing newlines or terminal control characters to mangle the
+   status output or even change the state of a terminal.
+
+   Filenames are now checked and quoted if they contain non-printable or non-Unicode
+   characters.
+
+   https://github.com/restic/restic/issues/2260
+   https://github.com/restic/restic/issues/4191
+   https://github.com/restic/restic/pull/4192
+
+ * Bugfix #4211: Make `dump` interpret `--host` and `--path` correctly
+
+   A regression in restic 0.15.0 caused `dump` to confuse its `--host=<host>` and
+   `--path=<path>` options: it looked for snapshots with paths called `<host>` from hosts
+   called `<path>`. It now treats the options as intended.
+
+   https://github.com/restic/restic/issues/4211
+   https://github.com/restic/restic/pull/4212
+
+ * Bugfix #4239: Correct number of blocks reported in mount point
+
+   Restic mount points reported an incorrect number of 512-byte (POSIX standard) blocks for
+   files and links due to a rounding bug. In particular, empty files were reported as taking one
+   block instead of zero.
+
+   The rounding is now fixed: the number of blocks reported is the file size (or link target size)
+   divided by 512 and rounded up to a whole number.
+
+   https://github.com/restic/restic/issues/4239
+   https://github.com/restic/restic/pull/4240
+
+ * Bugfix #4253: Minimize risk of spurious filesystem loops with `mount`
+
+   When a backup contains a directory that has the same name as its parent, say `a/b/b`, and the GNU
+   `find` command was run on this backup in a restic mount, `find` would refuse to traverse the
+   lowest `b` directory, instead printing `File system loop detected`. This was due to the way the
+   restic mount command generates inode numbers for directories in the mount point.
+
+   The rule for generating these inode numbers was changed in 0.15.0. It has now been changed again
+   to avoid this issue. A perfect rule does not exist, but the probability of this behavior
+   occurring is now extremely small.
+
+   When it does occur, the mount point is not broken, and scripts that traverse the mount point
+   should work as long as they don't rely on inode numbers for detecting filesystem loops.
+
+   https://github.com/restic/restic/issues/4253
+   https://github.com/restic/restic/pull/4255
+
+ * Enhancement #4180: Add release binaries for riscv64 architecture on Linux
+
+   Builds for the `riscv64` architecture on Linux are now included in the release binaries.
+
+   https://github.com/restic/restic/pull/4180
+
+ * Enhancement #4219: Upgrade Minio to version 7.0.49
+
+   The upgraded version now allows use of the `ap-southeast-4` region (Melbourne).
+
+   https://github.com/restic/restic/pull/4219
+
+
+Changelog for restic 0.15.1 (2023-01-30)
+=======================================
+
+The following sections list the changes in restic 0.15.1 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #3750: Remove `b2_download_file_by_name: 404` warning from B2 backend
+ * Fix #4147: Make `prune --quiet` not print progress bar
+ * Fix #4163: Make `self-update --output` work with new filename on Windows
+ * Fix #4167: Add missing ETA in `backup` progress bar
+ * Enh #4143: Ignore empty lock files
+
+Details
+-------
+
+ * Bugfix #3750: Remove `b2_download_file_by_name: 404` warning from B2 backend
+
+   In some cases the B2 backend could print `b2_download_file_by_name: 404: : b2.b2err`
+   warnings. These are only debug messages and can be safely ignored.
+
+   Restic now uses an updated library for accessing B2, which removes the warning.
+
+   https://github.com/restic/restic/issues/3750
+   https://github.com/restic/restic/issues/4144
+   https://github.com/restic/restic/pull/4146
+
+ * Bugfix #4147: Make `prune --quiet` not print progress bar
+
+   A regression in restic 0.15.0 caused `prune --quiet` to show a progress bar while deciding how
+   to process each pack files. This has now been fixed.
+
+   https://github.com/restic/restic/issues/4147
+   https://github.com/restic/restic/pull/4153
+
+ * Bugfix #4163: Make `self-update --output` work with new filename on Windows
+
+   Since restic 0.14.0 the `self-update` command did not work when a custom output filename was
+   specified via the `--output` option. This has now been fixed.
+
+   As a workaround, either use an older restic version to run the self-update or create an empty
+   file with the output filename before updating e.g. using CMD:
+
+   `type nul > new-file.exe` `restic self-update --output new-file.exe`
+
+   https://github.com/restic/restic/pull/4163
+   https://forum.restic.net/t/self-update-windows-started-failing-after-release-of-0-15/5836
+
+ * Bugfix #4167: Add missing ETA in `backup` progress bar
+
+   A regression in restic 0.15.0 caused the ETA to be missing from the progress bar displayed by the
+   `backup` command. This has now been fixed.
+
+   https://github.com/restic/restic/pull/4167
+
+ * Enhancement #4143: Ignore empty lock files
+
+   With restic 0.15.0 the checks for stale locks became much stricter than before. In particular,
+   empty or unreadable locks were no longer silently ignored. This made restic to complain with
+   `Load(<lock/1234567812>, 0, 0) returned error, retrying after 552.330144ms:
+   load(<lock/1234567812>): invalid data returned` and fail in the end.
+
+   The error message is now clarified and the implementation changed to ignore empty lock files
+   which are sometimes created as the result of a failed uploads on some backends.
+
+   Please note that unreadable lock files still have to cleaned up manually. To do so, you can run
+   `restic unlock --remove-all` which removes all existing lock files. But first make sure that
+   no other restic process is currently using the repository.
+
+   https://github.com/restic/restic/issues/4143
+   https://github.com/restic/restic/pull/4152
+
+
 Changelog for restic 0.15.0 (2023-01-12)
 =======================================
 

CONTRIBUTING.md

@@ -58,6 +58,19 @@ Please be aware that the debug log file will contain potentially sensitive
 things like file and directory names, so please either redact it before
 uploading it somewhere or post only the parts that are really relevant.
 
+If restic gets stuck, please also include a stacktrace in the description.
+On non-Windows systems, you can send a SIGQUIT signal to restic or press
+`Ctrl-\` to achieve the same result. This causes restic to print a stacktrace
+and then exit immediatelly. This will not damage your repository, however,
+it might be necessary to manually clean up stale lock files using
+`restic unlock`.
+
+On Windows, please set the environment variable `RESTIC_DEBUG_STACKTRACE_SIGINT`
+to `true` and press `Ctrl-C` to create a stacktrace.
+
+If you think restic uses too much memory or a too large cache directory, then
+please include the output of `restic stats --mode debug`.
+
 
 Development Environment
 =======================
@@ -78,10 +91,40 @@ Then use the `go` tool to build restic:
     $ ./restic version
     restic 0.14.0-dev (compiled manually) compiled with go1.19 on linux/amd64
 
+To create a debug build use:
+
+    $ go build -tags debug ./cmd/restic
+
 You can run all tests with the following command:
 
     $ go test ./...
 
+
+Performance and Memory Usage Issues
+===================================
+
+Debug builds of restic support the `--block-profile`, `--cpu-profile`,
+`--mem-profile`, and `--trace-profile` options which collect performance data
+that later on can be analyzed using the go tools:
+
+    $ restic --cpu-profile . [...]
+    $ go tool pprof -http localhost:12345 cpu.pprof
+
+To analyze a trace profile use `go tool trace -http=localhost:12345 trace.out`.
+
+As the memory usage of restic changes over time, it may be useful to capture a
+snapshot of the current heap. This is possible using then `--listen-profile`
+option. Then while restic runs you can query and afterwards analyze the heap statistics.
+
+    $ restic --listen-profile localhost:12345 [...]
+    $ curl http://localhost:12345/debug/pprof/heap -o heap.pprof
+    $ go tool pprof -http localhost:12345 heap.pprof
+
+Further useful tools are setting the environment variable `GODEBUG=gctrace=1`,
+which provides information about garbage collector runs. For a graphical variant
+combine this with gcvis.
+
+
 Providing Patches
 =================
 

README.md

@@ -95,7 +95,7 @@ release. Instructions on how to do that are contained in the
 News
 ----
 
-You can follow the restic project on Twitter [@resticbackup](https://twitter.com/resticbackup) or by subscribing to
+You can follow the restic project on Mastodon [@resticbackup](https://fosstodon.org/@restic) or by subscribing to
 the [project blog](https://restic.net/blog/).
 
 License

VERSION

@@ -1 +1 @@
-0.15.0
+0.16.1

build.go

@@ -380,6 +380,12 @@ func main() {
 		}
 	}
 
+	solarisMinVersion := GoVersion{Major: 1, Minor: 20, Patch: 0}
+	if env["GOARCH"] == "solaris" && !goVersion.AtLeast(solarisMinVersion) {
+		fmt.Fprintf(os.Stderr, "Detected version %s is too old, restic requires at least %s for Solaris\n", goVersion, solarisMinVersion)
+		os.Exit(1)
+	}
+
 	verbosePrintf("detected Go version %v\n", goVersion)
 
 	preserveSymbols := false

changelog/0.15.1_2023-01-30/issue-3750

@@ -0,0 +1,10 @@
+Bugfix: Remove `b2_download_file_by_name: 404` warning from B2 backend
+
+In some cases the B2 backend could print `b2_download_file_by_name: 404: :
+b2.b2err` warnings. These are only debug messages and can be safely ignored.
+
+Restic now uses an updated library for accessing B2, which removes the warning.
+
+https://github.com/restic/restic/issues/3750
+https://github.com/restic/restic/issues/4144
+https://github.com/restic/restic/pull/4146

changelog/0.15.1_2023-01-30/issue-4147

@@ -0,0 +1,7 @@
+Bugfix: Make `prune --quiet` not print progress bar
+
+A regression in restic 0.15.0 caused `prune --quiet` to show a progress bar
+while deciding how to process each pack files. This has now been fixed.
+
+https://github.com/restic/restic/issues/4147
+https://github.com/restic/restic/pull/4153

changelog/0.15.1_2023-01-30/pull-4152

@@ -0,0 +1,19 @@
+Enhancement: Ignore empty lock files
+
+With restic 0.15.0 the checks for stale locks became much stricter than before.
+In particular, empty or unreadable locks were no longer silently ignored. This
+made restic to complain with `Load(<lock/1234567812>, 0, 0) returned error,
+retrying after 552.330144ms: load(<lock/1234567812>): invalid data returned`
+and fail in the end.
+
+The error message is now clarified and the implementation changed to ignore
+empty lock files which are sometimes created as the result of a failed uploads
+on some backends.
+
+Please note that unreadable lock files still have to cleaned up manually. To do
+so, you can run `restic unlock --remove-all` which removes all existing lock
+files. But first make sure that no other restic process is currently using the
+repository.
+
+https://github.com/restic/restic/issues/4143
+https://github.com/restic/restic/pull/4152

changelog/0.15.1_2023-01-30/pull-4163

@@ -0,0 +1,13 @@
+Bugfix: Make `self-update --output` work with new filename on Windows
+
+Since restic 0.14.0 the `self-update` command did not work when a custom output
+filename was specified via the `--output` option. This has now been fixed.
+
+As a workaround, either use an older restic version to run the self-update or
+create an empty file with the output filename before updating e.g. using CMD:
+
+`type nul > new-file.exe`
+`restic self-update --output new-file.exe`
+
+https://github.com/restic/restic/pull/4163
+https://forum.restic.net/t/self-update-windows-started-failing-after-release-of-0-15/5836

changelog/0.15.1_2023-01-30/pull-4167

@@ -0,0 +1,6 @@
+Bugfix: Add missing ETA in `backup` progress bar
+
+A regression in restic 0.15.0 caused the ETA to be missing from the progress
+bar displayed by the `backup` command. This has now been fixed.
+
+https://github.com/restic/restic/pull/4167

changelog/0.15.2_2023-04-24/issue-2260

@@ -0,0 +1,12 @@
+Bugfix: Sanitize filenames printed by `backup` during processing
+
+The `backup` command would previously not sanitize the filenames it printed
+during processing, potentially causing newlines or terminal control characters
+to mangle the status output or even change the state of a terminal.
+
+Filenames are now checked and quoted if they contain non-printable or
+non-Unicode characters.
+
+https://github.com/restic/restic/issues/2260
+https://github.com/restic/restic/issues/4191
+https://github.com/restic/restic/pull/4192

changelog/0.15.2_2023-04-24/issue-4211

@@ -0,0 +1,8 @@
+Bugfix: Make `dump` interpret `--host` and `--path` correctly
+
+A regression in restic 0.15.0 caused `dump` to confuse its `--host=<host>` and
+`--path=<path>` options: it looked for snapshots with paths called `<host>`
+from hosts called `<path>`. It now treats the options as intended.
+
+https://github.com/restic/restic/issues/4211
+https://github.com/restic/restic/pull/4212

changelog/0.15.2_2023-04-24/issue-4239

@@ -0,0 +1,11 @@
+Bugfix: Correct number of blocks reported in mount point
+
+Restic mount points reported an incorrect number of 512-byte (POSIX standard)
+blocks for files and links due to a rounding bug. In particular, empty files
+were reported as taking one block instead of zero.
+
+The rounding is now fixed: the number of blocks reported is the file size
+(or link target size) divided by 512 and rounded up to a whole number.
+
+https://github.com/restic/restic/issues/4239
+https://github.com/restic/restic/pull/4240

changelog/0.15.2_2023-04-24/issue-4253

@@ -0,0 +1,18 @@
+Bugfix: Minimize risk of spurious filesystem loops with `mount`
+
+When a backup contains a directory that has the same name as its parent, say
+`a/b/b`, and the GNU `find` command was run on this backup in a restic mount,
+`find` would refuse to traverse the lowest `b` directory, instead printing
+`File system loop detected`. This was due to the way the restic mount command
+generates inode numbers for directories in the mount point.
+
+The rule for generating these inode numbers was changed in 0.15.0. It has
+now been changed again to avoid this issue. A perfect rule does not exist,
+but the probability of this behavior occurring is now extremely small.
+
+When it does occur, the mount point is not broken, and scripts that traverse
+the mount point should work as long as they don't rely on inode numbers for
+detecting filesystem loops.
+
+https://github.com/restic/restic/issues/4253
+https://github.com/restic/restic/pull/4255

changelog/0.15.2_2023-04-24/issue-4275

@@ -0,0 +1,4 @@
+Security: Update golang.org/x/net to address CVE-2022-41723
+
+https://github.com/restic/restic/issues/4275
+https://github.com/restic/restic/pull/4213

changelog/0.15.2_2023-04-24/pull-4180

@@ -0,0 +1,6 @@
+Enhancement: Add release binaries for riscv64 architecture on Linux
+
+Builds for the `riscv64` architecture on Linux are now included in the
+release binaries.
+
+https://github.com/restic/restic/pull/4180

changelog/0.15.2_2023-04-24/pull-4219

@@ -0,0 +1,5 @@
+Enhancement: Upgrade Minio to version 7.0.49
+
+The upgraded version now allows use of the `ap-southeast-4` region (Melbourne).
+
+https://github.com/restic/restic/pull/4219

changelog/0.16.0_2023-07-31/issue-1495

@@ -0,0 +1,7 @@
+Enhancement: Sort snapshots by timestamp in `restic find`
+
+The `find` command used to print snapshots in an arbitrary order. Restic now
+prints snapshots sorted by timestamp.
+
+https://github.com/restic/restic/issues/1495
+https://github.com/restic/restic/pull/4409

changelog/0.16.0_2023-07-31/issue-1759

@@ -0,0 +1,21 @@
+Enhancement: Add `repair index` and `repair snapshots` commands
+
+The `rebuild-index` command has been renamed to `repair index`. The old name
+will still work, but is deprecated.
+
+When a snapshot was damaged, the only option up to now was to completely forget
+the snapshot, even if only some unimportant files in it were damaged and other
+files were still fine.
+
+Restic now has a `repair snapshots` command, which can salvage any non-damaged
+files and parts of files in the snapshots by removing damaged directories and
+missing file contents. Please note that the damaged data may still be lost
+and see the "Troubleshooting" section in the documentation for more details.
+
+https://github.com/restic/restic/issues/1759
+https://github.com/restic/restic/issues/1714
+https://github.com/restic/restic/issues/1798
+https://github.com/restic/restic/issues/2334
+https://github.com/restic/restic/pull/2876
+https://forum.restic.net/t/corrupted-repo-how-to-repair/799
+https://forum.restic.net/t/recovery-options-for-damaged-repositories/1571

changelog/0.16.0_2023-07-31/issue-1926

@@ -0,0 +1,8 @@
+Enhancement: Allow certificate paths to be passed through environment variables
+
+Restic will now read paths to certificates from the environment variables
+`RESTIC_CACERT` or `RESTIC_TLS_CLIENT_CERT` if `--cacert` or `--tls-client-cert`
+are not specified. 
+
+https://github.com/restic/restic/issues/1926  
+https://github.com/restic/restic/pull/4384

changelog/0.16.0_2023-07-31/issue-2359

@@ -0,0 +1,11 @@
+Enhancement: Provide multi-platform Docker images
+
+The official Docker images are now built for the architectures linux/386,
+linux/amd64, linux/arm and linux/arm64.
+
+As an alternative to the Docker Hub, the Docker images are also
+available on ghcr.io, the GitHub Container Registry.
+
+https://github.com/restic/restic/issues/2359
+https://github.com/restic/restic/issues/4269
+https://github.com/restic/restic/pull/4364

changelog/0.16.0_2023-07-31/issue-2468

@@ -0,0 +1,10 @@
+Enhancement: Add support for non-global Azure clouds
+
+The `azure` backend previously only supported storages using the global domain
+`core.windows.net`. This meant that backups to other domains such as Azure
+China (`core.chinacloudapi.cn`) or Azure Germany (`core.cloudapi.de`) were
+not supported. Restic now allows overriding the global domain using the 
+environment variable `AZURE_ENDPOINT_SUFFIX`.
+
+https://github.com/restic/restic/issues/2468
+https://github.com/restic/restic/pull/4387

changelog/0.16.0_2023-07-31/issue-2565

@@ -0,0 +1,10 @@
+Bugfix: Support "unlimited" in `forget --keep-*` options
+
+Restic would previously forget snapshots that should have been kept when a
+negative value was passed to the `--keep-*` options. Negative values are now
+forbidden. To keep all snapshots, the special value `unlimited` is now
+supported. For example, `--keep-monthly unlimited` will keep all monthly
+snapshots.
+
+https://github.com/restic/restic/issues/2565
+https://github.com/restic/restic/pull/4234

changelog/0.16.0_2023-07-31/issue-3311

@@ -0,0 +1,12 @@
+Bugfix: Support non-UTF8 paths as symlink target
+
+Earlier restic versions did not correctly `backup` and `restore` symlinks that
+contain a non-UTF8 target. Note that this only affected systems that still use
+a non-Unicode encoding for filesystem paths.
+
+The repository format is now extended to add support for such symlinks. Please
+note that snapshots must have been created with at least restic version 0.16.0
+for `restore` to correctly handle non-UTF8 symlink targets when restoring them.
+
+https://github.com/restic/restic/issues/3311
+https://github.com/restic/restic/pull/3802

changelog/0.16.0_2023-07-31/issue-3328

@@ -0,0 +1,9 @@
+Enhancement: Reduce memory usage by up to 25%
+
+The in-memory index has been optimized to be more garbage collection friendly.
+Restic now defaults to `GOGC=50` to run the Go garbage collector more
+frequently.
+
+https://github.com/restic/restic/issues/3328
+https://github.com/restic/restic/pull/4352
+https://github.com/restic/restic/pull/4353

changelog/0.16.0_2023-07-31/issue-3397

@@ -0,0 +1,11 @@
+Enhancement: Improve accuracy of ETA displayed during backup
+
+Restic's `backup` command displayed an ETA that did not adapt when the rate of
+progress made during the backup changed during the course of the backup.
+
+Restic now uses recent progress when computing the ETA. It is important to
+realize that the estimate may still be wrong, because restic cannot predict
+the future, but the hope is that the ETA will be more accurate in most cases.
+
+https://github.com/restic/restic/issues/3397
+https://github.com/restic/restic/pull/3563

changelog/0.16.0_2023-07-31/issue-3624

@@ -0,0 +1,9 @@
+Enhancement: Keep oldest snapshot when there are not enough snapshots
+
+The `forget` command now additionally preserves the oldest snapshot if fewer
+snapshots than allowed by the `--keep-*` parameters would otherwise be kept.
+This maximizes the amount of history kept within the specified limits.
+
+https://github.com/restic/restic/issues/3624
+https://github.com/restic/restic/pull/4366
+https://forum.restic.net/t/keeping-yearly-snapshots-policy-when-backup-began-during-the-year/4670/2

changelog/0.16.0_2023-07-31/issue-3698

@@ -0,0 +1,8 @@
+Enhancement: Add support for Managed / Workload Identity to `azure` backend
+
+Restic now additionally supports authenticating to Azure using Workload
+Identity or Managed Identity credentials, which are automatically injected in
+several environments such as a managed Kubernetes cluster.
+
+https://github.com/restic/restic/issues/3698
+https://github.com/restic/restic/pull/4029

changelog/0.16.0_2023-07-31/issue-3871

@@ -0,0 +1,22 @@
+Enhancement: Support `<snapshot>:<subfolder>` syntax to select subfolders
+
+Commands like `diff` or `restore` always worked with the full snapshot. This
+did not allow comparing only a specific subfolder or only restoring that folder
+(`restore --include subfolder` filters the restored files, but still creates the
+directories included in `subfolder`).
+
+The commands `diff`, `dump`, `ls` and `restore` now support the
+`<snapshot>:<subfolder>` syntax, where `snapshot` is the ID of a snapshot (or
+the string `latest`) and `subfolder` is a path within the snapshot. The
+commands will then only work with the specified path of the snapshot. The
+`subfolder` must be a path to a folder as returned by `ls`. Two examples:
+
+`restic restore -t target latest:/some/path`
+`restic diff 12345678:/some/path 90abcef:/some/path`
+
+For debugging purposes, the `cat` command now supports `cat tree
+<snapshot>:<subfolder>` to return the directory metadata for the given
+subfolder.
+
+https://github.com/restic/restic/issues/3871
+https://github.com/restic/restic/pull/4334

changelog/0.16.0_2023-07-31/issue-3941

@@ -0,0 +1,17 @@
+Enhancement: Support `--group-by` for backup parent selection
+
+Previously, the `backup` command by default selected the parent snapshot based
+on the hostname and the backup targets. When the backup path list changed, the
+`backup` command was unable to determine a suitable parent snapshot and had to
+read all files again.
+
+The new `--group-by` option for the `backup` command allows filtering snapshots
+for the parent selection by `host`, `paths` and `tags`. It defaults to
+`host,paths` which selects the latest snapshot with hostname and paths matching
+those of the backup run. This matches the behavior of prior restic versions.
+
+The new `--group-by` option should be set to the same value as passed to
+`forget --group-by`.
+
+https://github.com/restic/restic/issues/3941
+https://github.com/restic/restic/pull/4081

changelog/0.16.0_2023-07-31/issue-4130

@@ -0,0 +1,9 @@
+Enhancement: Cancel current command if cache becomes unusable
+
+If the cache directory was removed or ran out of space while restic was
+running, this would previously cause further caching attempts to fail and
+thereby drastically slow down the command execution. Now, the currently running
+command is instead canceled.
+
+https://github.com/restic/restic/issues/4130
+https://github.com/restic/restic/pull/4166

changelog/0.16.0_2023-07-31/issue-4159

@@ -0,0 +1,12 @@
+Enhancement: Add `--human-readable` option to `ls` and `find` commands
+
+Previously, when using the `-l` option with the `ls` and `find` commands, the
+displayed size was always in bytes, without an option for a more human readable
+format such as MiB or GiB.
+
+The new `--human-readable` option will convert longer size values into more 
+human friendly values with an appropriate suffix depending on the output size.
+For example, a size of `14680064` will be shown as `14.000 MiB`.
+
+https://github.com/restic/restic/issues/4159
+https://github.com/restic/restic/pull/4351

changelog/0.16.0_2023-07-31/issue-4188

@@ -0,0 +1,8 @@
+Enhancement: Include restic version in snapshot metadata
+
+The restic version used to backup a snapshot is now included in its metadata
+and shown when inspecting a snapshot using `restic cat snapshot <snapshotID>`
+or `restic snapshots --json`.
+
+https://github.com/restic/restic/issues/4188
+https://github.com/restic/restic/pull/4378

changelog/0.16.0_2023-07-31/issue-4199

@@ -0,0 +1,9 @@
+Bugfix: Avoid lock refresh issues on slow network connections
+
+On network connections with a low upload speed, backups and other operations
+could fail with the error message `Fatal: failed to refresh lock in time`.
+
+This has now been fixed by reworking the lock refresh handling.
+
+https://github.com/restic/restic/issues/4199
+https://github.com/restic/restic/pull/4304

changelog/0.16.0_2023-07-31/issue-426

@@ -0,0 +1,14 @@
+Enhancement: Show progress bar during restore
+
+The `restore` command now shows a progress report while restoring files.
+
+Example: `[0:42]  5.76%  23 files 12.98 MiB, total 3456 files 23.54 GiB`
+
+JSON output is now also supported.
+
+https://github.com/restic/restic/issues/426
+https://github.com/restic/restic/issues/3413
+https://github.com/restic/restic/issues/3627
+https://github.com/restic/restic/pull/3991
+https://github.com/restic/restic/pull/4314
+https://forum.restic.net/t/progress-bar-for-restore/5210

changelog/0.16.0_2023-07-31/issue-4274

@@ -0,0 +1,11 @@
+Bugfix: Improve lock refresh handling after standby
+
+If the restic process was stopped or the host running restic entered standby
+during a long running operation such as a backup, this previously resulted in
+the operation failing with `Fatal: failed to refresh lock in time`.
+
+This has now been fixed such that restic first checks whether it is safe to
+continue the current operation and only throws an error if not.
+
+https://github.com/restic/restic/issues/4274
+https://github.com/restic/restic/pull/4374

changelog/0.16.0_2023-07-31/issue-4375

@@ -0,0 +1,8 @@
+Enhancement: Add support for extended attributes on symlinks
+
+Restic now supports extended attributes on symlinks when backing up,
+restoring, or FUSE-mounting snapshots. This includes, for example, the
+`security.selinux` xattr on Linux distributions that use SELinux.
+
+https://github.com/restic/restic/issues/4375
+https://github.com/restic/restic/pull/4379

changelog/0.16.0_2023-07-31/issue-719

@@ -0,0 +1,8 @@
+Enhancement: Add `--retry-lock` option
+
+This option allows specifying a duration for which restic will wait if the
+repository is already locked.
+
+https://github.com/restic/restic/issues/719
+https://github.com/restic/restic/pull/2214
+https://github.com/restic/restic/pull/4107

changelog/0.16.0_2023-07-31/pull-3261

@@ -0,0 +1,8 @@
+Enhancement: Reduce file fragmentation for local backend
+
+Before this change, local backend files could become fragmented.
+Now restic will try to preallocate space for pack files to avoid
+their fragmentation.
+
+https://github.com/restic/restic/issues/2679
+https://github.com/restic/restic/pull/3261

changelog/0.16.0_2023-07-31/pull-4176

@@ -0,0 +1,7 @@
+Change: Fix JSON message type of `scan_finished` for the `backup` command
+
+Restic incorrectly set the `message_type` of the `scan_finished` message to
+`status` instead of `verbose_status`. This has now been corrected so that
+the messages report the correct type.
+
+https://github.com/restic/restic/pull/4176

changelog/0.16.0_2023-07-31/pull-4201

@@ -0,0 +1,7 @@
+Change: Require Go 1.20 for Solaris builds
+
+Building restic on Solaris now requires Go 1.20, as the library used to access
+Azure uses the mmap syscall, which is only available on Solaris starting from
+Go 1.20. All other platforms however continue to build with Go 1.18.
+
+https://github.com/restic/restic/pull/4201

changelog/0.16.0_2023-07-31/pull-4220

@@ -0,0 +1,6 @@
+Enhancement: Add `jq` binary to Docker image
+
+The Docker image now contains `jq`, which can be useful to process JSON data
+output by restic.
+
+https://github.com/restic/restic/pull/4220

changelog/0.16.0_2023-07-31/pull-4226

@@ -0,0 +1,7 @@
+Enhancement: Allow specifying region of new buckets in the `gs` backend
+
+Previously, buckets used by the Google Cloud Storage backend would always get
+created in the "us" region. It is now possible to specify the region where a
+bucket should be created by using the `-o gs.region=us` option.
+
+https://github.com/restic/restic/pull/4226

changelog/0.16.0_2023-07-31/pull-4318

@@ -0,0 +1,8 @@
+Bugfix: Correctly clean up status bar output of the `backup` command
+
+Due to a regression in restic 0.15.2, the status bar of the `backup` command
+could leave some output behind. This happened if filenames were printed that
+are wider than the current terminal width. This has now been fixed.
+
+https://github.com/restic/restic/issues/4319
+https://github.com/restic/restic/pull/4318

changelog/0.16.0_2023-07-31/pull-4333

@@ -0,0 +1,3 @@
+Bugfix: `generate` and `init` no longer silently ignore unexpected arguments
+
+https://github.com/restic/restic/pull/4333

changelog/0.16.0_2023-07-31/pull-4400

@@ -0,0 +1,8 @@
+Bugfix: Ignore missing folders in `rest` backend
+
+If a repository accessed via the REST backend was missing folders, then restic
+would fail with an error while trying to list the data in the repository. This
+has been now fixed.
+
+https://github.com/restic/restic/pull/4400
+https://github.com/restic/rest-server/issues/235

changelog/0.16.1_2023-10-24/issue-4128

@@ -0,0 +1,9 @@
+Enhancement: Automatically set `GOMAXPROCS` in resource-constrained containers
+
+When running restic in a Linux container with CPU-usage limits, restic now
+automatically adjusts `GOMAXPROCS`. This helps to reduce the memory consumption
+on hosts with many CPU cores.
+
+https://github.com/restic/restic/issues/4128
+https://github.com/restic/restic/pull/4485
+https://github.com/restic/restic/pull/4531

changelog/0.16.1_2023-10-24/issue-4513

@@ -0,0 +1,8 @@
+Bugfix: Make `key list` command honor `--no-lock`
+
+The `key list` command now supports the `--no-lock` options. This allows
+determining which keys a repo can be accessed by without the need for having
+write access (e.g., read-only sftp access, filesystem snapshot).
+
+https://github.com/restic/restic/issues/4513
+https://github.com/restic/restic/pull/4514

changelog/0.16.1_2023-10-24/issue-4516

@@ -0,0 +1,8 @@
+Bugfix: Do not try to load password on command line autocomplete
+
+The command line autocompletion previously tried to load the repository
+password. This could cause the autocompletion not to work. Now, this step gets
+skipped.
+
+https://github.com/restic/restic/issues/4516
+https://github.com/restic/restic/pull/4526

changelog/0.16.1_2023-10-24/issue-4523

@@ -0,0 +1,22 @@
+Bugfix: Update zstd library to fix possible data corruption at max. compression
+
+In restic 0.16.0, backups where the compression level was set to `max` (using
+`--compression max`) could in rare and very specific circumstances result in
+data corruption due to a bug in the library used for compressing data.
+
+Restic now uses the latest version of the library used to compress data, which
+includes a fix for this issue. Please note that the `auto` compression level
+(which restic uses by default) was never affected, and even if you used `max`
+compression, chances of being affected by this issue were very small.
+
+To check a repository for any corruption, run `restic check --read-data`. This
+will download and verify the whole repository and can be used at any time to
+completely verify the integrity of a repository. If the `check` command detects
+anomalies, follow the suggested steps.
+
+To simplify any needed repository repair and minimize data loss, there is also
+a new and experimental `repair packs` command that salvages all valid data from
+the affected pack files (see `restic help repair packs` for more information).
+
+https://github.com/restic/restic/issues/4523
+https://github.com/restic/restic/pull/4530

changelog/0.16.1_2023-10-24/pull-299

@@ -0,0 +1,7 @@
+Enhancement: Show progress bar while loading the index
+
+Restic did not provide any feedback while loading index files. Now, there is a
+progress bar that shows the index loading progress.
+
+https://github.com/restic/restic/issues/229
+https://github.com/restic/restic/pull/4419

changelog/0.16.1_2023-10-24/pull-4480

@@ -0,0 +1,11 @@
+Enhancement: Allow setting REST password and username via environment variables
+
+Previously, it was only possible to specify the REST-server username and
+password in the repository URL, or by using the `--repository-file` option.
+This meant it was not possible to use authentication in contexts where the
+repository URL is stored in publicly accessible way.
+
+Restic now allows setting the username and password using the
+`RESTIC_REST_USERNAME` and `RESTIC_REST_PASSWORD` variables.
+
+https://github.com/restic/restic/pull/4480

changelog/0.16.1_2023-10-24/pull-4511

@@ -0,0 +1,7 @@
+Enhancement: Include inode numbers in JSON output for `find` and `ls` commands
+
+Restic used to omit the inode numbers in the JSON messages emitted for nodes by
+the `ls` command as well as for matches by the `find` command. It now includes
+those values whenever they are available.
+
+https://github.com/restic/restic/pull/4511

changelog/0.16.1_2023-10-24/pull-4519

@@ -0,0 +1,12 @@
+Enhancement: Add config option to set SFTP command arguments
+
+When using the `sftp` backend, scenarios where a custom identity file was
+needed for the SSH connection, required the full command to be specified:
+`-o sftp.command='ssh user@host:port -i /ssh/my_private_key -s sftp'`
+
+Now, the `-o sftp.args=...` option can be passed to restic to specify
+custom arguments for the SSH command executed by the SFTP backend.
+This simplifies the above example to `-o sftp.args='-i /ssh/my_private_key'`.
+
+https://github.com/restic/restic/pull/4519
+https://github.com/restic/restic/issues/4241

changelog/0.16.1_2023-10-24/pull-4532

@@ -0,0 +1,8 @@
+Change: Update dependencies and require Go 1.19 or newer
+
+We have updated all dependencies. Since some libraries require newer Go
+standard library features, support for Go 1.18 has been dropped, which means
+that restic now requires at least Go 1.19 to build.
+
+https://github.com/restic/restic/pull/4532
+https://github.com/restic/restic/pull/4533

changelog/TEMPLATE

@@ -1,16 +1,17 @@
 # The first line must start with Bugfix:, Enhancement: or Change:,
-# including the colon. Use present tense. Remove lines starting with '#'
-# from this template.
+# including the colon. Use present tense and the imperative mood. Remove
+# lines starting with '#' from this template.
 Enhancement: Allow custom bar in the foo command
 
 # Describe the problem in the past tense, the new behavior in the present
 # tense. Mention the affected commands, backends, operating systems, etc.
 # Focus on user-facing behavior, not the implementation.
+# Use "Restic now ..." instead of "We have changed ...".
 
 Restic foo always used the system-wide bar when deciding how to frob an
-item in the baz backend. It now permits selecting the bar with --bar or
-the environment variable RESTIC_BAR. The system-wide bar is still the
-default.
+item in the `baz` backend. It now permits selecting the bar with `--bar`
+or the environment variable `RESTIC_BAR`. The system-wide bar is still
+the default.
 
 # The last section is a list of issue, PR and forum URLs.
 # The first issue ID determines the filename for the changelog entry:

cmd/restic/cleanup.go

@@ -62,6 +62,12 @@ func CleanupHandler(c <-chan os.Signal) {
 		debug.Log("signal %v received, cleaning up", s)
 		Warnf("%ssignal %v received, cleaning up\n", clearLine(0), s)
 
+		if val, _ := os.LookupEnv("RESTIC_DEBUG_STACKTRACE_SIGINT"); val != "" {
+			_, _ = os.Stderr.WriteString("\n--- STACKTRACE START ---\n\n")
+			_, _ = os.Stderr.WriteString(debug.DumpStacktrace())
+			_, _ = os.Stderr.WriteString("\n--- STACKTRACE END ---\n")
+		}
+
 		code := 0
 
 		if s == syscall.SIGINT {
@@ -78,5 +84,6 @@ func CleanupHandler(c <-chan os.Signal) {
 // given exit code.
 func Exit(code int) {
 	code = RunCleanupHandlers(code)
+	debug.Log("exiting with status code %d", code)
 	os.Exit(code)
 }

cmd/restic/cmd_backup.go

@@ -25,6 +25,7 @@ import (
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/textfile"
+	"github.com/restic/restic/internal/ui"
 	"github.com/restic/restic/internal/ui/backup"
 	"github.com/restic/restic/internal/ui/termstatus"
 )
@@ -71,6 +72,14 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
 			term.Run(cancelCtx)
 		}()
 
+		// use the terminal for stdout/stderr
+		prevStdout, prevStderr := globalOptions.stdout, globalOptions.stderr
+		defer func() {
+			globalOptions.stdout, globalOptions.stderr = prevStdout, prevStderr
+		}()
+		stdioWrapper := ui.NewStdioWrapper(term)
+		globalOptions.stdout, globalOptions.stderr = stdioWrapper.Stdout(), stdioWrapper.Stderr()
+
 		return runBackup(ctx, backupOptions, globalOptions, term, args)
 	},
 }
@@ -80,6 +89,7 @@ type BackupOptions struct {
 	excludePatternOptions
 
 	Parent            string
+	GroupBy           restic.SnapshotGroupByOptions
 	Force             bool
 	ExcludeOtherFS    bool
 	ExcludeIfPresent  []string
@@ -111,7 +121,9 @@ func init() {
 	cmdRoot.AddCommand(cmdBackup)
 
 	f := cmdBackup.Flags()
-	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: last snapshot in the repository that has the same target files/directories, and is not newer than the snapshot time)")
+	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: latest snapshot in the group determined by --group-by and not newer than the timestamp determined by --time)")
+	backupOptions.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true}
+	f.VarP(&backupOptions.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
 	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
 
 	initExcludePatternOptions(f, &backupOptions.excludePatternOptions)
@@ -296,7 +308,7 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
 
 // collectRejectByNameFuncs returns a list of all functions which may reject data
 // from being saved in a snapshot based on path only
-func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, targets []string) (fs []RejectByNameFunc, err error) {
+func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (fs []RejectByNameFunc, err error) {
 	// exclude restic cache
 	if repo.Cache != nil {
 		f, err := rejectResticCache(repo)
@@ -331,7 +343,7 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, t
 
 // collectRejectFuncs returns a list of all functions which may reject data
 // from being saved in a snapshot based on path and file info
-func collectRejectFuncs(opts BackupOptions, repo *repository.Repository, targets []string) (fs []RejectFunc, err error) {
+func collectRejectFuncs(opts BackupOptions, targets []string) (fs []RejectFunc, err error) {
 	// allowed devices
 	if opts.ExcludeOtherFS && !opts.Stdin {
 		f, err := rejectByDevice(targets)
@@ -430,7 +442,18 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
 	if snName == "" {
 		snName = "latest"
 	}
-	sn, err := restic.FindFilteredSnapshot(ctx, repo.Backend(), repo, []string{opts.Host}, []restic.TagList{}, targets, &timeStampLimit, snName)
+	f := restic.SnapshotFilter{TimestampLimit: timeStampLimit}
+	if opts.GroupBy.Host {
+		f.Hosts = []string{opts.Host}
+	}
+	if opts.GroupBy.Path {
+		f.Paths = targets
+	}
+	if opts.GroupBy.Tag {
+		f.Tags = []restic.TagList{opts.Tags.Flatten()}
+	}
+
+	sn, _, err := f.FindLatest(ctx, repo.Backend(), repo, snName)
 	// Snapshot not found is ok if no explicit parent was set
 	if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) {
 		err = nil
@@ -474,40 +497,32 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 	}
 	progressReporter := backup.NewProgress(progressPrinter,
 		calculateProgressInterval(!gopts.Quiet, gopts.JSON))
+	defer progressReporter.Done()
 
 	if opts.DryRun {
 		repo.SetDryRun()
 	}
 
-	// use the terminal for stdout/stderr
-	prevStdout, prevStderr := gopts.stdout, gopts.stderr
-	defer func() {
-		gopts.stdout, gopts.stderr = prevStdout, prevStderr
-	}()
-	gopts.stdout, gopts.stderr = progressPrinter.Stdout(), progressPrinter.Stderr()
-
-	wg, wgCtx := errgroup.WithContext(ctx)
-	cancelCtx, cancel := context.WithCancel(wgCtx)
-	defer cancel()
-	wg.Go(func() error { progressReporter.Run(cancelCtx); return nil })
-
 	if !gopts.JSON {
 		progressPrinter.V("lock repository")
 	}
-	lock, ctx, err := lockRepo(ctx, repo)
-	defer unlockRepo(lock)
-	if err != nil {
-		return err
+	if !opts.DryRun {
+		var lock *restic.Lock
+		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
 	}
 
 	// rejectByNameFuncs collect functions that can reject items from the backup based on path only
-	rejectByNameFuncs, err := collectRejectByNameFuncs(opts, repo, targets)
+	rejectByNameFuncs, err := collectRejectByNameFuncs(opts, repo)
 	if err != nil {
 		return err
 	}
 
 	// rejectFuncs collect functions that can reject items from the backup based on path and file info
-	rejectFuncs, err := collectRejectFuncs(opts, repo, targets)
+	rejectFuncs, err := collectRejectFuncs(opts, targets)
 	if err != nil {
 		return err
 	}
@@ -531,7 +546,10 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 	if !gopts.JSON {
 		progressPrinter.V("load index files")
 	}
-	err = repo.LoadIndex(ctx)
+
+	bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
+
+	err = repo.LoadIndex(ctx, bar)
 	if err != nil {
 		return err
 	}
@@ -588,6 +606,10 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 		targets = []string{filename}
 	}
 
+	wg, wgCtx := errgroup.WithContext(ctx)
+	cancelCtx, cancel := context.WithCancel(wgCtx)
+	defer cancel()
+
 	if !opts.NoScan {
 		sc := archiver.NewScanner(targetFS)
 		sc.SelectByName = selectByNameFilter
@@ -629,6 +651,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 		Time:           timeStamp,
 		Hostname:       opts.Host,
 		ParentSnapshot: parentSnapshot,
+		ProgramVersion: "restic " + version,
 	}
 
 	if !gopts.JSON {

cmd/restic/cmd_backup_integration_test.go

@@ -0,0 +1,569 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/restic/restic/internal/fs"
+	"github.com/restic/restic/internal/restic"
+	rtest "github.com/restic/restic/internal/test"
+	"github.com/restic/restic/internal/ui/termstatus"
+)
+
+func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts BackupOptions, gopts GlobalOptions) error {
+	return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
+		t.Logf("backing up %v in %v", target, dir)
+		if dir != "" {
+			cleanup := rtest.Chdir(t, dir)
+			defer cleanup()
+		}
+
+		opts.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true}
+		return runBackup(ctx, opts, gopts, term, target)
+	})
+}
+
+func testRunBackup(t testing.TB, dir string, target []string, opts BackupOptions, gopts GlobalOptions) {
+	err := testRunBackupAssumeFailure(t, dir, target, opts, gopts)
+	rtest.Assert(t, err == nil, "Error while backing up")
+}
+
+func TestBackup(t *testing.T) {
+	testBackup(t, false)
+}
+
+func TestBackupWithFilesystemSnapshots(t *testing.T) {
+	if runtime.GOOS == "windows" && fs.HasSufficientPrivilegesForVSS() == nil {
+		testBackup(t, true)
+	}
+}
+
+func testBackup(t *testing.T, useFsSnapshot bool) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{UseFsSnapshot: useFsSnapshot}
+
+	// first backup
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	testListSnapshots(t, env.gopts, 1)
+
+	testRunCheck(t, env.gopts)
+	stat1 := dirStats(env.repo)
+
+	// second backup, implicit incremental
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	snapshotIDs := testListSnapshots(t, env.gopts, 2)
+
+	stat2 := dirStats(env.repo)
+	if stat2.size > stat1.size+stat1.size/10 {
+		t.Error("repository size has grown by more than 10 percent")
+	}
+	t.Logf("repository grown by %d bytes", stat2.size-stat1.size)
+
+	testRunCheck(t, env.gopts)
+	// third backup, explicit incremental
+	opts.Parent = snapshotIDs[0].String()
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	snapshotIDs = testListSnapshots(t, env.gopts, 3)
+
+	stat3 := dirStats(env.repo)
+	if stat3.size > stat1.size+stat1.size/10 {
+		t.Error("repository size has grown by more than 10 percent")
+	}
+	t.Logf("repository grown by %d bytes", stat3.size-stat2.size)
+
+	// restore all backups and compare
+	for i, snapshotID := range snapshotIDs {
+		restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
+		t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir)
+		testRunRestore(t, env.gopts, restoredir, snapshotID)
+		diff := directoriesContentsDiff(env.testdata, filepath.Join(restoredir, "testdata"))
+		rtest.Assert(t, diff == "", "directories are not equal: %v", diff)
+	}
+
+	testRunCheck(t, env.gopts)
+}
+
+func TestBackupWithRelativePath(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{}
+
+	// first backup
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	firstSnapshotID := testListSnapshots(t, env.gopts, 1)[0]
+
+	// second backup, implicit incremental
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+
+	// that the correct parent snapshot was used
+	latestSn, _ := testRunSnapshots(t, env.gopts)
+	rtest.Assert(t, latestSn != nil, "missing latest snapshot")
+	rtest.Assert(t, latestSn.Parent != nil && latestSn.Parent.Equal(firstSnapshotID), "second snapshot selected unexpected parent %v instead of %v", latestSn.Parent, firstSnapshotID)
+}
+
+func TestBackupParentSelection(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{}
+
+	// first backup
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata/0/0"}, opts, env.gopts)
+	firstSnapshotID := testListSnapshots(t, env.gopts, 1)[0]
+
+	// second backup, sibling path
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata/0/tests"}, opts, env.gopts)
+	testListSnapshots(t, env.gopts, 2)
+
+	// third backup, incremental for the first backup
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata/0/0"}, opts, env.gopts)
+
+	// test that the correct parent snapshot was used
+	latestSn, _ := testRunSnapshots(t, env.gopts)
+	rtest.Assert(t, latestSn != nil, "missing latest snapshot")
+	rtest.Assert(t, latestSn.Parent != nil && latestSn.Parent.Equal(firstSnapshotID), "third snapshot selected unexpected parent %v instead of %v", latestSn.Parent, firstSnapshotID)
+}
+
+func TestDryRunBackup(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{}
+	dryOpts := BackupOptions{DryRun: true}
+
+	// dry run before first backup
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
+	snapshotIDs := testListSnapshots(t, env.gopts, 0)
+	packIDs := testRunList(t, "packs", env.gopts)
+	rtest.Assert(t, len(packIDs) == 0,
+		"expected no data, got %v", snapshotIDs)
+	indexIDs := testRunList(t, "index", env.gopts)
+	rtest.Assert(t, len(indexIDs) == 0,
+		"expected no index, got %v", snapshotIDs)
+
+	// first backup
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	snapshotIDs = testListSnapshots(t, env.gopts, 1)
+	packIDs = testRunList(t, "packs", env.gopts)
+	indexIDs = testRunList(t, "index", env.gopts)
+
+	// dry run between backups
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
+	snapshotIDsAfter := testListSnapshots(t, env.gopts, 1)
+	rtest.Equals(t, snapshotIDs, snapshotIDsAfter)
+	dataIDsAfter := testRunList(t, "packs", env.gopts)
+	rtest.Equals(t, packIDs, dataIDsAfter)
+	indexIDsAfter := testRunList(t, "index", env.gopts)
+	rtest.Equals(t, indexIDs, indexIDsAfter)
+
+	// second backup, implicit incremental
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	snapshotIDs = testListSnapshots(t, env.gopts, 2)
+	packIDs = testRunList(t, "packs", env.gopts)
+	indexIDs = testRunList(t, "index", env.gopts)
+
+	// another dry run
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
+	snapshotIDsAfter = testListSnapshots(t, env.gopts, 2)
+	rtest.Equals(t, snapshotIDs, snapshotIDsAfter)
+	dataIDsAfter = testRunList(t, "packs", env.gopts)
+	rtest.Equals(t, packIDs, dataIDsAfter)
+	indexIDsAfter = testRunList(t, "index", env.gopts)
+	rtest.Equals(t, indexIDs, indexIDsAfter)
+}
+
+func TestBackupNonExistingFile(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+
+	_ = withRestoreGlobalOptions(func() error {
+		globalOptions.stderr = io.Discard
+
+		p := filepath.Join(env.testdata, "0", "0", "9")
+		dirs := []string{
+			filepath.Join(p, "0"),
+			filepath.Join(p, "1"),
+			filepath.Join(p, "nonexisting"),
+			filepath.Join(p, "5"),
+		}
+
+		opts := BackupOptions{}
+
+		testRunBackup(t, "", dirs, opts, env.gopts)
+		return nil
+	})
+}
+
+func TestBackupSelfHealing(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testRunInit(t, env.gopts)
+
+	p := filepath.Join(env.testdata, "test/test")
+	rtest.OK(t, os.MkdirAll(filepath.Dir(p), 0755))
+	rtest.OK(t, appendRandomData(p, 5))
+
+	opts := BackupOptions{}
+
+	testRunBackup(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
+	testRunCheck(t, env.gopts)
+
+	// remove all data packs
+	removePacksExcept(env.gopts, t, restic.NewIDSet(), false)
+
+	testRunRebuildIndex(t, env.gopts)
+	// now the repo is also missing the data blob in the index; check should report this
+	testRunCheckMustFail(t, env.gopts)
+
+	// second backup should report an error but "heal" this situation
+	err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
+	rtest.Assert(t, err != nil,
+		"backup should have reported an error")
+	testRunCheck(t, env.gopts)
+}
+
+func TestBackupTreeLoadError(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testRunInit(t, env.gopts)
+	p := filepath.Join(env.testdata, "test/test")
+	rtest.OK(t, os.MkdirAll(filepath.Dir(p), 0755))
+	rtest.OK(t, appendRandomData(p, 5))
+
+	opts := BackupOptions{}
+	// Backup a subdirectory first, such that we can remove the tree pack for the subdirectory
+	testRunBackup(t, env.testdata, []string{"test"}, opts, env.gopts)
+
+	r, err := OpenRepository(context.TODO(), env.gopts)
+	rtest.OK(t, err)
+	rtest.OK(t, r.LoadIndex(context.TODO(), nil))
+	treePacks := restic.NewIDSet()
+	r.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
+		if pb.Type == restic.TreeBlob {
+			treePacks.Insert(pb.PackID)
+		}
+	})
+
+	testRunBackup(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
+	testRunCheck(t, env.gopts)
+
+	// delete the subdirectory pack first
+	for id := range treePacks {
+		rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()}))
+	}
+	testRunRebuildIndex(t, env.gopts)
+	// now the repo is missing the tree blob in the index; check should report this
+	testRunCheckMustFail(t, env.gopts)
+	// second backup should report an error but "heal" this situation
+	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
+	rtest.Assert(t, err != nil, "backup should have reported an error for the subdirectory")
+	testRunCheck(t, env.gopts)
+
+	// remove all tree packs
+	removePacksExcept(env.gopts, t, restic.NewIDSet(), true)
+	testRunRebuildIndex(t, env.gopts)
+	// now the repo is also missing the data blob in the index; check should report this
+	testRunCheckMustFail(t, env.gopts)
+	// second backup should report an error but "heal" this situation
+	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
+	rtest.Assert(t, err != nil, "backup should have reported an error")
+	testRunCheck(t, env.gopts)
+}
+
+var backupExcludeFilenames = []string{
+	"testfile1",
+	"foo.tar.gz",
+	"private/secret/passwords.txt",
+	"work/source/test.c",
+}
+
+func TestBackupExclude(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testRunInit(t, env.gopts)
+
+	datadir := filepath.Join(env.base, "testdata")
+
+	for _, filename := range backupExcludeFilenames {
+		fp := filepath.Join(datadir, filename)
+		rtest.OK(t, os.MkdirAll(filepath.Dir(fp), 0755))
+
+		f, err := os.Create(fp)
+		rtest.OK(t, err)
+
+		fmt.Fprint(f, filename)
+		rtest.OK(t, f.Close())
+	}
+
+	snapshots := make(map[string]struct{})
+
+	opts := BackupOptions{}
+
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	snapshots, snapshotID := lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts))
+	files := testRunLs(t, env.gopts, snapshotID)
+	rtest.Assert(t, includes(files, "/testdata/foo.tar.gz"),
+		"expected file %q in first snapshot, but it's not included", "foo.tar.gz")
+
+	opts.Excludes = []string{"*.tar.gz"}
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	snapshots, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts))
+	files = testRunLs(t, env.gopts, snapshotID)
+	rtest.Assert(t, !includes(files, "/testdata/foo.tar.gz"),
+		"expected file %q not in first snapshot, but it's included", "foo.tar.gz")
+
+	opts.Excludes = []string{"*.tar.gz", "private/secret"}
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	_, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts))
+	files = testRunLs(t, env.gopts, snapshotID)
+	rtest.Assert(t, !includes(files, "/testdata/foo.tar.gz"),
+		"expected file %q not in first snapshot, but it's included", "foo.tar.gz")
+	rtest.Assert(t, !includes(files, "/testdata/private/secret/passwords.txt"),
+		"expected file %q not in first snapshot, but it's included", "passwords.txt")
+}
+
+func TestBackupErrors(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		return
+	}
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+
+	// Assume failure
+	inaccessibleFile := filepath.Join(env.testdata, "0", "0", "9", "0")
+	rtest.OK(t, os.Chmod(inaccessibleFile, 0000))
+	defer func() {
+		rtest.OK(t, os.Chmod(inaccessibleFile, 0644))
+	}()
+	opts := BackupOptions{}
+	err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	rtest.Assert(t, err != nil, "Assumed failure, but no error occurred.")
+	rtest.Assert(t, err == ErrInvalidSourceData, "Wrong error returned")
+	testListSnapshots(t, env.gopts, 1)
+}
+
+const (
+	incrementalFirstWrite  = 10 * 1042 * 1024
+	incrementalSecondWrite = 1 * 1042 * 1024
+	incrementalThirdWrite  = 1 * 1042 * 1024
+)
+
+func TestIncrementalBackup(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testRunInit(t, env.gopts)
+
+	datadir := filepath.Join(env.base, "testdata")
+	testfile := filepath.Join(datadir, "testfile")
+
+	rtest.OK(t, appendRandomData(testfile, incrementalFirstWrite))
+
+	opts := BackupOptions{}
+
+	testRunBackup(t, "", []string{datadir}, opts, env.gopts)
+	testRunCheck(t, env.gopts)
+	stat1 := dirStats(env.repo)
+
+	rtest.OK(t, appendRandomData(testfile, incrementalSecondWrite))
+
+	testRunBackup(t, "", []string{datadir}, opts, env.gopts)
+	testRunCheck(t, env.gopts)
+	stat2 := dirStats(env.repo)
+	if stat2.size-stat1.size > incrementalFirstWrite {
+		t.Errorf("repository size has grown by more than %d bytes", incrementalFirstWrite)
+	}
+	t.Logf("repository grown by %d bytes", stat2.size-stat1.size)
+
+	rtest.OK(t, appendRandomData(testfile, incrementalThirdWrite))
+
+	testRunBackup(t, "", []string{datadir}, opts, env.gopts)
+	testRunCheck(t, env.gopts)
+	stat3 := dirStats(env.repo)
+	if stat3.size-stat2.size > incrementalFirstWrite {
+		t.Errorf("repository size has grown by more than %d bytes", incrementalFirstWrite)
+	}
+	t.Logf("repository grown by %d bytes", stat3.size-stat2.size)
+}
+
+func TestBackupTags(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{}
+
+	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
+	testRunCheck(t, env.gopts)
+	newest, _ := testRunSnapshots(t, env.gopts)
+
+	if newest == nil {
+		t.Fatal("expected a backup, got nil")
+	}
+
+	rtest.Assert(t, len(newest.Tags) == 0,
+		"expected no tags, got %v", newest.Tags)
+	parent := newest
+
+	opts.Tags = restic.TagLists{[]string{"NL"}}
+	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
+	testRunCheck(t, env.gopts)
+	newest, _ = testRunSnapshots(t, env.gopts)
+
+	if newest == nil {
+		t.Fatal("expected a backup, got nil")
+	}
+
+	rtest.Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "NL",
+		"expected one NL tag, got %v", newest.Tags)
+	// Tagged backup should have untagged backup as parent.
+	rtest.Assert(t, parent.ID.Equal(*newest.Parent),
+		"expected parent to be %v, got %v", parent.ID, newest.Parent)
+}
+
+func TestBackupProgramVersion(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts)
+	newest, _ := testRunSnapshots(t, env.gopts)
+
+	if newest == nil {
+		t.Fatal("expected a backup, got nil")
+	}
+	resticVersion := "restic " + version
+	rtest.Assert(t, newest.ProgramVersion == resticVersion,
+		"expected %v, got %v", resticVersion, newest.ProgramVersion)
+}
+
+func TestQuietBackup(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{}
+
+	env.gopts.Quiet = false
+	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
+	testListSnapshots(t, env.gopts, 1)
+
+	testRunCheck(t, env.gopts)
+
+	env.gopts.Quiet = true
+	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
+	testListSnapshots(t, env.gopts, 2)
+
+	testRunCheck(t, env.gopts)
+}
+
+func TestHardLink(t *testing.T) {
+	// this test assumes a test set with a single directory containing hard linked files
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	datafile := filepath.Join("testdata", "test.hl.tar.gz")
+	fd, err := os.Open(datafile)
+	if os.IsNotExist(err) {
+		t.Skipf("unable to find data file %q, skipping", datafile)
+		return
+	}
+	rtest.OK(t, err)
+	rtest.OK(t, fd.Close())
+
+	testRunInit(t, env.gopts)
+
+	rtest.SetupTarTestFixture(t, env.testdata, datafile)
+
+	linkTests := createFileSetPerHardlink(env.testdata)
+
+	opts := BackupOptions{}
+
+	// first backup
+	testRunBackup(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
+	snapshotIDs := testListSnapshots(t, env.gopts, 1)
+
+	testRunCheck(t, env.gopts)
+
+	// restore all backups and compare
+	for i, snapshotID := range snapshotIDs {
+		restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
+		t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir)
+		testRunRestore(t, env.gopts, restoredir, snapshotID)
+		diff := directoriesContentsDiff(env.testdata, filepath.Join(restoredir, "testdata"))
+		rtest.Assert(t, diff == "", "directories are not equal %v", diff)
+
+		linkResults := createFileSetPerHardlink(filepath.Join(restoredir, "testdata"))
+		rtest.Assert(t, linksEqual(linkTests, linkResults),
+			"links are not equal")
+	}
+
+	testRunCheck(t, env.gopts)
+}
+
+func linksEqual(source, dest map[uint64][]string) bool {
+	for _, vs := range source {
+		found := false
+		for kd, vd := range dest {
+			if linkEqual(vs, vd) {
+				delete(dest, kd)
+				found = true
+				break
+			}
+		}
+		if !found {
+			return false
+		}
+	}
+
+	return len(dest) == 0
+}
+
+func linkEqual(source, dest []string) bool {
+	// equal if sliced are equal without considering order
+	if source == nil && dest == nil {
+		return true
+	}
+
+	if source == nil || dest == nil {
+		return false
+	}
+
+	if len(source) != len(dest) {
+		return false
+	}
+
+	for i := range source {
+		found := false
+		for j := range dest {
+			if source[i] == dest[j] {
+				found = true
+				break
+			}
+		}
+		if !found {
+			return false
+		}
+	}
+
+	return true
+}

cmd/restic/cmd_cache.go

@@ -155,7 +155,7 @@ func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
 		})
 	}
 
-	_ = tab.Write(gopts.stdout)
+	_ = tab.Write(globalOptions.stdout)
 	Printf("%d cache dirs in %s\n", len(dirs), cachedir)
 
 	return nil

cmd/restic/cmd_cat.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"strings"
 
 	"github.com/spf13/cobra"
 
@@ -13,7 +14,7 @@ import (
 )
 
 var cmdCat = &cobra.Command{
-	Use:   "cat [flags] [pack|blob|snapshot|index|key|masterkey|config|lock] ID",
+	Use:   "cat [flags] [masterkey|config|pack ID|blob ID|snapshot ID|index ID|key ID|lock ID|tree snapshot:subfolder]",
 	Short: "Print internal objects to stdout",
 	Long: `
 The "cat" command is used to print internal objects to stdout.
@@ -33,9 +34,34 @@ func init() {
 	cmdRoot.AddCommand(cmdCat)
 }
 
+func validateCatArgs(args []string) error {
+	var allowedCmds = []string{"config", "index", "snapshot", "key", "masterkey", "lock", "pack", "blob", "tree"}
+
+	if len(args) < 1 {
+		return errors.Fatal("type not specified")
+	}
+
+	validType := false
+	for _, v := range allowedCmds {
+		if v == args[0] {
+			validType = true
+			break
+		}
+	}
+	if !validType {
+		return errors.Fatalf("invalid type %q, must be one of [%s]", args[0], strings.Join(allowedCmds, "|"))
+	}
+
+	if args[0] != "masterkey" && args[0] != "config" && len(args) != 2 {
+		return errors.Fatal("ID not specified")
+	}
+
+	return nil
+}
+
 func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
-	if len(args) < 1 || (args[0] != "masterkey" && args[0] != "config" && len(args) != 2) {
-		return errors.Fatal("type or ID not specified")
+	if err := validateCatArgs(args); err != nil {
+		return err
 	}
 
 	repo, err := OpenRepository(ctx, gopts)
@@ -45,7 +71,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 
 	if !gopts.NoLock {
 		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo)
+		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -55,7 +81,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 	tpe := args[0]
 
 	var id restic.ID
-	if tpe != "masterkey" && tpe != "config" && tpe != "snapshot" {
+	if tpe != "masterkey" && tpe != "config" && tpe != "snapshot" && tpe != "tree" {
 		id, err = restic.ParseID(args[1])
 		if err != nil {
 			return errors.Fatalf("unable to parse ID: %v\n", err)
@@ -72,7 +98,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 		Println(string(buf))
 		return nil
 	case "index":
-		buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id, nil)
+		buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id)
 		if err != nil {
 			return err
 		}
@@ -80,7 +106,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 		Println(string(buf))
 		return nil
 	case "snapshot":
-		sn, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1])
+		sn, _, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1])
 		if err != nil {
 			return errors.Fatalf("could not find snapshot: %v\n", err)
 		}
@@ -143,7 +169,8 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 		return err
 
 	case "blob":
-		err = repo.LoadIndex(ctx)
+		bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+		err = repo.LoadIndex(ctx, bar)
 		if err != nil {
 			return err
 		}
@@ -165,6 +192,30 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 
 		return errors.Fatal("blob not found")
 
+	case "tree":
+		sn, subfolder, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1])
+		if err != nil {
+			return errors.Fatalf("could not find snapshot: %v\n", err)
+		}
+
+		bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+		err = repo.LoadIndex(ctx, bar)
+		if err != nil {
+			return err
+		}
+
+		sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
+		if err != nil {
+			return err
+		}
+
+		buf, err := repo.LoadBlob(ctx, restic.TreeBlob, *sn.Tree, nil)
+		if err != nil {
+			return err
+		}
+		_, err = globalOptions.stdout.Write(buf)
+		return err
+
 	default:
 		return errors.Fatal("invalid type")
 	}

cmd/restic/cmd_cat_test.go

@@ -0,0 +1,30 @@
+package main
+
+import (
+	"strings"
+	"testing"
+
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func TestCatArgsValidation(t *testing.T) {
+	for _, test := range []struct {
+		args []string
+		err  string
+	}{
+		{[]string{}, "Fatal: type not specified"},
+		{[]string{"masterkey"}, ""},
+		{[]string{"invalid"}, `Fatal: invalid type "invalid"`},
+		{[]string{"snapshot"}, "Fatal: ID not specified"},
+		{[]string{"snapshot", "12345678"}, ""},
+	} {
+		t.Run("", func(t *testing.T) {
+			err := validateCatArgs(test.args)
+			if test.err == "" {
+				rtest.Assert(t, err == nil, "unexpected error %q", err)
+			} else {
+				rtest.Assert(t, strings.Contains(err.Error(), test.err), "unexpected error expected %q to contain %q", err, test.err)
+			}
+		})
+	}
+}

cmd/restic/cmd_check.go

@@ -16,6 +16,7 @@ import (
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui"
 )
 
 var cmdCheck = &cobra.Command{
@@ -65,7 +66,7 @@ func init() {
 		// MarkDeprecated only returns an error when the flag is not found
 		panic(err)
 	}
-	f.BoolVar(&checkOptions.WithCache, "with-cache", false, "use the cache")
+	f.BoolVar(&checkOptions.WithCache, "with-cache", false, "use existing cache, only read uncached data from repository")
 }
 
 func checkFlags(opts CheckOptions) error {
@@ -97,7 +98,7 @@ func checkFlags(opts CheckOptions) error {
 			}
 
 		} else {
-			fileSize, err := parseSizeStr(opts.ReadDataSubset)
+			fileSize, err := ui.ParseBytes(opts.ReadDataSubset)
 			if err != nil {
 				return argumentError
 			}
@@ -211,7 +212,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	if !gopts.NoLock {
 		Verbosef("create exclusive lock for repository\n")
 		var lock *restic.Lock
-		lock, ctx, err = lockRepoExclusive(ctx, repo)
+		lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -225,7 +226,8 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	}
 
 	Verbosef("load indexes\n")
-	hints, errs := chkr.LoadIndex(ctx)
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	hints, errs := chkr.LoadIndex(ctx, bar)
 
 	errorsFound := false
 	suggestIndexRebuild := false
@@ -245,7 +247,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	}
 
 	if suggestIndexRebuild {
-		Printf("Duplicate packs/old indexes are non-critical, you can run `restic rebuild-index' to correct this.\n")
+		Printf("Duplicate packs/old indexes are non-critical, you can run `restic repair index' to correct this.\n")
 	}
 	if mixedFound {
 		Printf("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")
@@ -328,11 +330,28 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 
 		go chkr.ReadPacks(ctx, packs, p, errChan)
 
+		var salvagePacks restic.IDs
+
 		for err := range errChan {
 			errorsFound = true
 			Warnf("%v\n", err)
+			if err, ok := err.(*checker.ErrPackData); ok {
+				if strings.Contains(err.Error(), "wrong data returned, hash is") {
+					salvagePacks = append(salvagePacks, err.PackID)
+				}
+			}
 		}
 		p.Done()
+
+		if len(salvagePacks) > 0 {
+			Warnf("\nThe repository contains pack files with damaged blobs. These blobs must be removed to repair the repository. This can be done using the following commands:\n\n")
+			var strIds []string
+			for _, id := range salvagePacks {
+				strIds = append(strIds, id.String())
+			}
+			Warnf("RESTIC_FEATURES=repair-packs-v1 restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIds, " "))
+			Warnf("Corrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n")
+		}
 	}
 
 	switch {
@@ -363,7 +382,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 			if repoSize == 0 {
 				return errors.Fatal("Cannot read from a repository having size 0")
 			}
-			subsetSize, _ := parseSizeStr(opts.ReadDataSubset)
+			subsetSize, _ := ui.ParseBytes(opts.ReadDataSubset)
 			if subsetSize > repoSize {
 				subsetSize = repoSize
 			}

cmd/restic/cmd_check_integration_test.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"context"
+	"testing"
+
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func testRunCheck(t testing.TB, gopts GlobalOptions) {
+	t.Helper()
+	output, err := testRunCheckOutput(gopts, true)
+	if err != nil {
+		t.Error(output)
+		t.Fatalf("unexpected error: %+v", err)
+	}
+}
+
+func testRunCheckMustFail(t testing.TB, gopts GlobalOptions) {
+	t.Helper()
+	_, err := testRunCheckOutput(gopts, false)
+	rtest.Assert(t, err != nil, "expected non nil error after check of damaged repository")
+}
+
+func testRunCheckOutput(gopts GlobalOptions, checkUnused bool) (string, error) {
+	buf, err := withCaptureStdout(func() error {
+		opts := CheckOptions{
+			ReadData:    true,
+			CheckUnused: checkUnused,
+		}
+		return runCheck(context.TODO(), opts, gopts, nil)
+	})
+	return buf.String(), err
+}

cmd/restic/cmd_copy.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"golang.org/x/sync/errgroup"
@@ -39,7 +40,7 @@ new destination repository using the "init" command.
 // CopyOptions bundles all options for the copy command.
 type CopyOptions struct {
 	secondaryRepoOptions
-	snapshotFilterOptions
+	restic.SnapshotFilter
 }
 
 var copyOptions CopyOptions
@@ -49,7 +50,7 @@ func init() {
 
 	f := cmdCopy.Flags()
 	initSecondaryRepoOptions(f, &copyOptions.secondaryRepoOptions, "destination", "to copy snapshots from")
-	initMultiSnapshotFilterOptions(f, &copyOptions.snapshotFilterOptions, true)
+	initMultiSnapshotFilter(f, &copyOptions.SnapshotFilter, true)
 }
 
 func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []string) error {
@@ -74,14 +75,14 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
 
 	if !gopts.NoLock {
 		var srcLock *restic.Lock
-		srcLock, ctx, err = lockRepo(ctx, srcRepo)
+		srcLock, ctx, err = lockRepo(ctx, srcRepo, gopts.RetryLock, gopts.JSON)
 		defer unlockRepo(srcLock)
 		if err != nil {
 			return err
 		}
 	}
 
-	dstLock, ctx, err := lockRepo(ctx, dstRepo)
+	dstLock, ctx, err := lockRepo(ctx, dstRepo, gopts.RetryLock, gopts.JSON)
 	defer unlockRepo(dstLock)
 	if err != nil {
 		return err
@@ -98,17 +99,18 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
 	}
 
 	debug.Log("Loading source index")
-	if err := srcRepo.LoadIndex(ctx); err != nil {
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	if err := srcRepo.LoadIndex(ctx, bar); err != nil {
 		return err
 	}
-
+	bar = newIndexProgress(gopts.Quiet, gopts.JSON)
 	debug.Log("Loading destination index")
-	if err := dstRepo.LoadIndex(ctx); err != nil {
+	if err := dstRepo.LoadIndex(ctx, bar); err != nil {
 		return err
 	}
 
 	dstSnapshotByOriginal := make(map[restic.ID][]*restic.Snapshot)
-	for sn := range FindFilteredSnapshots(ctx, dstSnapshotLister, dstRepo, opts.Hosts, opts.Tags, opts.Paths, nil) {
+	for sn := range FindFilteredSnapshots(ctx, dstSnapshotLister, dstRepo, &opts.SnapshotFilter, nil) {
 		if sn.Original != nil && !sn.Original.IsNull() {
 			dstSnapshotByOriginal[*sn.Original] = append(dstSnapshotByOriginal[*sn.Original], sn)
 		}
@@ -119,8 +121,7 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
 	// remember already processed trees across all snapshots
 	visitedTrees := restic.NewIDSet()
 
-	for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) {
-
+	for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, &opts.SnapshotFilter, args) {
 		// check whether the destination has a snapshot with the same persistent ID which has similar snapshot fields
 		srcOriginal := *sn.ID()
 		if sn.Original != nil {
@@ -237,5 +238,8 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep
 	bar := newProgressMax(!quiet, uint64(len(packList)), "packs copied")
 	_, err = repository.Repack(ctx, srcRepo, dstRepo, packList, copyBlobs, bar)
 	bar.Done()
-	return err
+	if err != nil {
+		return errors.Fatal(err.Error())
+	}
+	return nil
 }

cmd/restic/cmd_copy_integration_test.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"testing"
+
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func testRunCopy(t testing.TB, srcGopts GlobalOptions, dstGopts GlobalOptions) {
+	gopts := srcGopts
+	gopts.Repo = dstGopts.Repo
+	gopts.password = dstGopts.password
+	copyOpts := CopyOptions{
+		secondaryRepoOptions: secondaryRepoOptions{
+			Repo:     srcGopts.Repo,
+			password: srcGopts.password,
+		},
+	}
+
+	rtest.OK(t, runCopy(context.TODO(), copyOpts, gopts, nil))
+}
+
+func TestCopy(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+	env2, cleanup2 := withTestEnvironment(t)
+	defer cleanup2()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{}
+	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9")}, opts, env.gopts)
+	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "2")}, opts, env.gopts)
+	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "3")}, opts, env.gopts)
+	testRunCheck(t, env.gopts)
+
+	testRunInit(t, env2.gopts)
+	testRunCopy(t, env.gopts, env2.gopts)
+
+	snapshotIDs := testListSnapshots(t, env.gopts, 3)
+	copiedSnapshotIDs := testListSnapshots(t, env2.gopts, 3)
+
+	// Check that the copies size seems reasonable
+	stat := dirStats(env.repo)
+	stat2 := dirStats(env2.repo)
+	sizeDiff := int64(stat.size) - int64(stat2.size)
+	if sizeDiff < 0 {
+		sizeDiff = -sizeDiff
+	}
+	rtest.Assert(t, sizeDiff < int64(stat.size)/50, "expected less than 2%% size difference: %v vs. %v",
+		stat.size, stat2.size)
+
+	// Check integrity of the copy
+	testRunCheck(t, env2.gopts)
+
+	// Check that the copied snapshots have the same tree contents as the old ones (= identical tree hash)
+	origRestores := make(map[string]struct{})
+	for i, snapshotID := range snapshotIDs {
+		restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
+		origRestores[restoredir] = struct{}{}
+		testRunRestore(t, env.gopts, restoredir, snapshotID)
+	}
+	for i, snapshotID := range copiedSnapshotIDs {
+		restoredir := filepath.Join(env2.base, fmt.Sprintf("restore%d", i))
+		testRunRestore(t, env2.gopts, restoredir, snapshotID)
+		foundMatch := false
+		for cmpdir := range origRestores {
+			diff := directoriesContentsDiff(restoredir, cmpdir)
+			if diff == "" {
+				delete(origRestores, cmpdir)
+				foundMatch = true
+			}
+		}
+
+		rtest.Assert(t, foundMatch, "found no counterpart for snapshot %v", snapshotID)
+	}
+
+	rtest.Assert(t, len(origRestores) == 0, "found not copied snapshots")
+}
+
+func TestCopyIncremental(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+	env2, cleanup2 := withTestEnvironment(t)
+	defer cleanup2()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{}
+	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9")}, opts, env.gopts)
+	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "2")}, opts, env.gopts)
+	testRunCheck(t, env.gopts)
+
+	testRunInit(t, env2.gopts)
+	testRunCopy(t, env.gopts, env2.gopts)
+
+	testListSnapshots(t, env.gopts, 2)
+	testListSnapshots(t, env2.gopts, 2)
+
+	// Check that the copies size seems reasonable
+	testRunCheck(t, env2.gopts)
+
+	// check that no snapshots are copied, as there are no new ones
+	testRunCopy(t, env.gopts, env2.gopts)
+	testRunCheck(t, env2.gopts)
+	testListSnapshots(t, env2.gopts, 2)
+
+	// check that only new snapshots are copied
+	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "3")}, opts, env.gopts)
+	testRunCopy(t, env.gopts, env2.gopts)
+	testRunCheck(t, env2.gopts)
+	testListSnapshots(t, env.gopts, 3)
+	testListSnapshots(t, env2.gopts, 3)
+
+	// also test the reverse direction
+	testRunCopy(t, env2.gopts, env.gopts)
+	testRunCheck(t, env.gopts)
+	testListSnapshots(t, env.gopts, 3)
+}
+
+func TestCopyUnstableJSON(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+	env2, cleanup2 := withTestEnvironment(t)
+	defer cleanup2()
+
+	// contains a symlink created using `ln -s '../i/'$'\355\246\361''d/samba' broken-symlink`
+	datafile := filepath.Join("testdata", "copy-unstable-json.tar.gz")
+	rtest.SetupTarTestFixture(t, env.base, datafile)
+
+	testRunInit(t, env2.gopts)
+	testRunCopy(t, env.gopts, env2.gopts)
+	testRunCheck(t, env2.gopts)
+	testListSnapshots(t, env2.gopts, 1)
+}

cmd/restic/cmd_debug.go

@@ -134,7 +134,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
 }
 
 func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error {
-	return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
+	return index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
 		Printf("index_id: %v\n", id)
 		if err != nil {
 			return err
@@ -156,7 +156,7 @@ func runDebugDump(ctx context.Context, gopts GlobalOptions, args []string) error
 
 	if !gopts.NoLock {
 		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo)
+		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -167,20 +167,20 @@ func runDebugDump(ctx context.Context, gopts GlobalOptions, args []string) error
 
 	switch tpe {
 	case "indexes":
-		return dumpIndexes(ctx, repo, gopts.stdout)
+		return dumpIndexes(ctx, repo, globalOptions.stdout)
 	case "snapshots":
-		return debugPrintSnapshots(ctx, repo, gopts.stdout)
+		return debugPrintSnapshots(ctx, repo, globalOptions.stdout)
 	case "packs":
-		return printPacks(ctx, repo, gopts.stdout)
+		return printPacks(ctx, repo, globalOptions.stdout)
 	case "all":
 		Printf("snapshots:\n")
-		err := debugPrintSnapshots(ctx, repo, gopts.stdout)
+		err := debugPrintSnapshots(ctx, repo, globalOptions.stdout)
 		if err != nil {
 			return err
 		}
 
 		Printf("\nindexes:\n")
-		err = dumpIndexes(ctx, repo, gopts.stdout)
+		err = dumpIndexes(ctx, repo, globalOptions.stdout)
 		if err != nil {
 			return err
 		}
@@ -462,14 +462,15 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er
 
 	if !gopts.NoLock {
 		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo)
+		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
 		}
 	}
 
-	err = repo.LoadIndex(ctx)
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	err = repo.LoadIndex(ctx, bar)
 	if err != nil {
 		return err
 	}

cmd/restic/cmd_diff.go

@@ -16,7 +16,7 @@ import (
 )
 
 var cmdDiff = &cobra.Command{
-	Use:   "diff [flags] snapshot-ID snapshot-ID",
+	Use:   "diff [flags] snapshotID snapshotID",
 	Short: "Show differences between two snapshots",
 	Long: `
 The "diff" command shows differences from the first to the second snapshot. The
@@ -29,6 +29,10 @@ directory:
 * M  The file's content was modified
 * T  The type was changed, e.g. a file was made a symlink
 
+To only compare files in specific subfolders, you can use the
+"<snapshotID>:<subfolder>" syntax, where "subfolder" is a path within the
+snapshot.
+
 EXIT STATUS
 ===========
 
@@ -54,12 +58,12 @@ func init() {
 	f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata")
 }
 
-func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, error) {
-	sn, err := restic.FindSnapshot(ctx, be, repo, desc)
+func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, string, error) {
+	sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc)
 	if err != nil {
-		return nil, errors.Fatal(err.Error())
+		return nil, "", errors.Fatal(err.Error())
 	}
-	return sn, err
+	return sn, subfolder, err
 }
 
 // Comparer collects all things needed to compare two snapshots.
@@ -334,7 +338,7 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
 
 	if !gopts.NoLock {
 		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo)
+		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -346,12 +350,12 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
 	if err != nil {
 		return err
 	}
-	sn1, err := loadSnapshot(ctx, be, repo, args[0])
+	sn1, subfolder1, err := loadSnapshot(ctx, be, repo, args[0])
 	if err != nil {
 		return err
 	}
 
-	sn2, err := loadSnapshot(ctx, be, repo, args[1])
+	sn2, subfolder2, err := loadSnapshot(ctx, be, repo, args[1])
 	if err != nil {
 		return err
 	}
@@ -359,8 +363,8 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
 	if !gopts.JSON {
 		Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
 	}
-
-	if err = repo.LoadIndex(ctx); err != nil {
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	if err = repo.LoadIndex(ctx, bar); err != nil {
 		return err
 	}
 
@@ -372,6 +376,16 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
 		return errors.Errorf("snapshot %v has nil tree", sn2.ID().Str())
 	}
 
+	sn1.Tree, err = restic.FindTreeDirectory(ctx, repo, sn1.Tree, subfolder1)
+	if err != nil {
+		return err
+	}
+
+	sn2.Tree, err = restic.FindTreeDirectory(ctx, repo, sn2.Tree, subfolder2)
+	if err != nil {
+		return err
+	}
+
 	c := &Comparer{
 		repo: repo,
 		opts: diffOptions,
@@ -381,7 +395,7 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
 	}
 
 	if gopts.JSON {
-		enc := json.NewEncoder(gopts.stdout)
+		enc := json.NewEncoder(globalOptions.stdout)
 		c.printChange = func(change *Change) {
 			err := enc.Encode(change)
 			if err != nil {
@@ -415,7 +429,7 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
 	updateBlobs(repo, stats.BlobsAfter.Sub(both).Sub(stats.BlobsCommon), &stats.Added)
 
 	if gopts.JSON {
-		err := json.NewEncoder(gopts.stdout).Encode(stats)
+		err := json.NewEncoder(globalOptions.stdout).Encode(stats)
 		if err != nil {
 			Warnf("JSON encode failed: %v\n", err)
 		}

cmd/restic/cmd_diff_integration_test.go

@@ -0,0 +1,193 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func testRunDiffOutput(gopts GlobalOptions, firstSnapshotID string, secondSnapshotID string) (string, error) {
+	buf, err := withCaptureStdout(func() error {
+		opts := DiffOptions{
+			ShowMetadata: false,
+		}
+		return runDiff(context.TODO(), opts, gopts, []string{firstSnapshotID, secondSnapshotID})
+	})
+	return buf.String(), err
+}
+
+func copyFile(dst string, src string) error {
+	srcFile, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+
+	dstFile, err := os.Create(dst)
+	if err != nil {
+		// ignore subsequent errors
+		_ = srcFile.Close()
+		return err
+	}
+
+	_, err = io.Copy(dstFile, srcFile)
+	if err != nil {
+		// ignore subsequent errors
+		_ = srcFile.Close()
+		_ = dstFile.Close()
+		return err
+	}
+
+	err = srcFile.Close()
+	if err != nil {
+		// ignore subsequent errors
+		_ = dstFile.Close()
+		return err
+	}
+
+	err = dstFile.Close()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+var diffOutputRegexPatterns = []string{
+	"-.+modfile",
+	"M.+modfile1",
+	"\\+.+modfile2",
+	"\\+.+modfile3",
+	"\\+.+modfile4",
+	"-.+submoddir",
+	"-.+submoddir.subsubmoddir",
+	"\\+.+submoddir2",
+	"\\+.+submoddir2.subsubmoddir",
+	"Files: +2 new, +1 removed, +1 changed",
+	"Dirs: +3 new, +2 removed",
+	"Data Blobs: +2 new, +1 removed",
+	"Added: +7[0-9]{2}\\.[0-9]{3} KiB",
+	"Removed: +2[0-9]{2}\\.[0-9]{3} KiB",
+}
+
+func setupDiffRepo(t *testing.T) (*testEnvironment, func(), string, string) {
+	env, cleanup := withTestEnvironment(t)
+	testRunInit(t, env.gopts)
+
+	datadir := filepath.Join(env.base, "testdata")
+	testdir := filepath.Join(datadir, "testdir")
+	subtestdir := filepath.Join(testdir, "subtestdir")
+	testfile := filepath.Join(testdir, "testfile")
+
+	rtest.OK(t, os.Mkdir(testdir, 0755))
+	rtest.OK(t, os.Mkdir(subtestdir, 0755))
+	rtest.OK(t, appendRandomData(testfile, 256*1024))
+
+	moddir := filepath.Join(datadir, "moddir")
+	submoddir := filepath.Join(moddir, "submoddir")
+	subsubmoddir := filepath.Join(submoddir, "subsubmoddir")
+	modfile := filepath.Join(moddir, "modfile")
+	rtest.OK(t, os.Mkdir(moddir, 0755))
+	rtest.OK(t, os.Mkdir(submoddir, 0755))
+	rtest.OK(t, os.Mkdir(subsubmoddir, 0755))
+	rtest.OK(t, copyFile(modfile, testfile))
+	rtest.OK(t, appendRandomData(modfile+"1", 256*1024))
+
+	snapshots := make(map[string]struct{})
+	opts := BackupOptions{}
+	testRunBackup(t, "", []string{datadir}, opts, env.gopts)
+	snapshots, firstSnapshotID := lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts))
+
+	rtest.OK(t, os.Rename(modfile, modfile+"3"))
+	rtest.OK(t, os.Rename(submoddir, submoddir+"2"))
+	rtest.OK(t, appendRandomData(modfile+"1", 256*1024))
+	rtest.OK(t, appendRandomData(modfile+"2", 256*1024))
+	rtest.OK(t, os.Mkdir(modfile+"4", 0755))
+
+	testRunBackup(t, "", []string{datadir}, opts, env.gopts)
+	_, secondSnapshotID := lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts))
+
+	return env, cleanup, firstSnapshotID, secondSnapshotID
+}
+
+func TestDiff(t *testing.T) {
+	env, cleanup, firstSnapshotID, secondSnapshotID := setupDiffRepo(t)
+	defer cleanup()
+
+	// quiet suppresses the diff output except for the summary
+	env.gopts.Quiet = false
+	_, err := testRunDiffOutput(env.gopts, "", secondSnapshotID)
+	rtest.Assert(t, err != nil, "expected error on invalid snapshot id")
+
+	out, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
+	rtest.OK(t, err)
+
+	for _, pattern := range diffOutputRegexPatterns {
+		r, err := regexp.Compile(pattern)
+		rtest.Assert(t, err == nil, "failed to compile regexp %v", pattern)
+		rtest.Assert(t, r.MatchString(out), "expected pattern %v in output, got\n%v", pattern, out)
+	}
+
+	// check quiet output
+	env.gopts.Quiet = true
+	outQuiet, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
+	rtest.OK(t, err)
+
+	rtest.Assert(t, len(outQuiet) < len(out), "expected shorter output on quiet mode %v vs. %v", len(outQuiet), len(out))
+}
+
+type typeSniffer struct {
+	MessageType string `json:"message_type"`
+}
+
+func TestDiffJSON(t *testing.T) {
+	env, cleanup, firstSnapshotID, secondSnapshotID := setupDiffRepo(t)
+	defer cleanup()
+
+	// quiet suppresses the diff output except for the summary
+	env.gopts.Quiet = false
+	env.gopts.JSON = true
+	out, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
+	rtest.OK(t, err)
+
+	var stat DiffStatsContainer
+	var changes int
+
+	scanner := bufio.NewScanner(strings.NewReader(out))
+	for scanner.Scan() {
+		line := scanner.Text()
+		var sniffer typeSniffer
+		rtest.OK(t, json.Unmarshal([]byte(line), &sniffer))
+		switch sniffer.MessageType {
+		case "change":
+			changes++
+		case "statistics":
+			rtest.OK(t, json.Unmarshal([]byte(line), &stat))
+		default:
+			t.Fatalf("unexpected message type %v", sniffer.MessageType)
+		}
+	}
+	rtest.Equals(t, 9, changes)
+	rtest.Assert(t, stat.Added.Files == 2 && stat.Added.Dirs == 3 && stat.Added.DataBlobs == 2 &&
+		stat.Removed.Files == 1 && stat.Removed.Dirs == 2 && stat.Removed.DataBlobs == 1 &&
+		stat.ChangedFiles == 1, "unexpected statistics")
+
+	// check quiet output
+	env.gopts.Quiet = true
+	outQuiet, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
+	rtest.OK(t, err)
+
+	stat = DiffStatsContainer{}
+	rtest.OK(t, json.Unmarshal([]byte(outQuiet), &stat))
+	rtest.Assert(t, stat.Added.Files == 2 && stat.Added.Dirs == 3 && stat.Added.DataBlobs == 2 &&
+		stat.Removed.Files == 1 && stat.Removed.Dirs == 2 && stat.Removed.DataBlobs == 1 &&
+		stat.ChangedFiles == 1, "unexpected statistics")
+	rtest.Assert(t, stat.SourceSnapshot == firstSnapshotID && stat.TargetSnapshot == secondSnapshotID, "unexpected snapshot ids")
+}

cmd/restic/cmd_dump.go

@@ -24,9 +24,13 @@ single file is selected, it prints its contents to stdout. Folders are output
 as a tar (default) or zip file containing the contents of the specified folder.
 Pass "/" as file name to dump the whole snapshot as an archive file.
 
-The special snapshot "latest" can be used to use the latest snapshot in the
+The special snapshotID "latest" can be used to use the latest snapshot in the
 repository.
 
+To include the folder content at the root of the archive, you can use the
+"<snapshotID>:<subfolder>" syntax, where "subfolder" is a path within the
+snapshot.
+
 EXIT STATUS
 ===========
 
@@ -40,7 +44,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 
 // DumpOptions collects all options for the dump command.
 type DumpOptions struct {
-	snapshotFilterOptions
+	restic.SnapshotFilter
 	Archive string
 }
 
@@ -50,7 +54,7 @@ func init() {
 	cmdRoot.AddCommand(cmdDump)
 
 	flags := cmdDump.Flags()
-	initSingleSnapshotFilterOptions(flags, &dumpOptions.snapshotFilterOptions)
+	initSingleSnapshotFilter(flags, &dumpOptions.SnapshotFilter)
 	flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"")
 }
 
@@ -132,19 +136,29 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []
 
 	if !gopts.NoLock {
 		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo)
+		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
 		}
 	}
 
-	sn, err := restic.FindFilteredSnapshot(ctx, repo.Backend(), repo, opts.Paths, opts.Tags, opts.Hosts, nil, snapshotIDString)
+	sn, subfolder, err := (&restic.SnapshotFilter{
+		Hosts: opts.Hosts,
+		Paths: opts.Paths,
+		Tags:  opts.Tags,
+	}).FindLatest(ctx, repo.Backend(), repo, snapshotIDString)
 	if err != nil {
 		return errors.Fatalf("failed to find snapshot: %v", err)
 	}
 
-	err = repo.LoadIndex(ctx)
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	err = repo.LoadIndex(ctx, bar)
+	if err != nil {
+		return err
+	}
+
+	sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
 	if err != nil {
 		return err
 	}

cmd/restic/cmd_find.go

@@ -51,7 +51,8 @@ type FindOptions struct {
 	PackID, ShowPackID bool
 	CaseInsensitive    bool
 	ListLong           bool
-	snapshotFilterOptions
+	HumanReadable      bool
+	restic.SnapshotFilter
 }
 
 var findOptions FindOptions
@@ -69,8 +70,9 @@ func init() {
 	f.BoolVar(&findOptions.ShowPackID, "show-pack-id", false, "display the pack-ID the blobs belong to (with --blob or --tree)")
 	f.BoolVarP(&findOptions.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern")
 	f.BoolVarP(&findOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
+	f.BoolVar(&findOptions.HumanReadable, "human-readable", false, "print sizes in human readable format")
 
-	initMultiSnapshotFilterOptions(f, &findOptions.snapshotFilterOptions, true)
+	initMultiSnapshotFilter(f, &findOptions.SnapshotFilter, true)
 }
 
 type findPattern struct {
@@ -104,12 +106,13 @@ func parseTime(str string) (time.Time, error) {
 }
 
 type statefulOutput struct {
-	ListLong bool
-	JSON     bool
-	inuse    bool
-	newsn    *restic.Snapshot
-	oldsn    *restic.Snapshot
-	hits     int
+	ListLong      bool
+	HumanReadable bool
+	JSON          bool
+	inuse         bool
+	newsn         *restic.Snapshot
+	oldsn         *restic.Snapshot
+	hits          int
 }
 
 func (s *statefulOutput) PrintPatternJSON(path string, node *restic.Node) {
@@ -123,7 +126,6 @@ func (s *statefulOutput) PrintPatternJSON(path string, node *restic.Node) {
 
 		// Make the following attributes disappear
 		Name               byte `json:"name,omitempty"`
-		Inode              byte `json:"inode,omitempty"`
 		ExtendedAttributes byte `json:"extended_attributes,omitempty"`
 		Device             byte `json:"device,omitempty"`
 		Content            byte `json:"content,omitempty"`
@@ -164,7 +166,7 @@ func (s *statefulOutput) PrintPatternNormal(path string, node *restic.Node) {
 		s.oldsn = s.newsn
 		Verbosef("Found matching entries in snapshot %s from %s\n", s.oldsn.ID().Str(), s.oldsn.Time.Local().Format(TimeFormat))
 	}
-	Println(formatNode(path, node, s.ListLong))
+	Println(formatNode(path, node, s.ListLong, s.HumanReadable))
 }
 
 func (s *statefulOutput) PrintPattern(path string, node *restic.Node) {
@@ -501,7 +503,7 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
 	return packIDs
 }
 
-func (f *Finder) findObjectPack(ctx context.Context, id string, t restic.BlobType) {
+func (f *Finder) findObjectPack(id string, t restic.BlobType) {
 	idx := f.repo.Index()
 
 	rid, err := restic.ParseID(id)
@@ -524,13 +526,13 @@ func (f *Finder) findObjectPack(ctx context.Context, id string, t restic.BlobTyp
 	}
 }
 
-func (f *Finder) findObjectsPacks(ctx context.Context) {
+func (f *Finder) findObjectsPacks() {
 	for i := range f.blobIDs {
-		f.findObjectPack(ctx, i, restic.DataBlob)
+		f.findObjectPack(i, restic.DataBlob)
 	}
 
 	for i := range f.treeIDs {
-		f.findObjectPack(ctx, i, restic.TreeBlob)
+		f.findObjectPack(i, restic.TreeBlob)
 	}
 }
 
@@ -575,7 +577,7 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
 
 	if !gopts.NoLock {
 		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo)
+		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -586,15 +588,15 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
 	if err != nil {
 		return err
 	}
-
-	if err = repo.LoadIndex(ctx); err != nil {
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	if err = repo.LoadIndex(ctx, bar); err != nil {
 		return err
 	}
 
 	f := &Finder{
 		repo:        repo,
 		pat:         pat,
-		out:         statefulOutput{ListLong: opts.ListLong, JSON: globalOptions.JSON},
+		out:         statefulOutput{ListLong: opts.ListLong, HumanReadable: opts.HumanReadable, JSON: gopts.JSON},
 		ignoreTrees: restic.NewIDSet(),
 	}
 
@@ -618,7 +620,16 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
 		}
 	}
 
-	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, opts.Snapshots) {
+	var filteredSnapshots []*restic.Snapshot
+	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, opts.Snapshots) {
+		filteredSnapshots = append(filteredSnapshots, sn)
+	}
+
+	sort.Slice(filteredSnapshots, func(i, j int) bool {
+		return filteredSnapshots[i].Time.Before(filteredSnapshots[j].Time)
+	})
+
+	for _, sn := range filteredSnapshots {
 		if f.blobIDs != nil || f.treeIDs != nil {
 			if err = f.findIDs(ctx, sn); err != nil && err.Error() != "OK" {
 				return err
@@ -632,7 +643,7 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
 	f.out.Finish()
 
 	if opts.ShowPackID && (f.blobIDs != nil || f.treeIDs != nil) {
-		f.findObjectsPacks(ctx)
+		f.findObjectsPacks()
 	}
 
 	return nil

cmd/restic/cmd_find_integration_test.go

@@ -0,0 +1,87 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func testRunFind(t testing.TB, wantJSON bool, gopts GlobalOptions, pattern string) []byte {
+	buf, err := withCaptureStdout(func() error {
+		gopts.JSON = wantJSON
+
+		opts := FindOptions{}
+		return runFind(context.TODO(), opts, gopts, []string{pattern})
+	})
+	rtest.OK(t, err)
+	return buf.Bytes()
+}
+
+func TestFind(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	datafile := testSetupBackupData(t, env)
+	opts := BackupOptions{}
+
+	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
+	testRunCheck(t, env.gopts)
+
+	results := testRunFind(t, false, env.gopts, "unexistingfile")
+	rtest.Assert(t, len(results) == 0, "unexisting file found in repo (%v)", datafile)
+
+	results = testRunFind(t, false, env.gopts, "testfile")
+	lines := strings.Split(string(results), "\n")
+	rtest.Assert(t, len(lines) == 2, "expected one file found in repo (%v)", datafile)
+
+	results = testRunFind(t, false, env.gopts, "testfile*")
+	lines = strings.Split(string(results), "\n")
+	rtest.Assert(t, len(lines) == 4, "expected three files found in repo (%v)", datafile)
+}
+
+type testMatch struct {
+	Path        string    `json:"path,omitempty"`
+	Permissions string    `json:"permissions,omitempty"`
+	Size        uint64    `json:"size,omitempty"`
+	Date        time.Time `json:"date,omitempty"`
+	UID         uint32    `json:"uid,omitempty"`
+	GID         uint32    `json:"gid,omitempty"`
+}
+
+type testMatches struct {
+	Hits       int         `json:"hits,omitempty"`
+	SnapshotID string      `json:"snapshot,omitempty"`
+	Matches    []testMatch `json:"matches,omitempty"`
+}
+
+func TestFindJSON(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	datafile := testSetupBackupData(t, env)
+	opts := BackupOptions{}
+
+	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
+	testRunCheck(t, env.gopts)
+
+	results := testRunFind(t, true, env.gopts, "unexistingfile")
+	matches := []testMatches{}
+	rtest.OK(t, json.Unmarshal(results, &matches))
+	rtest.Assert(t, len(matches) == 0, "expected no match in repo (%v)", datafile)
+
+	results = testRunFind(t, true, env.gopts, "testfile")
+	rtest.OK(t, json.Unmarshal(results, &matches))
+	rtest.Assert(t, len(matches) == 1, "expected a single snapshot in repo (%v)", datafile)
+	rtest.Assert(t, len(matches[0].Matches) == 1, "expected a single file to match (%v)", datafile)
+	rtest.Assert(t, matches[0].Hits == 1, "expected hits to show 1 match (%v)", datafile)
+
+	results = testRunFind(t, true, env.gopts, "testfile*")
+	rtest.OK(t, json.Unmarshal(results, &matches))
+	rtest.Assert(t, len(matches) == 1, "expected a single snapshot in repo (%v)", datafile)
+	rtest.Assert(t, len(matches[0].Matches) == 3, "expected 3 files to match (%v)", datafile)
+	rtest.Assert(t, matches[0].Hits == 3, "expected hits to show 3 matches (%v)", datafile)
+}

cmd/restic/cmd_forget.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"io"
+	"strconv"
 
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
@@ -36,14 +37,49 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 	},
 }
 
+type ForgetPolicyCount int
+
+var ErrNegativePolicyCount = errors.New("negative values not allowed, use 'unlimited' instead")
+
+func (c *ForgetPolicyCount) Set(s string) error {
+	switch s {
+	case "unlimited":
+		*c = -1
+	default:
+		val, err := strconv.ParseInt(s, 10, 0)
+		if err != nil {
+			return err
+		}
+		if val < 0 {
+			return ErrNegativePolicyCount
+		}
+		*c = ForgetPolicyCount(val)
+	}
+
+	return nil
+}
+
+func (c *ForgetPolicyCount) String() string {
+	switch *c {
+	case -1:
+		return "unlimited"
+	default:
+		return strconv.FormatInt(int64(*c), 10)
+	}
+}
+
+func (c *ForgetPolicyCount) Type() string {
+	return "n"
+}
+
 // ForgetOptions collects all options for the forget command.
 type ForgetOptions struct {
-	Last          int
-	Hourly        int
-	Daily         int
-	Weekly        int
-	Monthly       int
-	Yearly        int
+	Last          ForgetPolicyCount
+	Hourly        ForgetPolicyCount
+	Daily         ForgetPolicyCount
+	Weekly        ForgetPolicyCount
+	Monthly       ForgetPolicyCount
+	Yearly        ForgetPolicyCount
 	Within        restic.Duration
 	WithinHourly  restic.Duration
 	WithinDaily   restic.Duration
@@ -52,11 +88,11 @@ type ForgetOptions struct {
 	WithinYearly  restic.Duration
 	KeepTags      restic.TagLists
 
-	snapshotFilterOptions
+	restic.SnapshotFilter
 	Compact bool
 
 	// Grouping
-	GroupBy string
+	GroupBy restic.SnapshotGroupByOptions
 	DryRun  bool
 	Prune   bool
 }
@@ -67,12 +103,12 @@ func init() {
 	cmdRoot.AddCommand(cmdForget)
 
 	f := cmdForget.Flags()
-	f.IntVarP(&forgetOptions.Last, "keep-last", "l", 0, "keep the last `n` snapshots")
-	f.IntVarP(&forgetOptions.Hourly, "keep-hourly", "H", 0, "keep the last `n` hourly snapshots")
-	f.IntVarP(&forgetOptions.Daily, "keep-daily", "d", 0, "keep the last `n` daily snapshots")
-	f.IntVarP(&forgetOptions.Weekly, "keep-weekly", "w", 0, "keep the last `n` weekly snapshots")
-	f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots")
-	f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots")
+	f.VarP(&forgetOptions.Last, "keep-last", "l", "keep the last `n` snapshots (use 'unlimited' to keep all snapshots)")
+	f.VarP(&forgetOptions.Hourly, "keep-hourly", "H", "keep the last `n` hourly snapshots (use 'unlimited' to keep all hourly snapshots)")
+	f.VarP(&forgetOptions.Daily, "keep-daily", "d", "keep the last `n` daily snapshots (use 'unlimited' to keep all daily snapshots)")
+	f.VarP(&forgetOptions.Weekly, "keep-weekly", "w", "keep the last `n` weekly snapshots (use 'unlimited' to keep all weekly snapshots)")
+	f.VarP(&forgetOptions.Monthly, "keep-monthly", "m", "keep the last `n` monthly snapshots (use 'unlimited' to keep all monthly snapshots)")
+	f.VarP(&forgetOptions.Yearly, "keep-yearly", "y", "keep the last `n` yearly snapshots (use 'unlimited' to keep all yearly snapshots)")
 	f.VarP(&forgetOptions.Within, "keep-within", "", "keep snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
 	f.VarP(&forgetOptions.WithinHourly, "keep-within-hourly", "", "keep hourly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
 	f.VarP(&forgetOptions.WithinDaily, "keep-within-daily", "", "keep daily snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
@@ -81,7 +117,7 @@ func init() {
 	f.VarP(&forgetOptions.WithinYearly, "keep-within-yearly", "", "keep yearly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
 	f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)")
 
-	initMultiSnapshotFilterOptions(f, &forgetOptions.snapshotFilterOptions, false)
+	initMultiSnapshotFilter(f, &forgetOptions.SnapshotFilter, false)
 	f.StringArrayVar(&forgetOptions.Hosts, "hostname", nil, "only consider snapshots with the given `hostname` (can be specified multiple times)")
 	err := f.MarkDeprecated("hostname", "use --host")
 	if err != nil {
@@ -90,8 +126,8 @@ func init() {
 	}
 
 	f.BoolVarP(&forgetOptions.Compact, "compact", "c", false, "use compact output format")
-
-	f.StringVarP(&forgetOptions.GroupBy, "group-by", "g", "host,paths", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
+	forgetOptions.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true}
+	f.VarP(&forgetOptions.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
 	f.BoolVarP(&forgetOptions.DryRun, "dry-run", "n", false, "do not delete anything, just print what would be done")
 	f.BoolVar(&forgetOptions.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed")
 
@@ -99,8 +135,29 @@ func init() {
 	addPruneOptions(cmdForget)
 }
 
+func verifyForgetOptions(opts *ForgetOptions) error {
+	if opts.Last < -1 || opts.Hourly < -1 || opts.Daily < -1 || opts.Weekly < -1 ||
+		opts.Monthly < -1 || opts.Yearly < -1 {
+		return errors.Fatal("negative values other than -1 are not allowed for --keep-*")
+	}
+
+	for _, d := range []restic.Duration{opts.Within, opts.WithinHourly, opts.WithinDaily,
+		opts.WithinMonthly, opts.WithinWeekly, opts.WithinYearly} {
+		if d.Hours < 0 || d.Days < 0 || d.Months < 0 || d.Years < 0 {
+			return errors.Fatal("durations containing negative values are not allowed for --keep-within*")
+		}
+	}
+
+	return nil
+}
+
 func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, args []string) error {
-	err := verifyPruneOptions(&pruneOptions)
+	err := verifyForgetOptions(&opts)
+	if err != nil {
+		return err
+	}
+
+	err = verifyPruneOptions(&pruneOptions)
 	if err != nil {
 		return err
 	}
@@ -116,7 +173,7 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg
 
 	if !opts.DryRun || !gopts.NoLock {
 		var lock *restic.Lock
-		lock, ctx, err = lockRepoExclusive(ctx, repo)
+		lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -126,7 +183,7 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg
 	var snapshots restic.Snapshots
 	removeSnIDs := restic.NewIDSet()
 
-	for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) {
+	for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) {
 		snapshots = append(snapshots, sn)
 	}
 
@@ -144,12 +201,12 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg
 		}
 
 		policy := restic.ExpirePolicy{
-			Last:          opts.Last,
-			Hourly:        opts.Hourly,
-			Daily:         opts.Daily,
-			Weekly:        opts.Weekly,
-			Monthly:       opts.Monthly,
-			Yearly:        opts.Yearly,
+			Last:          int(opts.Last),
+			Hourly:        int(opts.Hourly),
+			Daily:         int(opts.Daily),
+			Weekly:        int(opts.Weekly),
+			Monthly:       int(opts.Monthly),
+			Yearly:        int(opts.Yearly),
 			Within:        opts.Within,
 			WithinHourly:  opts.WithinHourly,
 			WithinDaily:   opts.WithinDaily,
@@ -172,7 +229,7 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg
 
 			for k, snapshotGroup := range snapshotGroups {
 				if gopts.Verbose >= 1 && !gopts.JSON {
-					err = PrintSnapshotGroupHeader(gopts.stdout, k)
+					err = PrintSnapshotGroupHeader(globalOptions.stdout, k)
 					if err != nil {
 						return err
 					}
@@ -229,7 +286,7 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg
 	}
 
 	if gopts.JSON && len(jsonGroups) > 0 {
-		err = printJSONForget(gopts.stdout, jsonGroups)
+		err = printJSONForget(globalOptions.stdout, jsonGroups)
 		if err != nil {
 			return err
 		}

cmd/restic/cmd_forget_integration_test.go

@@ -0,0 +1,13 @@
+package main
+
+import (
+	"context"
+	"testing"
+
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
+	opts := ForgetOptions{}
+	rtest.OK(t, runForget(context.TODO(), opts, gopts, args))
+}

cmd/restic/cmd_forget_test.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/restic/restic/internal/restic"
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func TestForgetPolicyValues(t *testing.T) {
+	testCases := []struct {
+		input string
+		value ForgetPolicyCount
+		err   string
+	}{
+		{"0", ForgetPolicyCount(0), ""},
+		{"1", ForgetPolicyCount(1), ""},
+		{"unlimited", ForgetPolicyCount(-1), ""},
+		{"", ForgetPolicyCount(0), "strconv.ParseInt: parsing \"\": invalid syntax"},
+		{"-1", ForgetPolicyCount(0), ErrNegativePolicyCount.Error()},
+		{"abc", ForgetPolicyCount(0), "strconv.ParseInt: parsing \"abc\": invalid syntax"},
+	}
+	for _, testCase := range testCases {
+		t.Run("", func(t *testing.T) {
+			var count ForgetPolicyCount
+			err := count.Set(testCase.input)
+
+			if testCase.err != "" {
+				rtest.Assert(t, err != nil, "should have returned error for input %+v", testCase.input)
+				rtest.Equals(t, testCase.err, err.Error())
+			} else {
+				rtest.Assert(t, err == nil, "expected no error for input %+v, got %v", testCase.input, err)
+				rtest.Equals(t, testCase.value, count)
+				rtest.Equals(t, testCase.input, count.String())
+			}
+		})
+	}
+}
+
+func TestForgetOptionValues(t *testing.T) {
+	const negValErrorMsg = "Fatal: negative values other than -1 are not allowed for --keep-*"
+	const negDurationValErrorMsg = "Fatal: durations containing negative values are not allowed for --keep-within*"
+	testCases := []struct {
+		input    ForgetOptions
+		errorMsg string
+	}{
+		{ForgetOptions{Last: 1}, ""},
+		{ForgetOptions{Hourly: 1}, ""},
+		{ForgetOptions{Daily: 1}, ""},
+		{ForgetOptions{Weekly: 1}, ""},
+		{ForgetOptions{Monthly: 1}, ""},
+		{ForgetOptions{Yearly: 1}, ""},
+		{ForgetOptions{Last: 0}, ""},
+		{ForgetOptions{Hourly: 0}, ""},
+		{ForgetOptions{Daily: 0}, ""},
+		{ForgetOptions{Weekly: 0}, ""},
+		{ForgetOptions{Monthly: 0}, ""},
+		{ForgetOptions{Yearly: 0}, ""},
+		{ForgetOptions{Last: -1}, ""},
+		{ForgetOptions{Hourly: -1}, ""},
+		{ForgetOptions{Daily: -1}, ""},
+		{ForgetOptions{Weekly: -1}, ""},
+		{ForgetOptions{Monthly: -1}, ""},
+		{ForgetOptions{Yearly: -1}, ""},
+		{ForgetOptions{Last: -2}, negValErrorMsg},
+		{ForgetOptions{Hourly: -2}, negValErrorMsg},
+		{ForgetOptions{Daily: -2}, negValErrorMsg},
+		{ForgetOptions{Weekly: -2}, negValErrorMsg},
+		{ForgetOptions{Monthly: -2}, negValErrorMsg},
+		{ForgetOptions{Yearly: -2}, negValErrorMsg},
+		{ForgetOptions{Within: restic.ParseDurationOrPanic("1y2m3d3h")}, ""},
+		{ForgetOptions{WithinHourly: restic.ParseDurationOrPanic("1y2m3d3h")}, ""},
+		{ForgetOptions{WithinDaily: restic.ParseDurationOrPanic("1y2m3d3h")}, ""},
+		{ForgetOptions{WithinWeekly: restic.ParseDurationOrPanic("1y2m3d3h")}, ""},
+		{ForgetOptions{WithinMonthly: restic.ParseDurationOrPanic("2y4m6d8h")}, ""},
+		{ForgetOptions{WithinYearly: restic.ParseDurationOrPanic("2y4m6d8h")}, ""},
+		{ForgetOptions{Within: restic.ParseDurationOrPanic("-1y2m3d3h")}, negDurationValErrorMsg},
+		{ForgetOptions{WithinHourly: restic.ParseDurationOrPanic("1y-2m3d3h")}, negDurationValErrorMsg},
+		{ForgetOptions{WithinDaily: restic.ParseDurationOrPanic("1y2m-3d3h")}, negDurationValErrorMsg},
+		{ForgetOptions{WithinWeekly: restic.ParseDurationOrPanic("1y2m3d-3h")}, negDurationValErrorMsg},
+		{ForgetOptions{WithinMonthly: restic.ParseDurationOrPanic("-2y4m6d8h")}, negDurationValErrorMsg},
+		{ForgetOptions{WithinYearly: restic.ParseDurationOrPanic("2y-4m6d8h")}, negDurationValErrorMsg},
+	}
+
+	for _, testCase := range testCases {
+		err := verifyForgetOptions(&testCase.input)
+		if testCase.errorMsg != "" {
+			rtest.Assert(t, err != nil, "should have returned error for input %+v", testCase.input)
+			rtest.Equals(t, testCase.errorMsg, err.Error())
+		} else {
+			rtest.Assert(t, err == nil, "expected no error for input %+v", testCase.input)
+		}
+	}
+}

cmd/restic/cmd_generate.go

@@ -63,26 +63,38 @@ func writeManpages(dir string) error {
 }
 
 func writeBashCompletion(file string) error {
-	Verbosef("writing bash completion file to %v\n", file)
+	if stdoutIsTerminal() {
+		Verbosef("writing bash completion file to %v\n", file)
+	}
 	return cmdRoot.GenBashCompletionFile(file)
 }
 
 func writeFishCompletion(file string) error {
-	Verbosef("writing fish completion file to %v\n", file)
+	if stdoutIsTerminal() {
+		Verbosef("writing fish completion file to %v\n", file)
+	}
 	return cmdRoot.GenFishCompletionFile(file, true)
 }
 
 func writeZSHCompletion(file string) error {
-	Verbosef("writing zsh completion file to %v\n", file)
+	if stdoutIsTerminal() {
+		Verbosef("writing zsh completion file to %v\n", file)
+	}
 	return cmdRoot.GenZshCompletionFile(file)
 }
 
 func writePowerShellCompletion(file string) error {
-	Verbosef("writing powershell completion file to %v\n", file)
+	if stdoutIsTerminal() {
+		Verbosef("writing powershell completion file to %v\n", file)
+	}
 	return cmdRoot.GenPowerShellCompletionFile(file)
 }
 
-func runGenerate(cmd *cobra.Command, args []string) error {
+func runGenerate(_ *cobra.Command, args []string) error {
+	if len(args) > 0 {
+		return errors.Fatal("the generate command expects no arguments, only options - please see `restic help generate` for usage and flags")
+	}
+
 	if genOpts.ManDir != "" {
 		err := writeManpages(genOpts.ManDir)
 		if err != nil {

cmd/restic/cmd_init.go

@@ -50,6 +50,10 @@ func init() {
 }
 
 func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []string) error {
+	if len(args) > 0 {
+		return errors.Fatal("the init command expects no arguments, only options - please see `restic help init` for usage and flags")
+	}
+
 	var version uint
 	if opts.RepositoryVersion == "latest" || opts.RepositoryVersion == "" {
 		version = restic.MaxRepoVersion
@@ -83,9 +87,9 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []
 		return err
 	}
 
-	be, err := create(ctx, repo, gopts.extended)
+	be, err := create(ctx, repo, gopts, gopts.extended)
 	if err != nil {
-		return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
+		return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.backends, gopts.Repo), err)
 	}
 
 	s, err := repository.New(be, repository.Options{
@@ -93,16 +97,21 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []
 		PackSize:    gopts.PackSize * 1024 * 1024,
 	})
 	if err != nil {
-		return err
+		return errors.Fatal(err.Error())
 	}
 
 	err = s.Init(ctx, version, gopts.password, chunkerPolynomial)
 	if err != nil {
-		return errors.Fatalf("create key in repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
+		return errors.Fatalf("create key in repository at %s failed: %v\n", location.StripPassword(gopts.backends, gopts.Repo), err)
 	}
 
 	if !gopts.JSON {
-		Verbosef("created restic repository %v at %s\n", s.Config().ID[:10], location.StripPassword(gopts.Repo))
+		Verbosef("created restic repository %v at %s", s.Config().ID[:10], location.StripPassword(gopts.backends, gopts.Repo))
+		if opts.CopyChunkerParameters && chunkerPolynomial != nil {
+			Verbosef(" with chunker parameters copied from secondary repository\n")
+		} else {
+			Verbosef("\n")
+		}
 		Verbosef("\n")
 		Verbosef("Please note that knowledge of your password is required to access\n")
 		Verbosef("the repository. Losing your password means that your data is\n")
@@ -112,9 +121,9 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []
 		status := initSuccess{
 			MessageType: "initialized",
 			ID:          s.Config().ID,
-			Repository:  location.StripPassword(gopts.Repo),
+			Repository:  location.StripPassword(gopts.backends, gopts.Repo),
 		}
-		return json.NewEncoder(gopts.stdout).Encode(status)
+		return json.NewEncoder(globalOptions.stdout).Encode(status)
 	}
 
 	return nil

cmd/restic/cmd_init_integration_test.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"context"
+	"testing"
+
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func testRunInit(t testing.TB, opts GlobalOptions) {
+	repository.TestUseLowSecurityKDFParameters(t)
+	restic.TestDisableCheckPolynomial(t)
+	restic.TestSetLockTimeout(t, 0)
+
+	rtest.OK(t, runInit(context.TODO(), InitOptions{}, opts, nil))
+	t.Logf("repository initialized at %v", opts.Repo)
+}
+
+func TestInitCopyChunkerParams(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+	env2, cleanup2 := withTestEnvironment(t)
+	defer cleanup2()
+
+	testRunInit(t, env2.gopts)
+
+	initOpts := InitOptions{
+		secondaryRepoOptions: secondaryRepoOptions{
+			Repo:     env2.gopts.Repo,
+			password: env2.gopts.password,
+		},
+	}
+	rtest.Assert(t, runInit(context.TODO(), initOpts, env.gopts, nil) != nil, "expected invalid init options to fail")
+
+	initOpts.CopyChunkerParameters = true
+	rtest.OK(t, runInit(context.TODO(), initOpts, env.gopts, nil))
+
+	repo, err := OpenRepository(context.TODO(), env.gopts)
+	rtest.OK(t, err)
+
+	otherRepo, err := OpenRepository(context.TODO(), env2.gopts)
+	rtest.OK(t, err)
+
+	rtest.Assert(t, repo.Config().ChunkerPolynomial == otherRepo.Config().ChunkerPolynomial,
+		"expected equal chunker polynomials, got %v expected %v", repo.Config().ChunkerPolynomial,
+		otherRepo.Config().ChunkerPolynomial)
+}

cmd/restic/cmd_key.go

@@ -212,15 +212,18 @@ func runKey(ctx context.Context, gopts GlobalOptions, args []string) error {
 
 	switch args[0] {
 	case "list":
-		lock, ctx, err := lockRepo(ctx, repo)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
+		if !gopts.NoLock {
+			var lock *restic.Lock
+			lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
+			defer unlockRepo(lock)
+			if err != nil {
+				return err
+			}
 		}
 
 		return listKeys(ctx, repo, gopts)
 	case "add":
-		lock, ctx, err := lockRepo(ctx, repo)
+		lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -228,7 +231,7 @@ func runKey(ctx context.Context, gopts GlobalOptions, args []string) error {
 
 		return addKey(ctx, repo, gopts)
 	case "remove":
-		lock, ctx, err := lockRepoExclusive(ctx, repo)
+		lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -241,7 +244,7 @@ func runKey(ctx context.Context, gopts GlobalOptions, args []string) error {
 
 		return deleteKey(ctx, repo, id)
 	case "passwd":
-		lock, ctx, err := lockRepoExclusive(ctx, repo)
+		lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err

cmd/restic/cmd_key_integration_test.go

@@ -0,0 +1,145 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"regexp"
+	"testing"
+
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func testRunKeyListOtherIDs(t testing.TB, gopts GlobalOptions) []string {
+	buf, err := withCaptureStdout(func() error {
+		return runKey(context.TODO(), gopts, []string{"list"})
+	})
+	rtest.OK(t, err)
+
+	scanner := bufio.NewScanner(buf)
+	exp := regexp.MustCompile(`^ ([a-f0-9]+) `)
+
+	IDs := []string{}
+	for scanner.Scan() {
+		if id := exp.FindStringSubmatch(scanner.Text()); id != nil {
+			IDs = append(IDs, id[1])
+		}
+	}
+
+	return IDs
+}
+
+func testRunKeyAddNewKey(t testing.TB, newPassword string, gopts GlobalOptions) {
+	testKeyNewPassword = newPassword
+	defer func() {
+		testKeyNewPassword = ""
+	}()
+
+	rtest.OK(t, runKey(context.TODO(), gopts, []string{"add"}))
+}
+
+func testRunKeyAddNewKeyUserHost(t testing.TB, gopts GlobalOptions) {
+	testKeyNewPassword = "john's geheimnis"
+	defer func() {
+		testKeyNewPassword = ""
+		keyUsername = ""
+		keyHostname = ""
+	}()
+
+	rtest.OK(t, cmdKey.Flags().Parse([]string{"--user=john", "--host=example.com"}))
+
+	t.Log("adding key for john@example.com")
+	rtest.OK(t, runKey(context.TODO(), gopts, []string{"add"}))
+
+	repo, err := OpenRepository(context.TODO(), gopts)
+	rtest.OK(t, err)
+	key, err := repository.SearchKey(context.TODO(), repo, testKeyNewPassword, 2, "")
+	rtest.OK(t, err)
+
+	rtest.Equals(t, "john", key.Username)
+	rtest.Equals(t, "example.com", key.Hostname)
+}
+
+func testRunKeyPasswd(t testing.TB, newPassword string, gopts GlobalOptions) {
+	testKeyNewPassword = newPassword
+	defer func() {
+		testKeyNewPassword = ""
+	}()
+
+	rtest.OK(t, runKey(context.TODO(), gopts, []string{"passwd"}))
+}
+
+func testRunKeyRemove(t testing.TB, gopts GlobalOptions, IDs []string) {
+	t.Logf("remove %d keys: %q\n", len(IDs), IDs)
+	for _, id := range IDs {
+		rtest.OK(t, runKey(context.TODO(), gopts, []string{"remove", id}))
+	}
+}
+
+func TestKeyAddRemove(t *testing.T) {
+	passwordList := []string{
+		"OnnyiasyatvodsEvVodyawit",
+		"raicneirvOjEfEigonOmLasOd",
+	}
+
+	env, cleanup := withTestEnvironment(t)
+	// must list keys more than once
+	env.gopts.backendTestHook = nil
+	defer cleanup()
+
+	testRunInit(t, env.gopts)
+
+	testRunKeyPasswd(t, "geheim2", env.gopts)
+	env.gopts.password = "geheim2"
+	t.Logf("changed password to %q", env.gopts.password)
+
+	for _, newPassword := range passwordList {
+		testRunKeyAddNewKey(t, newPassword, env.gopts)
+		t.Logf("added new password %q", newPassword)
+		env.gopts.password = newPassword
+		testRunKeyRemove(t, env.gopts, testRunKeyListOtherIDs(t, env.gopts))
+	}
+
+	env.gopts.password = passwordList[len(passwordList)-1]
+	t.Logf("testing access with last password %q\n", env.gopts.password)
+	rtest.OK(t, runKey(context.TODO(), env.gopts, []string{"list"}))
+	testRunCheck(t, env.gopts)
+
+	testRunKeyAddNewKeyUserHost(t, env.gopts)
+}
+
+type emptySaveBackend struct {
+	restic.Backend
+}
+
+func (b *emptySaveBackend) Save(ctx context.Context, h restic.Handle, _ restic.RewindReader) error {
+	return b.Backend.Save(ctx, h, restic.NewByteReader([]byte{}, nil))
+}
+
+func TestKeyProblems(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testRunInit(t, env.gopts)
+	env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
+		return &emptySaveBackend{r}, nil
+	}
+
+	testKeyNewPassword = "geheim2"
+	defer func() {
+		testKeyNewPassword = ""
+	}()
+
+	err := runKey(context.TODO(), env.gopts, []string{"passwd"})
+	t.Log(err)
+	rtest.Assert(t, err != nil, "expected passwd change to fail")
+
+	err = runKey(context.TODO(), env.gopts, []string{"add"})
+	t.Log(err)
+	rtest.Assert(t, err != nil, "expected key adding to fail")
+
+	t.Logf("testing access with initial password %q\n", env.gopts.password)
+	rtest.OK(t, runKey(context.TODO(), env.gopts, []string{"list"}))
+	testRunCheck(t, env.gopts)
+}

cmd/restic/cmd_list.go

@@ -31,19 +31,19 @@ func init() {
 	cmdRoot.AddCommand(cmdList)
 }
 
-func runList(ctx context.Context, cmd *cobra.Command, opts GlobalOptions, args []string) error {
+func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args []string) error {
 	if len(args) != 1 {
 		return errors.Fatal("type not specified, usage: " + cmd.Use)
 	}
 
-	repo, err := OpenRepository(ctx, opts)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
 
-	if !opts.NoLock && args[0] != "locks" {
+	if !gopts.NoLock && args[0] != "locks" {
 		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo)
+		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -63,7 +63,7 @@ func runList(ctx context.Context, cmd *cobra.Command, opts GlobalOptions, args [
 	case "locks":
 		t = restic.LockFile
 	case "blobs":
-		return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
+		return index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
 			if err != nil {
 				return err
 			}

cmd/restic/cmd_list_integration_test.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"io"
+	"testing"
+
+	"github.com/restic/restic/internal/restic"
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func testRunList(t testing.TB, tpe string, opts GlobalOptions) restic.IDs {
+	buf, err := withCaptureStdout(func() error {
+		return runList(context.TODO(), cmdList, opts, []string{tpe})
+	})
+	rtest.OK(t, err)
+	return parseIDsFromReader(t, buf)
+}
+
+func parseIDsFromReader(t testing.TB, rd io.Reader) restic.IDs {
+	t.Helper()
+	IDs := restic.IDs{}
+	sc := bufio.NewScanner(rd)
+
+	for sc.Scan() {
+		id, err := restic.ParseID(sc.Text())
+		if err != nil {
+			t.Logf("parse id %v: %v", sc.Text(), err)
+			continue
+		}
+
+		IDs = append(IDs, id)
+	}
+
+	return IDs
+}
+
+func testListSnapshots(t testing.TB, opts GlobalOptions, expected int) restic.IDs {
+	t.Helper()
+	snapshotIDs := testRunList(t, "snapshots", opts)
+	rtest.Assert(t, len(snapshotIDs) == expected, "expected %v snapshot, got %v", expected, snapshotIDs)
+	return snapshotIDs
+}

cmd/restic/cmd_ls.go

@@ -49,8 +49,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 // LsOptions collects all options for the ls command.
 type LsOptions struct {
 	ListLong bool
-	snapshotFilterOptions
-	Recursive bool
+	restic.SnapshotFilter
+	Recursive     bool
+	HumanReadable bool
 }
 
 var lsOptions LsOptions
@@ -59,9 +60,10 @@ func init() {
 	cmdRoot.AddCommand(cmdLs)
 
 	flags := cmdLs.Flags()
-	initSingleSnapshotFilterOptions(flags, &lsOptions.snapshotFilterOptions)
+	initSingleSnapshotFilter(flags, &lsOptions.SnapshotFilter)
 	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
 	flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories")
+	flags.BoolVar(&lsOptions.HumanReadable, "human-readable", false, "print sizes in human readable format")
 }
 
 type lsSnapshot struct {
@@ -85,6 +87,7 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
 		ModTime     time.Time   `json:"mtime,omitempty"`
 		AccessTime  time.Time   `json:"atime,omitempty"`
 		ChangeTime  time.Time   `json:"ctime,omitempty"`
+		Inode       uint64      `json:"inode,omitempty"`
 		StructType  string      `json:"struct_type"` // "node"
 
 		size uint64 // Target for Size pointer.
@@ -100,6 +103,7 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
 		ModTime:     node.ModTime,
 		AccessTime:  node.AccessTime,
 		ChangeTime:  node.ChangeTime,
+		Inode:       node.Inode,
 		StructType:  "node",
 	}
 	// Always print size for regular files, even when empty,
@@ -171,7 +175,8 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		return err
 	}
 
-	if err = repo.LoadIndex(ctx); err != nil {
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	if err = repo.LoadIndex(ctx, bar); err != nil {
 		return err
 	}
 
@@ -181,7 +186,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 	)
 
 	if gopts.JSON {
-		enc := json.NewEncoder(gopts.stdout)
+		enc := json.NewEncoder(globalOptions.stdout)
 
 		printSnapshot = func(sn *restic.Snapshot) {
 			err := enc.Encode(lsSnapshot{
@@ -206,11 +211,20 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 			Verbosef("snapshot %s of %v filtered by %v at %s):\n", sn.ID().Str(), sn.Paths, dirs, sn.Time)
 		}
 		printNode = func(path string, node *restic.Node) {
-			Printf("%s\n", formatNode(path, node, lsOptions.ListLong))
+			Printf("%s\n", formatNode(path, node, lsOptions.ListLong, lsOptions.HumanReadable))
 		}
 	}
 
-	sn, err := restic.FindFilteredSnapshot(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, nil, args[0])
+	sn, subfolder, err := (&restic.SnapshotFilter{
+		Hosts: opts.Hosts,
+		Paths: opts.Paths,
+		Tags:  opts.Tags,
+	}).FindLatest(ctx, snapshotLister, repo, args[0])
+	if err != nil {
+		return err
+	}
+
+	sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
 	if err != nil {
 		return err
 	}

cmd/restic/cmd_ls_integration_test.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
+	buf, err := withCaptureStdout(func() error {
+		gopts.Quiet = true
+		opts := LsOptions{}
+		return runLs(context.TODO(), opts, gopts, []string{snapshotID})
+	})
+	rtest.OK(t, err)
+	return strings.Split(buf.String(), "\n")
+}

cmd/restic/cmd_migrate.go

@@ -122,7 +122,7 @@ func runMigrate(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, a
 		return err
 	}
 
-	lock, ctx, err := lockRepoExclusive(ctx, repo)
+	lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
 	defer unlockRepo(lock)
 	if err != nil {
 		return err

cmd/restic/cmd_mount.go

@@ -77,7 +77,7 @@ type MountOptions struct {
 	OwnerRoot            bool
 	AllowOther           bool
 	NoDefaultPermissions bool
-	snapshotFilterOptions
+	restic.SnapshotFilter
 	TimeTemplate  string
 	PathTemplates []string
 }
@@ -92,7 +92,7 @@ func init() {
 	mountFlags.BoolVar(&mountOptions.AllowOther, "allow-other", false, "allow other users to access the data in the mounted directory")
 	mountFlags.BoolVar(&mountOptions.NoDefaultPermissions, "no-default-permissions", false, "for 'allow-other', ignore Unix permissions and allow users to read all snapshot files")
 
-	initMultiSnapshotFilterOptions(mountFlags, &mountOptions.snapshotFilterOptions, true)
+	initMultiSnapshotFilter(mountFlags, &mountOptions.SnapshotFilter, true)
 
 	mountFlags.StringArrayVar(&mountOptions.PathTemplates, "path-template", nil, "set `template` for path names (can be specified multiple times)")
 	mountFlags.StringVar(&mountOptions.TimeTemplate, "snapshot-template", time.RFC3339, "set `template` to use for snapshot dirs")
@@ -123,14 +123,15 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args
 
 	if !gopts.NoLock {
 		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo)
+		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
 		}
 	}
 
-	err = repo.LoadIndex(ctx)
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	err = repo.LoadIndex(ctx, bar)
 	if err != nil {
 		return err
 	}
@@ -180,9 +181,7 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args
 
 	cfg := fuse.Config{
 		OwnerIsRoot:   opts.OwnerRoot,
-		Hosts:         opts.Hosts,
-		Tags:          opts.Tags,
-		Paths:         opts.Paths,
+		Filter:        opts.SnapshotFilter,
 		TimeTemplate:  opts.TimeTemplate,
 		PathTemplates: opts.PathTemplates,
 	}

cmd/restic/cmd_mount_integration_test.go

@@ -12,6 +12,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
@@ -63,7 +64,7 @@ func testRunMount(t testing.TB, gopts GlobalOptions, dir string, wg *sync.WaitGr
 	rtest.OK(t, runMount(context.TODO(), opts, gopts, []string{dir}))
 }
 
-func testRunUmount(t testing.TB, gopts GlobalOptions, dir string) {
+func testRunUmount(t testing.TB, dir string) {
 	var err error
 	for i := 0; i < mountWait; i++ {
 		if err = umount(dir); err == nil {
@@ -94,7 +95,7 @@ func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Reposit
 	go testRunMount(t, global, mountpoint, &wg)
 	waitForMount(t, mountpoint)
 	defer wg.Wait()
-	defer testRunUmount(t, global, mountpoint)
+	defer testRunUmount(t, mountpoint)
 
 	if !snapshotsDirExists(t, mountpoint) {
 		t.Fatal(`virtual directory "snapshots" doesn't exist`)
@@ -159,6 +160,11 @@ func TestMount(t *testing.T) {
 		t.Skip("Skipping fuse tests")
 	}
 
+	debugEnabled := debug.TestLogToStderr(t)
+	if debugEnabled {
+		defer debug.TestDisableLog(t)
+	}
+
 	env, cleanup := withTestEnvironment(t)
 	// must list snapshots more than once
 	env.gopts.backendTestHook = nil

cmd/restic/cmd_prune.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"math"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -80,7 +81,7 @@ func addPruneOptions(c *cobra.Command) {
 func verifyPruneOptions(opts *PruneOptions) error {
 	opts.MaxRepackBytes = math.MaxUint64
 	if len(opts.MaxRepackSize) > 0 {
-		size, err := parseSizeStr(opts.MaxRepackSize)
+		size, err := ui.ParseBytes(opts.MaxRepackSize)
 		if err != nil {
 			return err
 		}
@@ -123,7 +124,7 @@ func verifyPruneOptions(opts *PruneOptions) error {
 		}
 
 	default:
-		size, err := parseSizeStr(maxUnused)
+		size, err := ui.ParseBytes(maxUnused)
 		if err != nil {
 			return errors.Fatalf("invalid number of bytes %q for --max-unused: %v", opts.MaxUnused, err)
 		}
@@ -167,7 +168,7 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error
 		opts.unsafeRecovery = true
 	}
 
-	lock, ctx, err := lockRepoExclusive(ctx, repo)
+	lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
 	defer unlockRepo(lock)
 	if err != nil {
 		return err
@@ -186,7 +187,8 @@ func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOption
 
 	Verbosef("loading indexes...\n")
 	// loading the index before the snapshots is ok, as we use an exclusive lock here
-	err := repo.LoadIndex(ctx)
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	err := repo.LoadIndex(ctx, bar)
 	if err != nil {
 		return err
 	}
@@ -205,6 +207,9 @@ func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOption
 		return err
 	}
 
+	// Trigger GC to reset garbage collection threshold
+	runtime.GC()
+
 	return doPrune(ctx, opts, gopts, repo, plan)
 }
 
@@ -473,7 +478,7 @@ func decidePackAction(ctx context.Context, opts PruneOptions, repo restic.Reposi
 	}
 
 	// loop over all packs and decide what to do
-	bar := newProgressMax(quiet, uint64(len(indexPack)), "packs processed")
+	bar := newProgressMax(!quiet, uint64(len(indexPack)), "packs processed")
 	err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
 		p, ok := indexPack[id]
 		if !ok {
@@ -488,7 +493,7 @@ func decidePackAction(ctx context.Context, opts PruneOptions, repo restic.Reposi
 			// Pack size does not fit and pack is needed => error
 			// If the pack is not needed, this is no error, the pack can
 			// and will be simply removed, see below.
-			Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic rebuild-index'.\n",
+			Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic repair index'.\n",
 				id.Str(), p.unusedSize+p.usedSize, packSize)
 			return errorSizeNotMatching
 		}
@@ -729,7 +734,7 @@ func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo r
 		_, err := repository.Repack(ctx, repo, repo, plan.repackPacks, plan.keepBlobs, bar)
 		bar.Done()
 		if err != nil {
-			return errors.Fatalf("%s", err)
+			return errors.Fatal(err.Error())
 		}
 
 		// Also remove repacked packs

cmd/restic/cmd_prune_integration_test.go

@@ -0,0 +1,221 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"path/filepath"
+	"testing"
+
+	"github.com/restic/restic/internal/restic"
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) {
+	oldHook := gopts.backendTestHook
+	gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return newListOnceBackend(r), nil }
+	defer func() {
+		gopts.backendTestHook = oldHook
+	}()
+	rtest.OK(t, runPrune(context.TODO(), opts, gopts))
+}
+
+func TestPrune(t *testing.T) {
+	testPruneVariants(t, false)
+	testPruneVariants(t, true)
+}
+
+func testPruneVariants(t *testing.T, unsafeNoSpaceRecovery bool) {
+	suffix := ""
+	if unsafeNoSpaceRecovery {
+		suffix = "-recovery"
+	}
+	t.Run("0"+suffix, func(t *testing.T) {
+		opts := PruneOptions{MaxUnused: "0%", unsafeRecovery: unsafeNoSpaceRecovery}
+		checkOpts := CheckOptions{ReadData: true, CheckUnused: true}
+		testPrune(t, opts, checkOpts)
+	})
+
+	t.Run("50"+suffix, func(t *testing.T) {
+		opts := PruneOptions{MaxUnused: "50%", unsafeRecovery: unsafeNoSpaceRecovery}
+		checkOpts := CheckOptions{ReadData: true}
+		testPrune(t, opts, checkOpts)
+	})
+
+	t.Run("unlimited"+suffix, func(t *testing.T) {
+		opts := PruneOptions{MaxUnused: "unlimited", unsafeRecovery: unsafeNoSpaceRecovery}
+		checkOpts := CheckOptions{ReadData: true}
+		testPrune(t, opts, checkOpts)
+	})
+
+	t.Run("CachableOnly"+suffix, func(t *testing.T) {
+		opts := PruneOptions{MaxUnused: "5%", RepackCachableOnly: true, unsafeRecovery: unsafeNoSpaceRecovery}
+		checkOpts := CheckOptions{ReadData: true}
+		testPrune(t, opts, checkOpts)
+	})
+	t.Run("Small", func(t *testing.T) {
+		opts := PruneOptions{MaxUnused: "unlimited", RepackSmall: true}
+		checkOpts := CheckOptions{ReadData: true, CheckUnused: true}
+		testPrune(t, opts, checkOpts)
+	})
+}
+
+func createPrunableRepo(t *testing.T, env *testEnvironment) {
+	testSetupBackupData(t, env)
+	opts := BackupOptions{}
+
+	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9")}, opts, env.gopts)
+	firstSnapshot := testListSnapshots(t, env.gopts, 1)[0]
+
+	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "2")}, opts, env.gopts)
+	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "3")}, opts, env.gopts)
+	testListSnapshots(t, env.gopts, 3)
+
+	testRunForgetJSON(t, env.gopts)
+	testRunForget(t, env.gopts, firstSnapshot.String())
+}
+
+func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) {
+	buf, err := withCaptureStdout(func() error {
+		gopts.JSON = true
+		opts := ForgetOptions{
+			DryRun: true,
+			Last:   1,
+		}
+		return runForget(context.TODO(), opts, gopts, args)
+	})
+	rtest.OK(t, err)
+
+	var forgets []*ForgetGroup
+	rtest.OK(t, json.Unmarshal(buf.Bytes(), &forgets))
+
+	rtest.Assert(t, len(forgets) == 1,
+		"Expected 1 snapshot group, got %v", len(forgets))
+	rtest.Assert(t, len(forgets[0].Keep) == 1,
+		"Expected 1 snapshot to be kept, got %v", len(forgets[0].Keep))
+	rtest.Assert(t, len(forgets[0].Remove) == 2,
+		"Expected 2 snapshots to be removed, got %v", len(forgets[0].Remove))
+}
+
+func testPrune(t *testing.T, pruneOpts PruneOptions, checkOpts CheckOptions) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	createPrunableRepo(t, env)
+	testRunPrune(t, env.gopts, pruneOpts)
+	rtest.OK(t, runCheck(context.TODO(), checkOpts, env.gopts, nil))
+}
+
+var pruneDefaultOptions = PruneOptions{MaxUnused: "5%"}
+
+func TestPruneWithDamagedRepository(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	datafile := filepath.Join("testdata", "backup-data.tar.gz")
+	testRunInit(t, env.gopts)
+
+	rtest.SetupTarTestFixture(t, env.testdata, datafile)
+	opts := BackupOptions{}
+
+	// create and delete snapshot to create unused blobs
+	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "2")}, opts, env.gopts)
+	firstSnapshot := testListSnapshots(t, env.gopts, 1)[0]
+	testRunForget(t, env.gopts, firstSnapshot.String())
+
+	oldPacks := listPacks(env.gopts, t)
+
+	// create new snapshot, but lose all data
+	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "3")}, opts, env.gopts)
+	testListSnapshots(t, env.gopts, 1)
+	removePacksExcept(env.gopts, t, oldPacks, false)
+
+	oldHook := env.gopts.backendTestHook
+	env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return newListOnceBackend(r), nil }
+	defer func() {
+		env.gopts.backendTestHook = oldHook
+	}()
+	// prune should fail
+	rtest.Assert(t, runPrune(context.TODO(), pruneDefaultOptions, env.gopts) == errorPacksMissing,
+		"prune should have reported index not complete error")
+}
+
+// Test repos for edge cases
+func TestEdgeCaseRepos(t *testing.T) {
+	opts := CheckOptions{}
+
+	// repo where index is completely missing
+	// => check and prune should fail
+	t.Run("no-index", func(t *testing.T) {
+		testEdgeCaseRepo(t, "repo-index-missing.tar.gz", opts, pruneDefaultOptions, false, false)
+	})
+
+	// repo where an existing and used blob is missing from the index
+	// => check and prune should fail
+	t.Run("index-missing-blob", func(t *testing.T) {
+		testEdgeCaseRepo(t, "repo-index-missing-blob.tar.gz", opts, pruneDefaultOptions, false, false)
+	})
+
+	// repo where a blob is missing
+	// => check and prune should fail
+	t.Run("missing-data", func(t *testing.T) {
+		testEdgeCaseRepo(t, "repo-data-missing.tar.gz", opts, pruneDefaultOptions, false, false)
+	})
+
+	// repo where blobs which are not needed are missing or in invalid pack files
+	// => check should fail and prune should repair this
+	t.Run("missing-unused-data", func(t *testing.T) {
+		testEdgeCaseRepo(t, "repo-unused-data-missing.tar.gz", opts, pruneDefaultOptions, false, true)
+	})
+
+	// repo where data exists that is not referenced
+	// => check and prune should fully work
+	t.Run("unreferenced-data", func(t *testing.T) {
+		testEdgeCaseRepo(t, "repo-unreferenced-data.tar.gz", opts, pruneDefaultOptions, true, true)
+	})
+
+	// repo where an obsolete index still exists
+	// => check and prune should fully work
+	t.Run("obsolete-index", func(t *testing.T) {
+		testEdgeCaseRepo(t, "repo-obsolete-index.tar.gz", opts, pruneDefaultOptions, true, true)
+	})
+
+	// repo which contains mixed (data/tree) packs
+	// => check and prune should fully work
+	t.Run("mixed-packs", func(t *testing.T) {
+		testEdgeCaseRepo(t, "repo-mixed.tar.gz", opts, pruneDefaultOptions, true, true)
+	})
+
+	// repo which contains duplicate blobs
+	// => checking for unused data should report an error and prune resolves the
+	// situation
+	opts = CheckOptions{
+		ReadData:    true,
+		CheckUnused: true,
+	}
+	t.Run("duplicates", func(t *testing.T) {
+		testEdgeCaseRepo(t, "repo-duplicates.tar.gz", opts, pruneDefaultOptions, false, true)
+	})
+}
+
+func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, optionsPrune PruneOptions, checkOK, pruneOK bool) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	datafile := filepath.Join("testdata", tarfile)
+	rtest.SetupTarTestFixture(t, env.base, datafile)
+
+	if checkOK {
+		testRunCheck(t, env.gopts)
+	} else {
+		rtest.Assert(t, runCheck(context.TODO(), optionsCheck, env.gopts, nil) != nil,
+			"check should have reported an error")
+	}
+
+	if pruneOK {
+		testRunPrune(t, env.gopts, optionsPrune)
+		testRunCheck(t, env.gopts)
+	} else {
+		rtest.Assert(t, runPrune(context.TODO(), optionsPrune, env.gopts) != nil,
+			"prune should have reported an error")
+	}
+}

cmd/restic/cmd_recover.go

@@ -46,7 +46,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
 		return err
 	}
 
-	lock, ctx, err := lockRepo(ctx, repo)
+	lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
 	defer unlockRepo(lock)
 	if err != nil {
 		return err
@@ -58,7 +58,8 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
 	}
 
 	Verbosef("load index files\n")
-	if err = repo.LoadIndex(ctx); err != nil {
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	if err = repo.LoadIndex(ctx, bar); err != nil {
 		return err
 	}
 
@@ -73,7 +74,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
 	})
 
 	Verbosef("load %d trees\n", len(trees))
-	bar := newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
+	bar = newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
 	for id := range trees {
 		tree, err := restic.LoadTree(ctx, repo, id)
 		if err != nil {

cmd/restic/cmd_repair.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"github.com/spf13/cobra"
+)
+
+var cmdRepair = &cobra.Command{
+	Use:   "repair",
+	Short: "Repair the repository",
+}
+
+func init() {
+	cmdRoot.AddCommand(cmdRepair)
+}

cmd/restic/cmd_repair_index.go

@@ -7,15 +7,15 @@ import (
 	"github.com/restic/restic/internal/pack"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
-
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 )
 
-var cmdRebuildIndex = &cobra.Command{
-	Use:   "rebuild-index [flags]",
+var cmdRepairIndex = &cobra.Command{
+	Use:   "index [flags]",
 	Short: "Build a new index",
 	Long: `
-The "rebuild-index" command creates a new index based on the pack files in the
+The "repair index" command creates a new index based on the pack files in the
 repository.
 
 EXIT STATUS
@@ -25,40 +25,52 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runRebuildIndex(cmd.Context(), rebuildIndexOptions, globalOptions)
+		return runRebuildIndex(cmd.Context(), repairIndexOptions, globalOptions)
 	},
 }
 
-// RebuildIndexOptions collects all options for the rebuild-index command.
-type RebuildIndexOptions struct {
+var cmdRebuildIndex = &cobra.Command{
+	Use:               "rebuild-index [flags]",
+	Short:             cmdRepairIndex.Short,
+	Long:              cmdRepairIndex.Long,
+	Deprecated:        `Use "repair index" instead`,
+	DisableAutoGenTag: true,
+	RunE:              cmdRepairIndex.RunE,
+}
+
+// RepairIndexOptions collects all options for the repair index command.
+type RepairIndexOptions struct {
 	ReadAllPacks bool
 }
 
-var rebuildIndexOptions RebuildIndexOptions
+var repairIndexOptions RepairIndexOptions
 
 func init() {
+	cmdRepair.AddCommand(cmdRepairIndex)
+	// add alias for old name
 	cmdRoot.AddCommand(cmdRebuildIndex)
-	f := cmdRebuildIndex.Flags()
-	f.BoolVar(&rebuildIndexOptions.ReadAllPacks, "read-all-packs", false, "read all pack files to generate new index from scratch")
 
+	for _, f := range []*pflag.FlagSet{cmdRepairIndex.Flags(), cmdRebuildIndex.Flags()} {
+		f.BoolVar(&repairIndexOptions.ReadAllPacks, "read-all-packs", false, "read all pack files to generate new index from scratch")
+	}
 }
 
-func runRebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOptions) error {
+func runRebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions) error {
 	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
 
-	lock, ctx, err := lockRepoExclusive(ctx, repo)
+	lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
 	defer unlockRepo(lock)
 	if err != nil {
 		return err
 	}
 
-	return rebuildIndex(ctx, opts, gopts, repo, restic.NewIDSet())
+	return rebuildIndex(ctx, opts, gopts, repo)
 }
 
-func rebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error {
+func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions, repo *repository.Repository) error {
 	var obsoleteIndexes restic.IDs
 	packSizeFromList := make(map[restic.ID]int64)
 	packSizeFromIndex := make(map[restic.ID]int64)
@@ -76,7 +88,7 @@ func rebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOpt
 	} else {
 		Verbosef("loading indexes...\n")
 		mi := index.NewMasterIndex()
-		err := index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
+		err := index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
 			if err != nil {
 				Warnf("removing invalid index %v: %v\n", id, err)
 				obsoleteIndexes = append(obsoleteIndexes, id)
@@ -130,7 +142,7 @@ func rebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOpt
 
 	if len(packSizeFromList) > 0 {
 		Verbosef("reading pack files\n")
-		bar := newProgressMax(!globalOptions.Quiet, uint64(len(packSizeFromList)), "packs")
+		bar := newProgressMax(!gopts.Quiet, uint64(len(packSizeFromList)), "packs")
 		invalidFiles, err := repo.CreateIndexFromPacks(ctx, packSizeFromList, bar)
 		bar.Done()
 		if err != nil {

cmd/restic/cmd_repair_index_integration_test.go

@@ -0,0 +1,140 @@
+package main
+
+import (
+	"context"
+	"io"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/index"
+	"github.com/restic/restic/internal/restic"
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func testRunRebuildIndex(t testing.TB, gopts GlobalOptions) {
+	rtest.OK(t, withRestoreGlobalOptions(func() error {
+		globalOptions.stdout = io.Discard
+		return runRebuildIndex(context.TODO(), RepairIndexOptions{}, gopts)
+	}))
+}
+
+func testRebuildIndex(t *testing.T, backendTestHook backendWrapper) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	datafile := filepath.Join("..", "..", "internal", "checker", "testdata", "duplicate-packs-in-index-test-repo.tar.gz")
+	rtest.SetupTarTestFixture(t, env.base, datafile)
+
+	out, err := testRunCheckOutput(env.gopts, false)
+	if !strings.Contains(out, "contained in several indexes") {
+		t.Fatalf("did not find checker hint for packs in several indexes")
+	}
+
+	if err != nil {
+		t.Fatalf("expected no error from checker for test repository, got %v", err)
+	}
+
+	if !strings.Contains(out, "restic repair index") {
+		t.Fatalf("did not find hint for repair index command")
+	}
+
+	env.gopts.backendTestHook = backendTestHook
+	testRunRebuildIndex(t, env.gopts)
+
+	env.gopts.backendTestHook = nil
+	out, err = testRunCheckOutput(env.gopts, false)
+	if len(out) != 0 {
+		t.Fatalf("expected no output from the checker, got: %v", out)
+	}
+
+	if err != nil {
+		t.Fatalf("expected no error from checker after repair index, got: %v", err)
+	}
+}
+
+func TestRebuildIndex(t *testing.T) {
+	testRebuildIndex(t, nil)
+}
+
+func TestRebuildIndexAlwaysFull(t *testing.T) {
+	indexFull := index.IndexFull
+	defer func() {
+		index.IndexFull = indexFull
+	}()
+	index.IndexFull = func(*index.Index, bool) bool { return true }
+	testRebuildIndex(t, nil)
+}
+
+// indexErrorBackend modifies the first index after reading.
+type indexErrorBackend struct {
+	restic.Backend
+	lock     sync.Mutex
+	hasErred bool
+}
+
+func (b *indexErrorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
+	return b.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error {
+		// protect hasErred
+		b.lock.Lock()
+		defer b.lock.Unlock()
+		if !b.hasErred && h.Type == restic.IndexFile {
+			b.hasErred = true
+			return consumer(errorReadCloser{rd})
+		}
+		return consumer(rd)
+	})
+}
+
+type errorReadCloser struct {
+	io.Reader
+}
+
+func (erd errorReadCloser) Read(p []byte) (int, error) {
+	n, err := erd.Reader.Read(p)
+	if n > 0 {
+		p[0] ^= 1
+	}
+	return n, err
+}
+
+func TestRebuildIndexDamage(t *testing.T) {
+	testRebuildIndex(t, func(r restic.Backend) (restic.Backend, error) {
+		return &indexErrorBackend{
+			Backend: r,
+		}, nil
+	})
+}
+
+type appendOnlyBackend struct {
+	restic.Backend
+}
+
+// called via repo.Backend().Remove()
+func (b *appendOnlyBackend) Remove(_ context.Context, h restic.Handle) error {
+	return errors.Errorf("Failed to remove %v", h)
+}
+
+func TestRebuildIndexFailsOnAppendOnly(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	datafile := filepath.Join("..", "..", "internal", "checker", "testdata", "duplicate-packs-in-index-test-repo.tar.gz")
+	rtest.SetupTarTestFixture(t, env.base, datafile)
+
+	err := withRestoreGlobalOptions(func() error {
+		globalOptions.stdout = io.Discard
+
+		env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
+			return &appendOnlyBackend{r}, nil
+		}
+		return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts)
+	})
+
+	if err == nil {
+		t.Error("expected rebuildIndex to fail")
+	}
+	t.Log(err)
+}

cmd/restic/cmd_repair_packs.go

@@ -0,0 +1,158 @@
+package main
+
+import (
+	"context"
+	"io"
+	"os"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
+	"github.com/spf13/cobra"
+	"golang.org/x/sync/errgroup"
+)
+
+var cmdRepairPacks = &cobra.Command{
+	Use:   "packs [packIDs...]",
+	Short: "Salvage damaged pack files",
+	Long: `
+WARNING: The CLI for this command is experimental and will likely change in the future!
+
+The "repair packs" command extracts intact blobs from the specified pack files, rebuilds
+the index to remove the damaged pack files and removes the pack files from the repository.
+
+EXIT STATUS
+===========
+
+Exit status is 0 if the command was successful, and non-zero if there was any error.
+`,
+	DisableAutoGenTag: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runRepairPacks(cmd.Context(), globalOptions, args)
+	},
+}
+
+func init() {
+	cmdRepair.AddCommand(cmdRepairPacks)
+}
+
+func runRepairPacks(ctx context.Context, gopts GlobalOptions, args []string) error {
+	// FIXME discuss and add proper feature flag mechanism
+	flag, _ := os.LookupEnv("RESTIC_FEATURES")
+	if flag != "repair-packs-v1" {
+		return errors.Fatal("This command is experimental and may change/be removed without notice between restic versions. " +
+			"Set the environment variable 'RESTIC_FEATURES=repair-packs-v1' to enable it.")
+	}
+
+	ids := restic.NewIDSet()
+	for _, arg := range args {
+		id, err := restic.ParseID(arg)
+		if err != nil {
+			return err
+		}
+		ids.Insert(id)
+	}
+	if len(ids) == 0 {
+		return errors.Fatal("no ids specified")
+	}
+
+	repo, err := OpenRepository(ctx, gopts)
+	if err != nil {
+		return err
+	}
+
+	lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	return repairPacks(ctx, gopts, repo, ids)
+}
+
+func repairPacks(ctx context.Context, gopts GlobalOptions, repo *repository.Repository, ids restic.IDSet) error {
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	err := repo.LoadIndex(ctx, bar)
+	if err != nil {
+		return errors.Fatalf("%s", err)
+	}
+
+	Warnf("saving backup copies of pack files in current folder\n")
+	for id := range ids {
+		f, err := os.OpenFile("pack-"+id.String(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o666)
+		if err != nil {
+			return errors.Fatalf("%s", err)
+		}
+
+		err = repo.Backend().Load(ctx, restic.Handle{Type: restic.PackFile, Name: id.String()}, 0, 0, func(rd io.Reader) error {
+			_, err := f.Seek(0, 0)
+			if err != nil {
+				return err
+			}
+			_, err = io.Copy(f, rd)
+			return err
+		})
+		if err != nil {
+			return errors.Fatalf("%s", err)
+		}
+	}
+
+	wg, wgCtx := errgroup.WithContext(ctx)
+	repo.StartPackUploader(wgCtx, wg)
+	repo.DisableAutoIndexUpdate()
+
+	Warnf("salvaging intact data from specified pack files\n")
+	bar = newProgressMax(!gopts.Quiet, uint64(len(ids)), "pack files")
+	defer bar.Done()
+
+	wg.Go(func() error {
+		// examine all data the indexes have for the pack file
+		for b := range repo.Index().ListPacks(wgCtx, ids) {
+			blobs := b.Blobs
+			if len(blobs) == 0 {
+				Warnf("no blobs found for pack %v\n", b.PackID)
+				bar.Add(1)
+				continue
+			}
+
+			err = repository.StreamPack(wgCtx, repo.Backend().Load, repo.Key(), b.PackID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error {
+				if err != nil {
+					// Fallback path
+					buf, err = repo.LoadBlob(wgCtx, blob.Type, blob.ID, nil)
+					if err != nil {
+						Warnf("failed to load blob %v: %v\n", blob.ID, err)
+						return nil
+					}
+				}
+				id, _, _, err := repo.SaveBlob(wgCtx, blob.Type, buf, restic.ID{}, true)
+				if !id.Equal(blob.ID) {
+					panic("pack id mismatch during upload")
+				}
+				return err
+			})
+			if err != nil {
+				return err
+			}
+			bar.Add(1)
+		}
+		return repo.Flush(wgCtx)
+	})
+
+	if err := wg.Wait(); err != nil {
+		return errors.Fatalf("%s", err)
+	}
+	bar.Done()
+
+	// remove salvaged packs from index
+	err = rebuildIndexFiles(ctx, gopts, repo, ids, nil)
+	if err != nil {
+		return errors.Fatalf("%s", err)
+	}
+
+	// cleanup
+	Warnf("removing salvaged pack files\n")
+	DeleteFiles(ctx, gopts, repo, ids, restic.PackFile)
+
+	Warnf("\nUse `restic repair snapshots --forget` to remove the corrupted data blobs from all snapshots\n")
+	return nil
+}