wip

check: Suggest usage of `repair packs` if pack files are damaged

.github/ISSUE_TEMPLATE/Bug.md

@@ -32,23 +32,30 @@ Output of `restic version`
 --------------------------
 
 
-How did you run restic exactly?
--------------------------------
+
+What backend/service did you use to store the repository?
+---------------------------------------------------------
+
+
+
+Problem description / Steps to reproduce
+----------------------------------------
 
 <!--
 This section should include at least:
 
+ * A description of the problem you are having with restic.
+
  * The complete command line and any environment variables you used to
    configure restic's backend access. Make sure to replace sensitive values!
 
  * The output of the commands, what restic prints gives may give us much
    information to diagnose the problem!
+
+ * The more time you spend describing an easy way to reproduce the behavior (if
+   this is possible), the easier it is for the project developers to fix it!
 -->
 
-What backend/server/service did you use to store the repository?
-----------------------------------------------------------------
-
-
 
 Expected behavior
 -----------------
@@ -65,22 +72,12 @@ In this section, please try to concentrate on observations, so only describe
 what you observed directly.
 -->
 
-Steps to reproduce the behavior
--------------------------------
-
-<!--
-The more time you spend describing an easy way to reproduce the behavior (if
-this is possible), the easier it is for the project developers to fix it!
--->
-
 Do you have any idea what may have caused this?
 -----------------------------------------------
 
-
-
-Do you have an idea how to solve the issue?
--------------------------------------------
-
+<!--
+Did something noteworthy happen on your system, Internet connection, backend services, etc?
+-->
 
 
 Did restic help you today? Did it make you happy in any way?

.github/workflows/docker.yml

@@ -22,10 +22,10 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Log in to the Container registry
-        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        uses: docker/login-action@3d58c274f17dffee475a5520cbe67f0a882c4dbb
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -42,10 +42,17 @@ jobs:
             type=semver,pattern={{major}}.{{minor}}
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226
+
+      - name: Ensure consistent binaries
+        run: |
+          echo "removing git directory for consistency with release binaries"
+          rm -rf .git
+        # remove VCS information from release builds, keep VCS for nightly builds on master
+        if: github.ref != 'refs/heads/master'
 
       - name: Build and push Docker image
         uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4

.github/workflows/tests.yml

@@ -13,7 +13,7 @@ permissions:
   contents: read
 
 env:
-  latest_go: "1.20.x"
+  latest_go: "1.22.x"
   GO111MODULE: on
 
 jobs:
@@ -23,34 +23,39 @@ jobs:
         # list of jobs to run:
         include:
           - job_name: Windows
-            go: 1.20.x
+            go: 1.22.x
             os: windows-latest
 
           - job_name: macOS
-            go: 1.20.x
+            go: 1.22.x
             os: macOS-latest
             test_fuse: false
 
           - job_name: Linux
-            go: 1.20.x
+            go: 1.22.x
             os: ubuntu-latest
             test_cloud_backends: true
             test_fuse: true
             check_changelog: true
 
           - job_name: Linux (race)
-            go: 1.20.x
+            go: 1.22.x
             os: ubuntu-latest
             test_fuse: true
             test_opts: "-race"
 
           - job_name: Linux
-            go: 1.19.x
+            go: 1.21.x
             os: ubuntu-latest
             test_fuse: true
 
           - job_name: Linux
-            go: 1.18.x
+            go: 1.20.x
+            os: ubuntu-latest
+            test_fuse: true
+
+          - job_name: Linux
+            go: 1.19.x
             os: ubuntu-latest
             test_fuse: true
 
@@ -62,7 +67,7 @@ jobs:
 
     steps:
       - name: Set up Go ${{ matrix.go }}
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go }}
 
@@ -135,7 +140,7 @@ jobs:
         if: matrix.os == 'windows-latest'
 
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Build with build.go
         run: |
@@ -226,12 +231,12 @@ jobs:
 
     steps:
       - name: Set up Go ${{ env.latest_go }}
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ env.latest_go }}
 
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Cross-compile for subset ${{ matrix.subset }}
         run: |
@@ -244,18 +249,18 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set up Go ${{ env.latest_go }}
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ env.latest_go }}
 
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.52.2
+          version: v1.56.1
           args: --verbose --timeout 5m
 
         # only run golangci-lint for pull requests, otherwise ALL hints get
@@ -289,7 +294,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Docker meta
         id: meta
@@ -309,10 +314,10 @@ jobs:
             type=sha
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Build and push
         id: docker_build

.gitignore

@@ -1,3 +1,4 @@
+/.idea
 /restic
 /restic.exe
 /.vagrant

.golangci.yml

@@ -35,6 +35,9 @@ linters:
     # parse and typecheck code
     - typecheck
 
+    # ensure that http response bodies are closed
+    - bodyclose
+
 issues:
   # don't use the default exclude rules, this hides (among others) ignored
   # errors from Close() calls
@@ -51,3 +54,8 @@ issues:
     # staticcheck: there's no easy way to replace these packages
     - "SA1019: \"golang.org/x/crypto/poly1305\" is deprecated"
     - "SA1019: \"golang.org/x/crypto/openpgp\" is deprecated"
+
+  exclude-rules:
+    # revive: ignore unused parameters in tests
+    - path: (_test\.go|testing\.go|backend/.*/tests\.go)
+      text: "unused-parameter:"

.readthedocs.yaml

@@ -0,0 +1,22 @@
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+# Build HTMLZip
+formats:
+  - htmlzip
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: doc/conf.py
+
+# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+  - requirements: doc/requirements.txt

CONTRIBUTING.md

@@ -6,7 +6,8 @@ Ways to Help Out
 Thank you for your contribution! Please **open an issue first** (or add a
 comment to an existing issue) if you plan to work on any code or add a new
 feature. This way, duplicate work is prevented and we can discuss your ideas
-and design first.
+and design first. Small bugfixes are an exception to this rule, just open a
+pull request in this case.
 
 There are several ways you can help us out. First of all code contributions and
 bug fixes are most welcome. However even "minor" details as fixing spelling
@@ -61,7 +62,7 @@ uploading it somewhere or post only the parts that are really relevant.
 If restic gets stuck, please also include a stacktrace in the description.
 On non-Windows systems, you can send a SIGQUIT signal to restic or press
 `Ctrl-\` to achieve the same result. This causes restic to print a stacktrace
-and then exit immediatelly. This will not damage your repository, however,
+and then exit immediately. This will not damage your repository, however,
 it might be necessary to manually clean up stale lock files using
 `restic unlock`.
 

README.md

@@ -95,7 +95,7 @@ release. Instructions on how to do that are contained in the
 News
 ----
 
-You can follow the restic project on Twitter [@resticbackup](https://twitter.com/resticbackup) or by subscribing to
+You can follow the restic project on Mastodon [@resticbackup](https://fosstodon.org/@restic) or by subscribing to
 the [project blog](https://restic.net/blog/).
 
 License

VERSION

@@ -1 +1 @@
-0.16.0
+0.16.4

changelog/0.10.0_2020-09-19/pull-2668

@@ -1,6 +1,6 @@
 Bugfix: Don't abort the stats command when data blobs are missing
 
-Runing the stats command in the blobs-per-file mode on a repository with
+Running the stats command in the blobs-per-file mode on a repository with
 missing data blobs previously resulted in a crash.
 
 https://github.com/restic/restic/pull/2668

changelog/0.11.0_2020-11-05/issue-1756

@@ -1,6 +1,6 @@
 Bugfix: Mark repository files as read-only when using the local backend
 
-Files stored in a local repository were marked as writeable on the
+Files stored in a local repository were marked as writable on the
 filesystem for non-Windows systems, which did not prevent accidental file
 modifications outside of restic. In addition, the local backend did not work
 with certain filesystems and network mounts which do not permit modifications

changelog/0.11.0_2020-11-05/issue-340

@@ -5,7 +5,7 @@ another process using an exclusive lock through a filesystem snapshot. Restic
 was unable to backup those files before. This update enables backing up these
 files.
 
-This needs to be enabled explicitely using the --use-fs-snapshot option of the
+This needs to be enabled explicitly using the --use-fs-snapshot option of the
 backup command.
 
 https://github.com/restic/restic/issues/340

changelog/0.12.0_2021-02-14/pull-3106

@@ -2,7 +2,7 @@ Enhancement: Parallelize scan of snapshot content in `copy` and `prune`
 
 The `copy` and `prune` commands used to traverse the directories of
 snapshots one by one to find used data. This snapshot traversal is
-now parallized which can speed up this step several times.
+now parallelized which can speed up this step several times.
 
 In addition the `check` command now reports how many snapshots have
 already been processed.

changelog/0.14.0_2022-08-25/pull-3475

@@ -3,7 +3,7 @@ Enhancement: Allow limiting IO concurrency for local and SFTP backend
 Restic did not support limiting the IO concurrency / number of connections for
 accessing repositories stored using the local or SFTP backends. The number of
 connections is now limited as for other backends, and can be configured via the
-the `-o local.connections=2` and `-o sftp.connections=5` options. This ensures
-that restic does not overwhelm the backend with concurrent IO operations.
+`-o local.connections=2` and `-o sftp.connections=5` options. This ensures that
+restic does not overwhelm the backend with concurrent IO operations.
 
 https://github.com/restic/restic/pull/3475

changelog/0.16.1_2023-10-24/issue-4128

@@ -0,0 +1,9 @@
+Enhancement: Automatically set `GOMAXPROCS` in resource-constrained containers
+
+When running restic in a Linux container with CPU-usage limits, restic now
+automatically adjusts `GOMAXPROCS`. This helps to reduce the memory consumption
+on hosts with many CPU cores.
+
+https://github.com/restic/restic/issues/4128
+https://github.com/restic/restic/pull/4485
+https://github.com/restic/restic/pull/4531

changelog/0.16.1_2023-10-24/issue-4513

@@ -0,0 +1,8 @@
+Bugfix: Make `key list` command honor `--no-lock`
+
+The `key list` command now supports the `--no-lock` options. This allows
+determining which keys a repo can be accessed by without the need for having
+write access (e.g., read-only sftp access, filesystem snapshot).
+
+https://github.com/restic/restic/issues/4513
+https://github.com/restic/restic/pull/4514

changelog/0.16.1_2023-10-24/issue-4516

@@ -0,0 +1,8 @@
+Bugfix: Do not try to load password on command line autocomplete
+
+The command line autocompletion previously tried to load the repository
+password. This could cause the autocompletion not to work. Now, this step gets
+skipped.
+
+https://github.com/restic/restic/issues/4516
+https://github.com/restic/restic/pull/4526

changelog/0.16.1_2023-10-24/issue-4523

@@ -0,0 +1,22 @@
+Bugfix: Update zstd library to fix possible data corruption at max. compression
+
+In restic 0.16.0, backups where the compression level was set to `max` (using
+`--compression max`) could in rare and very specific circumstances result in
+data corruption due to a bug in the library used for compressing data.
+
+Restic now uses the latest version of the library used to compress data, which
+includes a fix for this issue. Please note that the `auto` compression level
+(which restic uses by default) was never affected, and even if you used `max`
+compression, chances of being affected by this issue were very small.
+
+To check a repository for any corruption, run `restic check --read-data`. This
+will download and verify the whole repository and can be used at any time to
+completely verify the integrity of a repository. If the `check` command detects
+anomalies, follow the suggested steps.
+
+To simplify any needed repository repair and minimize data loss, there is also
+a new and experimental `repair packs` command that salvages all valid data from
+the affected pack files (see `restic help repair packs` for more information).
+
+https://github.com/restic/restic/issues/4523
+https://github.com/restic/restic/pull/4530

changelog/0.16.1_2023-10-24/pull-299

@@ -0,0 +1,7 @@
+Enhancement: Show progress bar while loading the index
+
+Restic did not provide any feedback while loading index files. Now, there is a
+progress bar that shows the index loading progress.
+
+https://github.com/restic/restic/issues/229
+https://github.com/restic/restic/pull/4419

changelog/0.16.1_2023-10-24/pull-4480

@@ -0,0 +1,11 @@
+Enhancement: Allow setting REST password and username via environment variables
+
+Previously, it was only possible to specify the REST-server username and
+password in the repository URL, or by using the `--repository-file` option.
+This meant it was not possible to use authentication in contexts where the
+repository URL is stored in publicly accessible way.
+
+Restic now allows setting the username and password using the
+`RESTIC_REST_USERNAME` and `RESTIC_REST_PASSWORD` variables.
+
+https://github.com/restic/restic/pull/4480

changelog/0.16.1_2023-10-24/pull-4511

@@ -0,0 +1,7 @@
+Enhancement: Include inode numbers in JSON output for `find` and `ls` commands
+
+Restic used to omit the inode numbers in the JSON messages emitted for nodes by
+the `ls` command as well as for matches by the `find` command. It now includes
+those values whenever they are available.
+
+https://github.com/restic/restic/pull/4511

changelog/0.16.1_2023-10-24/pull-4519

@@ -0,0 +1,12 @@
+Enhancement: Add config option to set SFTP command arguments
+
+When using the `sftp` backend, scenarios where a custom identity file was
+needed for the SSH connection, required the full command to be specified:
+`-o sftp.command='ssh user@host:port -i /ssh/my_private_key -s sftp'`
+
+Now, the `-o sftp.args=...` option can be passed to restic to specify
+custom arguments for the SSH command executed by the SFTP backend.
+This simplifies the above example to `-o sftp.args='-i /ssh/my_private_key'`.
+
+https://github.com/restic/restic/pull/4519
+https://github.com/restic/restic/issues/4241

changelog/0.16.1_2023-10-24/pull-4532

@@ -0,0 +1,8 @@
+Change: Update dependencies and require Go 1.19 or newer
+
+We have updated all dependencies. Since some libraries require newer Go
+standard library features, support for Go 1.18 has been dropped, which means
+that restic now requires at least Go 1.19 to build.
+
+https://github.com/restic/restic/pull/4532
+https://github.com/restic/restic/pull/4533

changelog/0.16.2_2023-10-29/issue-4540

@@ -0,0 +1,9 @@
+Bugfix: Restore ARMv5 support for ARM binaries
+
+The official release binaries for restic 0.16.1 were accidentally built to
+require ARMv7. The build process is now updated to restore support for ARMv5.
+
+Please note that restic 0.17.0 will drop support for ARMv5 and require at least
+ARMv6.
+
+https://github.com/restic/restic/issues/4540

changelog/0.16.2_2023-10-29/pull-4545

@@ -0,0 +1,8 @@
+Bugfix: Repair documentation build on Read the Docs
+
+For restic 0.16.1, no documentation was available at
+https://restic.readthedocs.io/ .
+
+The documentation build process is now updated to work again.
+
+https://github.com/restic/restic/pull/4545

changelog/0.16.3_2024-01-14/issue-4560

@@ -0,0 +1,14 @@
+Bugfix: Improve errors for irregular files on Windows
+
+Since Go 1.21, most filesystem reparse points on Windows are considered to be
+irregular files. This caused restic to show an `error: invalid node type ""`
+error message for those files.
+
+This error message has now been improved and includes the relevant file path:
+`error: nodeFromFileInfo path/to/file: unsupported file type "irregular"`.
+As irregular files are not required to behave like regular files, it is not
+possible to provide a generic way to back up those files.
+
+https://github.com/restic/restic/issues/4560
+https://github.com/restic/restic/pull/4620
+https://forum.restic.net/t/windows-backup-error-invalid-node-type/6875

changelog/0.16.3_2024-01-14/issue-4574

@@ -0,0 +1,11 @@
+Bugfix: Support backup of deduplicated files on Windows again
+
+With the official release builds of restic 0.16.1 and 0.16.2, it was not
+possible to back up files that were deduplicated by the corresponding
+Windows Server feature. This also applied to restic versions built using
+Go 1.21.0-1.21.4.
+
+The Go version used to build restic has now been updated to fix this.
+
+https://github.com/restic/restic/issues/4574
+https://github.com/restic/restic/pull/4621

changelog/0.16.3_2024-01-14/issue-4612

@@ -0,0 +1,11 @@
+Bugfix: Improve error handling for `rclone` backend
+
+Since restic 0.16.0, if rclone encountered an error while listing files,
+this could in rare circumstances cause restic to assume that there are no
+files. Although unlikely, this situation could result in data loss if it
+were to happen right when the `prune` command is listing existing snapshots.
+
+Error handling has now been improved to detect and work around this case.
+
+https://github.com/restic/restic/issues/4612
+https://github.com/restic/restic/pull/4618

changelog/0.16.3_2024-01-14/pull-4624

@@ -0,0 +1,11 @@
+Bugfix: Correct `restore` progress information if an error occurs
+
+If an error occurred while restoring a snapshot, this could cause the `restore`
+progress bar to show incorrect information. In addition, if a data file could
+not be loaded completely, then errors would also be reported for some already
+restored files.
+
+Error reporting of the `restore` command has now been made more accurate.
+
+https://github.com/restic/restic/pull/4624
+https://forum.restic.net/t/errors-restoring-with-restic-on-windows-server-s3/6943

changelog/0.16.3_2024-01-14/pull-4626

@@ -0,0 +1,11 @@
+Bugfix: Improve reliability of restoring large files
+
+In some cases restic failed to restore large files that frequently contain the
+same file chunk. In combination with certain backends, this could result in
+network connection timeouts that caused incomplete restores.
+
+Restic now includes special handling for such file chunks to ensure reliable
+restores.
+
+https://github.com/restic/restic/pull/4626
+https://forum.restic.net/t/errors-restoring-with-restic-on-windows-server-s3/6943

changelog/0.16.4_2024-02-04/issue-4529

@@ -0,0 +1,18 @@
+Enhancement: Add extra verification of data integrity before upload
+
+Hardware issues, or a bug in restic or its dependencies, could previously cause
+corruption in the files restic created and stored in the repository. Detecting
+such corruption previously required explicitly running the `check --read-data`
+or `check --read-data-subset` commands.
+
+To further ensure data integrity, even in the case of hardware issues or
+software bugs, restic now performs additional verification of the files about
+to be uploaded to the repository.
+
+These extra checks will increase CPU usage during backups. They can therefore,
+if absolutely necessary, be disabled using the `--no-extra-verify` global
+option. Please note that this should be combined with more active checking
+using the previously mentioned check commands.
+
+https://github.com/restic/restic/issues/4529
+https://github.com/restic/restic/pull/4681

changelog/0.16.4_2024-02-04/issue-4677

@@ -0,0 +1,19 @@
+Bugfix: Downgrade zstd library to fix rare data corruption at max. compression
+
+In restic 0.16.3, backups where the compression level was set to `max` (using
+`--compression max`) could in rare and very specific circumstances result in
+data corruption due to a bug in the library used for compressing data. Restic
+0.16.1 and 0.16.2 were not affected.
+
+Restic now uses the previous version of the library used to compress data, the
+same version used by restic 0.16.2. Please note that the `auto` compression
+level (which restic uses by default) was never affected, and even if you used
+`max` compression, chances of being affected by this issue are small.
+
+To check a repository for any corruption, run `restic check --read-data`. This
+will download and verify the whole repository and can be used at any time to
+completely verify the integrity of a repository. If the `check` command detects
+anomalies, follow the suggested steps.
+
+https://github.com/restic/restic/issues/4677
+https://github.com/restic/restic/pull/4679

changelog/0.8.0_2017-11-26/pull-1040

@@ -3,7 +3,7 @@ Enhancement: Add local metadata cache
 We've added a local cache for metadata so that restic doesn't need to load
 all metadata (snapshots, indexes, ...) from the repo each time it starts. By
 default the cache is active, but there's a new global option `--no-cache`
-that can be used to disable the cache. By deafult, the cache a standard
+that can be used to disable the cache. By default, the cache a standard
 cache folder for the OS, which can be overridden with `--cache-dir`.  The
 cache will automatically populate, indexes and snapshots are saved as they
 are loaded. Cache directories for repos that haven't been used recently can

changelog/0.8.0_2017-11-26/pull-1319

@@ -1,6 +1,6 @@
 Enhancement: Make `check` print `no errors found` explicitly
 
-The `check` command now explicetly prints `No errors were found` when no errors
+The `check` command now explicitly prints `No errors were found` when no errors
 could be found.
 
 https://github.com/restic/restic/pull/1319

changelog/0.8.2_2018-02-17/issue-1506

@@ -1,4 +1,4 @@
-Bugfix: Limit bandwith at the http.RoundTripper for HTTP based backends
+Bugfix: Limit bandwidth at the http.RoundTripper for HTTP based backends
 
 https://github.com/restic/restic/issues/1506
 https://github.com/restic/restic/pull/1511

changelog/0.8.2_2018-02-17/pull-1595

@@ -1,7 +1,7 @@
 Bugfix: backup: Remove bandwidth display
 
 This commit removes the bandwidth displayed during backup process. It is
-misleading and seldomly correct, because it's neither the "read
+misleading and seldom correct, because it's neither the "read
 bandwidth" (only for the very first backup) nor the "upload bandwidth".
 Many users are confused about (and rightly so), c.f. #1581, #1033, #1591
 

changelog/0.8.3_2018-02-26/pull-1623

@@ -6,7 +6,7 @@ that means making a request (e.g. via HTTP) and returning an error when the
 file already exists.
 
 This is not accurate, the file could have been created between the HTTP request
-testing for it, and when writing starts, so we've relaxed this requeriment,
+testing for it, and when writing starts, so we've relaxed this requirement,
 which saves one additional HTTP request per newly added file.
 
 https://github.com/restic/restic/pull/1623

changelog/0.9.0_2018-05-21/pull-1735

@@ -1,4 +1,4 @@
-Enhancement: Allow keeping a time range of snaphots
+Enhancement: Allow keeping a time range of snapshots
 
 We've added the `--keep-within` option to the `forget` command. It instructs
 restic to keep all snapshots within the given duration since the newest

changelog/0.9.3_2018-10-13/pull-1876

@@ -1,7 +1,7 @@
 Enhancement: Display reason why forget keeps snapshots
 
 We've added a column to the list of snapshots `forget` keeps which details the
-reasons to keep a particuliar snapshot. This makes debugging policies for
+reasons to keep a particular snapshot. This makes debugging policies for
 forget much easier. Please remember to always try things out with `--dry-run`!
 
 https://github.com/restic/restic/pull/1876

changelog/0.9.6_2019-11-22/issue-2179

@@ -9,7 +9,7 @@ file should be noticed, and the modified file will be backed up. The ctime check
 will be disabled if the --ignore-inode flag was given.
 
 If this change causes problems for you, please open an issue, and we can look in
-to adding a seperate flag to disable just the ctime check.
+to adding a separate flag to disable just the ctime check.
 
 https://github.com/restic/restic/issues/2179
 https://github.com/restic/restic/pull/2212

changelog/CHANGELOG.tmpl

@@ -1,18 +1,21 @@
-{{- range $changes := . }}{{ with $changes -}}
-Changelog for restic {{ .Version }} ({{ .Date }})
-=======================================
+# Table of Contents
 
+{{ range . -}}
+  * [Changelog for {{ .Version }}](#changelog-for-restic-{{ .Version | replace "." ""}}-{{ .Date | lower -}})
+{{ end -}}
+
+{{- range $changes := . }}{{ with $changes }}
+
+# Changelog for restic {{ .Version }} ({{ .Date }})
 The following sections list the changes in restic {{ .Version }} relevant to
 restic users. The changes are ordered by importance.
 
-Summary
--------
+## Summary
 {{ range $entry := .Entries }}{{ with $entry }}
  * {{ .TypeShort }} #{{ .PrimaryID }}: {{ .Title }}
 {{- end }}{{ end }}
 
-Details
--------
+## Details
 {{ range $entry := .Entries }}{{ with $entry }}
  * {{ .Type }} #{{ .PrimaryID }}: {{ .Title }}
 {{ range $par := .Paragraphs }}
@@ -27,6 +30,5 @@ Details
 {{ range $url := .OtherURLs }}
    {{ $url -}}
 {{ end }}
-{{ end }}{{ end }}
-
+{{ end }}{{ end -}}
 {{ end }}{{ end -}}

changelog/unreleased/issue-4251

@@ -0,0 +1,16 @@
+Enhancement: Support reading backup from a commands's standard output
+
+The `backup` command now supports the `--stdin-from-command` option. When using
+this option, the arguments to `backup` are interpreted as a command instead of
+paths to back up. `backup` then executes the given command and stores the
+standard output from it in the backup, similar to the what the `--stdin` option
+does. This also enables restic to verify that the command completes with exit
+code zero. A non-zero exit code causes the backup to fail.
+
+Note that the `--stdin` option does not have to be specified at the same time,
+and that the `--stdin-filename` option also applies to `--stdin-from-command`.
+
+Example: `restic backup --stdin-from-command --stdin-filename dump.sql mysqldump [...]`
+
+https://github.com/restic/restic/issues/4251
+https://github.com/restic/restic/pull/4410

changelog/unreleased/issue-4472

@@ -0,0 +1,18 @@
+Enhancement: Allow AWS Assume Role to be used for S3 backend
+
+Previously only credentials discovered via the Minio discovery methods
+were used to authenticate.
+
+However, there are many circumstances where the discovered credentials have
+lower permissions and need to assume a specific role. This is now possible
+using the following new environment variables.
+
+- RESTIC_AWS_ASSUME_ROLE_ARN
+- RESTIC_AWS_ASSUME_ROLE_SESSION_NAME
+- RESTIC_AWS_ASSUME_ROLE_EXTERNAL_ID
+- RESTIC_AWS_ASSUME_ROLE_REGION (defaults to us-east-1)
+- RESTIC_AWS_ASSUME_ROLE_POLICY
+- RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT
+
+https://github.com/restic/restic/issues/4472
+https://github.com/restic/restic/pull/4474

changelog/unreleased/issue-4515

@@ -0,0 +1,8 @@
+Change: Don't retry to load files that don't exist
+
+Restic used to always retry to load files. It now only retries to load
+files if they exist.
+
+https://github.com/restic/restic/issues/4515
+https://github.com/restic/restic/issues/1523
+https://github.com/restic/restic/pull/4520

changelog/unreleased/issue-4540

@@ -0,0 +1,6 @@
+Change: Require at least ARMv6 for ARM binaries
+
+The official release binaries of restic now require at least ARMv6 support for ARM platforms.
+
+https://github.com/restic/restic/issues/4540
+https://github.com/restic/restic/pull/4542

changelog/unreleased/issue-4547

@@ -0,0 +1,7 @@
+Enhancement: Add support for `--json` option to `version` command
+
+Restic now supports outputting restic version and used go version, OS and
+architecture via JSON when using the version command.
+
+https://github.com/restic/restic/issues/4547
+https://github.com/restic/restic/pull/4553

changelog/unreleased/issue-4549

@@ -0,0 +1,11 @@
+Enhancement: Add `--ncdu` option to `ls` command
+
+NCDU (NCurses Disk Usage) is a tool to analyse disk usage of directories.
+It has an option to save a directory tree and analyse it later.
+The `ls` command now supports the `--ncdu` option which outputs information
+about a snapshot in the NCDU format.
+
+You can use it as follows: `restic ls latest --ncdu | ncdu -f -`
+
+https://github.com/restic/restic/issues/4549
+https://github.com/restic/restic/pull/4550

changelog/unreleased/issue-4583

@@ -0,0 +1,12 @@
+Enhancement: Ignore s3.storage-class for metadata if archive tier is specified
+
+There is no official cold storage support in restic, use this option at your
+own risk.
+
+Restic always stored all files on s3 using the specified `s3.storage-class`.
+Now, restic will store metadata using a non-archive storage tier to avoid
+problems when accessing a repository. To restore any data, it is still
+necessary to manually warm up the required data beforehand.
+
+https://github.com/restic/restic/issues/4583
+https://github.com/restic/restic/pull/4584

changelog/unreleased/issue-4656

@@ -0,0 +1,7 @@
+Bugfix: Properly report the ID of newly added keys
+
+`restic key add` now reports the ID of a newly added key. This simplifies
+selecting a specific key using the `--key-hint key` option.
+
+https://github.com/restic/restic/issues/4656
+https://github.com/restic/restic/pull/4657

changelog/unreleased/issue-4676

@@ -0,0 +1,8 @@
+Enhancement: Move key add, list, remove and passwd as separate sub-commands
+
+Restic now provides usage documentation for the `key` command. Each sub-command;
+`add`, `list`, `remove` and `passwd` now have their own sub-command documentation
+which can be invoked using `restic key <add|list|remove|passwd> --help`.
+
+https://github.com/restic/restic/issues/4676
+https://github.com/restic/restic/pull/4685

changelog/unreleased/issue-4678

@@ -0,0 +1,8 @@
+Enhancement: Add --target flag to the dump command
+
+Restic `dump` always printed to the standard output. It now permits to select a
+`--target` file to write the output to.
+
+https://github.com/restic/restic/issues/4678
+https://github.com/restic/restic/pull/4682
+https://github.com/restic/restic/pull/4692

changelog/unreleased/pull-4503

@@ -0,0 +1,7 @@
+Bugfix: Correct hardlink handling in `stats` command
+
+If files on different devices had the same inode id, then the `stats` command
+did not correctly calculate the snapshot size. This has been fixed.
+
+https://github.com/restic/restic/pull/4503
+https://forum.restic.net/t/possible-bug-in-stats/6461/8

changelog/unreleased/pull-4526

@@ -0,0 +1,11 @@
+Enhancement: Add bitrot detection to `diff` command
+
+The output of the `diff` command now includes the modifier `?` for files
+to indicate bitrot in backed up files. It will appear whenever there is a
+difference in content while the metadata is exactly the same. Since files with
+unchanged metadata are normally not read again when creating a backup, the
+detection is only effective if the right-hand side of the diff has been created
+with "backup --force".
+
+https://github.com/restic/restic/issues/805
+https://github.com/restic/restic/pull/4526

changelog/unreleased/pull-4573

@@ -0,0 +1,5 @@
+Enhancement: Add `--new-host` and `--new-time` options to `rewrite` command
+
+`restic rewrite` now allows rewriting the host and / or time metadata of a snapshot.
+
+https://github.com/restic/restic/pull/4573

changelog/unreleased/pull-4590

@@ -0,0 +1,7 @@
+Enhancement: `mount` tests mountpoint existence before opening the repository
+
+The restic `mount` command now checks for the existence of the
+mountpoint before opening the repository, leading to quicker error
+detection.
+
+https://github.com/restic/restic/pull/4590

changelog/unreleased/pull-4615

@@ -0,0 +1,6 @@
+Bugfix: `find` ignored directories in some cases
+
+In some cases, the `find` command ignored empty or moved directories. This has
+been fixed.
+
+https://github.com/restic/restic/pull/4615

changelog/unreleased/pull-4644

@@ -0,0 +1,10 @@
+Enhancement: Improve `repair packs` command
+
+The `repair packs` command has been improved to also be able to process
+truncated pack files. The `check --read-data` command will provide instructions
+on using the command if necessary to repair a repository. See the guide at
+https://restic.readthedocs.io/en/stable/077_troubleshooting.html for further
+instructions.
+
+https://github.com/restic/restic/pull/4644
+https://github.com/restic/restic/pull/4655

cmd/restic/cmd_backup.go

@@ -12,7 +12,6 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -25,7 +24,6 @@ import (
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/textfile"
-	"github.com/restic/restic/internal/ui"
 	"github.com/restic/restic/internal/ui/backup"
 	"github.com/restic/restic/internal/ui/termstatus"
 )
@@ -44,7 +42,7 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was a fatal error (no snapshot created).
 Exit status is 3 if some source data could not be read (incomplete snapshot created).
 `,
-	PreRun: func(cmd *cobra.Command, args []string) {
+	PreRun: func(_ *cobra.Command, _ []string) {
 		if backupOptions.Host == "" {
 			hostname, err := os.Hostname()
 			if err != nil {
@@ -56,31 +54,9 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
 	},
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		ctx := cmd.Context()
-		var wg sync.WaitGroup
-		cancelCtx, cancel := context.WithCancel(ctx)
-		defer func() {
-			// shutdown termstatus
-			cancel()
-			wg.Wait()
-		}()
-
-		term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet)
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			term.Run(cancelCtx)
-		}()
-
-		// use the terminal for stdout/stderr
-		prevStdout, prevStderr := globalOptions.stdout, globalOptions.stderr
-		defer func() {
-			globalOptions.stdout, globalOptions.stderr = prevStdout, prevStderr
-		}()
-		stdioWrapper := ui.NewStdioWrapper(term)
-		globalOptions.stdout, globalOptions.stderr = stdioWrapper.Stdout(), stdioWrapper.Stderr()
-
-		return runBackup(ctx, backupOptions, globalOptions, term, args)
+		term, cancel := setupTermstatus()
+		defer cancel()
+		return runBackup(cmd.Context(), backupOptions, globalOptions, term, args)
 	},
 }
 
@@ -97,6 +73,7 @@ type BackupOptions struct {
 	ExcludeLargerThan string
 	Stdin             bool
 	StdinFilename     string
+	StdinCommand      bool
 	Tags              restic.TagLists
 	Host              string
 	FilesFrom         []string
@@ -134,6 +111,7 @@ func init() {
 	f.StringVar(&backupOptions.ExcludeLargerThan, "exclude-larger-than", "", "max `size` of the files to be backed up (allowed suffixes: k/K, m/M, g/G, t/T)")
 	f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
 	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "`filename` to use when reading from stdin")
+	f.BoolVar(&backupOptions.StdinCommand, "stdin-from-command", false, "interpret arguments as command to execute and store its stdout")
 	f.Var(&backupOptions.Tags, "tag", "add `tags` for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times)")
 	f.UintVar(&backupOptions.ReadConcurrency, "read-concurrency", 0, "read `n` files concurrently (default: $RESTIC_READ_CONCURRENCY or 2)")
 	f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag")
@@ -148,7 +126,7 @@ func init() {
 	f.StringArrayVar(&backupOptions.FilesFromRaw, "files-from-raw", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
 	f.StringVar(&backupOptions.TimeStamp, "time", "", "`time` of the backup (ex. '2012-11-01 22:08:41') (default: now)")
 	f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories")
-	f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number changes when checking for modified files")
+	f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number and ctime changes when checking for modified files")
 	f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files")
 	f.BoolVarP(&backupOptions.DryRun, "dry-run", "n", false, "do not upload or write any data, just show what would be done")
 	f.BoolVar(&backupOptions.NoScan, "no-scan", false, "do not run scanner to estimate size of backup")
@@ -287,7 +265,7 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	if opts.Stdin {
+	if opts.Stdin || opts.StdinCommand {
 		if len(opts.FilesFrom) > 0 {
 			return errors.Fatal("--stdin and --files-from cannot be used together")
 		}
@@ -298,7 +276,7 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
 			return errors.Fatal("--stdin and --files-from-raw cannot be used together")
 		}
 
-		if len(args) > 0 {
+		if len(args) > 0 && !opts.StdinCommand {
 			return errors.Fatal("--stdin was specified and files/dirs were listed as arguments")
 		}
 	}
@@ -366,7 +344,7 @@ func collectRejectFuncs(opts BackupOptions, targets []string) (fs []RejectFunc,
 
 // collectTargets returns a list of target files/dirs from several sources.
 func collectTargets(opts BackupOptions, args []string) (targets []string, err error) {
-	if opts.Stdin {
+	if opts.Stdin || opts.StdinCommand {
 		return nil, nil
 	}
 
@@ -433,7 +411,7 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 
 // parent returns the ID of the parent snapshot. If there is none, nil is
 // returned.
-func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string, timeStampLimit time.Time) (*restic.Snapshot, error) {
+func findParentSnapshot(ctx context.Context, repo restic.ListerLoaderUnpacked, opts BackupOptions, targets []string, timeStampLimit time.Time) (*restic.Snapshot, error) {
 	if opts.Force {
 		return nil, nil
 	}
@@ -453,7 +431,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
 		f.Tags = []restic.TagList{opts.Tags.Flatten()}
 	}
 
-	sn, _, err := f.FindLatest(ctx, repo.Backend(), repo, snName)
+	sn, _, err := f.FindLatest(ctx, repo, repo, snName)
 	// Snapshot not found is ok if no explicit parent was set
 	if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) {
 		err = nil
@@ -546,7 +524,10 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 	if !gopts.JSON {
 		progressPrinter.V("load index files")
 	}
-	err = repo.LoadIndex(ctx)
+
+	bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
+
+	err = repo.LoadIndex(ctx, bar)
 	if err != nil {
 		return err
 	}
@@ -589,16 +570,24 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 		defer localVss.DeleteSnapshots()
 		targetFS = localVss
 	}
-	if opts.Stdin {
+
+	if opts.Stdin || opts.StdinCommand {
 		if !gopts.JSON {
 			progressPrinter.V("read data from stdin")
 		}
 		filename := path.Join("/", opts.StdinFilename)
+		var source io.ReadCloser = os.Stdin
+		if opts.StdinCommand {
+			source, err = fs.NewCommandReader(ctx, args, globalOptions.stderr)
+			if err != nil {
+				return err
+			}
+		}
 		targetFS = &fs.Reader{
 			ModTime:    timeStamp,
 			Name:       filename,
 			Mode:       0644,
-			ReadCloser: os.Stdin,
+			ReadCloser: source,
 		}
 		targets = []string{filename}
 	}
@@ -620,14 +609,20 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 		wg.Go(func() error { return sc.Scan(cancelCtx, targets) })
 	}
 
-	arch := archiver.New(repo, targetFS, archiver.Options{ReadConcurrency: backupOptions.ReadConcurrency})
+	arch := archiver.New(repo, targetFS, archiver.Options{ReadConcurrency: opts.ReadConcurrency})
 	arch.SelectByName = selectByNameFilter
 	arch.Select = selectFilter
 	arch.WithAtime = opts.WithAtime
 	success := true
 	arch.Error = func(item string, err error) error {
 		success = false
-		return progressReporter.Error(item, err)
+		reterr := progressReporter.Error(item, err)
+		// If we receive a fatal error during the execution of the snapshot,
+		// we abort the snapshot.
+		if reterr == nil && errors.IsFatal(err) {
+			reterr = err
+		}
+		return reterr
 	}
 	arch.CompleteItem = progressReporter.CompleteItem
 	arch.StartFile = progressReporter.StartFile

cmd/restic/cmd_backup_integration_test.go

@@ -9,6 +9,7 @@ import (
 	"runtime"
 	"testing"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
@@ -252,7 +253,7 @@ func TestBackupTreeLoadError(t *testing.T) {
 
 	r, err := OpenRepository(context.TODO(), env.gopts)
 	rtest.OK(t, err)
-	rtest.OK(t, r.LoadIndex(context.TODO()))
+	rtest.OK(t, r.LoadIndex(context.TODO(), nil))
 	treePacks := restic.NewIDSet()
 	r.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
 		if pb.Type == restic.TreeBlob {
@@ -265,7 +266,7 @@ func TestBackupTreeLoadError(t *testing.T) {
 
 	// delete the subdirectory pack first
 	for id := range treePacks {
-		rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()}))
+		rtest.OK(t, r.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()}))
 	}
 	testRunRebuildIndex(t, env.gopts)
 	// now the repo is missing the tree blob in the index; check should report this
@@ -567,3 +568,72 @@ func linkEqual(source, dest []string) bool {
 
 	return true
 }
+
+func TestStdinFromCommand(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{
+		StdinCommand:  true,
+		StdinFilename: "stdin",
+	}
+
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; print('something'); sys.exit(0)"}, opts, env.gopts)
+	testListSnapshots(t, env.gopts, 1)
+
+	testRunCheck(t, env.gopts)
+}
+
+func TestStdinFromCommandNoOutput(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{
+		StdinCommand:  true,
+		StdinFilename: "stdin",
+	}
+
+	err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; sys.exit(0)"}, opts, env.gopts)
+	rtest.Assert(t, err != nil && err.Error() == "at least one source file could not be read", "No data error expected")
+	testListSnapshots(t, env.gopts, 1)
+
+	testRunCheck(t, env.gopts)
+}
+
+func TestStdinFromCommandFailExitCode(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{
+		StdinCommand:  true,
+		StdinFilename: "stdin",
+	}
+
+	err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; print('test'); sys.exit(1)"}, opts, env.gopts)
+	rtest.Assert(t, err != nil, "Expected error while backing up")
+
+	testListSnapshots(t, env.gopts, 0)
+
+	testRunCheck(t, env.gopts)
+}
+
+func TestStdinFromCommandFailNoOutputAndExitCode(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{
+		StdinCommand:  true,
+		StdinFilename: "stdin",
+	}
+
+	err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; sys.exit(1)"}, opts, env.gopts)
+	rtest.Assert(t, err != nil, "Expected error while backing up")
+
+	testListSnapshots(t, env.gopts, 0)
+
+	testRunCheck(t, env.gopts)
+}

cmd/restic/cmd_cache.go

@@ -28,7 +28,7 @@ EXIT STATUS
 Exit status is 0 if the command was successful, and non-zero if there was any error.
 `,
 	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
+	RunE: func(_ *cobra.Command, args []string) error {
 		return runCache(cacheOptions, globalOptions, args)
 	},
 }

cmd/restic/cmd_cat.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"strings"
 
 	"github.com/spf13/cobra"
 
@@ -33,9 +34,34 @@ func init() {
 	cmdRoot.AddCommand(cmdCat)
 }
 
+func validateCatArgs(args []string) error {
+	var allowedCmds = []string{"config", "index", "snapshot", "key", "masterkey", "lock", "pack", "blob", "tree"}
+
+	if len(args) < 1 {
+		return errors.Fatal("type not specified")
+	}
+
+	validType := false
+	for _, v := range allowedCmds {
+		if v == args[0] {
+			validType = true
+			break
+		}
+	}
+	if !validType {
+		return errors.Fatalf("invalid type %q, must be one of [%s]", args[0], strings.Join(allowedCmds, "|"))
+	}
+
+	if args[0] != "masterkey" && args[0] != "config" && len(args) != 2 {
+		return errors.Fatal("ID not specified")
+	}
+
+	return nil
+}
+
 func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
-	if len(args) < 1 || (args[0] != "masterkey" && args[0] != "config" && len(args) != 2) {
-		return errors.Fatal("type or ID not specified")
+	if err := validateCatArgs(args); err != nil {
+		return err
 	}
 
 	repo, err := OpenRepository(ctx, gopts)
@@ -80,7 +106,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 		Println(string(buf))
 		return nil
 	case "snapshot":
-		sn, _, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1])
+		sn, _, err := restic.FindSnapshot(ctx, repo, repo, args[1])
 		if err != nil {
 			return errors.Fatalf("could not find snapshot: %v\n", err)
 		}
@@ -128,7 +154,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 		return nil
 
 	case "pack":
-		h := restic.Handle{Type: restic.PackFile, Name: id.String()}
+		h := backend.Handle{Type: restic.PackFile, Name: id.String()}
 		buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h)
 		if err != nil {
 			return err
@@ -143,7 +169,8 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 		return err
 
 	case "blob":
-		err = repo.LoadIndex(ctx)
+		bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+		err = repo.LoadIndex(ctx, bar)
 		if err != nil {
 			return err
 		}
@@ -166,12 +193,13 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 		return errors.Fatal("blob not found")
 
 	case "tree":
-		sn, subfolder, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1])
+		sn, subfolder, err := restic.FindSnapshot(ctx, repo, repo, args[1])
 		if err != nil {
 			return errors.Fatalf("could not find snapshot: %v\n", err)
 		}
 
-		err = repo.LoadIndex(ctx)
+		bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+		err = repo.LoadIndex(ctx, bar)
 		if err != nil {
 			return err
 		}

cmd/restic/cmd_cat_test.go

@@ -0,0 +1,30 @@
+package main
+
+import (
+	"strings"
+	"testing"
+
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func TestCatArgsValidation(t *testing.T) {
+	for _, test := range []struct {
+		args []string
+		err  string
+	}{
+		{[]string{}, "Fatal: type not specified"},
+		{[]string{"masterkey"}, ""},
+		{[]string{"invalid"}, `Fatal: invalid type "invalid"`},
+		{[]string{"snapshot"}, "Fatal: ID not specified"},
+		{[]string{"snapshot", "12345678"}, ""},
+	} {
+		t.Run("", func(t *testing.T) {
+			err := validateCatArgs(test.args)
+			if test.err == "" {
+				rtest.Assert(t, err == nil, "unexpected error %q", err)
+			} else {
+				rtest.Assert(t, strings.Contains(err.Error(), test.err), "unexpected error expected %q to contain %q", err, test.err)
+			}
+		})
+	}
+}

cmd/restic/cmd_check.go

@@ -38,7 +38,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runCheck(cmd.Context(), checkOptions, globalOptions, args)
 	},
-	PreRunE: func(cmd *cobra.Command, args []string) error {
+	PreRunE: func(_ *cobra.Command, _ []string) error {
 		return checkFlags(checkOptions)
 	},
 }
@@ -226,7 +226,8 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	}
 
 	Verbosef("load indexes\n")
-	hints, errs := chkr.LoadIndex(ctx)
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	hints, errs := chkr.LoadIndex(ctx, bar)
 
 	errorsFound := false
 	suggestIndexRebuild := false
@@ -329,11 +330,26 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 
 		go chkr.ReadPacks(ctx, packs, p, errChan)
 
+		var salvagePacks restic.IDs
+
 		for err := range errChan {
 			errorsFound = true
 			Warnf("%v\n", err)
+			if err, ok := err.(*checker.ErrPackData); ok {
+				salvagePacks = append(salvagePacks, err.PackID)
+			}
 		}
 		p.Done()
+
+		if len(salvagePacks) > 0 {
+			Warnf("\nThe repository contains pack files with damaged blobs. These blobs must be removed to repair the repository. This can be done using the following commands. Please read the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html first.\n\n")
+			var strIDs []string
+			for _, id := range salvagePacks {
+				strIDs = append(strIDs, id.String())
+			}
+			Warnf("restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIDs, " "))
+			Warnf("Corrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n")
+		}
 	}
 
 	switch {
@@ -399,7 +415,7 @@ func selectPacksByBucket(allPacks map[restic.ID]int64, bucket, totalBuckets uint
 	return packs
 }
 
-// selectRandomPacksByPercentage selects the given percentage of packs which are randomly choosen.
+// selectRandomPacksByPercentage selects the given percentage of packs which are randomly chosen.
 func selectRandomPacksByPercentage(allPacks map[restic.ID]int64, percentage float64) map[restic.ID]int64 {
 	packCount := len(allPacks)
 	packsToCheck := int(float64(packCount) * (percentage / 100.0))

cmd/restic/cmd_check_test.go

@@ -71,7 +71,7 @@ func TestSelectPacksByBucket(t *testing.T) {
 	var testPacks = make(map[restic.ID]int64)
 	for i := 1; i <= 10; i++ {
 		id := restic.NewRandomID()
-		// ensure relevant part of generated id is reproducable
+		// ensure relevant part of generated id is reproducible
 		id[0] = byte(i)
 		testPacks[id] = 0
 	}
@@ -124,7 +124,7 @@ func TestSelectRandomPacksByPercentage(t *testing.T) {
 }
 
 func TestSelectNoRandomPacksByPercentage(t *testing.T) {
-	// that the a repository without pack files works
+	// that the repository without pack files works
 	var testPacks = make(map[restic.ID]int64)
 	selectedPacks := selectRandomPacksByPercentage(testPacks, 10.0)
 	rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs")
@@ -158,7 +158,7 @@ func TestSelectRandomPacksByFileSize(t *testing.T) {
 }
 
 func TestSelectNoRandomPacksByFileSize(t *testing.T) {
-	// that the a repository without pack files works
+	// that the repository without pack files works
 	var testPacks = make(map[restic.ID]int64)
 	selectedPacks := selectRandomPacksByFileSize(testPacks, 10, 500)
 	rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs")

cmd/restic/cmd_copy.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
@@ -88,23 +87,24 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
 		return err
 	}
 
-	srcSnapshotLister, err := backend.MemorizeList(ctx, srcRepo.Backend(), restic.SnapshotFile)
+	srcSnapshotLister, err := restic.MemorizeList(ctx, srcRepo, restic.SnapshotFile)
 	if err != nil {
 		return err
 	}
 
-	dstSnapshotLister, err := backend.MemorizeList(ctx, dstRepo.Backend(), restic.SnapshotFile)
+	dstSnapshotLister, err := restic.MemorizeList(ctx, dstRepo, restic.SnapshotFile)
 	if err != nil {
 		return err
 	}
 
 	debug.Log("Loading source index")
-	if err := srcRepo.LoadIndex(ctx); err != nil {
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	if err := srcRepo.LoadIndex(ctx, bar); err != nil {
 		return err
 	}
-
+	bar = newIndexProgress(gopts.Quiet, gopts.JSON)
 	debug.Log("Loading destination index")
-	if err := dstRepo.LoadIndex(ctx); err != nil {
+	if err := dstRepo.LoadIndex(ctx, bar); err != nil {
 		return err
 	}
 
@@ -126,11 +126,12 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
 		if sn.Original != nil {
 			srcOriginal = *sn.Original
 		}
+
 		if originalSns, ok := dstSnapshotByOriginal[srcOriginal]; ok {
 			isCopy := false
 			for _, originalSn := range originalSns {
 				if similarSnapshots(originalSn, sn) {
-					Verboseff("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
+					Verboseff("\n%v\n", sn)
 					Verboseff("skipping source snapshot %s, was already copied to snapshot %s\n", sn.ID().Str(), originalSn.ID().Str())
 					isCopy = true
 					break
@@ -140,7 +141,7 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
 				continue
 			}
 		}
-		Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
+		Verbosef("\n%v\n", sn)
 		Verbosef("  copy started, this may take a while...\n")
 		if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree, gopts.Quiet); err != nil {
 			return err

cmd/restic/cmd_debug.go

@@ -52,19 +52,23 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 	},
 }
 
-var tryRepair bool
-var repairByte bool
-var extractPack bool
-var reuploadBlobs bool
+type DebugExamineOptions struct {
+	TryRepair     bool
+	RepairByte    bool
+	ExtractPack   bool
+	ReuploadBlobs bool
+}
+
+var debugExamineOpts DebugExamineOptions
 
 func init() {
 	cmdRoot.AddCommand(cmdDebug)
 	cmdDebug.AddCommand(cmdDebugDump)
 	cmdDebug.AddCommand(cmdDebugExamine)
-	cmdDebugExamine.Flags().BoolVar(&extractPack, "extract-pack", false, "write blobs to the current directory")
-	cmdDebugExamine.Flags().BoolVar(&reuploadBlobs, "reupload-blobs", false, "reupload blobs to the repository")
-	cmdDebugExamine.Flags().BoolVar(&tryRepair, "try-repair", false, "try to repair broken blobs with single bit flips")
-	cmdDebugExamine.Flags().BoolVar(&repairByte, "repair-byte", false, "try to repair broken blobs by trying bytes")
+	cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.ExtractPack, "extract-pack", false, "write blobs to the current directory")
+	cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.ReuploadBlobs, "reupload-blobs", false, "reupload blobs to the repository")
+	cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.TryRepair, "try-repair", false, "try to repair broken blobs with single bit flips")
+	cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.RepairByte, "repair-byte", false, "try to repair broken blobs by trying bytes")
 }
 
 func prettyPrintJSON(wr io.Writer, item interface{}) error {
@@ -78,7 +82,7 @@ func prettyPrintJSON(wr io.Writer, item interface{}) error {
 }
 
 func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
-	return restic.ForAllSnapshots(ctx, repo.Backend(), repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error {
+	return restic.ForAllSnapshots(ctx, repo, repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error {
 		if err != nil {
 			return err
 		}
@@ -107,7 +111,7 @@ type Blob struct {
 func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
 
 	var m sync.Mutex
-	return restic.ParallelList(ctx, repo.Backend(), restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
+	return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
 		blobs, _, err := repo.ListPack(ctx, id, size)
 		if err != nil {
 			Warnf("error for pack %v: %v\n", id.Str(), err)
@@ -133,8 +137,8 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
 	})
 }
 
-func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error {
-	return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
+func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer) error {
+	return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
 		Printf("index_id: %v\n", id)
 		if err != nil {
 			return err
@@ -196,7 +200,7 @@ var cmdDebugExamine = &cobra.Command{
 	Short:             "Examine a pack file",
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runDebugExamine(cmd.Context(), globalOptions, args)
+		return runDebugExamine(cmd.Context(), globalOptions, debugExamineOpts, args)
 	},
 }
 
@@ -290,7 +294,7 @@ func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, by
 	})
 	err := wg.Wait()
 	if err != nil {
-		panic("all go rountines can only return nil")
+		panic("all go routines can only return nil")
 	}
 
 	if !found {
@@ -315,20 +319,20 @@ func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte {
 	return out
 }
 
-func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, list []restic.Blob) error {
+func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, packID restic.ID, list []restic.Blob) error {
 	dec, err := zstd.NewReader(nil)
 	if err != nil {
 		panic(err)
 	}
 	be := repo.Backend()
-	h := restic.Handle{
+	h := backend.Handle{
 		Name: packID.String(),
 		Type: restic.PackFile,
 	}
 
 	wg, ctx := errgroup.WithContext(ctx)
 
-	if reuploadBlobs {
+	if opts.ReuploadBlobs {
 		repo.StartPackUploader(ctx, wg)
 	}
 
@@ -356,8 +360,8 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li
 			filePrefix := ""
 			if err != nil {
 				Warnf("error decrypting blob: %v\n", err)
-				if tryRepair || repairByte {
-					plaintext = tryRepairWithBitflip(ctx, key, buf, repairByte)
+				if opts.TryRepair || opts.RepairByte {
+					plaintext = tryRepairWithBitflip(ctx, key, buf, opts.RepairByte)
 				}
 				if plaintext != nil {
 					outputPrefix = "repaired "
@@ -391,13 +395,13 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li
 				Printf("         successfully %vdecrypted blob (length %v), hash is %v, ID matches\n", outputPrefix, len(plaintext), id)
 				prefix = "correct-"
 			}
-			if extractPack {
+			if opts.ExtractPack {
 				err = storePlainBlob(id, filePrefix+prefix, plaintext)
 				if err != nil {
 					return err
 				}
 			}
-			if reuploadBlobs {
+			if opts.ReuploadBlobs {
 				_, _, _, err := repo.SaveBlob(ctx, blob.Type, plaintext, id, true)
 				if err != nil {
 					return err
@@ -406,7 +410,7 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li
 			}
 		}
 
-		if reuploadBlobs {
+		if opts.ReuploadBlobs {
 			return repo.Flush(ctx)
 		}
 		return nil
@@ -437,7 +441,7 @@ func storePlainBlob(id restic.ID, prefix string, plain []byte) error {
 	return nil
 }
 
-func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) error {
+func runDebugExamine(ctx context.Context, gopts GlobalOptions, opts DebugExamineOptions, args []string) error {
 	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
@@ -447,7 +451,7 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er
 	for _, name := range args {
 		id, err := restic.ParseID(name)
 		if err != nil {
-			id, err = restic.Find(ctx, repo.Backend(), restic.PackFile, name)
+			id, err = restic.Find(ctx, repo, restic.PackFile, name)
 			if err != nil {
 				Warnf("error: %v\n", err)
 				continue
@@ -469,13 +473,14 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er
 		}
 	}
 
-	err = repo.LoadIndex(ctx)
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	err = repo.LoadIndex(ctx, bar)
 	if err != nil {
 		return err
 	}
 
 	for _, id := range ids {
-		err := examinePack(ctx, repo, id)
+		err := examinePack(ctx, opts, repo, id)
 		if err != nil {
 			Warnf("error: %v\n", err)
 		}
@@ -486,10 +491,10 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er
 	return nil
 }
 
-func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) error {
+func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, id restic.ID) error {
 	Printf("examine %v\n", id)
 
-	h := restic.Handle{
+	h := backend.Handle{
 		Type: restic.PackFile,
 		Name: id.String(),
 	}
@@ -523,7 +528,7 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro
 
 		checkPackSize(blobs, fi.Size)
 
-		err = loadBlobs(ctx, repo, id, blobs)
+		err = loadBlobs(ctx, opts, repo, id, blobs)
 		if err != nil {
 			Warnf("error: %v\n", err)
 		} else {
@@ -541,7 +546,7 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro
 	checkPackSize(blobs, fi.Size)
 
 	if !blobsLoaded {
-		return loadBlobs(ctx, repo, id, blobs)
+		return loadBlobs(ctx, opts, repo, id, blobs)
 	}
 	return nil
 }

cmd/restic/cmd_diff.go

@@ -7,7 +7,6 @@ import (
 	"reflect"
 	"sort"
 
-	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
@@ -16,7 +15,7 @@ import (
 )
 
 var cmdDiff = &cobra.Command{
-	Use:   "diff [flags] snapshot-ID snapshot-ID",
+	Use:   "diff [flags] snapshotID snapshotID",
 	Short: "Show differences between two snapshots",
 	Long: `
 The "diff" command shows differences from the first to the second snapshot. The
@@ -28,6 +27,14 @@ directory:
 * U  The metadata (access mode, timestamps, ...) for the item was updated
 * M  The file's content was modified
 * T  The type was changed, e.g. a file was made a symlink
+* ?  Bitrot detected: The file's content has changed but all metadata is the same
+
+Metadata comparison will likely not work if a backup was created using the
+'--ignore-inode' or '--ignore-ctime' option.
+
+To only compare files in specific subfolders, you can use the
+"<snapshotID>:<subfolder>" syntax, where "subfolder" is a path within the
+snapshot.
 
 EXIT STATUS
 ===========
@@ -54,7 +61,7 @@ func init() {
 	f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata")
 }
 
-func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, string, error) {
+func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.LoaderUnpacked, desc string) (*restic.Snapshot, string, error) {
 	sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc)
 	if err != nil {
 		return nil, "", errors.Fatal(err.Error())
@@ -64,7 +71,7 @@ func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository,
 
 // Comparer collects all things needed to compare two snapshots.
 type Comparer struct {
-	repo        restic.Repository
+	repo        restic.BlobLoader
 	opts        DiffOptions
 	printChange func(change *Change)
 }
@@ -140,7 +147,7 @@ type DiffStatsContainer struct {
 }
 
 // updateBlobs updates the blob counters in the stats struct.
-func updateBlobs(repo restic.Repository, blobs restic.BlobSet, stats *DiffStat) {
+func updateBlobs(repo restic.Loader, blobs restic.BlobSet, stats *DiffStat) {
 	for h := range blobs {
 		switch h.Type {
 		case restic.DataBlob:
@@ -269,6 +276,16 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
 				!reflect.DeepEqual(node1.Content, node2.Content) {
 				mod += "M"
 				stats.ChangedFiles++
+
+				node1NilContent := *node1
+				node2NilContent := *node2
+				node1NilContent.Content = nil
+				node2NilContent.Content = nil
+				// the bitrot detection may not work if `backup --ignore-inode` or `--ignore-ctime` were used
+				if node1NilContent.Equals(node2NilContent) {
+					// probable bitrot detected
+					mod += "?"
+				}
 			} else if c.opts.ShowMetadata && !node1.Equals(*node2) {
 				mod += "U"
 			}
@@ -342,7 +359,7 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
 	}
 
 	// cache snapshots listing
-	be, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
+	be, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
 	if err != nil {
 		return err
 	}
@@ -359,8 +376,8 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
 	if !gopts.JSON {
 		Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
 	}
-
-	if err = repo.LoadIndex(ctx); err != nil {
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	if err = repo.LoadIndex(ctx, bar); err != nil {
 		return err
 	}
 
@@ -384,7 +401,7 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
 
 	c := &Comparer{
 		repo: repo,
-		opts: diffOptions,
+		opts: opts,
 		printChange: func(change *Change) {
 			Printf("%-5s%v\n", change.Modifier, change.Path)
 		},
@@ -401,7 +418,7 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
 	}
 
 	if gopts.Quiet {
-		c.printChange = func(change *Change) {}
+		c.printChange = func(_ *Change) {}
 	}
 
 	stats := &DiffStatsContainer{

cmd/restic/cmd_dump.go

@@ -24,9 +24,13 @@ single file is selected, it prints its contents to stdout. Folders are output
 as a tar (default) or zip file containing the contents of the specified folder.
 Pass "/" as file name to dump the whole snapshot as an archive file.
 
-The special snapshot "latest" can be used to use the latest snapshot in the
+The special snapshotID "latest" can be used to use the latest snapshot in the
 repository.
 
+To include the folder content at the root of the archive, you can use the
+"<snapshotID>:<subfolder>" syntax, where "subfolder" is a path within the
+snapshot.
+
 EXIT STATUS
 ===========
 
@@ -42,6 +46,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 type DumpOptions struct {
 	restic.SnapshotFilter
 	Archive string
+	Target  string
 }
 
 var dumpOptions DumpOptions
@@ -52,6 +57,7 @@ func init() {
 	flags := cmdDump.Flags()
 	initSingleSnapshotFilter(flags, &dumpOptions.SnapshotFilter)
 	flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"")
+	flags.StringVarP(&dumpOptions.Target, "target", "t", "", "write the output to target `path`")
 }
 
 func splitPath(p string) []string {
@@ -63,11 +69,11 @@ func splitPath(p string) []string {
 	return append(s, f)
 }
 
-func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string, d *dump.Dumper) error {
+func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoader, prefix string, pathComponents []string, d *dump.Dumper, canWriteArchiveFunc func() error) error {
 	// If we print / we need to assume that there are multiple nodes at that
 	// level in the tree.
 	if pathComponents[0] == "" {
-		if err := checkStdoutArchive(); err != nil {
+		if err := canWriteArchiveFunc(); err != nil {
 			return err
 		}
 		return d.DumpTree(ctx, tree, "/")
@@ -87,9 +93,9 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
 				if err != nil {
 					return errors.Wrapf(err, "cannot load subtree for %q", item)
 				}
-				return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d)
+				return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d, canWriteArchiveFunc)
 			case dump.IsDir(node):
-				if err := checkStdoutArchive(); err != nil {
+				if err := canWriteArchiveFunc(); err != nil {
 					return err
 				}
 				subtree, err := restic.LoadTree(ctx, repo, *node.Subtree)
@@ -143,12 +149,13 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []
 		Hosts: opts.Hosts,
 		Paths: opts.Paths,
 		Tags:  opts.Tags,
-	}).FindLatest(ctx, repo.Backend(), repo, snapshotIDString)
+	}).FindLatest(ctx, repo, repo, snapshotIDString)
 	if err != nil {
 		return errors.Fatalf("failed to find snapshot: %v", err)
 	}
 
-	err = repo.LoadIndex(ctx)
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	err = repo.LoadIndex(ctx, bar)
 	if err != nil {
 		return err
 	}
@@ -163,8 +170,24 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []
 		return errors.Fatalf("loading tree for snapshot %q failed: %v", snapshotIDString, err)
 	}
 
-	d := dump.New(opts.Archive, repo, os.Stdout)
-	err = printFromTree(ctx, tree, repo, "/", splittedPath, d)
+	outputFileWriter := os.Stdout
+	canWriteArchiveFunc := checkStdoutArchive
+
+	if opts.Target != "" {
+		file, err := os.Create(opts.Target)
+		if err != nil {
+			return fmt.Errorf("cannot dump to file: %w", err)
+		}
+		defer func() {
+			_ = file.Close()
+		}()
+
+		outputFileWriter = file
+		canWriteArchiveFunc = func() error { return nil }
+	}
+
+	d := dump.New(opts.Archive, repo, outputFileWriter)
+	err = printFromTree(ctx, tree, repo, "/", splittedPath, d, canWriteArchiveFunc)
 	if err != nil {
 		return errors.Fatalf("cannot dump file: %v", err)
 	}

cmd/restic/cmd_find.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/filter"
@@ -126,7 +125,6 @@ func (s *statefulOutput) PrintPatternJSON(path string, node *restic.Node) {
 
 		// Make the following attributes disappear
 		Name               byte `json:"name,omitempty"`
-		Inode              byte `json:"inode,omitempty"`
 		ExtendedAttributes byte `json:"extended_attributes,omitempty"`
 		Device             byte `json:"device,omitempty"`
 		Content            byte `json:"content,omitempty"`
@@ -246,13 +244,12 @@ func (s *statefulOutput) Finish() {
 
 // Finder bundles information needed to find a file or directory.
 type Finder struct {
-	repo        restic.Repository
-	pat         findPattern
-	out         statefulOutput
-	ignoreTrees restic.IDSet
-	blobIDs     map[string]struct{}
-	treeIDs     map[string]struct{}
-	itemsFound  int
+	repo       restic.Repository
+	pat        findPattern
+	out        statefulOutput
+	blobIDs    map[string]struct{}
+	treeIDs    map[string]struct{}
+	itemsFound int
 }
 
 func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error {
@@ -263,17 +260,17 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 	}
 
 	f.out.newsn = sn
-	return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
+	return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
 		if err != nil {
 			debug.Log("Error loading tree %v: %v", parentTreeID, err)
 
 			Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID())
 
-			return false, walker.ErrSkipNode
+			return walker.ErrSkipNode
 		}
 
 		if node == nil {
-			return false, nil
+			return nil
 		}
 
 		normalizedNodepath := nodepath
@@ -286,7 +283,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 		for _, pat := range f.pat.pattern {
 			found, err := filter.Match(pat, normalizedNodepath)
 			if err != nil {
-				return false, err
+				return err
 			}
 			if found {
 				foundMatch = true
@@ -294,16 +291,13 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 			}
 		}
 
-		var (
-			ignoreIfNoMatch = true
-			errIfNoMatch    error
-		)
+		var errIfNoMatch error
 		if node.Type == "dir" {
 			var childMayMatch bool
 			for _, pat := range f.pat.pattern {
 				mayMatch, err := filter.ChildMatch(pat, normalizedNodepath)
 				if err != nil {
-					return false, err
+					return err
 				}
 				if mayMatch {
 					childMayMatch = true
@@ -312,31 +306,28 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 			}
 
 			if !childMayMatch {
-				ignoreIfNoMatch = true
 				errIfNoMatch = walker.ErrSkipNode
-			} else {
-				ignoreIfNoMatch = false
 			}
 		}
 
 		if !foundMatch {
-			return ignoreIfNoMatch, errIfNoMatch
+			return errIfNoMatch
 		}
 
 		if !f.pat.oldest.IsZero() && node.ModTime.Before(f.pat.oldest) {
 			debug.Log("    ModTime is older than %s\n", f.pat.oldest)
-			return ignoreIfNoMatch, errIfNoMatch
+			return errIfNoMatch
 		}
 
 		if !f.pat.newest.IsZero() && node.ModTime.After(f.pat.newest) {
 			debug.Log("    ModTime is newer than %s\n", f.pat.newest)
-			return ignoreIfNoMatch, errIfNoMatch
+			return errIfNoMatch
 		}
 
 		debug.Log("    found match\n")
 		f.out.PrintPattern(nodepath, node)
-		return false, nil
-	})
+		return nil
+	}})
 }
 
 func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
@@ -347,17 +338,17 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 	}
 
 	f.out.newsn = sn
-	return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
+	return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
 		if err != nil {
 			debug.Log("Error loading tree %v: %v", parentTreeID, err)
 
 			Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID())
 
-			return false, walker.ErrSkipNode
+			return walker.ErrSkipNode
 		}
 
 		if node == nil {
-			return false, nil
+			return nil
 		}
 
 		if node.Type == "dir" && f.treeIDs != nil {
@@ -375,7 +366,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 				// looking for blobs)
 				if f.itemsFound >= len(f.treeIDs) && f.blobIDs == nil {
 					// Return an error to terminate the Walk
-					return true, errors.New("OK")
+					return errors.New("OK")
 				}
 			}
 		}
@@ -396,8 +387,8 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 			}
 		}
 
-		return false, nil
-	})
+		return nil
+	}})
 }
 
 var errAllPacksFound = errors.New("all packs found")
@@ -585,20 +576,19 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
 		}
 	}
 
-	snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
+	snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
 	if err != nil {
 		return err
 	}
-
-	if err = repo.LoadIndex(ctx); err != nil {
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	if err = repo.LoadIndex(ctx, bar); err != nil {
 		return err
 	}
 
 	f := &Finder{
-		repo:        repo,
-		pat:         pat,
-		out:         statefulOutput{ListLong: opts.ListLong, HumanReadable: opts.HumanReadable, JSON: gopts.JSON},
-		ignoreTrees: restic.NewIDSet(),
+		repo: repo,
+		pat:  pat,
+		out:  statefulOutput{ListLong: opts.ListLong, HumanReadable: opts.HumanReadable, JSON: gopts.JSON},
 	}
 
 	if opts.BlobID {

cmd/restic/cmd_forget.go

@@ -33,7 +33,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runForget(cmd.Context(), forgetOptions, globalOptions, args)
+		return runForget(cmd.Context(), forgetOptions, forgetPruneOptions, globalOptions, args)
 	},
 }
 
@@ -98,6 +98,7 @@ type ForgetOptions struct {
 }
 
 var forgetOptions ForgetOptions
+var forgetPruneOptions PruneOptions
 
 func init() {
 	cmdRoot.AddCommand(cmdForget)
@@ -132,7 +133,7 @@ func init() {
 	f.BoolVar(&forgetOptions.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed")
 
 	f.SortFlags = false
-	addPruneOptions(cmdForget)
+	addPruneOptions(cmdForget, &forgetPruneOptions)
 }
 
 func verifyForgetOptions(opts *ForgetOptions) error {
@@ -151,7 +152,7 @@ func verifyForgetOptions(opts *ForgetOptions) error {
 	return nil
 }
 
-func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, args []string) error {
+func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOptions, gopts GlobalOptions, args []string) error {
 	err := verifyForgetOptions(&opts)
 	if err != nil {
 		return err
@@ -183,7 +184,7 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg
 	var snapshots restic.Snapshots
 	removeSnIDs := restic.NewIDSet()
 
-	for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) {
+	for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) {
 		snapshots = append(snapshots, sn)
 	}
 

cmd/restic/cmd_forget_integration_test.go

@@ -9,5 +9,8 @@ import (
 
 func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
 	opts := ForgetOptions{}
-	rtest.OK(t, runForget(context.TODO(), opts, gopts, args))
+	pruneOpts := PruneOptions{
+		MaxUnused: "5%",
+	}
+	rtest.OK(t, runForget(context.TODO(), opts, pruneOpts, gopts, args))
 }

cmd/restic/cmd_generate.go

@@ -21,7 +21,9 @@ EXIT STATUS
 Exit status is 0 if the command was successful, and non-zero if there was any error.
 `,
 	DisableAutoGenTag: true,
-	RunE:              runGenerate,
+	RunE: func(_ *cobra.Command, args []string) error {
+		return runGenerate(genOpts, args)
+	},
 }
 
 type generateOptions struct {
@@ -90,48 +92,48 @@ func writePowerShellCompletion(file string) error {
 	return cmdRoot.GenPowerShellCompletionFile(file)
 }
 
-func runGenerate(_ *cobra.Command, args []string) error {
+func runGenerate(opts generateOptions, args []string) error {
 	if len(args) > 0 {
 		return errors.Fatal("the generate command expects no arguments, only options - please see `restic help generate` for usage and flags")
 	}
 
-	if genOpts.ManDir != "" {
-		err := writeManpages(genOpts.ManDir)
+	if opts.ManDir != "" {
+		err := writeManpages(opts.ManDir)
 		if err != nil {
 			return err
 		}
 	}
 
-	if genOpts.BashCompletionFile != "" {
-		err := writeBashCompletion(genOpts.BashCompletionFile)
+	if opts.BashCompletionFile != "" {
+		err := writeBashCompletion(opts.BashCompletionFile)
 		if err != nil {
 			return err
 		}
 	}
 
-	if genOpts.FishCompletionFile != "" {
-		err := writeFishCompletion(genOpts.FishCompletionFile)
+	if opts.FishCompletionFile != "" {
+		err := writeFishCompletion(opts.FishCompletionFile)
 		if err != nil {
 			return err
 		}
 	}
 
-	if genOpts.ZSHCompletionFile != "" {
-		err := writeZSHCompletion(genOpts.ZSHCompletionFile)
+	if opts.ZSHCompletionFile != "" {
+		err := writeZSHCompletion(opts.ZSHCompletionFile)
 		if err != nil {
 			return err
 		}
 	}
 
-	if genOpts.PowerShellCompletionFile != "" {
-		err := writePowerShellCompletion(genOpts.PowerShellCompletionFile)
+	if opts.PowerShellCompletionFile != "" {
+		err := writePowerShellCompletion(opts.PowerShellCompletionFile)
 		if err != nil {
 			return err
 		}
 	}
 
 	var empty generateOptions
-	if genOpts == empty {
+	if opts == empty {
 		return errors.Fatal("nothing to do, please specify at least one output file/dir")
 	}
 

cmd/restic/cmd_init.go

@@ -75,7 +75,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []
 		return err
 	}
 
-	repo, err := ReadRepo(gopts)
+	gopts.Repo, err = ReadRepo(gopts)
 	if err != nil {
 		return err
 	}
@@ -87,7 +87,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []
 		return err
 	}
 
-	be, err := create(ctx, repo, gopts, gopts.extended)
+	be, err := create(ctx, gopts.Repo, gopts, gopts.extended)
 	if err != nil {
 		return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.backends, gopts.Repo), err)
 	}

cmd/restic/cmd_key.go

@@ -1,262 +1,18 @@
 package main
 
 import (
-	"context"
-	"encoding/json"
-	"os"
-	"strings"
-	"sync"
-
-	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/repository"
-	"github.com/restic/restic/internal/restic"
-	"github.com/restic/restic/internal/ui/table"
-
 	"github.com/spf13/cobra"
 )
 
 var cmdKey = &cobra.Command{
-	Use:   "key [flags] [list|add|remove|passwd] [ID]",
+	Use:   "key",
 	Short: "Manage keys (passwords)",
 	Long: `
-The "key" command manages keys (passwords) for accessing the repository.
-
-EXIT STATUS
-===========
-
-Exit status is 0 if the command was successful, and non-zero if there was any error.
-`,
-	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return runKey(cmd.Context(), globalOptions, args)
-	},
+The "key" command allows you to set multiple access keys or passwords
+per repository.
+	`,
 }
 
-var (
-	newPasswordFile string
-	keyUsername     string
-	keyHostname     string
-)
-
 func init() {
 	cmdRoot.AddCommand(cmdKey)
-
-	flags := cmdKey.Flags()
-	flags.StringVarP(&newPasswordFile, "new-password-file", "", "", "`file` from which to read the new password")
-	flags.StringVarP(&keyUsername, "user", "", "", "the username for new keys")
-	flags.StringVarP(&keyHostname, "host", "", "", "the hostname for new keys")
-}
-
-func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions) error {
-	type keyInfo struct {
-		Current  bool   `json:"current"`
-		ID       string `json:"id"`
-		UserName string `json:"userName"`
-		HostName string `json:"hostName"`
-		Created  string `json:"created"`
-	}
-
-	var m sync.Mutex
-	var keys []keyInfo
-
-	err := restic.ParallelList(ctx, s.Backend(), restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
-		k, err := repository.LoadKey(ctx, s, id)
-		if err != nil {
-			Warnf("LoadKey() failed: %v\n", err)
-			return nil
-		}
-
-		key := keyInfo{
-			Current:  id == s.KeyID(),
-			ID:       id.Str(),
-			UserName: k.Username,
-			HostName: k.Hostname,
-			Created:  k.Created.Local().Format(TimeFormat),
-		}
-
-		m.Lock()
-		defer m.Unlock()
-		keys = append(keys, key)
-		return nil
-	})
-
-	if err != nil {
-		return err
-	}
-
-	if gopts.JSON {
-		return json.NewEncoder(globalOptions.stdout).Encode(keys)
-	}
-
-	tab := table.New()
-	tab.AddColumn(" ID", "{{if .Current}}*{{else}} {{end}}{{ .ID }}")
-	tab.AddColumn("User", "{{ .UserName }}")
-	tab.AddColumn("Host", "{{ .HostName }}")
-	tab.AddColumn("Created", "{{ .Created }}")
-
-	for _, key := range keys {
-		tab.AddRow(key)
-	}
-
-	return tab.Write(globalOptions.stdout)
-}
-
-// testKeyNewPassword is used to set a new password during integration testing.
-var testKeyNewPassword string
-
-func getNewPassword(gopts GlobalOptions) (string, error) {
-	if testKeyNewPassword != "" {
-		return testKeyNewPassword, nil
-	}
-
-	if newPasswordFile != "" {
-		return loadPasswordFromFile(newPasswordFile)
-	}
-
-	// Since we already have an open repository, temporary remove the password
-	// to prompt the user for the passwd.
-	newopts := gopts
-	newopts.password = ""
-
-	return ReadPasswordTwice(newopts,
-		"enter new password: ",
-		"enter password again: ")
-}
-
-func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOptions) error {
-	pw, err := getNewPassword(gopts)
-	if err != nil {
-		return err
-	}
-
-	id, err := repository.AddKey(ctx, repo, pw, keyUsername, keyHostname, repo.Key())
-	if err != nil {
-		return errors.Fatalf("creating new key failed: %v\n", err)
-	}
-
-	err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw)
-	if err != nil {
-		return err
-	}
-
-	Verbosef("saved new key as %s\n", id)
-
-	return nil
-}
-
-func deleteKey(ctx context.Context, repo *repository.Repository, id restic.ID) error {
-	if id == repo.KeyID() {
-		return errors.Fatal("refusing to remove key currently used to access repository")
-	}
-
-	h := restic.Handle{Type: restic.KeyFile, Name: id.String()}
-	err := repo.Backend().Remove(ctx, h)
-	if err != nil {
-		return err
-	}
-
-	Verbosef("removed key %v\n", id)
-	return nil
-}
-
-func changePassword(ctx context.Context, repo *repository.Repository, gopts GlobalOptions) error {
-	pw, err := getNewPassword(gopts)
-	if err != nil {
-		return err
-	}
-
-	id, err := repository.AddKey(ctx, repo, pw, "", "", repo.Key())
-	if err != nil {
-		return errors.Fatalf("creating new key failed: %v\n", err)
-	}
-	oldID := repo.KeyID()
-
-	err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw)
-	if err != nil {
-		return err
-	}
-
-	h := restic.Handle{Type: restic.KeyFile, Name: oldID.String()}
-	err = repo.Backend().Remove(ctx, h)
-	if err != nil {
-		return err
-	}
-
-	Verbosef("saved new key as %s\n", id)
-
-	return nil
-}
-
-func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repository, key *repository.Key, pw string) error {
-	// Verify new key to make sure it really works. A broken key can render the
-	// whole repository inaccessible
-	err := repo.SearchKey(ctx, pw, 0, key.ID().String())
-	if err != nil {
-		// the key is invalid, try to remove it
-		h := restic.Handle{Type: restic.KeyFile, Name: key.ID().String()}
-		_ = repo.Backend().Remove(ctx, h)
-		return errors.Fatalf("failed to access repository with new key: %v", err)
-	}
-	return nil
-}
-
-func runKey(ctx context.Context, gopts GlobalOptions, args []string) error {
-	if len(args) < 1 || (args[0] == "remove" && len(args) != 2) || (args[0] != "remove" && len(args) != 1) {
-		return errors.Fatal("wrong number of arguments")
-	}
-
-	repo, err := OpenRepository(ctx, gopts)
-	if err != nil {
-		return err
-	}
-
-	switch args[0] {
-	case "list":
-		lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-
-		return listKeys(ctx, repo, gopts)
-	case "add":
-		lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-
-		return addKey(ctx, repo, gopts)
-	case "remove":
-		lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-
-		id, err := restic.Find(ctx, repo.Backend(), restic.KeyFile, args[1])
-		if err != nil {
-			return err
-		}
-
-		return deleteKey(ctx, repo, id)
-	case "passwd":
-		lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-
-		return changePassword(ctx, repo, gopts)
-	}
-
-	return nil
-}
-
-func loadPasswordFromFile(pwdFile string) (string, error) {
-	s, err := os.ReadFile(pwdFile)
-	if os.IsNotExist(err) {
-		return "", errors.Fatalf("%s does not exist", pwdFile)
-	}
-	return strings.TrimSpace(string(s)), errors.Wrap(err, "Readfile")
 }

cmd/restic/cmd_key_add.go

@@ -0,0 +1,128 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/repository"
+	"github.com/spf13/cobra"
+)
+
+var cmdKeyAdd = &cobra.Command{
+	Use:   "add",
+	Short: "Add a new key (password) to the repository; returns the new key ID",
+	Long: `
+The "add" sub-command creates a new key and validates the key. Returns the new key ID.
+
+EXIT STATUS
+===========
+
+Exit status is 0 if the command is successful, and non-zero if there was any error.
+	`,
+	DisableAutoGenTag: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runKeyAdd(cmd.Context(), globalOptions, keyAddOpts, args)
+	},
+}
+
+type KeyAddOptions struct {
+	NewPasswordFile string
+	Username        string
+	Hostname        string
+}
+
+var keyAddOpts KeyAddOptions
+
+func init() {
+	cmdKey.AddCommand(cmdKeyAdd)
+
+	flags := cmdKeyAdd.Flags()
+	flags.StringVarP(&keyAddOpts.NewPasswordFile, "new-password-file", "", "", "`file` from which to read the new password")
+	flags.StringVarP(&keyAddOpts.Username, "user", "", "", "the username for new key")
+	flags.StringVarP(&keyAddOpts.Hostname, "host", "", "", "the hostname for new key")
+}
+
+func runKeyAdd(ctx context.Context, gopts GlobalOptions, opts KeyAddOptions, args []string) error {
+	if len(args) > 0 {
+		return fmt.Errorf("the key add command expects no arguments, only options - please see `restic help key add` for usage and flags")
+	}
+
+	repo, err := OpenRepository(ctx, gopts)
+	if err != nil {
+		return err
+	}
+
+	lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	return addKey(ctx, repo, gopts, opts)
+}
+
+func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, opts KeyAddOptions) error {
+	pw, err := getNewPassword(gopts, opts.NewPasswordFile)
+	if err != nil {
+		return err
+	}
+
+	id, err := repository.AddKey(ctx, repo, pw, opts.Username, opts.Hostname, repo.Key())
+	if err != nil {
+		return errors.Fatalf("creating new key failed: %v\n", err)
+	}
+
+	err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw)
+	if err != nil {
+		return err
+	}
+
+	Verbosef("saved new key with ID %s\n", id.ID())
+
+	return nil
+}
+
+// testKeyNewPassword is used to set a new password during integration testing.
+var testKeyNewPassword string
+
+func getNewPassword(gopts GlobalOptions, newPasswordFile string) (string, error) {
+	if testKeyNewPassword != "" {
+		return testKeyNewPassword, nil
+	}
+
+	if newPasswordFile != "" {
+		return loadPasswordFromFile(newPasswordFile)
+	}
+
+	// Since we already have an open repository, temporary remove the password
+	// to prompt the user for the passwd.
+	newopts := gopts
+	newopts.password = ""
+
+	return ReadPasswordTwice(newopts,
+		"enter new password: ",
+		"enter password again: ")
+}
+
+func loadPasswordFromFile(pwdFile string) (string, error) {
+	s, err := os.ReadFile(pwdFile)
+	if os.IsNotExist(err) {
+		return "", errors.Fatalf("%s does not exist", pwdFile)
+	}
+	return strings.TrimSpace(string(s)), errors.Wrap(err, "Readfile")
+}
+
+func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repository, key *repository.Key, pw string) error {
+	// Verify new key to make sure it really works. A broken key can render the
+	// whole repository inaccessible
+	err := repo.SearchKey(ctx, pw, 0, key.ID().String())
+	if err != nil {
+		// the key is invalid, try to remove it
+		_ = repository.RemoveKey(ctx, repo, key.ID())
+		return errors.Fatalf("failed to access repository with new key: %v", err)
+	}
+	return nil
+}

cmd/restic/cmd_key_integration_test.go

@@ -4,16 +4,17 @@ import (
 	"bufio"
 	"context"
 	"regexp"
+	"strings"
 	"testing"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/repository"
-	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 )
 
 func testRunKeyListOtherIDs(t testing.TB, gopts GlobalOptions) []string {
 	buf, err := withCaptureStdout(func() error {
-		return runKey(context.TODO(), gopts, []string{"list"})
+		return runKeyList(context.TODO(), gopts, []string{})
 	})
 	rtest.OK(t, err)
 
@@ -36,21 +37,20 @@ func testRunKeyAddNewKey(t testing.TB, newPassword string, gopts GlobalOptions)
 		testKeyNewPassword = ""
 	}()
 
-	rtest.OK(t, runKey(context.TODO(), gopts, []string{"add"}))
+	rtest.OK(t, runKeyAdd(context.TODO(), gopts, KeyAddOptions{}, []string{}))
 }
 
 func testRunKeyAddNewKeyUserHost(t testing.TB, gopts GlobalOptions) {
 	testKeyNewPassword = "john's geheimnis"
 	defer func() {
 		testKeyNewPassword = ""
-		keyUsername = ""
-		keyHostname = ""
 	}()
 
-	rtest.OK(t, cmdKey.Flags().Parse([]string{"--user=john", "--host=example.com"}))
-
 	t.Log("adding key for john@example.com")
-	rtest.OK(t, runKey(context.TODO(), gopts, []string{"add"}))
+	rtest.OK(t, runKeyAdd(context.TODO(), gopts, KeyAddOptions{
+		Username: "john",
+		Hostname: "example.com",
+	}, []string{}))
 
 	repo, err := OpenRepository(context.TODO(), gopts)
 	rtest.OK(t, err)
@@ -67,13 +67,13 @@ func testRunKeyPasswd(t testing.TB, newPassword string, gopts GlobalOptions) {
 		testKeyNewPassword = ""
 	}()
 
-	rtest.OK(t, runKey(context.TODO(), gopts, []string{"passwd"}))
+	rtest.OK(t, runKeyPasswd(context.TODO(), gopts, KeyPasswdOptions{}, []string{}))
 }
 
 func testRunKeyRemove(t testing.TB, gopts GlobalOptions, IDs []string) {
 	t.Logf("remove %d keys: %q\n", len(IDs), IDs)
 	for _, id := range IDs {
-		rtest.OK(t, runKey(context.TODO(), gopts, []string{"remove", id}))
+		rtest.OK(t, runKeyRemove(context.TODO(), gopts, []string{id}))
 	}
 }
 
@@ -103,18 +103,18 @@ func TestKeyAddRemove(t *testing.T) {
 
 	env.gopts.password = passwordList[len(passwordList)-1]
 	t.Logf("testing access with last password %q\n", env.gopts.password)
-	rtest.OK(t, runKey(context.TODO(), env.gopts, []string{"list"}))
+	rtest.OK(t, runKeyList(context.TODO(), env.gopts, []string{}))
 	testRunCheck(t, env.gopts)
 
 	testRunKeyAddNewKeyUserHost(t, env.gopts)
 }
 
 type emptySaveBackend struct {
-	restic.Backend
+	backend.Backend
 }
 
-func (b *emptySaveBackend) Save(ctx context.Context, h restic.Handle, _ restic.RewindReader) error {
-	return b.Backend.Save(ctx, h, restic.NewByteReader([]byte{}, nil))
+func (b *emptySaveBackend) Save(ctx context.Context, h backend.Handle, _ backend.RewindReader) error {
+	return b.Backend.Save(ctx, h, backend.NewByteReader([]byte{}, nil))
 }
 
 func TestKeyProblems(t *testing.T) {
@@ -122,7 +122,7 @@ func TestKeyProblems(t *testing.T) {
 	defer cleanup()
 
 	testRunInit(t, env.gopts)
-	env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
+	env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
 		return &emptySaveBackend{r}, nil
 	}
 
@@ -131,15 +131,45 @@ func TestKeyProblems(t *testing.T) {
 		testKeyNewPassword = ""
 	}()
 
-	err := runKey(context.TODO(), env.gopts, []string{"passwd"})
+	err := runKeyPasswd(context.TODO(), env.gopts, KeyPasswdOptions{}, []string{})
 	t.Log(err)
 	rtest.Assert(t, err != nil, "expected passwd change to fail")
 
-	err = runKey(context.TODO(), env.gopts, []string{"add"})
+	err = runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{}, []string{})
 	t.Log(err)
 	rtest.Assert(t, err != nil, "expected key adding to fail")
 
 	t.Logf("testing access with initial password %q\n", env.gopts.password)
-	rtest.OK(t, runKey(context.TODO(), env.gopts, []string{"list"}))
+	rtest.OK(t, runKeyList(context.TODO(), env.gopts, []string{}))
 	testRunCheck(t, env.gopts)
 }
+
+func TestKeyCommandInvalidArguments(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testRunInit(t, env.gopts)
+	env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
+		return &emptySaveBackend{r}, nil
+	}
+
+	err := runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{}, []string{"johndoe"})
+	t.Log(err)
+	rtest.Assert(t, err != nil && strings.Contains(err.Error(), "no arguments"), "unexpected error for key add: %v", err)
+
+	err = runKeyPasswd(context.TODO(), env.gopts, KeyPasswdOptions{}, []string{"johndoe"})
+	t.Log(err)
+	rtest.Assert(t, err != nil && strings.Contains(err.Error(), "no arguments"), "unexpected error for key passwd: %v", err)
+
+	err = runKeyList(context.TODO(), env.gopts, []string{"johndoe"})
+	t.Log(err)
+	rtest.Assert(t, err != nil && strings.Contains(err.Error(), "no arguments"), "unexpected error for key list: %v", err)
+
+	err = runKeyRemove(context.TODO(), env.gopts, []string{})
+	t.Log(err)
+	rtest.Assert(t, err != nil && strings.Contains(err.Error(), "one argument"), "unexpected error for key remove: %v", err)
+
+	err = runKeyRemove(context.TODO(), env.gopts, []string{"john", "doe"})
+	t.Log(err)
+	rtest.Assert(t, err != nil && strings.Contains(err.Error(), "one argument"), "unexpected error for key remove: %v", err)
+}

cmd/restic/cmd_key_list.go

@@ -0,0 +1,112 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sync"
+
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui/table"
+	"github.com/spf13/cobra"
+)
+
+var cmdKeyList = &cobra.Command{
+	Use:   "list",
+	Short: "List keys (passwords)",
+	Long: `
+The "list" sub-command lists all the keys (passwords) associated with the repository.
+Returns the key ID, username, hostname, created time and if it's the current key being
+used to access the repository.
+
+EXIT STATUS
+===========
+
+Exit status is 0 if the command is successful, and non-zero if there was any error.
+	`,
+	DisableAutoGenTag: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runKeyList(cmd.Context(), globalOptions, args)
+	},
+}
+
+func init() {
+	cmdKey.AddCommand(cmdKeyList)
+}
+
+func runKeyList(ctx context.Context, gopts GlobalOptions, args []string) error {
+	if len(args) > 0 {
+		return fmt.Errorf("the key list command expects no arguments, only options - please see `restic help key list` for usage and flags")
+	}
+
+	repo, err := OpenRepository(ctx, gopts)
+	if err != nil {
+		return err
+	}
+
+	if !gopts.NoLock {
+		var lock *restic.Lock
+		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	}
+
+	return listKeys(ctx, repo, gopts)
+}
+
+func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions) error {
+	type keyInfo struct {
+		Current  bool   `json:"current"`
+		ID       string `json:"id"`
+		UserName string `json:"userName"`
+		HostName string `json:"hostName"`
+		Created  string `json:"created"`
+	}
+
+	var m sync.Mutex
+	var keys []keyInfo
+
+	err := restic.ParallelList(ctx, s, restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, _ int64) error {
+		k, err := repository.LoadKey(ctx, s, id)
+		if err != nil {
+			Warnf("LoadKey() failed: %v\n", err)
+			return nil
+		}
+
+		key := keyInfo{
+			Current:  id == s.KeyID(),
+			ID:       id.Str(),
+			UserName: k.Username,
+			HostName: k.Hostname,
+			Created:  k.Created.Local().Format(TimeFormat),
+		}
+
+		m.Lock()
+		defer m.Unlock()
+		keys = append(keys, key)
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	if gopts.JSON {
+		return json.NewEncoder(globalOptions.stdout).Encode(keys)
+	}
+
+	tab := table.New()
+	tab.AddColumn(" ID", "{{if .Current}}*{{else}} {{end}}{{ .ID }}")
+	tab.AddColumn("User", "{{ .UserName }}")
+	tab.AddColumn("Host", "{{ .HostName }}")
+	tab.AddColumn("Created", "{{ .Created }}")
+
+	for _, key := range keys {
+		tab.AddRow(key)
+	}
+
+	return tab.Write(globalOptions.stdout)
+}

cmd/restic/cmd_key_passwd.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/repository"
+	"github.com/spf13/cobra"
+)
+
+var cmdKeyPasswd = &cobra.Command{
+	Use:   "passwd",
+	Short: "Change key (password); creates a new key ID and removes the old key ID, returns new key ID",
+	Long: `
+The "passwd" sub-command creates a new key, validates the key and remove the old key ID.
+Returns the new key ID. 
+
+EXIT STATUS
+===========
+
+Exit status is 0 if the command is successful, and non-zero if there was any error.
+	`,
+	DisableAutoGenTag: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runKeyPasswd(cmd.Context(), globalOptions, keyPasswdOpts, args)
+	},
+}
+
+type KeyPasswdOptions struct {
+	KeyAddOptions
+}
+
+var keyPasswdOpts KeyPasswdOptions
+
+func init() {
+	cmdKey.AddCommand(cmdKeyPasswd)
+
+	flags := cmdKeyPasswd.Flags()
+	flags.StringVarP(&keyPasswdOpts.NewPasswordFile, "new-password-file", "", "", "`file` from which to read the new password")
+	flags.StringVarP(&keyPasswdOpts.Username, "user", "", "", "the username for new key")
+	flags.StringVarP(&keyPasswdOpts.Hostname, "host", "", "", "the hostname for new key")
+}
+
+func runKeyPasswd(ctx context.Context, gopts GlobalOptions, opts KeyPasswdOptions, args []string) error {
+	if len(args) > 0 {
+		return fmt.Errorf("the key passwd command expects no arguments, only options - please see `restic help key passwd` for usage and flags")
+	}
+
+	repo, err := OpenRepository(ctx, gopts)
+	if err != nil {
+		return err
+	}
+
+	lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	return changePassword(ctx, repo, gopts, opts)
+}
+
+func changePassword(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, opts KeyPasswdOptions) error {
+	pw, err := getNewPassword(gopts, opts.NewPasswordFile)
+	if err != nil {
+		return err
+	}
+
+	id, err := repository.AddKey(ctx, repo, pw, "", "", repo.Key())
+	if err != nil {
+		return errors.Fatalf("creating new key failed: %v\n", err)
+	}
+	oldID := repo.KeyID()
+
+	err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw)
+	if err != nil {
+		return err
+	}
+
+	err = repository.RemoveKey(ctx, repo, oldID)
+	if err != nil {
+		return err
+	}
+
+	Verbosef("saved new key as %s\n", id)
+
+	return nil
+}

cmd/restic/cmd_key_remove.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
+	"github.com/spf13/cobra"
+)
+
+var cmdKeyRemove = &cobra.Command{
+	Use:   "remove [ID]",
+	Short: "Remove key ID (password) from the repository.",
+	Long: `
+The "remove" sub-command removes the selected key ID. The "remove" command does not allow
+removing the current key being used to access the repository. 
+
+EXIT STATUS
+===========
+
+Exit status is 0 if the command is successful, and non-zero if there was any error.
+	`,
+	DisableAutoGenTag: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runKeyRemove(cmd.Context(), globalOptions, args)
+	},
+}
+
+func init() {
+	cmdKey.AddCommand(cmdKeyRemove)
+}
+
+func runKeyRemove(ctx context.Context, gopts GlobalOptions, args []string) error {
+	if len(args) != 1 {
+		return fmt.Errorf("key remove expects one argument as the key id")
+	}
+
+	repo, err := OpenRepository(ctx, gopts)
+	if err != nil {
+		return err
+	}
+
+	lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	idPrefix := args[0]
+
+	return deleteKey(ctx, repo, idPrefix)
+}
+
+func deleteKey(ctx context.Context, repo *repository.Repository, idPrefix string) error {
+	id, err := restic.Find(ctx, repo, restic.KeyFile, idPrefix)
+	if err != nil {
+		return err
+	}
+
+	if id == repo.KeyID() {
+		return errors.Fatal("refusing to remove key currently used to access repository")
+	}
+
+	err = repository.RemoveKey(ctx, repo, id)
+	if err != nil {
+		return err
+	}
+
+	Verbosef("removed key %v\n", id)
+	return nil
+}

cmd/restic/cmd_list.go

@@ -23,7 +23,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runList(cmd.Context(), cmd, globalOptions, args)
+		return runList(cmd.Context(), globalOptions, args)
 	},
 }
 
@@ -31,9 +31,9 @@ func init() {
 	cmdRoot.AddCommand(cmdList)
 }
 
-func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args []string) error {
+func runList(ctx context.Context, gopts GlobalOptions, args []string) error {
 	if len(args) != 1 {
-		return errors.Fatal("type not specified, usage: " + cmd.Use)
+		return errors.Fatal("type not specified")
 	}
 
 	repo, err := OpenRepository(ctx, gopts)
@@ -63,7 +63,7 @@ func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args
 	case "locks":
 		t = restic.LockFile
 	case "blobs":
-		return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
+		return index.ForAllIndexes(ctx, repo, repo, func(_ restic.ID, idx *index.Index, _ bool, err error) error {
 			if err != nil {
 				return err
 			}
@@ -76,7 +76,7 @@ func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args
 		return errors.Fatal("invalid type")
 	}
 
-	return repo.List(ctx, t, func(id restic.ID, size int64) error {
+	return repo.List(ctx, t, func(id restic.ID, _ int64) error {
 		Printf("%s\n", id)
 		return nil
 	})

cmd/restic/cmd_list_integration_test.go

@@ -12,7 +12,7 @@ import (
 
 func testRunList(t testing.TB, tpe string, opts GlobalOptions) restic.IDs {
 	buf, err := withCaptureStdout(func() error {
-		return runList(context.TODO(), cmdList, opts, []string{tpe})
+		return runList(context.TODO(), opts, []string{tpe})
 	})
 	rtest.OK(t, err)
 	return parseIDsFromReader(t, buf)

cmd/restic/cmd_ls.go

@@ -3,13 +3,14 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"fmt"
+	"io"
 	"os"
 	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
 
-	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/restic"
@@ -52,6 +53,7 @@ type LsOptions struct {
 	restic.SnapshotFilter
 	Recursive     bool
 	HumanReadable bool
+	Ncdu          bool
 }
 
 var lsOptions LsOptions
@@ -64,16 +66,47 @@ func init() {
 	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
 	flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories")
 	flags.BoolVar(&lsOptions.HumanReadable, "human-readable", false, "print sizes in human readable format")
+	flags.BoolVar(&lsOptions.Ncdu, "ncdu", false, "output NCDU export format (pipe into 'ncdu -f -')")
 }
 
-type lsSnapshot struct {
-	*restic.Snapshot
-	ID         *restic.ID `json:"id"`
-	ShortID    string     `json:"short_id"`
-	StructType string     `json:"struct_type"` // "snapshot"
+type lsPrinter interface {
+	Snapshot(sn *restic.Snapshot)
+	Node(path string, node *restic.Node)
+	LeaveDir(path string)
+	Close()
+}
+
+type jsonLsPrinter struct {
+	enc *json.Encoder
+}
+
+func (p *jsonLsPrinter) Snapshot(sn *restic.Snapshot) {
+	type lsSnapshot struct {
+		*restic.Snapshot
+		ID         *restic.ID `json:"id"`
+		ShortID    string     `json:"short_id"`
+		StructType string     `json:"struct_type"` // "snapshot"
+	}
+
+	err := p.enc.Encode(lsSnapshot{
+		Snapshot:   sn,
+		ID:         sn.ID(),
+		ShortID:    sn.ID().Str(),
+		StructType: "snapshot",
+	})
+	if err != nil {
+		Warnf("JSON encode failed: %v\n", err)
+	}
 }
 
 // Print node in our custom JSON format, followed by a newline.
+func (p *jsonLsPrinter) Node(path string, node *restic.Node) {
+	err := lsNodeJSON(p.enc, path, node)
+	if err != nil {
+		Warnf("JSON encode failed: %v\n", err)
+	}
+}
+
 func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
 	n := &struct {
 		Name        string      `json:"name"`
@@ -87,6 +120,7 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
 		ModTime     time.Time   `json:"mtime,omitempty"`
 		AccessTime  time.Time   `json:"atime,omitempty"`
 		ChangeTime  time.Time   `json:"ctime,omitempty"`
+		Inode       uint64      `json:"inode,omitempty"`
 		StructType  string      `json:"struct_type"` // "node"
 
 		size uint64 // Target for Size pointer.
@@ -102,6 +136,7 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
 		ModTime:     node.ModTime,
 		AccessTime:  node.AccessTime,
 		ChangeTime:  node.ChangeTime,
+		Inode:       node.Inode,
 		StructType:  "node",
 	}
 	// Always print size for regular files, even when empty,
@@ -113,10 +148,117 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
 	return enc.Encode(n)
 }
 
+func (p *jsonLsPrinter) LeaveDir(_ string) {}
+func (p *jsonLsPrinter) Close()            {}
+
+type ncduLsPrinter struct {
+	out   io.Writer
+	depth int
+}
+
+// lsSnapshotNcdu prints a restic snapshot in Ncdu save format.
+// It opens the JSON list. Nodes are added with lsNodeNcdu and the list is closed by lsCloseNcdu.
+// Format documentation: https://dev.yorhel.nl/ncdu/jsonfmt
+func (p *ncduLsPrinter) Snapshot(sn *restic.Snapshot) {
+	const NcduMajorVer = 1
+	const NcduMinorVer = 2
+
+	snapshotBytes, err := json.Marshal(sn)
+	if err != nil {
+		Warnf("JSON encode failed: %v\n", err)
+	}
+	p.depth++
+	fmt.Fprintf(p.out, "[%d, %d, %s", NcduMajorVer, NcduMinorVer, string(snapshotBytes))
+}
+
+func lsNcduNode(_ string, node *restic.Node) ([]byte, error) {
+	type NcduNode struct {
+		Name   string `json:"name"`
+		Asize  uint64 `json:"asize"`
+		Dsize  uint64 `json:"dsize"`
+		Dev    uint64 `json:"dev"`
+		Ino    uint64 `json:"ino"`
+		NLink  uint64 `json:"nlink"`
+		NotReg bool   `json:"notreg"`
+		UID    uint32 `json:"uid"`
+		GID    uint32 `json:"gid"`
+		Mode   uint16 `json:"mode"`
+		Mtime  int64  `json:"mtime"`
+	}
+
+	outNode := NcduNode{
+		Name:   node.Name,
+		Asize:  node.Size,
+		Dsize:  node.Size,
+		Dev:    node.DeviceID,
+		Ino:    node.Inode,
+		NLink:  node.Links,
+		NotReg: node.Type != "dir" && node.Type != "file",
+		UID:    node.UID,
+		GID:    node.GID,
+		Mode:   uint16(node.Mode & os.ModePerm),
+		Mtime:  node.ModTime.Unix(),
+	}
+	//  bits according to inode(7) manpage
+	if node.Mode&os.ModeSetuid != 0 {
+		outNode.Mode |= 0o4000
+	}
+	if node.Mode&os.ModeSetgid != 0 {
+		outNode.Mode |= 0o2000
+	}
+	if node.Mode&os.ModeSticky != 0 {
+		outNode.Mode |= 0o1000
+	}
+
+	return json.Marshal(outNode)
+}
+
+func (p *ncduLsPrinter) Node(path string, node *restic.Node) {
+	out, err := lsNcduNode(path, node)
+	if err != nil {
+		Warnf("JSON encode failed: %v\n", err)
+	}
+
+	if node.Type == "dir" {
+		fmt.Fprintf(p.out, ",\n%s[\n%s%s", strings.Repeat("  ", p.depth), strings.Repeat("  ", p.depth+1), string(out))
+		p.depth++
+	} else {
+		fmt.Fprintf(p.out, ",\n%s%s", strings.Repeat("  ", p.depth), string(out))
+	}
+}
+
+func (p *ncduLsPrinter) LeaveDir(_ string) {
+	p.depth--
+	fmt.Fprintf(p.out, "\n%s]", strings.Repeat("  ", p.depth))
+}
+
+func (p *ncduLsPrinter) Close() {
+	fmt.Fprint(p.out, "\n]\n")
+}
+
+type textLsPrinter struct {
+	dirs          []string
+	ListLong      bool
+	HumanReadable bool
+}
+
+func (p *textLsPrinter) Snapshot(sn *restic.Snapshot) {
+	Verbosef("%v filtered by %v:\n", sn, p.dirs)
+}
+func (p *textLsPrinter) Node(path string, node *restic.Node) {
+	Printf("%s\n", formatNode(path, node, p.ListLong, p.HumanReadable))
+}
+
+func (p *textLsPrinter) LeaveDir(_ string) {}
+func (p *textLsPrinter) Close()            {}
+
 func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error {
 	if len(args) == 0 {
 		return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'")
 	}
+	if opts.Ncdu && gopts.JSON {
+		return errors.Fatal("only either '--json' or '--ncdu' can be specified")
+	}
 
 	// extract any specific directories to walk
 	var dirs []string
@@ -168,47 +310,31 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		return err
 	}
 
-	snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
+	snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
 	if err != nil {
 		return err
 	}
 
-	if err = repo.LoadIndex(ctx); err != nil {
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	if err = repo.LoadIndex(ctx, bar); err != nil {
 		return err
 	}
 
-	var (
-		printSnapshot func(sn *restic.Snapshot)
-		printNode     func(path string, node *restic.Node)
-	)
+	var printer lsPrinter
 
 	if gopts.JSON {
-		enc := json.NewEncoder(globalOptions.stdout)
-
-		printSnapshot = func(sn *restic.Snapshot) {
-			err := enc.Encode(lsSnapshot{
-				Snapshot:   sn,
-				ID:         sn.ID(),
-				ShortID:    sn.ID().Str(),
-				StructType: "snapshot",
-			})
-			if err != nil {
-				Warnf("JSON encode failed: %v\n", err)
-			}
+		printer = &jsonLsPrinter{
+			enc: json.NewEncoder(globalOptions.stdout),
 		}
-
-		printNode = func(path string, node *restic.Node) {
-			err := lsNodeJSON(enc, path, node)
-			if err != nil {
-				Warnf("JSON encode failed: %v\n", err)
-			}
+	} else if opts.Ncdu {
+		printer = &ncduLsPrinter{
+			out: globalOptions.stdout,
 		}
 	} else {
-		printSnapshot = func(sn *restic.Snapshot) {
-			Verbosef("snapshot %s of %v filtered by %v at %s):\n", sn.ID().Str(), sn.Paths, dirs, sn.Time)
-		}
-		printNode = func(path string, node *restic.Node) {
-			Printf("%s\n", formatNode(path, node, lsOptions.ListLong, lsOptions.HumanReadable))
+		printer = &textLsPrinter{
+			dirs:          dirs,
+			ListLong:      opts.ListLong,
+			HumanReadable: opts.HumanReadable,
 		}
 	}
 
@@ -226,44 +352,55 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		return err
 	}
 
-	printSnapshot(sn)
+	printer.Snapshot(sn)
 
-	err = walker.Walk(ctx, repo, *sn.Tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
+	processNode := func(_ restic.ID, nodepath string, node *restic.Node, err error) error {
 		if err != nil {
-			return false, err
+			return err
 		}
 		if node == nil {
-			return false, nil
+			return nil
 		}
 
 		if withinDir(nodepath) {
 			// if we're within a dir, print the node
-			printNode(nodepath, node)
+			printer.Node(nodepath, node)
 
 			// if recursive listing is requested, signal the walker that it
 			// should continue walking recursively
 			if opts.Recursive {
-				return false, nil
+				return nil
 			}
 		}
 
 		// if there's an upcoming match deeper in the tree (but we're not
 		// there yet), signal the walker to descend into any subdirs
 		if approachingMatchingTree(nodepath) {
-			return false, nil
+			return nil
 		}
 
 		// otherwise, signal the walker to not walk recursively into any
 		// subdirs
 		if node.Type == "dir" {
-			return false, walker.ErrSkipNode
+			return walker.ErrSkipNode
 		}
-		return false, nil
+		return nil
+	}
+
+	err = walker.Walk(ctx, repo, *sn.Tree, walker.WalkVisitor{
+		ProcessNode: processNode,
+		LeaveDir: func(path string) {
+			// the root path `/` has no corresponding node and is thus also skipped by processNode
+			if withinDir(path) && path != "/" {
+				printer.LeaveDir(path)
+			}
+		},
 	})
 
 	if err != nil {
 		return err
 	}
 
+	printer.Close()
 	return nil
 }

cmd/restic/cmd_ls_integration_test.go

@@ -2,18 +2,46 @@ package main
 
 import (
 	"context"
+	"encoding/json"
+	"path/filepath"
 	"strings"
 	"testing"
 
 	rtest "github.com/restic/restic/internal/test"
 )
 
-func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
+func testRunLsWithOpts(t testing.TB, gopts GlobalOptions, opts LsOptions, args []string) []byte {
 	buf, err := withCaptureStdout(func() error {
 		gopts.Quiet = true
-		opts := LsOptions{}
-		return runLs(context.TODO(), opts, gopts, []string{snapshotID})
+		return runLs(context.TODO(), opts, gopts, args)
 	})
 	rtest.OK(t, err)
-	return strings.Split(buf.String(), "\n")
+	return buf.Bytes()
+}
+
+func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
+	out := testRunLsWithOpts(t, gopts, LsOptions{}, []string{snapshotID})
+	return strings.Split(string(out), "\n")
+}
+
+func assertIsValidJSON(t *testing.T, data []byte) {
+	// Sanity check: output must be valid JSON.
+	var v interface{}
+	err := json.Unmarshal(data, &v)
+	rtest.OK(t, err)
+}
+
+func TestRunLsNcdu(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testRunInit(t, env.gopts)
+	opts := BackupOptions{}
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+
+	ncdu := testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest"})
+	assertIsValidJSON(t, ncdu)
+
+	ncdu = testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest", "/testdata"})
+	assertIsValidJSON(t, ncdu)
 }

cmd/restic/cmd_ls_test.go

@@ -11,78 +11,94 @@ import (
 	rtest "github.com/restic/restic/internal/test"
 )
 
+type lsTestNode struct {
+	path string
+	restic.Node
+}
+
+var lsTestNodes = []lsTestNode{
+	// Mode is omitted when zero.
+	// Permissions, by convention is "-" per mode bit
+	{
+		path: "/bar/baz",
+		Node: restic.Node{
+			Name: "baz",
+			Type: "file",
+			Size: 12345,
+			UID:  10000000,
+			GID:  20000000,
+
+			User:  "nobody",
+			Group: "nobodies",
+			Links: 1,
+		},
+	},
+
+	// Even empty files get an explicit size.
+	{
+		path: "/foo/empty",
+		Node: restic.Node{
+			Name: "empty",
+			Type: "file",
+			Size: 0,
+			UID:  1001,
+			GID:  1001,
+
+			User:  "not printed",
+			Group: "not printed",
+			Links: 0xF00,
+		},
+	},
+
+	// Non-regular files do not get a size.
+	// Mode is printed in decimal, including the type bits.
+	{
+		path: "/foo/link",
+		Node: restic.Node{
+			Name:       "link",
+			Type:       "symlink",
+			Mode:       os.ModeSymlink | 0777,
+			LinkTarget: "not printed",
+		},
+	},
+
+	{
+		path: "/some/directory",
+		Node: restic.Node{
+			Name:       "directory",
+			Type:       "dir",
+			Mode:       os.ModeDir | 0755,
+			ModTime:    time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC),
+			AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC),
+			ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC),
+		},
+	},
+
+	// Test encoding of setuid/setgid/sticky bit
+	{
+		path: "/some/sticky",
+		Node: restic.Node{
+			Name: "sticky",
+			Type: "dir",
+			Mode: os.ModeDir | 0755 | os.ModeSetuid | os.ModeSetgid | os.ModeSticky,
+		},
+	},
+}
+
 func TestLsNodeJSON(t *testing.T) {
-	for _, c := range []struct {
-		path string
-		restic.Node
-		expect string
-	}{
-		// Mode is omitted when zero.
-		// Permissions, by convention is "-" per mode bit
-		{
-			path: "/bar/baz",
-			Node: restic.Node{
-				Name: "baz",
-				Type: "file",
-				Size: 12345,
-				UID:  10000000,
-				GID:  20000000,
-
-				User:  "nobody",
-				Group: "nobodies",
-				Links: 1,
-			},
-			expect: `{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
-		},
-
-		// Even empty files get an explicit size.
-		{
-			path: "/foo/empty",
-			Node: restic.Node{
-				Name: "empty",
-				Type: "file",
-				Size: 0,
-				UID:  1001,
-				GID:  1001,
-
-				User:  "not printed",
-				Group: "not printed",
-				Links: 0xF00,
-			},
-			expect: `{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
-		},
-
-		// Non-regular files do not get a size.
-		// Mode is printed in decimal, including the type bits.
-		{
-			path: "/foo/link",
-			Node: restic.Node{
-				Name:       "link",
-				Type:       "symlink",
-				Mode:       os.ModeSymlink | 0777,
-				LinkTarget: "not printed",
-			},
-			expect: `{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
-		},
-
-		{
-			path: "/some/directory",
-			Node: restic.Node{
-				Name:       "directory",
-				Type:       "dir",
-				Mode:       os.ModeDir | 0755,
-				ModTime:    time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC),
-				AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC),
-				ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC),
-			},
-			expect: `{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`,
-		},
+	for i, expect := range []string{
+		`{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
+		`{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
+		`{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
+		`{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`,
+		`{"name":"sticky","type":"dir","path":"/some/sticky","uid":0,"gid":0,"mode":2161115629,"permissions":"dugtrwxr-xr-x","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
 	} {
+		c := lsTestNodes[i]
 		buf := new(bytes.Buffer)
 		enc := json.NewEncoder(buf)
 		err := lsNodeJSON(enc, c.path, &c.Node)
 		rtest.OK(t, err)
-		rtest.Equals(t, c.expect+"\n", buf.String())
+		rtest.Equals(t, expect+"\n", buf.String())
 
 		// Sanity check: output must be valid JSON.
 		var v interface{}
@@ -90,3 +106,54 @@ func TestLsNodeJSON(t *testing.T) {
 		rtest.OK(t, err)
 	}
 }
+
+func TestLsNcduNode(t *testing.T) {
+	for i, expect := range []string{
+		`{"name":"baz","asize":12345,"dsize":12345,"dev":0,"ino":0,"nlink":1,"notreg":false,"uid":10000000,"gid":20000000,"mode":0,"mtime":-62135596800}`,
+		`{"name":"empty","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":3840,"notreg":false,"uid":1001,"gid":1001,"mode":0,"mtime":-62135596800}`,
+		`{"name":"link","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":true,"uid":0,"gid":0,"mode":511,"mtime":-62135596800}`,
+		`{"name":"directory","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":493,"mtime":1577934245}`,
+		`{"name":"sticky","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":4077,"mtime":-62135596800}`,
+	} {
+		c := lsTestNodes[i]
+		out, err := lsNcduNode(c.path, &c.Node)
+		rtest.OK(t, err)
+		rtest.Equals(t, expect, string(out))
+
+		// Sanity check: output must be valid JSON.
+		var v interface{}
+		err = json.Unmarshal(out, &v)
+		rtest.OK(t, err)
+	}
+}
+
+func TestLsNcdu(t *testing.T) {
+	var buf bytes.Buffer
+	printer := &ncduLsPrinter{
+		out: &buf,
+	}
+
+	printer.Snapshot(&restic.Snapshot{
+		Hostname: "host",
+		Paths:    []string{"/example"},
+	})
+	printer.Node("/directory", &restic.Node{
+		Type: "dir",
+		Name: "directory",
+	})
+	printer.Node("/directory/data", &restic.Node{
+		Type: "file",
+		Name: "data",
+		Size: 42,
+	})
+	printer.LeaveDir("/directory")
+	printer.Close()
+
+	rtest.Equals(t, `[1, 2, {"time":"0001-01-01T00:00:00Z","tree":null,"paths":["/example"],"hostname":"host"},
+  [
+    {"name":"directory","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":-62135596800},
+    {"name":"data","asize":42,"dsize":42,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":-62135596800}
+  ]
+]
+`, buf.String())
+}

cmd/restic/cmd_mount.go

@@ -113,6 +113,15 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args
 		return errors.Fatal("wrong number of parameters")
 	}
 
+	mountpoint := args[0]
+
+	// Check the existence of the mount point at the earliest stage to
+	// prevent unnecessary computations while opening the repository.
+	if _, err := resticfs.Stat(mountpoint); errors.Is(err, os.ErrNotExist) {
+		Verbosef("Mountpoint %s doesn't exist\n", mountpoint)
+		return err
+	}
+
 	debug.Log("start mount")
 	defer debug.Log("finish mount")
 
@@ -130,17 +139,12 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args
 		}
 	}
 
-	err = repo.LoadIndex(ctx)
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	err = repo.LoadIndex(ctx, bar)
 	if err != nil {
 		return err
 	}
 
-	mountpoint := args[0]
-
-	if _, err := resticfs.Stat(mountpoint); errors.Is(err, os.ErrNotExist) {
-		Verbosef("Mountpoint %s doesn't exist\n", mountpoint)
-		return err
-	}
 	mountOptions := []systemFuse.MountOption{
 		systemFuse.ReadOnly(),
 		systemFuse.FSName("restic"),

cmd/restic/cmd_options.go

@@ -21,7 +21,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	Hidden:            true,
 	DisableAutoGenTag: true,
-	Run: func(cmd *cobra.Command, args []string) {
+	Run: func(_ *cobra.Command, _ []string) {
 		fmt.Printf("All Extended Options:\n")
 		var maxLen int
 		for _, opt := range options.List() {

cmd/restic/cmd_prune.go

@@ -15,6 +15,7 @@ import (
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/ui"
+	"github.com/restic/restic/internal/ui/progress"
 
 	"github.com/spf13/cobra"
 )
@@ -36,7 +37,7 @@ EXIT STATUS
 Exit status is 0 if the command was successful, and non-zero if there was any error.
 `,
 	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
+	RunE: func(cmd *cobra.Command, _ []string) error {
 		return runPrune(cmd.Context(), pruneOptions, globalOptions)
 	},
 }
@@ -66,10 +67,10 @@ func init() {
 	f := cmdPrune.Flags()
 	f.BoolVarP(&pruneOptions.DryRun, "dry-run", "n", false, "do not modify the repository, just print what would be done")
 	f.StringVarP(&pruneOptions.UnsafeNoSpaceRecovery, "unsafe-recover-no-free-space", "", "", "UNSAFE, READ THE DOCUMENTATION BEFORE USING! Try to recover a repository stuck with no free space. Do not use without trying out 'prune --max-repack-size 0' first.")
-	addPruneOptions(cmdPrune)
+	addPruneOptions(cmdPrune, &pruneOptions)
 }
 
-func addPruneOptions(c *cobra.Command) {
+func addPruneOptions(c *cobra.Command, pruneOptions *PruneOptions) {
 	f := c.Flags()
 	f.StringVar(&pruneOptions.MaxUnused, "max-unused", "5%", "tolerate given `limit` of unused data (absolute value in bytes with suffixes k/K, m/M, g/G, t/T, a value in % or the word 'unlimited')")
 	f.StringVar(&pruneOptions.MaxRepackSize, "max-repack-size", "", "maximum `size` to repack (allowed suffixes: k/K, m/M, g/G, t/T)")
@@ -100,7 +101,7 @@ func verifyPruneOptions(opts *PruneOptions) error {
 	// parse MaxUnused either as unlimited, a percentage, or an absolute number of bytes
 	switch {
 	case maxUnused == "unlimited":
-		opts.maxUnusedBytes = func(used uint64) uint64 {
+		opts.maxUnusedBytes = func(_ uint64) uint64 {
 			return math.MaxUint64
 		}
 
@@ -129,7 +130,7 @@ func verifyPruneOptions(opts *PruneOptions) error {
 			return errors.Fatalf("invalid number of bytes %q for --max-unused: %v", opts.MaxUnused, err)
 		}
 
-		opts.maxUnusedBytes = func(used uint64) uint64 {
+		opts.maxUnusedBytes = func(_ uint64) uint64 {
 			return uint64(size)
 		}
 	}
@@ -152,7 +153,7 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error
 		return err
 	}
 
-	if repo.Backend().Connections() < 2 {
+	if repo.Connections() < 2 {
 		return errors.Fatal("prune requires a backend connection limit of at least two")
 	}
 
@@ -187,7 +188,8 @@ func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOption
 
 	Verbosef("loading indexes...\n")
 	// loading the index before the snapshots is ok, as we use an exclusive lock here
-	err := repo.LoadIndex(ctx)
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	err := repo.LoadIndex(ctx, bar)
 	if err != nil {
 		return err
 	}
@@ -405,7 +407,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs re
 	})
 
 	// if duplicate blobs exist, those will be set to either "used" or "unused":
-	// - mark only one occurence of duplicate blobs as used
+	// - mark only one occurrence of duplicate blobs as used
 	// - if there are already some used blobs in a pack, possibly mark duplicates in this pack as "used"
 	// - if there are no used blobs in a pack, possibly mark duplicates as "unused"
 	if hasDuplicates {
@@ -414,7 +416,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs re
 			bh := blob.BlobHandle
 			count, ok := usedBlobs[bh]
 			// skip non-duplicate, aka. normal blobs
-			// count == 0 is used to mark that this was a duplicate blob with only a single occurence remaining
+			// count == 0 is used to mark that this was a duplicate blob with only a single occurrence remaining
 			if !ok || count == 1 {
 				return
 			}
@@ -423,7 +425,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs re
 			size := uint64(blob.Length)
 			switch {
 			case ip.usedBlobs > 0, count == 0:
-				// other used blobs in pack or "last" occurence ->  transition to used
+				// other used blobs in pack or "last" occurrence ->  transition to used
 				ip.usedSize += size
 				ip.usedBlobs++
 				ip.unusedSize -= size
@@ -433,7 +435,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs re
 				stats.blobs.used++
 				stats.size.duplicate -= size
 				stats.blobs.duplicate--
-				// let other occurences remain marked as unused
+				// let other occurrences remain marked as unused
 				usedBlobs[bh] = 1
 			default:
 				// remain unused and decrease counter
@@ -765,7 +767,7 @@ func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo r
 			return errors.Fatalf("%s", err)
 		}
 	} else if len(plan.ignorePacks) != 0 {
-		err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil)
+		err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil, false)
 		if err != nil {
 			return errors.Fatalf("%s", err)
 		}
@@ -777,7 +779,7 @@ func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo r
 	}
 
 	if opts.unsafeRecovery {
-		_, err = writeIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil)
+		err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil, true)
 		if err != nil {
 			return errors.Fatalf("%s", err)
 		}
@@ -787,29 +789,28 @@ func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo r
 	return nil
 }
 
-func writeIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) (restic.IDSet, error) {
+func rebuildIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs, skipDeletion bool) error {
 	Verbosef("rebuilding index\n")
 
 	bar := newProgressMax(!gopts.Quiet, 0, "packs processed")
-	obsoleteIndexes, err := repo.Index().Save(ctx, repo, removePacks, extraObsolete, bar)
-	bar.Done()
-	return obsoleteIndexes, err
-}
-
-func rebuildIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error {
-	obsoleteIndexes, err := writeIndexFiles(ctx, gopts, repo, removePacks, extraObsolete)
-	if err != nil {
-		return err
-	}
-
-	Verbosef("deleting obsolete index files\n")
-	return DeleteFilesChecked(ctx, gopts, repo, obsoleteIndexes, restic.IndexFile)
+	return repo.Index().Save(ctx, repo, removePacks, extraObsolete, restic.MasterIndexSaveOpts{
+		SaveProgress: bar,
+		DeleteProgress: func() *progress.Counter {
+			return newProgressMax(!gopts.Quiet, 0, "old indexes deleted")
+		},
+		DeleteReport: func(id restic.ID, _ error) {
+			if gopts.verbosity > 2 {
+				Verbosef("removed index %v\n", id.String())
+			}
+		},
+		SkipDeletion: skipDeletion,
+	})
 }
 
 func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (usedBlobs restic.CountedBlobSet, err error) {
 	var snapshotTrees restic.IDs
 	Verbosef("loading all snapshots...\n")
-	err = restic.ForAllSnapshots(ctx, repo.Backend(), repo, ignoreSnapshots,
+	err = restic.ForAllSnapshots(ctx, repo, repo, ignoreSnapshots,
 		func(id restic.ID, sn *restic.Snapshot, err error) error {
 			if err != nil {
 				debug.Log("failed to load snapshot %v (error %v)", id, err)

cmd/restic/cmd_prune_integration_test.go

@@ -6,13 +6,13 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/backend"
 	rtest "github.com/restic/restic/internal/test"
 )
 
 func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) {
 	oldHook := gopts.backendTestHook
-	gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return newListOnceBackend(r), nil }
+	gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil }
 	defer func() {
 		gopts.backendTestHook = oldHook
 	}()
@@ -81,7 +81,10 @@ func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) {
 			DryRun: true,
 			Last:   1,
 		}
-		return runForget(context.TODO(), opts, gopts, args)
+		pruneOpts := PruneOptions{
+			MaxUnused: "5%",
+		}
+		return runForget(context.TODO(), opts, pruneOpts, gopts, args)
 	})
 	rtest.OK(t, err)
 
@@ -130,7 +133,7 @@ func TestPruneWithDamagedRepository(t *testing.T) {
 	removePacksExcept(env.gopts, t, oldPacks, false)
 
 	oldHook := env.gopts.backendTestHook
-	env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return newListOnceBackend(r), nil }
+	env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil }
 	defer func() {
 		env.gopts.backendTestHook = oldHook
 	}()

cmd/restic/cmd_recover.go

@@ -5,7 +5,6 @@ import (
 	"os"
 	"time"
 
-	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
 	"github.com/spf13/cobra"
@@ -26,7 +25,7 @@ EXIT STATUS
 Exit status is 0 if the command was successful, and non-zero if there was any error.
 `,
 	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
+	RunE: func(cmd *cobra.Command, _ []string) error {
 		return runRecover(cmd.Context(), globalOptions)
 	},
 }
@@ -52,13 +51,14 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
 		return err
 	}
 
-	snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
+	snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
 	if err != nil {
 		return err
 	}
 
 	Verbosef("load index files\n")
-	if err = repo.LoadIndex(ctx); err != nil {
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	if err = repo.LoadIndex(ctx, bar); err != nil {
 		return err
 	}
 
@@ -73,7 +73,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
 	})
 
 	Verbosef("load %d trees\n", len(trees))
-	bar := newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
+	bar = newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
 	for id := range trees {
 		tree, err := restic.LoadTree(ctx, repo, id)
 		if err != nil {
@@ -91,7 +91,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
 	bar.Done()
 
 	Verbosef("load snapshots\n")
-	err = restic.ForAllSnapshots(ctx, snapshotLister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error {
+	err = restic.ForAllSnapshots(ctx, snapshotLister, repo, nil, func(_ restic.ID, sn *restic.Snapshot, _ error) error {
 		trees[*sn.Tree] = true
 		return nil
 	})
@@ -158,7 +158,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
 
 }
 
-func createSnapshot(ctx context.Context, name, hostname string, tags []string, repo restic.Repository, tree *restic.ID) error {
+func createSnapshot(ctx context.Context, name, hostname string, tags []string, repo restic.SaverUnpacked, tree *restic.ID) error {
 	sn, err := restic.NewSnapshot([]string{name}, tags, hostname, time.Now())
 	if err != nil {
 		return errors.Fatalf("unable to save snapshot: %v", err)

cmd/restic/cmd_repair_index.go

@@ -24,7 +24,7 @@ EXIT STATUS
 Exit status is 0 if the command was successful, and non-zero if there was any error.
 `,
 	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
+	RunE: func(cmd *cobra.Command, _ []string) error {
 		return runRebuildIndex(cmd.Context(), repairIndexOptions, globalOptions)
 	},
 }
@@ -78,7 +78,7 @@ func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOpti
 
 	if opts.ReadAllPacks {
 		// get list of old index files but start with empty index
-		err := repo.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error {
+		err := repo.List(ctx, restic.IndexFile, func(id restic.ID, _ int64) error {
 			obsoleteIndexes = append(obsoleteIndexes, id)
 			return nil
 		})
@@ -88,7 +88,7 @@ func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOpti
 	} else {
 		Verbosef("loading indexes...\n")
 		mi := index.NewMasterIndex()
-		err := index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
+		err := index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, _ bool, err error) error {
 			if err != nil {
 				Warnf("removing invalid index %v: %v\n", id, err)
 				obsoleteIndexes = append(obsoleteIndexes, id)
@@ -154,7 +154,7 @@ func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOpti
 		}
 	}
 
-	err = rebuildIndexFiles(ctx, gopts, repo, removePacks, obsoleteIndexes)
+	err = rebuildIndexFiles(ctx, gopts, repo, removePacks, obsoleteIndexes, false)
 	if err != nil {
 		return err
 	}

cmd/restic/cmd_repair_index_integration_test.go

@@ -8,6 +8,7 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/index"
 	"github.com/restic/restic/internal/restic"
@@ -70,12 +71,12 @@ func TestRebuildIndexAlwaysFull(t *testing.T) {
 
 // indexErrorBackend modifies the first index after reading.
 type indexErrorBackend struct {
-	restic.Backend
+	backend.Backend
 	lock     sync.Mutex
 	hasErred bool
 }
 
-func (b *indexErrorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
+func (b *indexErrorBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
 	return b.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error {
 		// protect hasErred
 		b.lock.Lock()
@@ -101,7 +102,7 @@ func (erd errorReadCloser) Read(p []byte) (int, error) {
 }
 
 func TestRebuildIndexDamage(t *testing.T) {
-	testRebuildIndex(t, func(r restic.Backend) (restic.Backend, error) {
+	testRebuildIndex(t, func(r backend.Backend) (backend.Backend, error) {
 		return &indexErrorBackend{
 			Backend: r,
 		}, nil
@@ -109,11 +110,11 @@ func TestRebuildIndexDamage(t *testing.T) {
 }
 
 type appendOnlyBackend struct {
-	restic.Backend
+	backend.Backend
 }
 
 // called via repo.Backend().Remove()
-func (b *appendOnlyBackend) Remove(_ context.Context, h restic.Handle) error {
+func (b *appendOnlyBackend) Remove(_ context.Context, h backend.Handle) error {
 	return errors.Errorf("Failed to remove %v", h)
 }
 
@@ -127,7 +128,7 @@ func TestRebuildIndexFailsOnAppendOnly(t *testing.T) {
 	err := withRestoreGlobalOptions(func() error {
 		globalOptions.stdout = io.Discard
 
-		env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
+		env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
 			return &appendOnlyBackend{r}, nil
 		}
 		return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts)

cmd/restic/cmd_repair_packs.go

@@ -0,0 +1,101 @@
+package main
+
+import (
+	"context"
+	"io"
+	"os"
+
+	"github.com/restic/restic/internal/backend"
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui/termstatus"
+	"github.com/spf13/cobra"
+)
+
+var cmdRepairPacks = &cobra.Command{
+	Use:   "packs [packIDs...]",
+	Short: "Salvage damaged pack files",
+	Long: `
+WARNING: The CLI for this command is experimental and will likely change in the future!
+
+The "repair packs" command extracts intact blobs from the specified pack files, rebuilds
+the index to remove the damaged pack files and removes the pack files from the repository.
+
+EXIT STATUS
+===========
+
+Exit status is 0 if the command was successful, and non-zero if there was any error.
+`,
+	DisableAutoGenTag: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		term, cancel := setupTermstatus()
+		defer cancel()
+		return runRepairPacks(cmd.Context(), globalOptions, term, args)
+	},
+}
+
+func init() {
+	cmdRepair.AddCommand(cmdRepairPacks)
+}
+
+func runRepairPacks(ctx context.Context, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
+	ids := restic.NewIDSet()
+	for _, arg := range args {
+		id, err := restic.ParseID(arg)
+		if err != nil {
+			return err
+		}
+		ids.Insert(id)
+	}
+	if len(ids) == 0 {
+		return errors.Fatal("no ids specified")
+	}
+
+	repo, err := OpenRepository(ctx, gopts)
+	if err != nil {
+		return err
+	}
+
+	lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	err = repo.LoadIndex(ctx, bar)
+	if err != nil {
+		return errors.Fatalf("%s", err)
+	}
+
+	printer := newTerminalProgressPrinter(gopts.verbosity, term)
+
+	printer.P("saving backup copies of pack files to current folder")
+	for id := range ids {
+		f, err := os.OpenFile("pack-"+id.String(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o666)
+		if err != nil {
+			return err
+		}
+
+		err = repo.Backend().Load(ctx, backend.Handle{Type: restic.PackFile, Name: id.String()}, 0, 0, func(rd io.Reader) error {
+			_, err := f.Seek(0, 0)
+			if err != nil {
+				return err
+			}
+			_, err = io.Copy(f, rd)
+			return err
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	err = repository.RepairPacks(ctx, repo, ids, printer)
+	if err != nil {
+		return errors.Fatalf("%s", err)
+	}
+
+	Warnf("\nUse `restic repair snapshots --forget` to remove the corrupted data blobs from all snapshots\n")
+	return nil
+}

cmd/restic/cmd_repair_snapshots.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 
-	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/walker"
@@ -84,12 +83,13 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt
 		repo.SetDryRun()
 	}
 
-	snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
+	snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
 	if err != nil {
 		return err
 	}
 
-	if err := repo.LoadIndex(ctx); err != nil {
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	if err := repo.LoadIndex(ctx, bar); err != nil {
 		return err
 	}
 
@@ -125,7 +125,7 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt
 			node.Size = newSize
 			return node
 		},
-		RewriteFailedTree: func(nodeID restic.ID, path string, _ error) (restic.ID, error) {
+		RewriteFailedTree: func(_ restic.ID, path string, _ error) (restic.ID, error) {
 			if path == "/" {
 				Verbosef("  dir %q: not readable\n", path)
 				// remove snapshots with invalid root node
@@ -144,11 +144,11 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt
 
 	changedCount := 0
 	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) {
-		Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
+		Verbosef("\n%v\n", sn)
 		changed, err := filterAndReplaceSnapshot(ctx, repo, sn,
 			func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) {
 				return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
-			}, opts.DryRun, opts.Forget, "repaired")
+			}, opts.DryRun, opts.Forget, nil, "repaired")
 		if err != nil {
 			return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err)
 		}

cmd/restic/cmd_restore.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/restic/restic/internal/debug"
@@ -25,9 +24,12 @@ var cmdRestore = &cobra.Command{
 The "restore" command extracts the data from a snapshot from the repository to
 a directory.
 
-The special snapshot "latest" can be used to restore the latest snapshot in the
+The special snapshotID "latest" can be used to restore the latest snapshot in the
 repository.
 
+To only restore a specific subfolder, you can use the "<snapshotID>:<subfolder>"
+syntax, where "subfolder" is a path within the snapshot.
+
 EXIT STATUS
 ===========
 
@@ -35,31 +37,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		ctx := cmd.Context()
-		var wg sync.WaitGroup
-		cancelCtx, cancel := context.WithCancel(ctx)
-		defer func() {
-			// shutdown termstatus
-			cancel()
-			wg.Wait()
-		}()
-
-		term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet)
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			term.Run(cancelCtx)
-		}()
-
-		// allow usage of warnf / verbosef
-		prevStdout, prevStderr := globalOptions.stdout, globalOptions.stderr
-		defer func() {
-			globalOptions.stdout, globalOptions.stderr = prevStdout, prevStderr
-		}()
-		stdioWrapper := ui.NewStdioWrapper(term)
-		globalOptions.stdout, globalOptions.stderr = stdioWrapper.Stdout(), stdioWrapper.Stderr()
-
-		return runRestore(ctx, restoreOptions, globalOptions, term, args)
+		term, cancel := setupTermstatus()
+		defer cancel()
+		return runRestore(cmd.Context(), restoreOptions, globalOptions, term, args)
 	},
 }
 
@@ -82,9 +62,9 @@ func init() {
 
 	flags := cmdRestore.Flags()
 	flags.StringArrayVarP(&restoreOptions.Exclude, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
-	flags.StringArrayVar(&restoreOptions.InsensitiveExclude, "iexclude", nil, "same as `--exclude` but ignores the casing of filenames")
+	flags.StringArrayVar(&restoreOptions.InsensitiveExclude, "iexclude", nil, "same as --exclude but ignores the casing of `pattern`")
 	flags.StringArrayVarP(&restoreOptions.Include, "include", "i", nil, "include a `pattern`, exclude everything else (can be specified multiple times)")
-	flags.StringArrayVar(&restoreOptions.InsensitiveInclude, "iinclude", nil, "same as `--include` but ignores the casing of filenames")
+	flags.StringArrayVar(&restoreOptions.InsensitiveInclude, "iinclude", nil, "same as --include but ignores the casing of `pattern`")
 	flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to")
 
 	initSingleSnapshotFilter(flags, &restoreOptions.SnapshotFilter)
@@ -165,12 +145,13 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
 		Hosts: opts.Hosts,
 		Paths: opts.Paths,
 		Tags:  opts.Tags,
-	}).FindLatest(ctx, repo.Backend(), repo, snapshotIDString)
+	}).FindLatest(ctx, repo, repo, snapshotIDString)
 	if err != nil {
 		return errors.Fatalf("failed to find snapshot: %v", err)
 	}
 
-	err = repo.LoadIndex(ctx)
+	bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
+	err = repo.LoadIndex(ctx, bar)
 	if err != nil {
 		return err
 	}
@@ -200,7 +181,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
 
 	excludePatterns := filter.ParsePatterns(opts.Exclude)
 	insensitiveExcludePatterns := filter.ParsePatterns(opts.InsensitiveExclude)
-	selectExcludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
+	selectExcludeFilter := func(item string, _ string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
 		matched, err := filter.List(excludePatterns, item)
 		if err != nil {
 			msg.E("error for exclude pattern: %v", err)
@@ -223,7 +204,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
 
 	includePatterns := filter.ParsePatterns(opts.Include)
 	insensitiveIncludePatterns := filter.ParsePatterns(opts.InsensitiveInclude)
-	selectIncludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
+	selectIncludeFilter := func(item string, _ string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
 		matched, childMayMatch, err := filter.ListWithChild(includePatterns, item)
 		if err != nil {
 			msg.E("error for include pattern: %v", err)

cmd/restic/cmd_rewrite.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/spf13/cobra"
 	"golang.org/x/sync/errgroup"
@@ -46,11 +47,42 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 	},
 }
 
+type snapshotMetadata struct {
+	Hostname string
+	Time     *time.Time
+}
+
+type snapshotMetadataArgs struct {
+	Hostname string
+	Time     string
+}
+
+func (sma snapshotMetadataArgs) empty() bool {
+	return sma.Hostname == "" && sma.Time == ""
+}
+
+func (sma snapshotMetadataArgs) convert() (*snapshotMetadata, error) {
+	if sma.empty() {
+		return nil, nil
+	}
+
+	var timeStamp *time.Time
+	if sma.Time != "" {
+		t, err := time.ParseInLocation(TimeFormat, sma.Time, time.Local)
+		if err != nil {
+			return nil, errors.Fatalf("error in time option: %v\n", err)
+		}
+		timeStamp = &t
+	}
+	return &snapshotMetadata{Hostname: sma.Hostname, Time: timeStamp}, nil
+}
+
 // RewriteOptions collects all options for the rewrite command.
 type RewriteOptions struct {
 	Forget bool
 	DryRun bool
 
+	Metadata snapshotMetadataArgs
 	restic.SnapshotFilter
 	excludePatternOptions
 }
@@ -63,11 +95,15 @@ func init() {
 	f := cmdRewrite.Flags()
 	f.BoolVarP(&rewriteOptions.Forget, "forget", "", false, "remove original snapshots after creating new ones")
 	f.BoolVarP(&rewriteOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done")
+	f.StringVar(&rewriteOptions.Metadata.Hostname, "new-host", "", "replace hostname")
+	f.StringVar(&rewriteOptions.Metadata.Time, "new-time", "", "replace time of the backup")
 
 	initMultiSnapshotFilter(f, &rewriteOptions.SnapshotFilter, true)
 	initExcludePatternOptions(f, &rewriteOptions.excludePatternOptions)
 }
 
+type rewriteFilterFunc func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error)
+
 func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, opts RewriteOptions) (bool, error) {
 	if sn.Tree == nil {
 		return false, errors.Errorf("snapshot %v has nil tree", sn.ID().Str())
@@ -78,33 +114,50 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 		return false, err
 	}
 
-	selectByName := func(nodepath string) bool {
-		for _, reject := range rejectByNameFuncs {
-			if reject(nodepath) {
-				return false
-			}
-		}
-		return true
+	metadata, err := opts.Metadata.convert()
+
+	if err != nil {
+		return false, err
 	}
 
-	rewriter := walker.NewTreeRewriter(walker.RewriteOpts{
-		RewriteNode: func(node *restic.Node, path string) *restic.Node {
-			if selectByName(path) {
-				return node
+	var filter rewriteFilterFunc
+
+	if len(rejectByNameFuncs) > 0 {
+		selectByName := func(nodepath string) bool {
+			for _, reject := range rejectByNameFuncs {
+				if reject(nodepath) {
+					return false
+				}
 			}
-			Verbosef(fmt.Sprintf("excluding %s\n", path))
-			return nil
-		},
-		DisableNodeCache: true,
-	})
+			return true
+		}
+
+		rewriter := walker.NewTreeRewriter(walker.RewriteOpts{
+			RewriteNode: func(node *restic.Node, path string) *restic.Node {
+				if selectByName(path) {
+					return node
+				}
+				Verbosef(fmt.Sprintf("excluding %s\n", path))
+				return nil
+			},
+			DisableNodeCache: true,
+		})
+
+		filter = func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) {
+			return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
+		}
+	} else {
+		filter = func(_ context.Context, sn *restic.Snapshot) (restic.ID, error) {
+			return *sn.Tree, nil
+		}
+	}
 
 	return filterAndReplaceSnapshot(ctx, repo, sn,
-		func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) {
-			return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
-		}, opts.DryRun, opts.Forget, "rewrite")
+		filter, opts.DryRun, opts.Forget, metadata, "rewrite")
 }
 
-func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *restic.Snapshot, filter func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error), dryRun bool, forget bool, addTag string) (bool, error) {
+func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *restic.Snapshot,
+	filter rewriteFilterFunc, dryRun bool, forget bool, newMetadata *snapshotMetadata, addTag string) (bool, error) {
 
 	wg, wgCtx := errgroup.WithContext(ctx)
 	repo.StartPackUploader(wgCtx, wg)
@@ -128,7 +181,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
 		if dryRun {
 			Verbosef("would delete empty snapshot\n")
 		} else {
-			h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
+			h := backend.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
 			if err = repo.Backend().Remove(ctx, h); err != nil {
 				return false, err
 			}
@@ -138,7 +191,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
 		return true, nil
 	}
 
-	if filteredTree == *sn.Tree {
+	if filteredTree == *sn.Tree && newMetadata == nil {
 		debug.Log("Snapshot %v not modified", sn)
 		return false, nil
 	}
@@ -151,6 +204,14 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
 			Verbosef("would remove old snapshot\n")
 		}
 
+		if newMetadata != nil && newMetadata.Time != nil {
+			Verbosef("would set time to %s\n", newMetadata.Time)
+		}
+
+		if newMetadata != nil && newMetadata.Hostname != "" {
+			Verbosef("would set hostname to %s\n", newMetadata.Hostname)
+		}
+
 		return true, nil
 	}
 
@@ -162,6 +223,16 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
 		sn.AddTags([]string{addTag})
 	}
 
+	if newMetadata != nil && newMetadata.Time != nil {
+		Verbosef("setting time to %s\n", *newMetadata.Time)
+		sn.Time = *newMetadata.Time
+	}
+
+	if newMetadata != nil && newMetadata.Hostname != "" {
+		Verbosef("setting host to %s\n", newMetadata.Hostname)
+		sn.Hostname = newMetadata.Hostname
+	}
+
 	// Save the new snapshot.
 	id, err := restic.SaveSnapshot(ctx, repo, sn)
 	if err != nil {
@@ -170,7 +241,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
 	Verbosef("saved new snapshot %v\n", id.Str())
 
 	if forget {
-		h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
+		h := backend.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
 		if err = repo.Backend().Remove(ctx, h); err != nil {
 			return false, err
 		}
@@ -181,8 +252,8 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
 }
 
 func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error {
-	if opts.excludePatternOptions.Empty() {
-		return errors.Fatal("Nothing to do: no excludes provided")
+	if opts.excludePatternOptions.Empty() && opts.Metadata.empty() {
+		return errors.Fatal("Nothing to do: no excludes provided and no new metadata provided")
 	}
 
 	repo, err := OpenRepository(ctx, gopts)
@@ -207,18 +278,19 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a
 		repo.SetDryRun()
 	}
 
-	snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
+	snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
 	if err != nil {
 		return err
 	}
 
-	if err = repo.LoadIndex(ctx); err != nil {
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	if err = repo.LoadIndex(ctx, bar); err != nil {
 		return err
 	}
 
 	changedCount := 0
 	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) {
-		Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
+		Verbosef("\n%v\n", sn)
 		changed, err := rewriteSnapshot(ctx, repo, sn, opts)
 		if err != nil {
 			return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err)

cmd/restic/cmd_rewrite_integration_test.go

@@ -9,12 +9,13 @@ import (
 	rtest "github.com/restic/restic/internal/test"
 )
 
-func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool) {
+func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool, metadata snapshotMetadataArgs) {
 	opts := RewriteOptions{
 		excludePatternOptions: excludePatternOptions{
 			Excludes: excludes,
 		},
-		Forget: forget,
+		Forget:   forget,
+		Metadata: metadata,
 	}
 
 	rtest.OK(t, runRewrite(context.TODO(), opts, gopts, nil))
@@ -38,7 +39,7 @@ func TestRewrite(t *testing.T) {
 	createBasicRewriteRepo(t, env)
 
 	// exclude some data
-	testRunRewriteExclude(t, env.gopts, []string{"3"}, false)
+	testRunRewriteExclude(t, env.gopts, []string{"3"}, false, snapshotMetadataArgs{Hostname: "", Time: ""})
 	snapshotIDs := testRunList(t, "snapshots", env.gopts)
 	rtest.Assert(t, len(snapshotIDs) == 2, "expected two snapshots, got %v", snapshotIDs)
 	testRunCheck(t, env.gopts)
@@ -50,7 +51,7 @@ func TestRewriteUnchanged(t *testing.T) {
 	snapshotID := createBasicRewriteRepo(t, env)
 
 	// use an exclude that will not exclude anything
-	testRunRewriteExclude(t, env.gopts, []string{"3dflkhjgdflhkjetrlkhjgfdlhkj"}, false)
+	testRunRewriteExclude(t, env.gopts, []string{"3dflkhjgdflhkjetrlkhjgfdlhkj"}, false, snapshotMetadataArgs{Hostname: "", Time: ""})
 	newSnapshotIDs := testRunList(t, "snapshots", env.gopts)
 	rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs)
 	rtest.Assert(t, snapshotID == newSnapshotIDs[0], "snapshot id changed unexpectedly")
@@ -63,11 +64,44 @@ func TestRewriteReplace(t *testing.T) {
 	snapshotID := createBasicRewriteRepo(t, env)
 
 	// exclude some data
-	testRunRewriteExclude(t, env.gopts, []string{"3"}, true)
-	newSnapshotIDs := testRunList(t, "snapshots", env.gopts)
-	rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs)
+	testRunRewriteExclude(t, env.gopts, []string{"3"}, true, snapshotMetadataArgs{Hostname: "", Time: ""})
+	newSnapshotIDs := testListSnapshots(t, env.gopts, 1)
 	rtest.Assert(t, snapshotID != newSnapshotIDs[0], "snapshot id should have changed")
 	// check forbids unused blobs, thus remove them first
 	testRunPrune(t, env.gopts, PruneOptions{MaxUnused: "0"})
 	testRunCheck(t, env.gopts)
 }
+
+func testRewriteMetadata(t *testing.T, metadata snapshotMetadataArgs) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+	createBasicRewriteRepo(t, env)
+	testRunRewriteExclude(t, env.gopts, []string{}, true, metadata)
+
+	repo, _ := OpenRepository(context.TODO(), env.gopts)
+	snapshots, err := restic.TestLoadAllSnapshots(context.TODO(), repo, nil)
+	rtest.OK(t, err)
+	rtest.Assert(t, len(snapshots) == 1, "expected one snapshot, got %v", len(snapshots))
+	newSnapshot := snapshots[0]
+
+	if metadata.Time != "" {
+		rtest.Assert(t, newSnapshot.Time.Format(TimeFormat) == metadata.Time, "New snapshot should have time %s", metadata.Time)
+	}
+
+	if metadata.Hostname != "" {
+		rtest.Assert(t, newSnapshot.Hostname == metadata.Hostname, "New snapshot should have host %s", metadata.Hostname)
+	}
+}
+
+func TestRewriteMetadata(t *testing.T) {
+	newHost := "new host"
+	newTime := "1999-01-01 11:11:11"
+
+	for _, metadata := range []snapshotMetadataArgs{
+		{Hostname: "", Time: newTime},
+		{Hostname: newHost, Time: ""},
+		{Hostname: newHost, Time: newTime},
+	} {
+		testRewriteMetadata(t, metadata)
+	}
+}

cmd/restic/cmd_snapshots.go

@@ -73,7 +73,7 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions
 	}
 
 	var snapshots restic.Snapshots
-	for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) {
+	for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) {
 		snapshots = append(snapshots, sn)
 	}
 	snapshotGroups, grouped, err := restic.GroupSnapshots(snapshots, opts.GroupBy)
@@ -290,7 +290,7 @@ func PrintSnapshotGroupHeader(stdout io.Writer, groupKeyJSON string) error {
 	return nil
 }
 
-// Snapshot helps to print Snaphots as JSON with their ID included.
+// Snapshot helps to print Snapshots as JSON with their ID included.
 type Snapshot struct {
 	*restic.Snapshot
 

cmd/restic/cmd_stats.go

@@ -8,10 +8,10 @@ import (
 	"strings"
 
 	"github.com/restic/chunker"
-	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/restorer"
 	"github.com/restic/restic/internal/ui"
 	"github.com/restic/restic/internal/ui/table"
 	"github.com/restic/restic/internal/walker"
@@ -94,12 +94,12 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args
 		}
 	}
 
-	snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
+	snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
 	if err != nil {
 		return err
 	}
-
-	if err = repo.LoadIndex(ctx); err != nil {
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	if err = repo.LoadIndex(ctx, bar); err != nil {
 		return err
 	}
 
@@ -189,7 +189,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args
 	return nil
 }
 
-func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo restic.Repository, opts StatsOptions, stats *statsContainer) error {
+func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo restic.Loader, opts StatsOptions, stats *statsContainer) error {
 	if snapshot.Tree == nil {
 		return fmt.Errorf("snapshot %s has nil tree", snapshot.ID().Str())
 	}
@@ -202,8 +202,10 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest
 		return restic.FindUsedBlobs(ctx, repo, restic.IDs{*snapshot.Tree}, stats.blobs, nil)
 	}
 
-	uniqueInodes := make(map[uint64]struct{})
-	err := walker.Walk(ctx, repo, *snapshot.Tree, restic.NewIDSet(), statsWalkTree(repo, opts, stats, uniqueInodes))
+	hardLinkIndex := restorer.NewHardlinkIndex[struct{}]()
+	err := walker.Walk(ctx, repo, *snapshot.Tree, walker.WalkVisitor{
+		ProcessNode: statsWalkTree(repo, opts, stats, hardLinkIndex),
+	})
 	if err != nil {
 		return fmt.Errorf("walking tree %s: %v", *snapshot.Tree, err)
 	}
@@ -211,13 +213,13 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest
 	return nil
 }
 
-func statsWalkTree(repo restic.Repository, opts StatsOptions, stats *statsContainer, uniqueInodes map[uint64]struct{}) walker.WalkFunc {
-	return func(parentTreeID restic.ID, npath string, node *restic.Node, nodeErr error) (bool, error) {
+func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer, hardLinkIndex *restorer.HardlinkIndex[struct{}]) walker.WalkFunc {
+	return func(parentTreeID restic.ID, npath string, node *restic.Node, nodeErr error) error {
 		if nodeErr != nil {
-			return true, nodeErr
+			return nodeErr
 		}
 		if node == nil {
-			return true, nil
+			return nil
 		}
 
 		if opts.countMode == countModeUniqueFilesByContents || opts.countMode == countModeBlobsPerFile {
@@ -247,7 +249,7 @@ func statsWalkTree(repo restic.Repository, opts StatsOptions, stats *statsContai
 							// is always a data blob since we're accessing it via a file's Content array
 							blobSize, found := repo.LookupBlobSize(blobID, restic.DataBlob)
 							if !found {
-								return true, fmt.Errorf("blob %s not found for tree %s", blobID, parentTreeID)
+								return fmt.Errorf("blob %s not found for tree %s", blobID, parentTreeID)
 							}
 
 							// count the blob's size, then add this blob by this
@@ -270,15 +272,13 @@ func statsWalkTree(repo restic.Repository, opts StatsOptions, stats *statsContai
 
 			// if inodes are present, only count each inode once
 			// (hard links do not increase restore size)
-			if _, ok := uniqueInodes[node.Inode]; !ok || node.Inode == 0 {
-				uniqueInodes[node.Inode] = struct{}{}
+			if !hardLinkIndex.Has(node.Inode, node.DeviceID) || node.Inode == 0 {
+				hardLinkIndex.Add(node.Inode, node.DeviceID, struct{}{})
 				stats.TotalSize += node.Size
 			}
-
-			return false, nil
 		}
 
-		return true, nil
+		return nil
 	}
 }
 
@@ -365,9 +365,9 @@ func statsDebug(ctx context.Context, repo restic.Repository) error {
 	return nil
 }
 
-func statsDebugFileType(ctx context.Context, repo restic.Repository, tpe restic.FileType) (*sizeHistogram, error) {
+func statsDebugFileType(ctx context.Context, repo restic.Lister, tpe restic.FileType) (*sizeHistogram, error) {
 	hist := newSizeHistogram(2 * repository.MaxPackSize)
-	err := repo.List(ctx, tpe, func(id restic.ID, size int64) error {
+	err := repo.List(ctx, tpe, func(_ restic.ID, size int64) error {
 		hist.Add(uint64(size))
 		return nil
 	})

cmd/restic/cmd_tag.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
@@ -12,7 +13,7 @@ import (
 )
 
 var cmdTag = &cobra.Command{
-	Use:   "tag [flags] [snapshot-ID ...]",
+	Use:   "tag [flags] [snapshotID ...]",
 	Short: "Modify tags on snapshots",
 	Long: `
 The "tag" command allows you to modify tags on exiting snapshots.
@@ -20,7 +21,7 @@ The "tag" command allows you to modify tags on exiting snapshots.
 You can either set/replace the entire set of tags on a snapshot, or
 add tags to/remove tags from the existing set.
 
-When no snapshot-ID is given, all snapshots matching the host, tag and path filter criteria are modified.
+When no snapshotID is given, all snapshots matching the host, tag and path filter criteria are modified.
 
 EXIT STATUS
 ===========
@@ -85,7 +86,7 @@ func changeTags(ctx context.Context, repo *repository.Repository, sn *restic.Sna
 		debug.Log("new snapshot saved as %v", id)
 
 		// Remove the old snapshot.
-		h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
+		h := backend.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
 		if err = repo.Backend().Remove(ctx, h); err != nil {
 			return false, err
 		}
@@ -119,7 +120,7 @@ func runTag(ctx context.Context, opts TagOptions, gopts GlobalOptions, args []st
 	}
 
 	changeCnt := 0
-	for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) {
+	for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) {
 		changed, err := changeTags(ctx, repo, sn, opts.SetTags.Flatten(), opts.AddTags.Flatten(), opts.RemoveTags.Flatten())
 		if err != nil {
 			Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", sn.ID(), err)