Merge pull request #5466 from y0n3d4/gentoo-install-info

Added Gentoo install info

.github/PULL_REQUEST_TEMPLATE.md

@@ -28,13 +28,15 @@ Checklist
 You do not need to check all the boxes below all at once. Feel free to take
 your time and add more commits. If you're done and ready for review, please
 check the last box. Enable a checkbox by replacing [ ] with [x].
+
+Please always follow these steps:
+- Read the [contribution guidelines](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#providing-patches).
+- Enable [maintainer edits](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/allowing-changes-to-a-pull-request-branch-created-from-a-fork).
+- Run `gofmt` on the code in all commits.
+- Format all commit messages in the same style as [the other commits in the repository](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#git-commits).
 -->
 
-- [ ] I have read the [contribution guidelines](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#providing-patches).
-- [ ] I have [enabled maintainer edits](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/allowing-changes-to-a-pull-request-branch-created-from-a-fork).
 - [ ] I have added tests for all code changes.
 - [ ] I have added documentation for relevant changes (in the manual).
 - [ ] There's a new file in `changelog/unreleased/` that describes the changes for our users (see [template](https://github.com/restic/restic/blob/master/changelog/TEMPLATE)).
-- [ ] I have run `gofmt` on the code in all commits.
-- [ ] All commit messages are formatted in the same style as [the other commits in the repo](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#git-commits).
 - [ ] I'm done! This pull request is ready for review.

.github/workflows/docker.yml

@@ -20,12 +20,16 @@ jobs:
       contents: read
       packages: write
 
+    outputs:
+      image: ${{ steps.image.outputs.image }}
+      digest: ${{ steps.build-and-push.outputs.digest }}
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
       - name: Log in to the Container registry
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -37,6 +41,7 @@ jobs:
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
+            type=sha
             type=ref,event=branch
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
@@ -55,6 +60,7 @@ jobs:
         if: github.ref != 'refs/heads/master'
 
       - name: Build and push Docker image
+        id: build-and-push
         uses: docker/build-push-action@15560696de535e4014efeff63c48f16952e52dd1
         with:
           push: true
@@ -64,3 +70,26 @@ jobs:
           pull: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+        
+      - name: Output image
+        id: image
+        run: |
+          # NOTE: Set the image as an output because the `env` context is not
+          # available to the inputs of a reusable workflow call.
+          image_name="${REGISTRY}/${IMAGE_NAME}"
+          echo "image=$image_name" >> "$GITHUB_OUTPUT"
+        
+  provenance:
+    needs: [build-and-push-image]
+    permissions:
+      actions: read # for detecting the Github Actions environment.
+      id-token: write # for creating OIDC tokens for signing.
+      packages: write # for uploading attestations.
+    if: github.repository == 'restic/restic'
+    uses: slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@v2.1.0
+    with:
+      image: ${{ needs.build-and-push-image.outputs.image }}
+      digest: ${{ needs.build-and-push-image.outputs.digest }}
+      registry-username: ${{ github.actor }}
+    secrets:
+      registry-password: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/tests.yml

@@ -13,7 +13,7 @@ permissions:
   contents: read
 
 env:
-  latest_go: "1.22.x"
+  latest_go: "1.24.x"
   GO111MODULE: on
 
 jobs:
@@ -23,39 +23,29 @@ jobs:
         # list of jobs to run:
         include:
           - job_name: Windows
-            go: 1.22.x
+            go: 1.24.x
             os: windows-latest
 
           - job_name: macOS
-            go: 1.22.x
+            go: 1.24.x
             os: macOS-latest
             test_fuse: false
 
           - job_name: Linux
-            go: 1.22.x
+            go: 1.24.x
             os: ubuntu-latest
             test_cloud_backends: true
             test_fuse: true
             check_changelog: true
 
           - job_name: Linux (race)
-            go: 1.22.x
+            go: 1.24.x
             os: ubuntu-latest
             test_fuse: true
             test_opts: "-race"
 
           - job_name: Linux
-            go: 1.21.x
-            os: ubuntu-latest
-            test_fuse: true
-
-          - job_name: Linux
-            go: 1.20.x
-            os: ubuntu-latest
-            test_fuse: true
-
-          - job_name: Linux
-            go: 1.19.x
+            go: 1.23.x
             os: ubuntu-latest
             test_fuse: true
 
@@ -195,7 +185,7 @@ jobs:
           # prepare credentials for Google Cloud Storage tests in a temp file
           export GOOGLE_APPLICATION_CREDENTIALS=$(mktemp --tmpdir restic-gcs-auth-XXXXXXX)
           echo $RESTIC_TEST_GS_APPLICATION_CREDENTIALS_B64 | base64 -d > $GOOGLE_APPLICATION_CREDENTIALS
-          go test -cover -parallel 4 ./internal/backend/...
+          go test -cover -parallel 5 -timeout 15m ./internal/backend/...
 
         # only run cloud backend tests for pull requests from and pushes to our
         # own repo, otherwise the secrets are not available
@@ -214,7 +204,6 @@ jobs:
 
   cross_compile:
     strategy:
-
       matrix:
         # run cross-compile in three batches parallel so the overall tests run faster
         subset:
@@ -264,7 +253,7 @@ jobs:
         uses: golangci/golangci-lint-action@v6
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.57.1
+          version: v1.64.8
           args: --verbose --timeout 5m
 
         # only run golangci-lint for pull requests, otherwise ALL hints get

.golangci.yml

@@ -56,6 +56,7 @@ issues:
     # staticcheck: there's no easy way to replace these packages
     - "SA1019: \"golang.org/x/crypto/poly1305\" is deprecated"
     - "SA1019: \"golang.org/x/crypto/openpgp\" is deprecated"
+    - "redefines-builtin-id:"
 
   exclude-rules:
     # revive: ignore unused parameters in tests

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Table of Contents
 
+* [Changelog for 0.18.0](#changelog-for-restic-0180-2025-03-27)
+* [Changelog for 0.17.3](#changelog-for-restic-0173-2024-11-08)
+* [Changelog for 0.17.2](#changelog-for-restic-0172-2024-10-27)
 * [Changelog for 0.17.1](#changelog-for-restic-0171-2024-09-05)
 * [Changelog for 0.17.0](#changelog-for-restic-0170-2024-07-26)
 * [Changelog for 0.16.5](#changelog-for-restic-0165-2024-07-01)
@@ -36,6 +39,539 @@
 * [Changelog for 0.6.0](#changelog-for-restic-060-2017-05-29)
 
 
+# Changelog for restic 0.18.0 (2025-03-27)
+The following sections list the changes in restic 0.18.0 relevant to
+restic users. The changes are ordered by importance.
+
+## Summary
+
+ * Sec #5291: Mitigate attack on content-defined chunking algorithm
+ * Fix #1843: Correctly restore long filepaths' timestamp on old Windows
+ * Fix #2165: Ignore disappeared backup source files
+ * Fix #5153: Include root tree when searching using `find --tree`
+ * Fix #5169: Prevent Windows VSS event log 8194 warnings for backup with fs snapshot
+ * Fix #5212: Fix duplicate data handling in `prune --max-unused`
+ * Fix #5249: Fix creation of oversized index by `repair index --read-all-packs`
+ * Fix #5259: Fix rare crash in command output
+ * Chg #4938: Update dependencies and require Go 1.23 or newer
+ * Chg #5162: Graduate feature flags
+ * Enh #1378: Add JSON support to `check` command
+ * Enh #2511: Support generating shell completions to stdout
+ * Enh #3697: Allow excluding online-only cloud files (e.g. OneDrive)
+ * Enh #4179: Add `sort` option to `ls` command
+ * Enh #4433: Change default sort order for `find` output
+ * Enh #4521: Add support for Microsoft Blob Storage access tiers
+ * Enh #4942: Add snapshot summary statistics to rewritten snapshots
+ * Enh #4948: Format exit errors as JSON when requested
+ * Enh #4983: Add SLSA provenance to GHCR container images
+ * Enh #5054: Enable compression for ZIP archives in `dump` command
+ * Enh #5081: Add retry mechanism for loading repository config
+ * Enh #5089: Allow including/excluding extended file attributes during `restore`
+ * Enh #5092: Show count of deleted files and directories during `restore`
+ * Enh #5109: Make small pack size configurable for `prune`
+ * Enh #5119: Add start and end timestamps to `backup` JSON output
+ * Enh #5131: Add DragonFlyBSD support
+ * Enh #5137: Make `tag` command print which snapshots were modified
+ * Enh #5141: Provide clear error message if AZURE_ACCOUNT_NAME is not set
+ * Enh #5173: Add experimental S3 cold storage support
+ * Enh #5174: Add xattr support for NetBSD 10+
+ * Enh #5251: Improve retry handling for flaky `rclone` backends
+ * Enh #5287: Make `recover` automatically rebuild index when needed
+
+## Details
+
+ * Security #5291: Mitigate attack on content-defined chunking algorithm
+
+   Restic uses [Rabin
+   Fingerprints](https://restic.net/blog/2015-09-12/restic-foundation1-cdc/) for
+   its content-defined chunker. The algorithm relies on a secret polynomial to
+   split files into chunks.
+
+   As shown in the paper "[Chunking Attacks on File Backup Services using
+   Content-Defined Chunking](https://eprint.iacr.org/2025/532.pdf)" by Boris
+   Alexeev, Colin Percival and Yan X Zhang, an attacker that can observe chunk
+   sizes for a known file can derive the secret polynomial. Knowledge of the
+   polynomial might in some cases allow an attacker to check whether certain large
+   files are stored in a repository.
+
+   A practical attack is nevertheless hard as restic merges multiple chunks into
+   opaque pack files and by default processes multiple files in parallel. This
+   likely prevents an attacker from matching pack files to the attacker-known file
+   and thereby prevents the attack.
+
+   Despite the low chances of a practical attack, restic now has added mitigation
+   that randomizes how chunks are assembled into pack files. This prevents
+   attackers from guessing which chunks are part of a pack file and thereby
+   prevents learning the chunk sizes.
+
+   https://github.com/restic/restic/issues/5291
+   https://github.com/restic/restic/pull/5295
+
+ * Bugfix #1843: Correctly restore long filepaths' timestamp on old Windows
+
+   The `restore` command now correctly restores timestamps for files with paths
+   longer than 256 characters on Windows versions prior to Windows 10 1607.
+
+   https://github.com/restic/restic/issues/1843
+   https://github.com/restic/restic/pull/5061
+
+ * Bugfix #2165: Ignore disappeared backup source files
+
+   The `backup` command now quietly skips files that are removed between directory
+   listing and backup, instead of printing errors like:
+
+   ```
+   error: lstat /some/file/name: no such file or directory
+   ```
+
+   https://github.com/restic/restic/issues/2165
+   https://github.com/restic/restic/issues/3098
+   https://github.com/restic/restic/pull/5143
+   https://github.com/restic/restic/pull/5145
+
+ * Bugfix #5153: Include root tree when searching using `find --tree`
+
+   The `restic find --tree` command did not find trees referenced by `restic
+   snapshot --json`. It now correctly includes the root tree when searching.
+
+   https://github.com/restic/restic/pull/5153
+
+ * Bugfix #5169: Prevent Windows VSS event log 8194 warnings for backup with fs snapshot
+
+   When running `backup` with the `--use-fs-snapshot` option in Windows with admin
+   rights, event logs like
+
+   ```
+   Volume Shadow Copy Service error: Unexpected error querying for the IVssWriterCallback interface.  hr = 0x80070005, Access is denied.
+   . This is often caused by incorrect security settings in either the writer or requester process. 
+   
+   Operation:
+      Gathering Writer Data
+   
+   Context:
+      Writer Class Id: {e8132975-6f93-4464-a53e-1050253ae220}
+      Writer Name: System Writer
+      Writer Instance ID: {54b151ac-d27d-4628-9cb0-2bc40959f50f}
+   ```
+
+   Are created several times even though the backup itself succeeds. This has now
+   been fixed.
+
+   https://github.com/restic/restic/issues/5169
+   https://github.com/restic/restic/pull/5170
+   https://forum.restic.net/t/windows-shadow-copy-snapshot-vss-unexpected-provider-error/3674/2
+
+ * Bugfix #5212: Fix duplicate data handling in `prune --max-unused`
+
+   The `prune --max-unused size` command did not correctly account for duplicate
+   data. If a repository contained a large amount of duplicate data, this could
+   previously result in pruning too little data. This has now been fixed.
+
+   https://github.com/restic/restic/pull/5212
+   https://forum.restic.net/t/restic-not-obeying-max-unused-parameter-on-prune/8879
+
+ * Bugfix #5249: Fix creation of oversized index by `repair index --read-all-packs`
+
+   Since restic 0.17.0, the new index created by `repair index --read-all-packs`
+   was written as a single large index. This significantly increased memory usage
+   while loading the index.
+
+   The index is now correctly split into multiple smaller indexes, and `repair
+   index` now also automatically splits oversized indexes.
+
+   https://github.com/restic/restic/pull/5249
+
+ * Bugfix #5259: Fix rare crash in command output
+
+   Some commands could in rare cases crash when trying to print status messages and
+   request retries at the same time, resulting in an error like the following:
+
+   ```
+   panic: runtime error: slice bounds out of range [468:156]
+   [...]
+   github.com/restic/restic/internal/ui/termstatus.(*lineWriter).Write(...)
+   	/restic/internal/ui/termstatus/stdio_wrapper.go:36 +0x136
+   ```
+
+   This has now been fixed.
+
+   https://github.com/restic/restic/issues/5259
+   https://github.com/restic/restic/pull/5300
+
+ * Change #4938: Update dependencies and require Go 1.23 or newer
+
+   We have updated all dependencies. Restic now requires Go 1.23 or newer to build.
+
+   This also disables support for TLS versions older than TLS 1.2. On Windows,
+   restic now requires at least Windows 10 or Windows Server 2016. On macOS, restic
+   now requires at least macOS 11 Big Sur.
+
+   https://github.com/restic/restic/pull/4938
+
+ * Change #5162: Graduate feature flags
+
+   The `deprecate-legacy-index`, `deprecate-s3-legacy-layout`,
+   `explicit-s3-anonymous-auth` and `safe-forget-keep-tags` features are now stable
+   and can no longer be disabled. The corresponding feature flags will be removed
+   in restic 0.19.0.
+
+   https://github.com/restic/restic/pull/5162
+
+ * Enhancement #1378: Add JSON support to `check` command
+
+   The `check` command now supports the `--json` option to output all statistics in
+   JSON format.
+
+   https://github.com/restic/restic/issues/1378
+   https://github.com/restic/restic/pull/5194
+
+ * Enhancement #2511: Support generating shell completions to stdout
+
+   The `generate` command now supports using `-` as the filename with the
+   `--[shell]-completion` option to write the generated output to stdout.
+
+   https://github.com/restic/restic/issues/2511
+   https://github.com/restic/restic/pull/5053
+
+ * Enhancement #3697: Allow excluding online-only cloud files (e.g. OneDrive)
+
+   Restic treated files synced using OneDrive Files On-Demand as though they were
+   regular files. This caused issues with VSS and could cause OneDrive to download
+   all files.
+
+   Restic now allows the user to exclude these files when backing up with the
+   `--exclude-cloud-files` option.
+
+   https://github.com/restic/restic/issues/3697
+   https://github.com/restic/restic/issues/4935
+   https://github.com/restic/restic/pull/4990
+
+ * Enhancement #4179: Add `sort` option to `ls` command
+
+   The `ls -l` command output can now be sorted using the new `--sort <field>`
+   option for the fields `name`, `size`, `time` (same as `mtime`), `mtime`,
+   `atime`, `ctime` and `extension`. A `--reverse` option is also available.
+
+   https://github.com/restic/restic/issues/4179
+   https://github.com/restic/restic/pull/5182
+
+ * Enhancement #4433: Change default sort order for `find` output
+
+   The `find` command now sorts snapshots from newest to oldest by default. The
+   previous oldest-to-newest order can be restored using the new `--reverse`
+   option.
+
+   https://github.com/restic/restic/issues/4433
+   https://github.com/restic/restic/pull/5184
+
+ * Enhancement #4521: Add support for Microsoft Blob Storage access tiers
+
+   The new `-o azure.access-tier=<tier>` option allows specifying the access tier
+   (`Hot`, `Cool` or `Cold`) for objects created in Microsoft Blob Storage. If
+   unspecified, the storage account's default tier is used.
+
+   There is no official `Archive` storage support in restic, use this option at
+   your own risk. To restore any data, it is necessary to manually warm up the
+   required data in the `Archive` tier.
+
+   https://github.com/restic/restic/issues/4521
+   https://github.com/restic/restic/pull/5046
+
+ * Enhancement #4942: Add snapshot summary statistics to rewritten snapshots
+
+   The `rewrite` command now supports a `--snapshot-summary` option to add
+   statistics data to snapshots. Only two fields in the summary will be non-zero:
+   `TotalFilesProcessed` and `TotalBytesProcessed`.
+
+   For snapshots rewritten using the `--exclude` options, the summary statistics
+   are updated accordingly.
+
+   https://github.com/restic/restic/issues/4942
+   https://github.com/restic/restic/pull/5185
+
+ * Enhancement #4948: Format exit errors as JSON when requested
+
+   Restic now formats error messages as JSON when the `--json` flag is used.
+
+   https://github.com/restic/restic/issues/4948
+   https://github.com/restic/restic/pull/4952
+
+ * Enhancement #4983: Add SLSA provenance to GHCR container images
+
+   Restic's GitHub Container Registry (GHCR) image build workflow now includes SLSA
+   (Supply-chain Levels for Software Artifacts) provenance generation.
+
+   Please see the restic documentation for more information about verifying SLSA
+   provenance.
+
+   https://github.com/restic/restic/issues/4983
+   https://github.com/restic/restic/pull/4999
+
+ * Enhancement #5054: Enable compression for ZIP archives in `dump` command
+
+   The `dump` command now compresses ZIP archives using the DEFLATE algorithm,
+   reducing the size of exported archives.
+
+   https://github.com/restic/restic/pull/5054
+
+ * Enhancement #5081: Add retry mechanism for loading repository config
+
+   Restic now retries loading the repository config file when opening a repository.
+   The `init` command now also retries backend operations.
+
+   https://github.com/restic/restic/issues/5081
+   https://github.com/restic/restic/pull/5095
+
+ * Enhancement #5089: Allow including/excluding extended file attributes during `restore`
+
+   The `restore` command now supports the `--exclude-xattr` and `--include-xattr`
+   options to control which extended file attributes will be restored. By default,
+   all attributes are restored.
+
+   https://github.com/restic/restic/issues/5089
+   https://github.com/restic/restic/pull/5129
+
+ * Enhancement #5092: Show count of deleted files and directories during `restore`
+
+   The `restore` command now reports the number of deleted files and directories,
+   both in the regular output and in the `files_deleted` field of the JSON output.
+
+   https://github.com/restic/restic/issues/5092
+   https://github.com/restic/restic/pull/5100
+
+ * Enhancement #5109: Make small pack size configurable for `prune`
+
+   The `prune` command now supports the `--repack-smaller-than` option that allows
+   repacking pack files smaller than a specified size.
+
+   https://github.com/restic/restic/issues/5109
+   https://github.com/restic/restic/pull/5183
+
+ * Enhancement #5119: Add start and end timestamps to `backup` JSON output
+
+   The JSON output of the `backup` command now includes `backup_start` and
+   `backup_end` timestamps, containing the start and end time of the backup.
+
+   https://github.com/restic/restic/pull/5119
+
+ * Enhancement #5131: Add DragonFlyBSD support
+
+   Restic can now be compiled on DragonflyBSD.
+
+   https://github.com/restic/restic/issues/5131
+   https://github.com/restic/restic/pull/5138
+
+ * Enhancement #5137: Make `tag` command print which snapshots were modified
+
+   The `tag` command now outputs which snapshots were modified along with their new
+   snapshot ID. The command supports the `--json` option for machine-readable
+   output.
+
+   https://github.com/restic/restic/issues/5137
+   https://github.com/restic/restic/pull/5144
+
+ * Enhancement #5141: Provide clear error message if AZURE_ACCOUNT_NAME is not set
+
+   If `AZURE_ACCOUNT_NAME` was not set, commands related to an Azure repository
+   would result in a misleading networking error. Restic now detect this and
+   provides a clear warning that the variable is not defined.
+
+   https://github.com/restic/restic/pull/5141
+
+ * Enhancement #5173: Add experimental S3 cold storage support
+
+   Introduce S3 backend options for transitioning pack files from cold to hot
+   storage on S3 and S3-compatible providers. Note: this only works for the
+   `prune`, `copy` and `restore` commands for now.
+
+   This experimental feature is gated behind the "s3-restore" feature flag.
+
+   https://github.com/restic/restic/issues/3202
+   https://github.com/restic/restic/issues/2504
+   https://github.com/restic/restic/pull/5173
+
+ * Enhancement #5174: Add xattr support for NetBSD 10+
+
+   Extended attribute support for `backup` and `restore` operations is now
+   available on NetBSD version 10 and later.
+
+   https://github.com/restic/restic/issues/5174
+   https://github.com/restic/restic/pull/5180
+
+ * Enhancement #5251: Improve retry handling for flaky `rclone` backends
+
+   Since restic 0.17.0, the backend retry mechanisms rely on backends correctly
+   reporting when a file does not exist. This is not always the case for some
+   `rclone` backends, which caused restic to stop retrying after the first failure.
+
+   For rclone, failed requests are now retried up to 5 times before giving up.
+
+   https://github.com/restic/restic/pull/5251
+
+ * Enhancement #5287: Make `recover` automatically rebuild index when needed
+
+   When trying to recover data from an interrupted snapshot, it was previously
+   necessary to manually run `repair index` before runnning `recover`. This now
+   happens automatically so that only `recover` is necessary.
+
+   https://github.com/restic/restic/issues/5287
+   https://github.com/restic/restic/pull/5296
+
+
+# Changelog for restic 0.17.3 (2024-11-08)
+The following sections list the changes in restic 0.17.3 relevant to
+restic users. The changes are ordered by importance.
+
+## Summary
+
+ * Fix #4971: Fix unusable `mount` on macOS Sonoma
+ * Fix #5003: Fix metadata errors during backup of removable disks on Windows
+ * Fix #5101: Do not retry load/list operation if SFTP connection is broken
+ * Fix #5107: Fix metadata error on Windows for backups using VSS
+ * Enh #5096: Allow `prune --dry-run` without lock
+
+## Details
+
+ * Bugfix #4971: Fix unusable `mount` on macOS Sonoma
+
+   On macOS Sonoma when using FUSE-T, it was not possible to access files in a
+   mounted repository. This issue is now resolved.
+
+   https://github.com/restic/restic/issues/4971
+   https://github.com/restic/restic/pull/5048
+
+ * Bugfix #5003: Fix metadata errors during backup of removable disks on Windows
+
+   Since restic 0.17.0, backing up removable disks on Windows could report errors
+   with retrieving metadata like shown below.
+
+   ```
+   error: incomplete metadata for d:\filename: get named security info failed with: Access is denied.
+   ```
+
+   This has now been fixed.
+
+   https://github.com/restic/restic/issues/5003
+   https://github.com/restic/restic/pull/5123
+   https://forum.restic.net/t/backing-up-a-folder-from-a-veracrypt-volume-brings-up-errors-since-restic-v17-0/8444
+
+ * Bugfix #5101: Do not retry load/list operation if SFTP connection is broken
+
+   When using restic with the SFTP backend, backend operations that load a file or
+   list files were retried even if the SFTP connection was broken. This has now
+   been fixed.
+
+   https://github.com/restic/restic/pull/5101
+   https://forum.restic.net/t/restic-hanging-on-backup/8559
+
+ * Bugfix #5107: Fix metadata error on Windows for backups using VSS
+
+   Since restic 0.17.2, when creating a backup on Windows using
+   `--use-fs-snapshot`, restic would report an error like the following:
+
+   ```
+   error: incomplete metadata for C:\: get EA failed while opening file handle for path \\?\GLOBALROOT\Device\HarddiskVolumeShadowCopyXX\, with: The process cannot access the file because it is being used by another process.
+   ```
+
+   This has now been fixed by correctly handling paths that refer to volume shadow
+   copy snapshots.
+
+   https://github.com/restic/restic/issues/5107
+   https://github.com/restic/restic/pull/5110
+   https://github.com/restic/restic/pull/5112
+
+ * Enhancement #5096: Allow `prune --dry-run` without lock
+
+   The `prune --dry-run --no-lock` now allows performing a dry-run without locking
+   the repository. Note that if the repository is modified concurrently, `prune`
+   may return inaccurate statistics or errors.
+
+   https://github.com/restic/restic/pull/5096
+
+
+# Changelog for restic 0.17.2 (2024-10-27)
+The following sections list the changes in restic 0.17.2 relevant to
+restic users. The changes are ordered by importance.
+
+## Summary
+
+ * Fix #4004: Support container-level SAS/SAT tokens for Azure backend
+ * Fix #5047: Resolve potential error during concurrent cache cleanup
+ * Fix #5050: Return error if `tag` fails to lock repository
+ * Fix #5057: Exclude irregular files from backups
+ * Fix #5063: Correctly `backup` extended metadata when using VSS on Windows
+
+## Details
+
+ * Bugfix #4004: Support container-level SAS/SAT tokens for Azure backend
+
+   Restic previously expected SAS/SAT tokens to be generated at the account level,
+   which prevented tokens created at the container level from being used to
+   initialize a repository. This caused an error when attempting to initialize a
+   repository with container-level tokens.
+
+   Restic now supports both account-level and container-level SAS/SAT tokens for
+   initializing a repository.
+
+   https://github.com/restic/restic/issues/4004
+   https://github.com/restic/restic/pull/5093
+
+ * Bugfix #5047: Resolve potential error during concurrent cache cleanup
+
+   When multiple restic processes ran concurrently, they could compete to remove
+   obsolete snapshots from the local backend cache, sometimes leading to a "no such
+   file or directory" error. Restic now suppresses this error to prevent issues
+   during cache cleanup.
+
+   https://github.com/restic/restic/pull/5047
+
+ * Bugfix #5050: Return error if `tag` fails to lock repository
+
+   Since restic 0.17.0, the `tag` command did not return an error when it failed to
+   open or lock the repository. This issue has now been fixed.
+
+   https://github.com/restic/restic/issues/5050
+   https://github.com/restic/restic/pull/5056
+
+ * Bugfix #5057: Exclude irregular files from backups
+
+   Since restic 0.17.1, files with the type `irregular` could mistakenly be
+   included in snapshots, especially when backing up special file types on Windows
+   that restic cannot process. This issue has now been fixed.
+
+   Previously, this bug caused the `check` command to report errors like the
+   following one:
+
+   ```
+     tree 12345678[...]: node "example.zip" with invalid type "irregular"
+   ```
+
+   To repair affected snapshots, upgrade to restic 0.17.2 and run:
+
+   ```
+   restic repair snapshots --forget
+   ```
+
+   This will remove the `irregular` files from the snapshots (creating a new
+   snapshot ID for each of the affected snapshots).
+
+   https://github.com/restic/restic/pull/5057
+   https://forum.restic.net/t/errors-found-by-check-1-invalid-type-irregular-2-ciphertext-verification-failed/8447/2
+
+ * Bugfix #5063: Correctly `backup` extended metadata when using VSS on Windows
+
+   On Windows, when creating a backup with the `--use-fs-snapshot` option, restic
+   read extended metadata from the original filesystem path instead of from the
+   snapshot. This could result in errors if files were removed during the backup
+   process.
+
+   This issue has now been resolved.
+
+   https://github.com/restic/restic/issues/5063
+   https://github.com/restic/restic/pull/5097
+   https://github.com/restic/restic/pull/5099
+
+
 # Changelog for restic 0.17.1 (2024-09-05)
 The following sections list the changes in restic 0.17.1 relevant to
 restic users. The changes are ordered by importance.

VERSION

@@ -1 +1 @@
-0.17.1
+0.18.0-dev

build.go

@@ -53,12 +53,14 @@ import (
 
 // config contains the configuration for the program to build.
 var config = Config{
-	Name:             "restic",                                 // name of the program executable and directory
-	Namespace:        "github.com/restic/restic",               // subdir of GOPATH, e.g. "github.com/foo/bar"
-	Main:             "./cmd/restic",                           // package name for the main package
-	DefaultBuildTags: []string{"selfupdate"},                   // specify build tags which are always used
-	Tests:            []string{"./..."},                        // tests to run
-	MinVersion:       GoVersion{Major: 1, Minor: 18, Patch: 0}, // minimum Go version supported
+	Name:      "restic",                   // name of the program executable and directory
+	Namespace: "github.com/restic/restic", // subdir of GOPATH, e.g. "github.com/foo/bar"
+	Main:      "./cmd/restic",             // package name for the main package
+	// disable_grpc_modules is necessary to reduce the binary size since cloud.google.com/go/storage v1.44.0
+	// see https://github.com/googleapis/google-cloud-go/issues/11448
+	DefaultBuildTags: []string{"selfupdate", "disable_grpc_modules"}, // specify build tags which are always used
+	Tests:            []string{"./..."},                              // tests to run
+	MinVersion:       GoVersion{Major: 1, Minor: 23, Patch: 0},       // minimum Go version supported
 }
 
 // Config configures the build.
@@ -298,19 +300,21 @@ func (v GoVersion) AtLeast(other GoVersion) bool {
 		return true
 	}
 
+	if v.Major > other.Major {
+		return true
+	}
 	if v.Major < other.Major {
 		return false
 	}
 
+	if v.Minor > other.Minor {
+		return true
+	}
 	if v.Minor < other.Minor {
 		return false
 	}
 
-	if v.Patch < other.Patch {
-		return false
-	}
-
-	return true
+	return v.Patch >= other.Patch
 }
 
 func (v GoVersion) String() string {
@@ -380,12 +384,6 @@ func main() {
 		}
 	}
 
-	solarisMinVersion := GoVersion{Major: 1, Minor: 20, Patch: 0}
-	if env["GOARCH"] == "solaris" && !goVersion.AtLeast(solarisMinVersion) {
-		fmt.Fprintf(os.Stderr, "Detected version %s is too old, restic requires at least %s for Solaris\n", goVersion, solarisMinVersion)
-		os.Exit(1)
-	}
-
 	verbosePrintf("detected Go version %v\n", goVersion)
 
 	preserveSymbols := false

changelog/0.17.2_2024-10-27/issue-4004

@@ -0,0 +1,12 @@
+Bugfix: Support container-level SAS/SAT tokens for Azure backend
+
+Restic previously expected SAS/SAT tokens to be generated at the account level,
+which prevented tokens created at the container level from being used to
+initialize a repository. This caused an error when attempting to initialize a
+repository with container-level tokens.
+
+Restic now supports both account-level and container-level SAS/SAT tokens for
+initializing a repository.
+
+https://github.com/restic/restic/issues/4004
+https://github.com/restic/restic/pull/5093

changelog/0.17.2_2024-10-27/issue-5050

@@ -0,0 +1,7 @@
+Bugfix: Return error if `tag` fails to lock repository
+
+Since restic 0.17.0, the `tag` command did not return an error when it failed
+to open or lock the repository. This issue has now been fixed.
+
+https://github.com/restic/restic/issues/5050
+https://github.com/restic/restic/pull/5056

changelog/0.17.2_2024-10-27/issue-5063

@@ -0,0 +1,12 @@
+Bugfix: Correctly `backup` extended metadata when using VSS on Windows
+
+On Windows, when creating a backup with the `--use-fs-snapshot` option, restic
+read extended metadata from the original filesystem path instead of from the
+snapshot. This could result in errors if files were removed during the backup
+process.
+
+This issue has now been resolved.
+
+https://github.com/restic/restic/issues/5063
+https://github.com/restic/restic/pull/5097
+https://github.com/restic/restic/pull/5099

changelog/0.17.2_2024-10-27/pull-5047

@@ -0,0 +1,8 @@
+Bugfix: Resolve potential error during concurrent cache cleanup
+
+When multiple restic processes ran concurrently, they could compete to remove
+obsolete snapshots from the local backend cache, sometimes leading to a "no
+such file or directory" error. Restic now suppresses this error to prevent
+issues during cache cleanup.
+
+https://github.com/restic/restic/pull/5047

changelog/0.17.2_2024-10-27/pull-5057

@@ -0,0 +1,24 @@
+Bugfix: Exclude irregular files from backups
+
+Since restic 0.17.1, files with the type `irregular` could mistakenly be included
+in snapshots, especially when backing up special file types on Windows that
+restic cannot process. This issue has now been fixed.
+
+Previously, this bug caused the `check` command to report errors like the
+following one:
+
+```
+  tree 12345678[...]: node "example.zip" with invalid type "irregular"
+```
+
+To repair affected snapshots, upgrade to restic 0.17.2 and run:
+
+```
+restic repair snapshots --forget
+```
+
+This will remove the `irregular` files from the snapshots (creating
+a new snapshot ID for each of the affected snapshots).
+
+https://github.com/restic/restic/pull/5057
+https://forum.restic.net/t/errors-found-by-check-1-invalid-type-irregular-2-ciphertext-verification-failed/8447/2

changelog/0.17.3_2024-11-08/issue-4971

@@ -0,0 +1,7 @@
+Bugfix: Fix unusable `mount` on macOS Sonoma
+
+On macOS Sonoma when using FUSE-T, it was not possible to access files in
+a mounted repository. This issue is now resolved.
+
+https://github.com/restic/restic/issues/4971
+https://github.com/restic/restic/pull/5048

changelog/0.17.3_2024-11-08/issue-5003

@@ -0,0 +1,14 @@
+Bugfix: Fix metadata errors during backup of removable disks on Windows
+
+Since restic 0.17.0, backing up removable disks on Windows could report
+errors with retrieving metadata like shown below.
+
+```
+error: incomplete metadata for d:\filename: get named security info failed with: Access is denied.
+```
+
+This has now been fixed.
+
+https://github.com/restic/restic/issues/5003
+https://github.com/restic/restic/pull/5123
+https://forum.restic.net/t/backing-up-a-folder-from-a-veracrypt-volume-brings-up-errors-since-restic-v17-0/8444

changelog/0.17.3_2024-11-08/issue-5107

@@ -0,0 +1,15 @@
+Bugfix: Fix metadata error on Windows for backups using VSS
+
+Since restic 0.17.2, when creating a backup on Windows using `--use-fs-snapshot`,
+restic would report an error like the following:
+
+```
+error: incomplete metadata for C:\: get EA failed while opening file handle for path \\?\GLOBALROOT\Device\HarddiskVolumeShadowCopyXX\, with: The process cannot access the file because it is being used by another process.
+```
+
+This has now been fixed by correctly handling paths that refer to volume
+shadow copy snapshots.
+
+https://github.com/restic/restic/issues/5107
+https://github.com/restic/restic/pull/5110
+https://github.com/restic/restic/pull/5112

changelog/0.17.3_2024-11-08/pull-5096

@@ -0,0 +1,8 @@
+Enhancement: Allow `prune --dry-run` without lock
+
+The `prune --dry-run --no-lock` now allows performing a dry-run
+without locking the repository. Note that if the repository is
+modified concurrently, `prune` may return inaccurate statistics
+or errors.
+
+https://github.com/restic/restic/pull/5096

changelog/0.17.3_2024-11-08/pull-5101

@@ -0,0 +1,8 @@
+Bugfix: Do not retry load/list operation if SFTP connection is broken
+
+When using restic with the SFTP backend, backend operations that load a
+file or list files were retried even if the SFTP connection was broken.
+This has now been fixed.
+
+https://github.com/restic/restic/pull/5101
+https://forum.restic.net/t/restic-hanging-on-backup/8559

changelog/0.18.0_2025-03-27/issue-1378

@@ -0,0 +1,7 @@
+Enhancement: Add JSON support to `check` command
+
+The `check` command now supports the `--json` option to output all statistics in 
+JSON format.
+
+https://github.com/restic/restic/issues/1378
+https://github.com/restic/restic/pull/5194

changelog/0.18.0_2025-03-27/issue-1843

@@ -0,0 +1,7 @@
+Bugfix: Correctly restore long filepaths' timestamp on old Windows
+
+The `restore` command now correctly restores timestamps for files with paths longer
+than 256 characters on Windows versions prior to Windows 10 1607.
+
+https://github.com/restic/restic/issues/1843
+https://github.com/restic/restic/pull/5061

changelog/0.18.0_2025-03-27/issue-2165

@@ -0,0 +1,13 @@
+Bugfix: Ignore disappeared backup source files
+
+The `backup` command now quietly skips files that are removed between directory
+listing and backup, instead of printing errors like:
+
+```
+error: lstat /some/file/name: no such file or directory
+```
+
+https://github.com/restic/restic/issues/2165
+https://github.com/restic/restic/issues/3098
+https://github.com/restic/restic/pull/5143
+https://github.com/restic/restic/pull/5145

changelog/0.18.0_2025-03-27/issue-2511

@@ -0,0 +1,7 @@
+Enhancement: Support generating shell completions to stdout
+
+The `generate` command now supports using `-` as the filename with the
+`--[shell]-completion` option to write the generated output to stdout.
+
+https://github.com/restic/restic/issues/2511
+https://github.com/restic/restic/pull/5053

changelog/0.18.0_2025-03-27/issue-3202

@@ -0,0 +1,11 @@
+Enhancement: Add experimental S3 cold storage support
+
+Introduce S3 backend options for transitioning pack files from cold to hot storage 
+on S3 and S3-compatible providers. Note: this only works for the `prune`, `copy` 
+and `restore` commands for now.
+
+This experimental feature is gated behind the "s3-restore" feature flag.
+
+https://github.com/restic/restic/pull/5173
+https://github.com/restic/restic/issues/3202
+https://github.com/restic/restic/issues/2504

changelog/0.18.0_2025-03-27/issue-3697

@@ -0,0 +1,12 @@
+Enhancement: Allow excluding online-only cloud files (e.g. OneDrive)
+
+Restic treated files synced using OneDrive Files On-Demand as though they
+were regular files. This caused issues with VSS and could cause OneDrive to
+download all files.
+
+Restic now allows the user to exclude these files when backing up with
+the `--exclude-cloud-files` option.
+
+https://github.com/restic/restic/issues/3697
+https://github.com/restic/restic/issues/4935
+https://github.com/restic/restic/pull/4990

changelog/0.18.0_2025-03-27/issue-4179

@@ -0,0 +1,8 @@
+Enhancement: Add `sort` option to `ls` command
+
+The `ls -l` command output can now be sorted using the new `--sort <field>`
+option for the fields `name`, `size`, `time` (same as `mtime`), `mtime`,
+`atime`, `ctime` and `extension`. A `--reverse` option is also available.
+
+https://github.com/restic/restic/issues/4179
+https://github.com/restic/restic/pull/5182

changelog/0.18.0_2025-03-27/issue-4433

@@ -0,0 +1,7 @@
+Enhancement: Change default sort order for `find` output
+
+The `find` command now sorts snapshots from newest to oldest by default. The
+previous oldest-to-newest order can be restored using the new `--reverse` option.
+
+https://github.com/restic/restic/issues/4433
+https://github.com/restic/restic/pull/5184

changelog/0.18.0_2025-03-27/issue-4521

@@ -0,0 +1,12 @@
+Enhancement: Add support for Microsoft Blob Storage access tiers
+
+The new `-o azure.access-tier=<tier>` option allows specifying the access tier
+(`Hot`, `Cool` or `Cold`) for objects created in Microsoft Blob Storage. If
+unspecified, the storage account's default tier is used.
+
+There is no official `Archive` storage support in restic, use this option at
+your own risk. To restore any data, it is necessary to manually warm up the
+required data in the `Archive` tier.
+
+https://github.com/restic/restic/issues/4521
+https://github.com/restic/restic/pull/5046

changelog/0.18.0_2025-03-27/issue-4942

@@ -0,0 +1,11 @@
+Enhancement: Add snapshot summary statistics to rewritten snapshots
+
+The `rewrite` command now supports a `--snapshot-summary` option to add
+statistics data to snapshots. Only two fields in the summary will be non-zero:
+`TotalFilesProcessed` and `TotalBytesProcessed`.
+
+For snapshots rewritten using the `--exclude` options, the summary
+statistics are updated accordingly.
+
+https://github.com/restic/restic/issues/4942
+https://github.com/restic/restic/pull/5185

changelog/0.18.0_2025-03-27/issue-4948

@@ -0,0 +1,6 @@
+Enhancement: Format exit errors as JSON when requested
+
+Restic now formats error messages as JSON when the `--json` flag is used.
+
+https://github.com/restic/restic/issues/4948
+https://github.com/restic/restic/pull/4952

changelog/0.18.0_2025-03-27/issue-4983

@@ -0,0 +1,10 @@
+Enhancement: Add SLSA provenance to GHCR container images
+
+Restic's GitHub Container Registry (GHCR) image build workflow now includes
+SLSA (Supply-chain Levels for Software Artifacts) provenance generation.
+
+Please see the restic documentation for more information about verifying SLSA
+provenance.
+
+https://github.com/restic/restic/issues/4983
+https://github.com/restic/restic/pull/4999

changelog/0.18.0_2025-03-27/issue-5081

@@ -0,0 +1,7 @@
+Enhancement: Add retry mechanism for loading repository config
+
+Restic now retries loading the repository config file when opening a repository.
+The `init` command now also retries backend operations.
+
+https://github.com/restic/restic/issues/5081
+https://github.com/restic/restic/pull/5095

changelog/0.18.0_2025-03-27/issue-5089

@@ -0,0 +1,8 @@
+Enhancement: Allow including/excluding extended file attributes during `restore`
+
+The `restore` command now supports the `--exclude-xattr` and `--include-xattr`
+options to control which extended file attributes will be restored. By default,
+all attributes are restored.
+
+https://github.com/restic/restic/issues/5089
+https://github.com/restic/restic/pull/5129

changelog/0.18.0_2025-03-27/issue-5092

@@ -0,0 +1,7 @@
+Enhancement: Show count of deleted files and directories during `restore`
+
+The `restore` command now reports the number of deleted files and directories,
+both in the regular output and in the `files_deleted` field of the JSON output.
+
+https://github.com/restic/restic/issues/5092
+https://github.com/restic/restic/pull/5100

changelog/0.18.0_2025-03-27/issue-5109

@@ -0,0 +1,7 @@
+Enhancement: Make small pack size configurable for `prune`
+
+The `prune` command now supports the `--repack-smaller-than` option that
+allows repacking pack files smaller than a specified size.
+
+https://github.com/restic/restic/issues/5109
+https://github.com/restic/restic/pull/5183

changelog/0.18.0_2025-03-27/issue-5131

@@ -0,0 +1,6 @@
+Enhancement: Add DragonFlyBSD support
+
+Restic can now be compiled on DragonflyBSD.
+
+https://github.com/restic/restic/issues/5131
+https://github.com/restic/restic/pull/5138

changelog/0.18.0_2025-03-27/issue-5137

@@ -0,0 +1,8 @@
+Enhancement: Make `tag` command print which snapshots were modified
+
+The `tag` command now outputs which snapshots were modified along with their
+new snapshot ID. The command supports the `--json` option for machine-readable
+output.
+
+https://github.com/restic/restic/issues/5137
+https://github.com/restic/restic/pull/5144

changelog/0.18.0_2025-03-27/issue-5174

@@ -0,0 +1,7 @@
+Enhancement: Add xattr support for NetBSD 10+
+
+Extended attribute support for `backup` and `restore` operations
+is now available on NetBSD version 10 and later.
+
+https://github.com/restic/restic/issues/5174
+https://github.com/restic/restic/pull/5180

changelog/0.18.0_2025-03-27/issue-5259

@@ -0,0 +1,16 @@
+Bugfix: Fix rare crash in command output
+
+Some commands could in rare cases crash when trying to print status messages
+and request retries at the same time, resulting in an error like the following:
+
+```
+panic: runtime error: slice bounds out of range [468:156]
+[...]
+github.com/restic/restic/internal/ui/termstatus.(*lineWriter).Write(...)
+	/restic/internal/ui/termstatus/stdio_wrapper.go:36 +0x136
+```
+
+This has now been fixed.
+
+https://github.com/restic/restic/issues/5259
+https://github.com/restic/restic/pull/5300

changelog/0.18.0_2025-03-27/issue-5287

@@ -0,0 +1,8 @@
+Enhancement: Make `recover` automatically rebuild index when needed
+
+When trying to recover data from an interrupted snapshot, it was previously
+necessary to manually run `repair index` before runnning `recover`. This now
+happens automatically so that only `recover` is necessary.
+
+https://github.com/restic/restic/issues/5287
+https://github.com/restic/restic/pull/5296

changelog/0.18.0_2025-03-27/issue-5291

@@ -0,0 +1,24 @@
+Security: Mitigate attack on content-defined chunking algorithm
+
+Restic uses [Rabin Fingerprints](https://restic.net/blog/2015-09-12/restic-foundation1-cdc/)
+for its content-defined chunker. The algorithm relies on a secret polynomial
+to split files into chunks.
+
+As shown in the paper "[Chunking Attacks on File Backup Services using Content-Defined Chunking](https://eprint.iacr.org/2025/532.pdf)"
+by Boris Alexeev, Colin Percival and Yan X Zhang, an
+attacker that can observe chunk sizes for a known file can derive the secret
+polynomial. Knowledge of the polynomial might in some cases allow an attacker
+to check whether certain large files are stored in a repository.
+
+A practical attack is nevertheless hard as restic merges multiple chunks into
+opaque pack files and by default processes multiple files in parallel. This
+likely prevents an attacker from matching pack files to the attacker-known file
+and thereby prevents the attack.
+
+Despite the low chances of a practical attack, restic now has added mitigation
+that randomizes how chunks are assembled into pack files. This prevents attackers
+from guessing which chunks are part of a pack file and thereby prevents learning
+the chunk sizes.
+
+https://github.com/restic/restic/issues/5291
+https://github.com/restic/restic/pull/5295

changelog/0.18.0_2025-03-27/pull-4938

@@ -0,0 +1,9 @@
+Change: Update dependencies and require Go 1.23 or newer
+
+We have updated all dependencies. Restic now requires Go 1.23 or newer to build.
+
+This also disables support for TLS versions older than TLS 1.2. On Windows,
+restic now requires at least Windows 10 or Windows Server 2016. On macOS,
+restic now requires at least macOS 11 Big Sur.
+
+https://github.com/restic/restic/pull/4938

changelog/0.18.0_2025-03-27/pull-5054

@@ -0,0 +1,6 @@
+Enhancement: Enable compression for ZIP archives in `dump` command
+
+The `dump` command now compresses ZIP archives using the DEFLATE algorithm,
+reducing the size of exported archives.
+
+https://github.com/restic/restic/pull/5054

changelog/0.18.0_2025-03-27/pull-5119

@@ -0,0 +1,6 @@
+Enhancement: Add start and end timestamps to `backup` JSON output
+
+The JSON output of the `backup` command now includes `backup_start` and
+`backup_end` timestamps, containing the start and end time of the backup.
+
+https://github.com/restic/restic/pull/5119

changelog/0.18.0_2025-03-27/pull-5141

@@ -0,0 +1,7 @@
+Enhancement: Provide clear error message if AZURE_ACCOUNT_NAME is not set
+
+If `AZURE_ACCOUNT_NAME` was not set, commands related to an Azure repository
+would result in a misleading networking error. Restic now detect this and
+provides a clear warning that the variable is not defined.
+
+https://github.com/restic/restic/pull/5141

changelog/0.18.0_2025-03-27/pull-5153

@@ -0,0 +1,7 @@
+Bugfix: Include root tree when searching using `find --tree`
+
+The `restic find --tree` command did not find trees referenced by
+`restic snapshot --json`. It now correctly includes the root tree
+when searching.
+
+https://github.com/restic/restic/pull/5153

changelog/0.18.0_2025-03-27/pull-5162

@@ -0,0 +1,8 @@
+Change: Graduate feature flags
+
+The `deprecate-legacy-index`, `deprecate-s3-legacy-layout`,
+`explicit-s3-anonymous-auth` and `safe-forget-keep-tags` features are
+now stable and can no longer be disabled. The corresponding feature flags
+will be removed in restic 0.19.0.
+
+https://github.com/restic/restic/pull/5162

changelog/0.18.0_2025-03-27/pull-5170

@@ -0,0 +1,22 @@
+Bugfix: Prevent Windows VSS event log 8194 warnings for backup with fs snapshot
+
+When running `backup` with the `--use-fs-snapshot` option in Windows with admin rights, event logs like
+
+```
+Volume Shadow Copy Service error: Unexpected error querying for the IVssWriterCallback interface.  hr = 0x80070005, Access is denied.
+. This is often caused by incorrect security settings in either the writer or requester process. 
+
+Operation:
+   Gathering Writer Data
+
+Context:
+   Writer Class Id: {e8132975-6f93-4464-a53e-1050253ae220}
+   Writer Name: System Writer
+   Writer Instance ID: {54b151ac-d27d-4628-9cb0-2bc40959f50f}
+```
+
+are created several times even though the backup itself succeeds. This has now been fixed.
+
+https://github.com/restic/restic/issues/5169
+https://github.com/restic/restic/pull/5170
+https://forum.restic.net/t/windows-shadow-copy-snapshot-vss-unexpected-provider-error/3674/2

changelog/0.18.0_2025-03-27/pull-5212

@@ -0,0 +1,8 @@
+Bugfix: Fix duplicate data handling in `prune --max-unused`
+
+The `prune --max-unused size` command did not correctly account for duplicate
+data. If a repository contained a large amount of duplicate data, this could
+previously result in pruning too little data. This has now been fixed.
+
+https://github.com/restic/restic/pull/5212
+https://forum.restic.net/t/restic-not-obeying-max-unused-parameter-on-prune/8879

changelog/0.18.0_2025-03-27/pull-5249

@@ -0,0 +1,10 @@
+Bugfix: Fix creation of oversized index by `repair index --read-all-packs`
+
+Since restic 0.17.0, the new index created by `repair index --read-all-packs` was
+written as a single large index. This significantly increased memory usage while
+loading the index.
+
+The index is now correctly split into multiple smaller indexes, and `repair index`
+now also automatically splits oversized indexes.
+
+https://github.com/restic/restic/pull/5249

changelog/0.18.0_2025-03-27/pull-5251

@@ -0,0 +1,9 @@
+Enhancement: Improve retry handling for flaky `rclone` backends
+
+Since restic 0.17.0, the backend retry mechanisms rely on backends correctly
+reporting when a file does not exist. This is not always the case for some
+`rclone` backends, which caused restic to stop retrying after the first failure.
+
+For rclone, failed requests are now retried up to 5 times before giving up.
+
+https://github.com/restic/restic/pull/5251

changelog/TEMPLATE

@@ -5,6 +5,8 @@ Enhancement: Allow custom bar in the foo command
 
 # Describe the problem in the past tense, the new behavior in the present
 # tense. Mention the affected commands, backends, operating systems, etc.
+# If the problem description just says that a feature was missing, then
+# only explain the new behavior.
 # Focus on user-facing behavior, not the implementation.
 # Use "Restic now ..." instead of "We have changed ...".
 

changelog/changelog-github.tmpl

@@ -15,7 +15,7 @@ Details
 {{ range $entry := .Entries }}{{ with $entry }}
  * {{ .Type }} #{{ .PrimaryID }}: {{ .Title }}
 {{ range $par := .Paragraphs }}
-   {{ $par }}
+{{ indent 3 $par }}
 {{ end }}
    {{ range $id := .Issues -}}
 {{ ` ` }}[#{{ $id }}](https://github.com/restic/restic/issues/{{ $id -}})

changelog/unreleased/issue-4728

@@ -0,0 +1,7 @@
+Enhancement: Added support for zstd compression levels `fastest` and `better`
+
+Restic now supports the zstd compression modes `fastest` and `better`. Set the
+environment variable `RESTIC_COMPRESSION` to `fastest` or `better` to use these
+compression levels. This can also be set with the `--compression` flag.
+
+https://github.com/restic/restic/issues/4728

changelog/unreleased/issue-4868

@@ -0,0 +1,14 @@
+Enhancement: Include repository id in filesystem name used by `mount`
+
+The filesystem created by restic's `mount` command now includes the repository
+id in the filesystem name. The repository id is printed by restic when opening
+a repository or can be looked up using `restic cat config`.
+
+```
+[restic-user@hostname restic]$ df ./test-mount/
+Filesystem        1K-blocks  Used Available Use% Mounted on
+restic:d3b07384d1         0     0         0    - /mnt/my-restic-repo
+```
+
+https://github.com/restic/restic/issues/4868
+https://github.com/restic/restic/pull/5243

changelog/unreleased/issue-5233

@@ -0,0 +1,8 @@
+Bugfix: forget command returns exit code 3 on partial removal of snapshots
+
+The `forget` command now returns exit code 3 when it fails to remove one or
+more snapshots. Previously, it returned exit code 0, which could lead to
+confusion if the command was used in a script.
+
+https://github.com/restic/restic/issues/5233
+https://github.com/restic/restic/pull/5322

changelog/unreleased/issue-5324

@@ -0,0 +1,14 @@
+Bugfix: Correctly handle `backup --stdin-filename` with directories
+
+In restic 0.18.0, the `backup` command failed if a filename that includes
+a least a directory was passed to `--stdin-filename`. For example,
+`--stdin-filename /foo/bar` resulted in the following error:
+
+```
+Fatal: unable to save snapshot: open /foo: no such file or directory
+```
+
+This has been fixed now.
+
+https://github.com/restic/restic/issues/5324
+https://github.com/restic/restic/pull/5356

changelog/unreleased/issue-5325

@@ -0,0 +1,7 @@
+Bugfix: Correctly handle `RESTIC_HOST` in `forget` command
+
+The `forget` command did not use the host name from the `RESTIC_HOST`
+environment variable. This has been fixed.
+
+https://github.com/restic/restic/issues/5325
+https://github.com/restic/restic/pull/5327

changelog/unreleased/issue-5342

@@ -0,0 +1,7 @@
+Bugfix: Ignore "chmod not supported" errors when writing files
+
+Restic 0.18.0 introduced a bug that caused "chmod xxx: operation not supported"
+errors to appear when writing to a local file repository that did not support
+chmod (like CIFS or WebDAV mounted via FUSE). Restic now ignores those errors.
+
+https://github.com/restic/restic/issues/5342

cmd/restic/cmd_backup.go

@@ -15,23 +15,29 @@ import (
 	"time"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 	"golang.org/x/sync/errgroup"
 
 	"github.com/restic/restic/internal/archiver"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/filter"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/textfile"
+	"github.com/restic/restic/internal/ui"
 	"github.com/restic/restic/internal/ui/backup"
 	"github.com/restic/restic/internal/ui/termstatus"
 )
 
-var cmdBackup = &cobra.Command{
-	Use:   "backup [flags] [FILE/DIR] ...",
-	Short: "Create a new backup of files and/or directories",
-	Long: `
+func newBackupCommand() *cobra.Command {
+	var opts BackupOptions
+
+	cmd := &cobra.Command{
+		Use:   "backup [flags] [FILE/DIR] ...",
+		Short: "Create a new backup of files and/or directories",
+		Long: `
 The "backup" command creates a new snapshot and saves the files and directories
 given as the arguments.
 
@@ -45,28 +51,32 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	PreRun: func(_ *cobra.Command, _ []string) {
-		if backupOptions.Host == "" {
-			hostname, err := os.Hostname()
-			if err != nil {
-				debug.Log("os.Hostname() returned err: %v", err)
-				return
+		PreRun: func(_ *cobra.Command, _ []string) {
+			if opts.Host == "" {
+				hostname, err := os.Hostname()
+				if err != nil {
+					debug.Log("os.Hostname() returned err: %v", err)
+					return
+				}
+				opts.Host = hostname
 			}
-			backupOptions.Host = hostname
-		}
-	},
-	GroupID:           cmdGroupDefault,
-	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		term, cancel := setupTermstatus()
-		defer cancel()
-		return runBackup(cmd.Context(), backupOptions, globalOptions, term, args)
-	},
+		},
+		GroupID:           cmdGroupDefault,
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			term, cancel := setupTermstatus()
+			defer cancel()
+			return runBackup(cmd.Context(), opts, globalOptions, term, args)
+		},
+	}
+
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 // BackupOptions bundles all options for the backup command.
 type BackupOptions struct {
-	excludePatternOptions
+	filter.ExcludePatternOptions
 
 	Parent            string
 	GroupBy           restic.SnapshotGroupByOptions
@@ -75,6 +85,7 @@ type BackupOptions struct {
 	ExcludeIfPresent  []string
 	ExcludeCaches     bool
 	ExcludeLargerThan string
+	ExcludeCloudFiles bool
 	Stdin             bool
 	StdinFilename     string
 	StdinCommand      bool
@@ -94,62 +105,60 @@ type BackupOptions struct {
 	SkipIfUnchanged   bool
 }
 
-var backupOptions BackupOptions
+func (opts *BackupOptions) AddFlags(f *pflag.FlagSet) {
+	f.StringVar(&opts.Parent, "parent", "", "use this parent `snapshot` (default: latest snapshot in the group determined by --group-by and not newer than the timestamp determined by --time)")
+	opts.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true}
+	f.VarP(&opts.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
+	f.BoolVarP(&opts.Force, "force", "f", false, `force re-reading the source files/directories (overrides the "parent" flag)`)
 
-// ErrInvalidSourceData is used to report an incomplete backup
-var ErrInvalidSourceData = errors.New("at least one source file could not be read")
+	opts.ExcludePatternOptions.Add(f)
 
-func init() {
-	cmdRoot.AddCommand(cmdBackup)
-
-	f := cmdBackup.Flags()
-	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: latest snapshot in the group determined by --group-by and not newer than the timestamp determined by --time)")
-	backupOptions.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true}
-	f.VarP(&backupOptions.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
-	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the source files/directories (overrides the "parent" flag)`)
-
-	initExcludePatternOptions(f, &backupOptions.excludePatternOptions)
-
-	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems, don't cross filesystem boundaries and subvolumes")
-	f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes `filename[:header]`, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)")
-	f.BoolVar(&backupOptions.ExcludeCaches, "exclude-caches", false, `excludes cache directories that are marked with a CACHEDIR.TAG file. See https://bford.info/cachedir/ for the Cache Directory Tagging Standard`)
-	f.StringVar(&backupOptions.ExcludeLargerThan, "exclude-larger-than", "", "max `size` of the files to be backed up (allowed suffixes: k/K, m/M, g/G, t/T)")
-	f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
-	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "`filename` to use when reading from stdin")
-	f.BoolVar(&backupOptions.StdinCommand, "stdin-from-command", false, "interpret arguments as command to execute and store its stdout")
-	f.Var(&backupOptions.Tags, "tag", "add `tags` for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times)")
-	f.UintVar(&backupOptions.ReadConcurrency, "read-concurrency", 0, "read `n` files concurrently (default: $RESTIC_READ_CONCURRENCY or 2)")
-	f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually (default: $RESTIC_HOST). To prevent an expensive rescan use the \"parent\" flag")
-	f.StringVar(&backupOptions.Host, "hostname", "", "set the `hostname` for the snapshot manually")
+	f.BoolVarP(&opts.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems, don't cross filesystem boundaries and subvolumes")
+	f.StringArrayVar(&opts.ExcludeIfPresent, "exclude-if-present", nil, "takes `filename[:header]`, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)")
+	f.BoolVar(&opts.ExcludeCaches, "exclude-caches", false, `excludes cache directories that are marked with a CACHEDIR.TAG file. See https://bford.info/cachedir/ for the Cache Directory Tagging Standard`)
+	f.StringVar(&opts.ExcludeLargerThan, "exclude-larger-than", "", "max `size` of the files to be backed up (allowed suffixes: k/K, m/M, g/G, t/T)")
+	f.BoolVar(&opts.Stdin, "stdin", false, "read backup from stdin")
+	f.StringVar(&opts.StdinFilename, "stdin-filename", "stdin", "`filename` to use when reading from stdin")
+	f.BoolVar(&opts.StdinCommand, "stdin-from-command", false, "interpret arguments as command to execute and store its stdout")
+	f.Var(&opts.Tags, "tag", "add `tags` for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times)")
+	f.UintVar(&opts.ReadConcurrency, "read-concurrency", 0, "read `n` files concurrently (default: $RESTIC_READ_CONCURRENCY or 2)")
+	f.StringVarP(&opts.Host, "host", "H", "", "set the `hostname` for the snapshot manually (default: $RESTIC_HOST). To prevent an expensive rescan use the \"parent\" flag")
+	f.StringVar(&opts.Host, "hostname", "", "set the `hostname` for the snapshot manually")
 	err := f.MarkDeprecated("hostname", "use --host")
 	if err != nil {
 		// MarkDeprecated only returns an error when the flag could not be found
 		panic(err)
 	}
-	f.StringArrayVar(&backupOptions.FilesFrom, "files-from", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
-	f.StringArrayVar(&backupOptions.FilesFromVerbatim, "files-from-verbatim", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
-	f.StringArrayVar(&backupOptions.FilesFromRaw, "files-from-raw", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
-	f.StringVar(&backupOptions.TimeStamp, "time", "", "`time` of the backup (ex. '2012-11-01 22:08:41') (default: now)")
-	f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories")
-	f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number and ctime changes when checking for modified files")
-	f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files")
-	f.BoolVarP(&backupOptions.DryRun, "dry-run", "n", false, "do not upload or write any data, just show what would be done")
-	f.BoolVar(&backupOptions.NoScan, "no-scan", false, "do not run scanner to estimate size of backup")
+	f.StringArrayVar(&opts.FilesFrom, "files-from", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
+	f.StringArrayVar(&opts.FilesFromVerbatim, "files-from-verbatim", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
+	f.StringArrayVar(&opts.FilesFromRaw, "files-from-raw", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
+	f.StringVar(&opts.TimeStamp, "time", "", "`time` of the backup (ex. '2012-11-01 22:08:41') (default: now)")
+	f.BoolVar(&opts.WithAtime, "with-atime", false, "store the atime for all files and directories")
+	f.BoolVar(&opts.IgnoreInode, "ignore-inode", false, "ignore inode number and ctime changes when checking for modified files")
+	f.BoolVar(&opts.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files")
+	f.BoolVarP(&opts.DryRun, "dry-run", "n", false, "do not upload or write any data, just show what would be done")
+	f.BoolVar(&opts.NoScan, "no-scan", false, "do not run scanner to estimate size of backup")
 	if runtime.GOOS == "windows" {
-		f.BoolVar(&backupOptions.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)")
+		f.BoolVar(&opts.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)")
+		f.BoolVar(&opts.ExcludeCloudFiles, "exclude-cloud-files", false, "excludes online-only cloud files (such as OneDrive Files On-Demand)")
 	}
-	f.BoolVar(&backupOptions.SkipIfUnchanged, "skip-if-unchanged", false, "skip snapshot creation if identical to parent snapshot")
+	f.BoolVar(&opts.SkipIfUnchanged, "skip-if-unchanged", false, "skip snapshot creation if identical to parent snapshot")
 
 	// parse read concurrency from env, on error the default value will be used
 	readConcurrency, _ := strconv.ParseUint(os.Getenv("RESTIC_READ_CONCURRENCY"), 10, 32)
-	backupOptions.ReadConcurrency = uint(readConcurrency)
+	opts.ReadConcurrency = uint(readConcurrency)
 
 	// parse host from env, if not exists or empty the default value will be used
 	if host := os.Getenv("RESTIC_HOST"); host != "" {
-		backupOptions.Host = host
+		opts.Host = host
 	}
 }
 
+var backupFSTestHook func(fs fs.FS) fs.FS
+
+// ErrInvalidSourceData is used to report an incomplete backup
+var ErrInvalidSourceData = errors.New("at least one source file could not be read")
+
 // filterExisting returns a slice of all existing items, or an error if no
 // items exist at all.
 func filterExisting(items []string) (result []string, err error) {
@@ -297,9 +306,9 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
 
 // collectRejectByNameFuncs returns a list of all functions which may reject data
 // from being saved in a snapshot based on path only
-func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (fs []RejectByNameFunc, err error) {
+func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (fs []archiver.RejectByNameFunc, err error) {
 	// exclude restic cache
-	if repo.Cache != nil {
+	if repo.Cache() != nil {
 		f, err := rejectResticCache(repo)
 		if err != nil {
 			return nil, err
@@ -308,23 +317,12 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (
 		fs = append(fs, f)
 	}
 
-	fsPatterns, err := opts.excludePatternOptions.CollectPatterns()
+	fsPatterns, err := opts.ExcludePatternOptions.CollectPatterns(Warnf)
 	if err != nil {
 		return nil, err
 	}
-	fs = append(fs, fsPatterns...)
-
-	if opts.ExcludeCaches {
-		opts.ExcludeIfPresent = append(opts.ExcludeIfPresent, "CACHEDIR.TAG:Signature: 8a477f597d28d172789f06886806bc55")
-	}
-
-	for _, spec := range opts.ExcludeIfPresent {
-		f, err := rejectIfPresent(spec)
-		if err != nil {
-			return nil, err
-		}
-
-		fs = append(fs, f)
+	for _, pat := range fsPatterns {
+		fs = append(fs, archiver.RejectByNameFunc(pat))
 	}
 
 	return fs, nil
@@ -332,25 +330,54 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (
 
 // collectRejectFuncs returns a list of all functions which may reject data
 // from being saved in a snapshot based on path and file info
-func collectRejectFuncs(opts BackupOptions, targets []string) (fs []RejectFunc, err error) {
+func collectRejectFuncs(opts BackupOptions, targets []string, fs fs.FS) (funcs []archiver.RejectFunc, err error) {
 	// allowed devices
-	if opts.ExcludeOtherFS && !opts.Stdin {
-		f, err := rejectByDevice(targets)
+	if opts.ExcludeOtherFS && !opts.Stdin && !opts.StdinCommand {
+		f, err := archiver.RejectByDevice(targets, fs)
 		if err != nil {
 			return nil, err
 		}
-		fs = append(fs, f)
+		funcs = append(funcs, f)
 	}
 
-	if len(opts.ExcludeLargerThan) != 0 && !opts.Stdin {
-		f, err := rejectBySize(opts.ExcludeLargerThan)
+	if len(opts.ExcludeLargerThan) != 0 && !opts.Stdin && !opts.StdinCommand {
+		maxSize, err := ui.ParseBytes(opts.ExcludeLargerThan)
 		if err != nil {
 			return nil, err
 		}
-		fs = append(fs, f)
+
+		f, err := archiver.RejectBySize(maxSize)
+		if err != nil {
+			return nil, err
+		}
+		funcs = append(funcs, f)
 	}
 
-	return fs, nil
+	if opts.ExcludeCloudFiles && !opts.Stdin && !opts.StdinCommand {
+		if runtime.GOOS != "windows" {
+			return nil, errors.Fatalf("exclude-cloud-files is only supported on Windows")
+		}
+		f, err := archiver.RejectCloudFiles(Warnf)
+		if err != nil {
+			return nil, err
+		}
+		funcs = append(funcs, f)
+	}
+
+	if opts.ExcludeCaches {
+		opts.ExcludeIfPresent = append(opts.ExcludeIfPresent, "CACHEDIR.TAG:Signature: 8a477f597d28d172789f06886806bc55")
+	}
+
+	for _, spec := range opts.ExcludeIfPresent {
+		f, err := archiver.RejectIfPresent(spec, Warnf)
+		if err != nil {
+			return nil, err
+		}
+
+		funcs = append(funcs, f)
+	}
+
+	return funcs, nil
 }
 
 // collectTargets returns a list of target files/dirs from several sources.
@@ -505,12 +532,6 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 		return err
 	}
 
-	// rejectFuncs collect functions that can reject items from the backup based on path and file info
-	rejectFuncs, err := collectRejectFuncs(opts, targets)
-	if err != nil {
-		return err
-	}
-
 	var parentSnapshot *restic.Snapshot
 	if !opts.Stdin {
 		parentSnapshot, err = findParentSnapshot(ctx, repo, opts, targets, timeStamp)
@@ -532,30 +553,11 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 	}
 
 	bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
-
 	err = repo.LoadIndex(ctx, bar)
 	if err != nil {
 		return err
 	}
 
-	selectByNameFilter := func(item string) bool {
-		for _, reject := range rejectByNameFuncs {
-			if reject(item) {
-				return false
-			}
-		}
-		return true
-	}
-
-	selectFilter := func(item string, fi os.FileInfo) bool {
-		for _, reject := range rejectFuncs {
-			if reject(item, fi) {
-				return false
-			}
-		}
-		return true
-	}
-
 	var targetFS fs.FS = fs.Local{}
 	if runtime.GOOS == "windows" && opts.UseFsSnapshot {
 		if err = fs.HasSufficientPrivilegesForVSS(); err != nil {
@@ -589,15 +591,29 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 				return err
 			}
 		}
-		targetFS = &fs.Reader{
-			ModTime:    timeStamp,
-			Name:       filename,
-			Mode:       0644,
-			ReadCloser: source,
+		targetFS, err = fs.NewReader(filename, source, fs.ReaderOptions{
+			ModTime: timeStamp,
+			Mode:    0644,
+		})
+		if err != nil {
+			return fmt.Errorf("failed to backup from stdin: %w", err)
 		}
 		targets = []string{filename}
 	}
 
+	if backupFSTestHook != nil {
+		targetFS = backupFSTestHook(targetFS)
+	}
+
+	// rejectFuncs collect functions that can reject items from the backup based on path and file info
+	rejectFuncs, err := collectRejectFuncs(opts, targets, targetFS)
+	if err != nil {
+		return err
+	}
+
+	selectByNameFilter := archiver.CombineRejectByNames(rejectByNameFuncs)
+	selectFilter := archiver.CombineRejects(rejectFuncs)
+
 	wg, wgCtx := errgroup.WithContext(ctx)
 	cancelCtx, cancel := context.WithCancel(wgCtx)
 	defer cancel()

cmd/restic/cmd_backup_integration_test.go

@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"testing"
+	"time"
 
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/restic"
@@ -30,7 +31,7 @@ func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts
 
 func testRunBackup(t testing.TB, dir string, target []string, opts BackupOptions, gopts GlobalOptions) {
 	err := testRunBackupAssumeFailure(t, dir, target, opts, gopts)
-	rtest.Assert(t, err == nil, "Error while backing up")
+	rtest.Assert(t, err == nil, "Error while backing up: %v", err)
 }
 
 func TestBackup(t *testing.T) {
@@ -51,14 +52,14 @@ func testBackup(t *testing.T, useFsSnapshot bool) {
 	opts := BackupOptions{UseFsSnapshot: useFsSnapshot}
 
 	// first backup
-	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
 	testListSnapshots(t, env.gopts, 1)
 
 	testRunCheck(t, env.gopts)
 	stat1 := dirStats(env.repo)
 
 	// second backup, implicit incremental
-	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
 	snapshotIDs := testListSnapshots(t, env.gopts, 2)
 
 	stat2 := dirStats(env.repo)
@@ -70,7 +71,7 @@ func testBackup(t *testing.T, useFsSnapshot bool) {
 	testRunCheck(t, env.gopts)
 	// third backup, explicit incremental
 	opts.Parent = snapshotIDs[0].String()
-	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
 	snapshotIDs = testListSnapshots(t, env.gopts, 3)
 
 	stat3 := dirStats(env.repo)
@@ -83,7 +84,7 @@ func testBackup(t *testing.T, useFsSnapshot bool) {
 	for i, snapshotID := range snapshotIDs {
 		restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
 		t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir)
-		testRunRestore(t, env.gopts, restoredir, snapshotID)
+		testRunRestore(t, env.gopts, restoredir, snapshotID.String()+":"+toPathInSnapshot(filepath.Dir(env.testdata)))
 		diff := directoriesContentsDiff(env.testdata, filepath.Join(restoredir, "testdata"))
 		rtest.Assert(t, diff == "", "directories are not equal: %v", diff)
 	}
@@ -91,6 +92,20 @@ func testBackup(t *testing.T, useFsSnapshot bool) {
 	testRunCheck(t, env.gopts)
 }
 
+func toPathInSnapshot(path string) string {
+	// use path as is on most platforms, but convert it on windows
+	if runtime.GOOS == "windows" {
+		// the path generated by the test is always local so take the shortcut
+		vol := filepath.VolumeName(path)
+		if vol[len(vol)-1] != ':' {
+			panic(fmt.Sprintf("unexpected path: %q", path))
+		}
+		path = vol[:len(vol)-1] + string(filepath.Separator) + path[len(vol)+1:]
+		path = filepath.ToSlash(path)
+	}
+	return path
+}
+
 func TestBackupWithRelativePath(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
@@ -111,6 +126,63 @@ func TestBackupWithRelativePath(t *testing.T) {
 	rtest.Assert(t, latestSn.Parent != nil && latestSn.Parent.Equal(firstSnapshotID), "second snapshot selected unexpected parent %v instead of %v", latestSn.Parent, firstSnapshotID)
 }
 
+type vssDeleteOriginalFS struct {
+	fs.FS
+	testdata   string
+	hasRemoved bool
+}
+
+func (f *vssDeleteOriginalFS) Lstat(name string) (*fs.ExtendedFileInfo, error) {
+	if !f.hasRemoved {
+		// call Lstat to trigger snapshot creation
+		_, _ = f.FS.Lstat(name)
+		// nuke testdata
+		var err error
+		for i := 0; i < 3; i++ {
+			// The CI sometimes runs into "The process cannot access the file because it is being used by another process" errors
+			// thus try a few times to remove the data
+			err = os.RemoveAll(f.testdata)
+			if err == nil {
+				break
+			}
+			time.Sleep(10 * time.Millisecond)
+		}
+		if err != nil {
+			return nil, err
+		}
+		f.hasRemoved = true
+	}
+	return f.FS.Lstat(name)
+}
+
+func TestBackupVSS(t *testing.T) {
+	if runtime.GOOS != "windows" || fs.HasSufficientPrivilegesForVSS() != nil {
+		t.Skip("vss fs test can only be run on windows with admin privileges")
+	}
+
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{UseFsSnapshot: true}
+
+	var testFS *vssDeleteOriginalFS
+	backupFSTestHook = func(fs fs.FS) fs.FS {
+		testFS = &vssDeleteOriginalFS{
+			FS:       fs,
+			testdata: env.testdata,
+		}
+		return testFS
+	}
+	defer func() {
+		backupFSTestHook = nil
+	}()
+
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	testListSnapshots(t, env.gopts, 1)
+	rtest.Equals(t, true, testFS.hasRemoved, "testdata was not removed")
+}
+
 func TestBackupParentSelection(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
@@ -293,12 +365,7 @@ func TestBackupExclude(t *testing.T) {
 	for _, filename := range backupExcludeFilenames {
 		fp := filepath.Join(datadir, filename)
 		rtest.OK(t, os.MkdirAll(filepath.Dir(fp), 0755))
-
-		f, err := os.Create(fp)
-		rtest.OK(t, err)
-
-		fmt.Fprint(f, filename)
-		rtest.OK(t, f.Close())
+		rtest.OK(t, os.WriteFile(fp, []byte(filename), 0o666))
 	}
 
 	snapshots := make(map[string]struct{})
@@ -499,7 +566,7 @@ func TestHardLink(t *testing.T) {
 	for i, snapshotID := range snapshotIDs {
 		restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
 		t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir)
-		testRunRestore(t, env.gopts, restoredir, snapshotID)
+		testRunRestore(t, env.gopts, restoredir, snapshotID.String())
 		diff := directoriesContentsDiff(env.testdata, filepath.Join(restoredir, "testdata"))
 		rtest.Assert(t, diff == "", "directories are not equal %v", diff)
 
@@ -565,12 +632,15 @@ func TestStdinFromCommand(t *testing.T) {
 
 	testSetupBackupData(t, env)
 	opts := BackupOptions{
-		StdinCommand:  true,
-		StdinFilename: "stdin",
+		StdinCommand: true,
+		// test that subdirectories are handled correctly
+		StdinFilename: "stdin/subdir/file",
 	}
 
 	testRunBackup(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; print('something'); sys.exit(0)"}, opts, env.gopts)
-	testListSnapshots(t, env.gopts, 1)
+	snapshots := testListSnapshots(t, env.gopts, 1)
+	files := testRunLs(t, env.gopts, snapshots[0].String())
+	rtest.Assert(t, includes(files, "/stdin/subdir/file"), "file %q missing from snapshot, got %v", "stdin/subdir/file", files)
 
 	testRunCheck(t, env.gopts)
 }

cmd/restic/cmd_backup_test.go

@@ -39,21 +39,24 @@ func TestCollectTargets(t *testing.T) {
 	f1, err := os.Create(filepath.Join(dir, "fromfile"))
 	rtest.OK(t, err)
 	// Empty lines should be ignored. A line starting with '#' is a comment.
-	fmt.Fprintf(f1, "\n%s*\n # here's a comment\n", f1.Name())
+	_, err = fmt.Fprintf(f1, "\n%s*\n # here's a comment\n", f1.Name())
+	rtest.OK(t, err)
 	rtest.OK(t, f1.Close())
 
 	f2, err := os.Create(filepath.Join(dir, "fromfile-verbatim"))
 	rtest.OK(t, err)
 	for _, filename := range []string{fooSpace, barStar} {
 		// Empty lines should be ignored. CR+LF is allowed.
-		fmt.Fprintf(f2, "%s\r\n\n", filepath.Join(dir, filename))
+		_, err = fmt.Fprintf(f2, "%s\r\n\n", filepath.Join(dir, filename))
+		rtest.OK(t, err)
 	}
 	rtest.OK(t, f2.Close())
 
 	f3, err := os.Create(filepath.Join(dir, "fromfile-raw"))
 	rtest.OK(t, err)
 	for _, filename := range []string{"baz", "quux"} {
-		fmt.Fprintf(f3, "%s\x00", filepath.Join(dir, filename))
+		_, err = fmt.Fprintf(f3, "%s\x00", filepath.Join(dir, filename))
+		rtest.OK(t, err)
 	}
 	rtest.OK(t, err)
 	rtest.OK(t, f3.Close())

cmd/restic/cmd_cache.go

@@ -10,16 +10,19 @@ import (
 
 	"github.com/restic/restic/internal/backend/cache"
 	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/ui"
 	"github.com/restic/restic/internal/ui/table"
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 )
 
-var cmdCache = &cobra.Command{
-	Use:   "cache",
-	Short: "Operate on local cache directories",
-	Long: `
+func newCacheCommand() *cobra.Command {
+	var opts CacheOptions
+
+	cmd := &cobra.Command{
+		Use:   "cache",
+		Short: "Operate on local cache directories",
+		Long: `
 The "cache" command allows listing and cleaning local cache directories.
 
 EXIT STATUS
@@ -28,11 +31,15 @@ EXIT STATUS
 Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 `,
-	GroupID:           cmdGroupDefault,
-	DisableAutoGenTag: true,
-	RunE: func(_ *cobra.Command, args []string) error {
-		return runCache(cacheOptions, globalOptions, args)
-	},
+		GroupID:           cmdGroupDefault,
+		DisableAutoGenTag: true,
+		RunE: func(_ *cobra.Command, args []string) error {
+			return runCache(opts, globalOptions, args)
+		},
+	}
+
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 // CacheOptions bundles all options for the snapshots command.
@@ -42,15 +49,10 @@ type CacheOptions struct {
 	NoSize  bool
 }
 
-var cacheOptions CacheOptions
-
-func init() {
-	cmdRoot.AddCommand(cmdCache)
-
-	f := cmdCache.Flags()
-	f.BoolVar(&cacheOptions.Cleanup, "cleanup", false, "remove old cache directories")
-	f.UintVar(&cacheOptions.MaxAge, "max-age", 30, "max age in `days` for cache directories to be considered old")
-	f.BoolVar(&cacheOptions.NoSize, "no-size", false, "do not output the size of the cache directories")
+func (opts *CacheOptions) AddFlags(f *pflag.FlagSet) {
+	f.BoolVar(&opts.Cleanup, "cleanup", false, "remove old cache directories")
+	f.UintVar(&opts.MaxAge, "max-age", 30, "max age in `days` for cache directories to be considered old")
+	f.BoolVar(&opts.NoSize, "no-size", false, "do not output the size of the cache directories")
 }
 
 func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
@@ -89,7 +91,7 @@ func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
 
 		for _, item := range oldDirs {
 			dir := filepath.Join(cachedir, item.Name())
-			err = fs.RemoveAll(dir)
+			err = os.RemoveAll(dir)
 			if err != nil {
 				Warnf("unable to remove %v: %v\n", dir, err)
 			}

cmd/restic/cmd_cat.go

@@ -14,10 +14,11 @@ import (
 
 var catAllowedCmds = []string{"config", "index", "snapshot", "key", "masterkey", "lock", "pack", "blob", "tree"}
 
-var cmdCat = &cobra.Command{
-	Use:   "cat [flags] [masterkey|config|pack ID|blob ID|snapshot ID|index ID|key ID|lock ID|tree snapshot:subfolder]",
-	Short: "Print internal objects to stdout",
-	Long: `
+func newCatCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "cat [flags] [masterkey|config|pack ID|blob ID|snapshot ID|index ID|key ID|lock ID|tree snapshot:subfolder]",
+		Short: "Print internal objects to stdout",
+		Long: `
 The "cat" command is used to print internal objects to stdout.
 
 EXIT STATUS
@@ -29,16 +30,14 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	GroupID:           cmdGroupDefault,
-	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return runCat(cmd.Context(), globalOptions, args)
-	},
-	ValidArgs: catAllowedCmds,
-}
-
-func init() {
-	cmdRoot.AddCommand(cmdCat)
+		GroupID:           cmdGroupDefault,
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runCat(cmd.Context(), globalOptions, args)
+		},
+		ValidArgs: catAllowedCmds,
+	}
+	return cmd
 }
 
 func validateCatArgs(args []string) error {

cmd/restic/cmd_check.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"math/rand"
 	"os"
 	"strconv"
@@ -10,11 +11,11 @@ import (
 	"time"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 
 	"github.com/restic/restic/internal/backend/cache"
 	"github.com/restic/restic/internal/checker"
 	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/ui"
@@ -22,10 +23,12 @@ import (
 	"github.com/restic/restic/internal/ui/termstatus"
 )
 
-var cmdCheck = &cobra.Command{
-	Use:   "check [flags]",
-	Short: "Check the repository for errors",
-	Long: `
+func newCheckCommand() *cobra.Command {
+	var opts CheckOptions
+	cmd := &cobra.Command{
+		Use:   "check [flags]",
+		Short: "Check the repository for errors",
+		Long: `
 The "check" command tests the repository for errors and reports any errors it
 finds. It can also be used to read all data and therefore simulate a restore.
 
@@ -41,16 +44,27 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	GroupID:           cmdGroupDefault,
-	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		term, cancel := setupTermstatus()
-		defer cancel()
-		return runCheck(cmd.Context(), checkOptions, globalOptions, args, term)
-	},
-	PreRunE: func(_ *cobra.Command, _ []string) error {
-		return checkFlags(checkOptions)
-	},
+		GroupID:           cmdGroupDefault,
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			term, cancel := setupTermstatus()
+			defer cancel()
+			summary, err := runCheck(cmd.Context(), opts, globalOptions, args, term)
+			if globalOptions.JSON {
+				if err != nil && summary.NumErrors == 0 {
+					summary.NumErrors = 1
+				}
+				term.Print(ui.ToJSONString(summary))
+			}
+			return err
+		},
+		PreRunE: func(_ *cobra.Command, _ []string) error {
+			return checkFlags(opts)
+		},
+	}
+
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 // CheckOptions bundles all options for the 'check' command.
@@ -61,14 +75,9 @@ type CheckOptions struct {
 	WithCache      bool
 }
 
-var checkOptions CheckOptions
-
-func init() {
-	cmdRoot.AddCommand(cmdCheck)
-
-	f := cmdCheck.Flags()
-	f.BoolVar(&checkOptions.ReadData, "read-data", false, "read all data blobs")
-	f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read a `subset` of data packs, specified as 'n/t' for specific part, or either 'x%' or 'x.y%' or a size in bytes with suffixes k/K, m/M, g/G, t/T for a random subset")
+func (opts *CheckOptions) AddFlags(f *pflag.FlagSet) {
+	f.BoolVar(&opts.ReadData, "read-data", false, "read all data blobs")
+	f.StringVar(&opts.ReadDataSubset, "read-data-subset", "", "read a `subset` of data packs, specified as 'n/t' for specific part, or either 'x%' or 'x.y%' or a size in bytes with suffixes k/K, m/M, g/G, t/T for a random subset")
 	var ignored bool
 	f.BoolVar(&ignored, "check-unused", false, "find unused blobs")
 	err := f.MarkDeprecated("check-unused", "`--check-unused` is deprecated and will be ignored")
@@ -76,7 +85,7 @@ func init() {
 		// MarkDeprecated only returns an error when the flag is not found
 		panic(err)
 	}
-	f.BoolVar(&checkOptions.WithCache, "with-cache", false, "use existing cache, only read uncached data from repository")
+	f.BoolVar(&opts.WithCache, "with-cache", false, "use existing cache, only read uncached data from repository")
 }
 
 func checkFlags(opts CheckOptions) error {
@@ -202,7 +211,7 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions, printer progress
 	printer.P("using temporary cache in %v\n", tempdir)
 
 	cleanup = func() {
-		err := fs.RemoveAll(tempdir)
+		err := os.RemoveAll(tempdir)
 		if err != nil {
 			printer.E("error removing temporary cache directory: %v\n", err)
 		}
@@ -211,12 +220,18 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions, printer progress
 	return cleanup
 }
 
-func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args []string, term *termstatus.Terminal) error {
+func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args []string, term *termstatus.Terminal) (checkSummary, error) {
+	summary := checkSummary{MessageType: "summary"}
 	if len(args) != 0 {
-		return errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags")
+		return summary, errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags")
 	}
 
-	printer := newTerminalProgressPrinter(gopts.verbosity, term)
+	var printer progress.Printer
+	if !gopts.JSON {
+		printer = newTerminalProgressPrinter(gopts.verbosity, term)
+	} else {
+		printer = newJSONErrorPrinter(term)
+	}
 
 	cleanup := prepareCheckCache(opts, &gopts, printer)
 	defer cleanup()
@@ -226,53 +241,43 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	}
 	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
-		return err
+		return summary, err
 	}
 	defer unlock()
 
 	chkr := checker.New(repo, opts.CheckUnused)
 	err = chkr.LoadSnapshots(ctx)
 	if err != nil {
-		return err
+		return summary, err
 	}
 
 	printer.P("load indexes\n")
 	bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
 	hints, errs := chkr.LoadIndex(ctx, bar)
 	if ctx.Err() != nil {
-		return ctx.Err()
+		return summary, ctx.Err()
 	}
 
 	errorsFound := false
-	suggestIndexRebuild := false
-	suggestLegacyIndexRebuild := false
-	mixedFound := false
 	for _, hint := range hints {
 		switch hint.(type) {
 		case *checker.ErrDuplicatePacks:
-			term.Print(hint.Error())
-			suggestIndexRebuild = true
-		case *checker.ErrOldIndexFormat:
-			printer.E("error: %v\n", hint)
-			suggestLegacyIndexRebuild = true
-			errorsFound = true
+			printer.S("%s", hint.Error())
+			summary.HintRepairIndex = true
 		case *checker.ErrMixedPack:
-			term.Print(hint.Error())
-			mixedFound = true
+			printer.S("%s", hint.Error())
+			summary.HintPrune = true
 		default:
 			printer.E("error: %v\n", hint)
 			errorsFound = true
 		}
 	}
 
-	if suggestIndexRebuild {
-		term.Print("Duplicate packs are non-critical, you can run `restic repair index' to correct this.\n")
+	if summary.HintRepairIndex {
+		printer.S("Duplicate packs are non-critical, you can run `restic repair index' to correct this.\n")
 	}
-	if suggestLegacyIndexRebuild {
-		printer.E("error: Found indexes using the legacy format, you must run `restic repair index' to correct this.\n")
-	}
-	if mixedFound {
-		term.Print("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")
+	if summary.HintPrune {
+		printer.S("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")
 	}
 
 	if len(errs) > 0 {
@@ -280,8 +285,10 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 			printer.E("error: %v\n", err)
 		}
 
+		summary.NumErrors += len(errs)
+		summary.HintRepairIndex = true
 		printer.E("\nThe repository index is damaged and must be repaired. You must run `restic repair index' to correct this.\n\n")
-		return errors.Fatal("repository contains errors")
+		return summary, errors.Fatal("repository contains errors")
 	}
 
 	orphanedPacks := 0
@@ -302,23 +309,24 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 					salvagePacks.Insert(packErr.ID)
 				}
 				errorsFound = true
+				summary.NumErrors++
 				printer.E("%v\n", err)
 			}
-		} else if err == checker.ErrLegacyLayout {
-			errorsFound = true
-			printer.E("error: repository still uses the S3 legacy layout\nYou must run `restic migrate s3legacy` to correct this.\n")
 		} else {
 			errorsFound = true
 			printer.E("%v\n", err)
 		}
 	}
 
-	if orphanedPacks > 0 && !errorsFound {
-		// hide notice if repository is damaged
-		printer.P("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks)
+	if orphanedPacks > 0 {
+		summary.HintPrune = true
+		if !errorsFound {
+			// hide notice if repository is damaged
+			printer.P("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks)
+		}
 	}
 	if ctx.Err() != nil {
-		return ctx.Err()
+		return summary, ctx.Err()
 	}
 
 	printer.P("check snapshots, trees and blobs\n")
@@ -328,7 +336,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		bar := newTerminalProgressMax(!gopts.Quiet, 0, "snapshots", term)
+		bar := printer.NewCounter("snapshots")
 		defer bar.Done()
 		chkr.Structure(ctx, bar, errChan)
 	}()
@@ -338,9 +346,11 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 		if e, ok := err.(*checker.TreeError); ok {
 			printer.E("error for tree %v:\n", e.ID.Str())
 			for _, treeErr := range e.Errors {
+				summary.NumErrors++
 				printer.E("  %v\n", treeErr)
 			}
 		} else {
+			summary.NumErrors++
 			printer.E("error: %v\n", err)
 		}
 	}
@@ -350,13 +360,13 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	// deadlocking in the case of errors.
 	wg.Wait()
 	if ctx.Err() != nil {
-		return ctx.Err()
+		return summary, ctx.Err()
 	}
 
 	if opts.CheckUnused {
 		unused, err := chkr.UnusedBlobs(ctx)
 		if err != nil {
-			return err
+			return summary, err
 		}
 		for _, id := range unused {
 			printer.P("unused blob %v\n", id)
@@ -365,15 +375,15 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	}
 
 	doReadData := func(packs map[restic.ID]int64) {
-		packCount := uint64(len(packs))
-
-		p := newTerminalProgressMax(!gopts.Quiet, packCount, "packs", term)
+		p := printer.NewCounter("packs")
+		p.SetMax(uint64(len(packs)))
 		errChan := make(chan error)
 
 		go chkr.ReadPacks(ctx, packs, p, errChan)
 
 		for err := range errChan {
 			errorsFound = true
+			summary.NumErrors++
 			printer.E("%v\n", err)
 			if err, ok := err.(*repository.ErrPackData); ok {
 				salvagePacks.Insert(err.PackID)
@@ -408,44 +418,43 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 				repoSize += size
 			}
 			if repoSize == 0 {
-				return errors.Fatal("Cannot read from a repository having size 0")
+				return summary, errors.Fatal("Cannot read from a repository having size 0")
 			}
 			subsetSize, _ := ui.ParseBytes(opts.ReadDataSubset)
 			if subsetSize > repoSize {
 				subsetSize = repoSize
 			}
 			packs = selectRandomPacksByFileSize(chkr.GetPacks(), subsetSize, repoSize)
-			printer.P("read %d bytes of data packs\n", subsetSize)
+			percentage := float64(subsetSize) / float64(repoSize) * 100.0
+			printer.P("read %d bytes (%.1f%%) of data packs\n", subsetSize, percentage)
 		}
 		if packs == nil {
-			return errors.Fatal("internal error: failed to select packs to check")
+			return summary, errors.Fatal("internal error: failed to select packs to check")
 		}
 		doReadData(packs)
 	}
 
 	if len(salvagePacks) > 0 {
 		printer.E("\nThe repository contains damaged pack files. These damaged files must be removed to repair the repository. This can be done using the following commands. Please read the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html first.\n\n")
-		var strIDs []string
 		for id := range salvagePacks {
-			strIDs = append(strIDs, id.String())
+			summary.BrokenPacks = append(summary.BrokenPacks, id.String())
 		}
-		printer.E("restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIDs, " "))
+		printer.E("restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(summary.BrokenPacks, " "))
 		printer.E("Damaged pack files can be caused by backend problems, hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n")
 	}
 
 	if ctx.Err() != nil {
-		return ctx.Err()
+		return summary, ctx.Err()
 	}
 
 	if errorsFound {
 		if len(salvagePacks) == 0 {
 			printer.E("\nThe repository is damaged and must be repaired. Please follow the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html .\n\n")
 		}
-		return errors.Fatal("repository contains errors")
+		return summary, errors.Fatal("repository contains errors")
 	}
 	printer.P("no errors were found\n")
-
-	return nil
+	return summary, nil
 }
 
 // selectPacksByBucket selects subsets of packs by ranges of buckets.
@@ -491,3 +500,42 @@ func selectRandomPacksByFileSize(allPacks map[restic.ID]int64, subsetSize int64,
 	packs := selectRandomPacksByPercentage(allPacks, subsetPercentage)
 	return packs
 }
+
+type checkSummary struct {
+	MessageType     string   `json:"message_type"` // "summary"
+	NumErrors       int      `json:"num_errors"`
+	BrokenPacks     []string `json:"broken_packs"`         // run "restic repair packs ID..." and "restic repair snapshots --forget" to remove damaged files
+	HintRepairIndex bool     `json:"suggest_repair_index"` // run "restic repair index"
+	HintPrune       bool     `json:"suggest_prune"`        // run "restic prune"
+}
+
+type checkError struct {
+	MessageType string `json:"message_type"` // "error"
+	Message     string `json:"message"`
+}
+
+type jsonErrorPrinter struct {
+	term ui.Terminal
+}
+
+func newJSONErrorPrinter(term ui.Terminal) *jsonErrorPrinter {
+	return &jsonErrorPrinter{
+		term: term,
+	}
+}
+
+func (*jsonErrorPrinter) NewCounter(_ string) *progress.Counter {
+	return nil
+}
+
+func (p *jsonErrorPrinter) E(msg string, args ...interface{}) {
+	status := checkError{
+		MessageType: "error",
+		Message:     fmt.Sprintf(msg, args...),
+	}
+	p.term.Error(ui.ToJSONString(status))
+}
+func (*jsonErrorPrinter) S(_ string, _ ...interface{})  {}
+func (*jsonErrorPrinter) P(_ string, _ ...interface{})  {}
+func (*jsonErrorPrinter) V(_ string, _ ...interface{})  {}
+func (*jsonErrorPrinter) VV(_ string, _ ...interface{}) {}

cmd/restic/cmd_check_integration_test.go

@@ -32,7 +32,8 @@ func testRunCheckOutput(gopts GlobalOptions, checkUnused bool) (string, error) {
 			ReadData:    true,
 			CheckUnused: checkUnused,
 		}
-		return runCheck(context.TODO(), opts, gopts, nil, term)
+		_, err := runCheck(context.TODO(), opts, gopts, nil, term)
+		return err
 	})
 	return buf.String(), err
 }

cmd/restic/cmd_copy.go

@@ -11,12 +11,15 @@ import (
 	"golang.org/x/sync/errgroup"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 )
 
-var cmdCopy = &cobra.Command{
-	Use:   "copy [flags] [snapshotID ...]",
-	Short: "Copy snapshots from one repository to another",
-	Long: `
+func newCopyCommand() *cobra.Command {
+	var opts CopyOptions
+	cmd := &cobra.Command{
+		Use:   "copy [flags] [snapshotID ...]",
+		Short: "Copy snapshots from one repository to another",
+		Long: `
 The "copy" command copies one or more snapshots from one repository to another.
 
 NOTE: This process will have to both download (read) and upload (write) the
@@ -40,11 +43,15 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	GroupID:           cmdGroupDefault,
-	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return runCopy(cmd.Context(), copyOptions, globalOptions, args)
-	},
+		GroupID:           cmdGroupDefault,
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runCopy(cmd.Context(), opts, globalOptions, args)
+		},
+	}
+
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 // CopyOptions bundles all options for the copy command.
@@ -53,14 +60,9 @@ type CopyOptions struct {
 	restic.SnapshotFilter
 }
 
-var copyOptions CopyOptions
-
-func init() {
-	cmdRoot.AddCommand(cmdCopy)
-
-	f := cmdCopy.Flags()
-	initSecondaryRepoOptions(f, &copyOptions.secondaryRepoOptions, "destination", "to copy snapshots from")
-	initMultiSnapshotFilter(f, &copyOptions.SnapshotFilter, true)
+func (opts *CopyOptions) AddFlags(f *pflag.FlagSet) {
+	opts.secondaryRepoOptions.AddFlags(f, "destination", "to copy snapshots from")
+	initMultiSnapshotFilter(f, &opts.SnapshotFilter, true)
 }
 
 func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []string) error {
@@ -237,7 +239,15 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep
 	}
 
 	bar := newProgressMax(!quiet, uint64(len(packList)), "packs copied")
-	_, err = repository.Repack(ctx, srcRepo, dstRepo, packList, copyBlobs, bar)
+	_, err = repository.Repack(
+		ctx,
+		srcRepo,
+		dstRepo,
+		packList,
+		copyBlobs,
+		bar,
+		func(msg string, args ...interface{}) { fmt.Printf(msg+"\n", args...) },
+	)
 	bar.Done()
 	if err != nil {
 		return errors.Fatal(err.Error())

cmd/restic/cmd_copy_integration_test.go

@@ -62,11 +62,11 @@ func TestCopy(t *testing.T) {
 	for i, snapshotID := range snapshotIDs {
 		restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
 		origRestores[restoredir] = struct{}{}
-		testRunRestore(t, env.gopts, restoredir, snapshotID)
+		testRunRestore(t, env.gopts, restoredir, snapshotID.String())
 	}
 	for i, snapshotID := range copiedSnapshotIDs {
 		restoredir := filepath.Join(env2.base, fmt.Sprintf("restore%d", i))
-		testRunRestore(t, env2.gopts, restoredir, snapshotID)
+		testRunRestore(t, env2.gopts, restoredir, snapshotID.String())
 		foundMatch := false
 		for cmpdir := range origRestores {
 			diff := directoriesContentsDiff(restoredir, cmpdir)

cmd/restic/cmd_debug.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/klauspost/compress/zstd"
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 	"golang.org/x/sync/errgroup"
 
 	"github.com/restic/restic/internal/crypto"
@@ -28,17 +29,29 @@ import (
 	"github.com/restic/restic/internal/restic"
 )
 
-var cmdDebug = &cobra.Command{
-	Use:               "debug",
-	Short:             "Debug commands",
-	GroupID:           cmdGroupDefault,
-	DisableAutoGenTag: true,
+func registerDebugCommand(cmd *cobra.Command) {
+	cmd.AddCommand(
+		newDebugCommand(),
+	)
 }
 
-var cmdDebugDump = &cobra.Command{
-	Use:   "dump [indexes|snapshots|all|packs]",
-	Short: "Dump data structures",
-	Long: `
+func newDebugCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:               "debug",
+		Short:             "Debug commands",
+		GroupID:           cmdGroupDefault,
+		DisableAutoGenTag: true,
+	}
+	cmd.AddCommand(newDebugDumpCommand())
+	cmd.AddCommand(newDebugExamineCommand())
+	return cmd
+}
+
+func newDebugDumpCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "dump [indexes|snapshots|all|packs]",
+		Short: "Dump data structures",
+		Long: `
 The "dump" command dumps data structures from the repository as JSON objects. It
 is used for debugging purposes only.
 
@@ -51,10 +64,28 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return runDebugDump(cmd.Context(), globalOptions, args)
-	},
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runDebugDump(cmd.Context(), globalOptions, args)
+		},
+	}
+	return cmd
+}
+
+func newDebugExamineCommand() *cobra.Command {
+	var opts DebugExamineOptions
+
+	cmd := &cobra.Command{
+		Use:               "examine pack-ID...",
+		Short:             "Examine a pack file",
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runDebugExamine(cmd.Context(), globalOptions, opts, args)
+		},
+	}
+
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 type DebugExamineOptions struct {
@@ -64,16 +95,11 @@ type DebugExamineOptions struct {
 	ReuploadBlobs bool
 }
 
-var debugExamineOpts DebugExamineOptions
-
-func init() {
-	cmdRoot.AddCommand(cmdDebug)
-	cmdDebug.AddCommand(cmdDebugDump)
-	cmdDebug.AddCommand(cmdDebugExamine)
-	cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.ExtractPack, "extract-pack", false, "write blobs to the current directory")
-	cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.ReuploadBlobs, "reupload-blobs", false, "reupload blobs to the repository")
-	cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.TryRepair, "try-repair", false, "try to repair broken blobs with single bit flips")
-	cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.RepairByte, "repair-byte", false, "try to repair broken blobs by trying bytes")
+func (opts *DebugExamineOptions) AddFlags(f *pflag.FlagSet) {
+	f.BoolVar(&opts.ExtractPack, "extract-pack", false, "write blobs to the current directory")
+	f.BoolVar(&opts.ReuploadBlobs, "reupload-blobs", false, "reupload blobs to the repository")
+	f.BoolVar(&opts.TryRepair, "try-repair", false, "try to repair broken blobs with single bit flips")
+	f.BoolVar(&opts.RepairByte, "repair-byte", false, "try to repair broken blobs by trying bytes")
 }
 
 func prettyPrintJSON(wr io.Writer, item interface{}) error {
@@ -92,7 +118,9 @@ func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io
 			return err
 		}
 
-		fmt.Fprintf(wr, "snapshot_id: %v\n", id)
+		if _, err := fmt.Fprintf(wr, "snapshot_id: %v\n", id); err != nil {
+			return err
+		}
 
 		return prettyPrintJSON(wr, snapshot)
 	})
@@ -143,7 +171,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
 }
 
 func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer) error {
-	return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
+	return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, err error) error {
 		Printf("index_id: %v\n", id)
 		if err != nil {
 			return err
@@ -192,16 +220,7 @@ func runDebugDump(ctx context.Context, gopts GlobalOptions, args []string) error
 	}
 }
 
-var cmdDebugExamine = &cobra.Command{
-	Use:               "examine pack-ID...",
-	Short:             "Examine a pack file",
-	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return runDebugExamine(cmd.Context(), globalOptions, debugExamineOpts, args)
-	},
-}
-
-func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, bytewise bool) []byte {
+func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool) []byte {
 	if bytewise {
 		Printf("        trying to repair blob by finding a broken byte\n")
 	} else {
@@ -300,7 +319,7 @@ func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, by
 	return fixed
 }
 
-func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte {
+func decryptUnsigned(k *crypto.Key, buf []byte) []byte {
 	// strip signature at the end
 	l := len(buf)
 	nonce, ct := buf[:16], buf[16:l-16]
@@ -351,13 +370,13 @@ func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Reposi
 			if err != nil {
 				Warnf("error decrypting blob: %v\n", err)
 				if opts.TryRepair || opts.RepairByte {
-					plaintext = tryRepairWithBitflip(ctx, key, buf, opts.RepairByte)
+					plaintext = tryRepairWithBitflip(key, buf, opts.RepairByte)
 				}
 				if plaintext != nil {
 					outputPrefix = "repaired "
 					filePrefix = "repaired-"
 				} else {
-					plaintext = decryptUnsigned(ctx, key, buf)
+					plaintext = decryptUnsigned(key, buf)
 					err = storePlainBlob(blob.ID, "damaged-", plaintext)
 					if err != nil {
 						return err

cmd/restic/cmd_debug_disabled.go

@@ -0,0 +1,9 @@
+//go:build !debug
+
+package main
+
+import "github.com/spf13/cobra"
+
+func registerDebugCommand(_ *cobra.Command) {
+	// No commands to register in non-debug mode
+}

cmd/restic/cmd_diff.go

@@ -12,12 +12,16 @@ import (
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/ui"
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 )
 
-var cmdDiff = &cobra.Command{
-	Use:   "diff [flags] snapshotID snapshotID",
-	Short: "Show differences between two snapshots",
-	Long: `
+func newDiffCommand() *cobra.Command {
+	var opts DiffOptions
+
+	cmd := &cobra.Command{
+		Use:   "diff [flags] snapshotID snapshotID",
+		Short: "Show differences between two snapshots",
+		Long: `
 The "diff" command shows differences from the first to the second snapshot. The
 first characters in each line display what has happened to a particular file or
 directory:
@@ -45,11 +49,15 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	GroupID:           cmdGroupDefault,
-	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return runDiff(cmd.Context(), diffOptions, globalOptions, args)
-	},
+		GroupID:           cmdGroupDefault,
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runDiff(cmd.Context(), opts, globalOptions, args)
+		},
+	}
+
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 // DiffOptions collects all options for the diff command.
@@ -57,13 +65,8 @@ type DiffOptions struct {
 	ShowMetadata bool
 }
 
-var diffOptions DiffOptions
-
-func init() {
-	cmdRoot.AddCommand(cmdDiff)
-
-	f := cmdDiff.Flags()
-	f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata")
+func (opts *DiffOptions) AddFlags(f *pflag.FlagSet) {
+	f.BoolVar(&opts.ShowMetadata, "metadata", false, "print changes in metadata")
 }
 
 func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.LoaderUnpacked, desc string) (*restic.Snapshot, string, error) {
@@ -108,9 +111,9 @@ func (s *DiffStat) Add(node *restic.Node) {
 	}
 
 	switch node.Type {
-	case "file":
+	case restic.NodeTypeFile:
 		s.Files++
-	case "dir":
+	case restic.NodeTypeDir:
 		s.Dirs++
 	default:
 		s.Others++
@@ -124,7 +127,7 @@ func addBlobs(bs restic.BlobSet, node *restic.Node) {
 	}
 
 	switch node.Type {
-	case "file":
+	case restic.NodeTypeFile:
 		for _, blob := range node.Content {
 			h := restic.BlobHandle{
 				ID:   blob,
@@ -132,7 +135,7 @@ func addBlobs(bs restic.BlobSet, node *restic.Node) {
 			}
 			bs.Insert(h)
 		}
-	case "dir":
+	case restic.NodeTypeDir:
 		h := restic.BlobHandle{
 			ID:   *node.Subtree,
 			Type: restic.TreeBlob,
@@ -184,14 +187,14 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b
 		}
 
 		name := path.Join(prefix, node.Name)
-		if node.Type == "dir" {
+		if node.Type == restic.NodeTypeDir {
 			name += "/"
 		}
 		c.printChange(NewChange(name, mode))
 		stats.Add(node)
 		addBlobs(blobs, node)
 
-		if node.Type == "dir" {
+		if node.Type == restic.NodeTypeDir {
 			err := c.printDir(ctx, mode, stats, blobs, name, *node.Subtree)
 			if err != nil && err != context.Canceled {
 				Warnf("error: %v\n", err)
@@ -216,7 +219,7 @@ func (c *Comparer) collectDir(ctx context.Context, blobs restic.BlobSet, id rest
 
 		addBlobs(blobs, node)
 
-		if node.Type == "dir" {
+		if node.Type == restic.NodeTypeDir {
 			err := c.collectDir(ctx, blobs, *node.Subtree)
 			if err != nil && err != context.Canceled {
 				Warnf("error: %v\n", err)
@@ -284,12 +287,12 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
 				mod += "T"
 			}
 
-			if node2.Type == "dir" {
+			if node2.Type == restic.NodeTypeDir {
 				name += "/"
 			}
 
-			if node1.Type == "file" &&
-				node2.Type == "file" &&
+			if node1.Type == restic.NodeTypeFile &&
+				node2.Type == restic.NodeTypeFile &&
 				!reflect.DeepEqual(node1.Content, node2.Content) {
 				mod += "M"
 				stats.ChangedFiles++
@@ -311,7 +314,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
 				c.printChange(NewChange(name, mod))
 			}
 
-			if node1.Type == "dir" && node2.Type == "dir" {
+			if node1.Type == restic.NodeTypeDir && node2.Type == restic.NodeTypeDir {
 				var err error
 				if (*node1.Subtree).Equal(*node2.Subtree) {
 					err = c.collectDir(ctx, stats.BlobsCommon, *node1.Subtree)
@@ -324,13 +327,13 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
 			}
 		case t1 && !t2:
 			prefix := path.Join(prefix, name)
-			if node1.Type == "dir" {
+			if node1.Type == restic.NodeTypeDir {
 				prefix += "/"
 			}
 			c.printChange(NewChange(prefix, "-"))
 			stats.Removed.Add(node1)
 
-			if node1.Type == "dir" {
+			if node1.Type == restic.NodeTypeDir {
 				err := c.printDir(ctx, "-", &stats.Removed, stats.BlobsBefore, prefix, *node1.Subtree)
 				if err != nil && err != context.Canceled {
 					Warnf("error: %v\n", err)
@@ -338,13 +341,13 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
 			}
 		case !t1 && t2:
 			prefix := path.Join(prefix, name)
-			if node2.Type == "dir" {
+			if node2.Type == restic.NodeTypeDir {
 				prefix += "/"
 			}
 			c.printChange(NewChange(prefix, "+"))
 			stats.Added.Add(node2)
 
-			if node2.Type == "dir" {
+			if node2.Type == restic.NodeTypeDir {
 				err := c.printDir(ctx, "+", &stats.Added, stats.BlobsAfter, prefix, *node2.Subtree)
 				if err != nil && err != context.Canceled {
 					Warnf("error: %v\n", err)
@@ -462,8 +465,8 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
 		Printf("Others:      %5d new, %5d removed\n", stats.Added.Others, stats.Removed.Others)
 		Printf("Data Blobs:  %5d new, %5d removed\n", stats.Added.DataBlobs, stats.Removed.DataBlobs)
 		Printf("Tree Blobs:  %5d new, %5d removed\n", stats.Added.TreeBlobs, stats.Removed.TreeBlobs)
-		Printf("  Added:   %-5s\n", ui.FormatBytes(uint64(stats.Added.Bytes)))
-		Printf("  Removed: %-5s\n", ui.FormatBytes(uint64(stats.Removed.Bytes)))
+		Printf("  Added:   %-5s\n", ui.FormatBytes(stats.Added.Bytes))
+		Printf("  Removed: %-5s\n", ui.FormatBytes(stats.Removed.Bytes))
 	}
 
 	return nil

cmd/restic/cmd_dump.go

@@ -13,12 +13,15 @@ import (
 	"github.com/restic/restic/internal/restic"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 )
 
-var cmdDump = &cobra.Command{
-	Use:   "dump [flags] snapshotID file",
-	Short: "Print a backed-up file to stdout",
-	Long: `
+func newDumpCommand() *cobra.Command {
+	var opts DumpOptions
+	cmd := &cobra.Command{
+		Use:   "dump [flags] snapshotID file",
+		Short: "Print a backed-up file to stdout",
+		Long: `
 The "dump" command extracts files from a snapshot from the repository. If a
 single file is selected, it prints its contents to stdout. Folders are output
 as a tar (default) or zip file containing the contents of the specified folder.
@@ -40,11 +43,15 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	GroupID:           cmdGroupDefault,
-	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return runDump(cmd.Context(), dumpOptions, globalOptions, args)
-	},
+		GroupID:           cmdGroupDefault,
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runDump(cmd.Context(), opts, globalOptions, args)
+		},
+	}
+
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 // DumpOptions collects all options for the dump command.
@@ -54,15 +61,10 @@ type DumpOptions struct {
 	Target  string
 }
 
-var dumpOptions DumpOptions
-
-func init() {
-	cmdRoot.AddCommand(cmdDump)
-
-	flags := cmdDump.Flags()
-	initSingleSnapshotFilter(flags, &dumpOptions.SnapshotFilter)
-	flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"")
-	flags.StringVarP(&dumpOptions.Target, "target", "t", "", "write the output to target `path`")
+func (opts *DumpOptions) AddFlags(f *pflag.FlagSet) {
+	initSingleSnapshotFilter(f, &opts.SnapshotFilter)
+	f.StringVarP(&opts.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"")
+	f.StringVarP(&opts.Target, "target", "t", "", "write the output to target `path`")
 }
 
 func splitPath(p string) []string {
@@ -95,15 +97,15 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoade
 		// first item it finds and dump that according to the switch case below.
 		if node.Name == pathComponents[0] {
 			switch {
-			case l == 1 && dump.IsFile(node):
+			case l == 1 && node.Type == restic.NodeTypeFile:
 				return d.WriteNode(ctx, node)
-			case l > 1 && dump.IsDir(node):
+			case l > 1 && node.Type == restic.NodeTypeDir:
 				subtree, err := restic.LoadTree(ctx, repo, *node.Subtree)
 				if err != nil {
 					return errors.Wrapf(err, "cannot load subtree for %q", item)
 				}
 				return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d, canWriteArchiveFunc)
-			case dump.IsDir(node):
+			case node.Type == restic.NodeTypeDir:
 				if err := canWriteArchiveFunc(); err != nil {
 					return err
 				}
@@ -114,7 +116,7 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoade
 				return d.DumpTree(ctx, subtree, item)
 			case l > 1:
 				return fmt.Errorf("%q should be a dir, but is a %q", item, node.Type)
-			case !dump.IsFile(node):
+			case node.Type != restic.NodeTypeFile:
 				return fmt.Errorf("%q should be a file, but is a %q", item, node.Type)
 			}
 		}

cmd/restic/cmd_features.go

@@ -10,10 +10,11 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var featuresCmd = &cobra.Command{
-	Use:   "features",
-	Short: "Print list of feature flags",
-	Long: `
+func newFeaturesCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "features",
+		Short: "Print list of feature flags",
+		Long: `
 The "features" command prints a list of supported feature flags.
 
 To pass feature flags to restic, set the RESTIC_FEATURES environment variable
@@ -31,29 +32,28 @@ EXIT STATUS
 Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 `,
-	GroupID:           cmdGroupAdvanced,
-	DisableAutoGenTag: true,
-	RunE: func(_ *cobra.Command, args []string) error {
-		if len(args) != 0 {
-			return errors.Fatal("the feature command expects no arguments")
-		}
+		GroupID:           cmdGroupAdvanced,
+		DisableAutoGenTag: true,
+		RunE: func(_ *cobra.Command, args []string) error {
+			if len(args) != 0 {
+				return errors.Fatal("the feature command expects no arguments")
+			}
 
-		fmt.Printf("All Feature Flags:\n")
-		flags := feature.Flag.List()
+			fmt.Printf("All Feature Flags:\n")
+			flags := feature.Flag.List()
 
-		tab := table.New()
-		tab.AddColumn("Name", "{{ .Name }}")
-		tab.AddColumn("Type", "{{ .Type }}")
-		tab.AddColumn("Default", "{{ .Default }}")
-		tab.AddColumn("Description", "{{ .Description }}")
+			tab := table.New()
+			tab.AddColumn("Name", "{{ .Name }}")
+			tab.AddColumn("Type", "{{ .Type }}")
+			tab.AddColumn("Default", "{{ .Default }}")
+			tab.AddColumn("Description", "{{ .Description }}")
 
-		for _, flag := range flags {
-			tab.AddRow(flag)
-		}
-		return tab.Write(globalOptions.stdout)
-	},
-}
-
-func init() {
-	cmdRoot.AddCommand(featuresCmd)
+			for _, flag := range flags {
+				tab.AddRow(flag)
+			}
+			return tab.Write(globalOptions.stdout)
+		},
+	}
+
+	return cmd
 }

cmd/restic/cmd_find.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
@@ -16,14 +17,19 @@ import (
 	"github.com/restic/restic/internal/walker"
 )
 
-var cmdFind = &cobra.Command{
-	Use:   "find [flags] PATTERN...",
-	Short: "Find a file, a directory or restic IDs",
-	Long: `
+func newFindCommand() *cobra.Command {
+	var opts FindOptions
+
+	cmd := &cobra.Command{
+		Use:   "find [flags] PATTERN...",
+		Short: "Find a file, a directory or restic IDs",
+		Long: `
 The "find" command searches for files or directories in snapshots stored in the
 repo.
-It can also be used to search for restic blobs or trees for troubleshooting.`,
-	Example: `restic find config.json
+It can also be used to search for restic blobs or trees for troubleshooting.
+The default sort option for the snapshots is youngest to oldest. To sort the
+output from oldest to youngest specify --reverse.`,
+		Example: `restic find config.json
 restic find --json "*.yml" "*.json"
 restic find --json --blob 420f620f b46ebe8a ddd38656
 restic find --show-pack-id --blob 420f620f
@@ -39,11 +45,15 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	GroupID:           cmdGroupDefault,
-	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return runFind(cmd.Context(), findOptions, globalOptions, args)
-	},
+		GroupID:           cmdGroupDefault,
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runFind(cmd.Context(), opts, globalOptions, args)
+		},
+	}
+
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 // FindOptions bundles all options for the find command.
@@ -56,27 +66,24 @@ type FindOptions struct {
 	CaseInsensitive    bool
 	ListLong           bool
 	HumanReadable      bool
+	Reverse            bool
 	restic.SnapshotFilter
 }
 
-var findOptions FindOptions
+func (opts *FindOptions) AddFlags(f *pflag.FlagSet) {
+	f.StringVarP(&opts.Oldest, "oldest", "O", "", "oldest modification date/time")
+	f.StringVarP(&opts.Newest, "newest", "N", "", "newest modification date/time")
+	f.StringArrayVarP(&opts.Snapshots, "snapshot", "s", nil, "snapshot `id` to search in (can be given multiple times)")
+	f.BoolVar(&opts.BlobID, "blob", false, "pattern is a blob-ID")
+	f.BoolVar(&opts.TreeID, "tree", false, "pattern is a tree-ID")
+	f.BoolVar(&opts.PackID, "pack", false, "pattern is a pack-ID")
+	f.BoolVar(&opts.ShowPackID, "show-pack-id", false, "display the pack-ID the blobs belong to (with --blob or --tree)")
+	f.BoolVarP(&opts.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern")
+	f.BoolVarP(&opts.Reverse, "reverse", "R", false, "reverse sort order oldest to newest")
+	f.BoolVarP(&opts.ListLong, "long", "l", false, "use a long listing format showing size and mode")
+	f.BoolVar(&opts.HumanReadable, "human-readable", false, "print sizes in human readable format")
 
-func init() {
-	cmdRoot.AddCommand(cmdFind)
-
-	f := cmdFind.Flags()
-	f.StringVarP(&findOptions.Oldest, "oldest", "O", "", "oldest modification date/time")
-	f.StringVarP(&findOptions.Newest, "newest", "N", "", "newest modification date/time")
-	f.StringArrayVarP(&findOptions.Snapshots, "snapshot", "s", nil, "snapshot `id` to search in (can be given multiple times)")
-	f.BoolVar(&findOptions.BlobID, "blob", false, "pattern is a blob-ID")
-	f.BoolVar(&findOptions.TreeID, "tree", false, "pattern is a tree-ID")
-	f.BoolVar(&findOptions.PackID, "pack", false, "pattern is a pack-ID")
-	f.BoolVar(&findOptions.ShowPackID, "show-pack-id", false, "display the pack-ID the blobs belong to (with --blob or --tree)")
-	f.BoolVarP(&findOptions.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern")
-	f.BoolVarP(&findOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
-	f.BoolVar(&findOptions.HumanReadable, "human-readable", false, "print sizes in human readable format")
-
-	initMultiSnapshotFilter(f, &findOptions.SnapshotFilter, true)
+	initMultiSnapshotFilter(f, &opts.SnapshotFilter, true)
 }
 
 type findPattern struct {
@@ -298,7 +305,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 		}
 
 		var errIfNoMatch error
-		if node.Type == "dir" {
+		if node.Type == restic.NodeTypeDir {
 			var childMayMatch bool
 			for _, pat := range f.pat.pattern {
 				mayMatch, err := filter.ChildMatch(pat, normalizedNodepath)
@@ -336,6 +343,26 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 	}})
 }
 
+func (f *Finder) findTree(treeID restic.ID, nodepath string) error {
+	found := false
+	if _, ok := f.treeIDs[treeID.String()]; ok {
+		found = true
+	} else if _, ok := f.treeIDs[treeID.Str()]; ok {
+		found = true
+	}
+	if found {
+		f.out.PrintObject("tree", treeID.String(), nodepath, "", f.out.newsn)
+		f.itemsFound++
+		// Terminate if we have found all trees (and we are not
+		// looking for blobs)
+		if f.itemsFound >= len(f.treeIDs) && f.blobIDs == nil {
+			// Return an error to terminate the Walk
+			return errors.New("OK")
+		}
+	}
+	return nil
+}
+
 func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 	debug.Log("searching IDs in snapshot %s", sn.ID())
 
@@ -354,30 +381,21 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 		}
 
 		if node == nil {
+			if nodepath == "/" {
+				if err := f.findTree(parentTreeID, "/"); err != nil {
+					return err
+				}
+			}
 			return nil
 		}
 
 		if node.Type == "dir" && f.treeIDs != nil {
-			treeID := node.Subtree
-			found := false
-			if _, ok := f.treeIDs[treeID.Str()]; ok {
-				found = true
-			} else if _, ok := f.treeIDs[treeID.String()]; ok {
-				found = true
-			}
-			if found {
-				f.out.PrintObject("tree", treeID.String(), nodepath, "", sn)
-				f.itemsFound++
-				// Terminate if we have found all trees (and we are not
-				// looking for blobs)
-				if f.itemsFound >= len(f.treeIDs) && f.blobIDs == nil {
-					// Return an error to terminate the Walk
-					return errors.New("OK")
-				}
+			if err := f.findTree(*node.Subtree, nodepath); err != nil {
+				return err
 			}
 		}
 
-		if node.Type == "file" && f.blobIDs != nil {
+		if node.Type == restic.NodeTypeFile && f.blobIDs != nil {
 			for _, id := range node.Content {
 				if ctx.Err() != nil {
 					return ctx.Err()
@@ -626,7 +644,10 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
 	}
 
 	sort.Slice(filteredSnapshots, func(i, j int) bool {
-		return filteredSnapshots[i].Time.Before(filteredSnapshots[j].Time)
+		if opts.Reverse {
+			return filteredSnapshots[i].Time.Before(filteredSnapshots[j].Time)
+		}
+		return filteredSnapshots[i].Time.After(filteredSnapshots[j].Time)
 	})
 
 	for _, sn := range filteredSnapshots {

cmd/restic/cmd_find_integration_test.go

@@ -10,11 +10,10 @@ import (
 	rtest "github.com/restic/restic/internal/test"
 )
 
-func testRunFind(t testing.TB, wantJSON bool, gopts GlobalOptions, pattern string) []byte {
+func testRunFind(t testing.TB, wantJSON bool, opts FindOptions, gopts GlobalOptions, pattern string) []byte {
 	buf, err := withCaptureStdout(func() error {
 		gopts.JSON = wantJSON
 
-		opts := FindOptions{}
 		return runFind(context.TODO(), opts, gopts, []string{pattern})
 	})
 	rtest.OK(t, err)
@@ -29,16 +28,15 @@ func TestFind(t *testing.T) {
 	opts := BackupOptions{}
 
 	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
-	testRunCheck(t, env.gopts)
 
-	results := testRunFind(t, false, env.gopts, "unexistingfile")
+	results := testRunFind(t, false, FindOptions{}, env.gopts, "unexistingfile")
 	rtest.Assert(t, len(results) == 0, "unexisting file found in repo (%v)", datafile)
 
-	results = testRunFind(t, false, env.gopts, "testfile")
+	results = testRunFind(t, false, FindOptions{}, env.gopts, "testfile")
 	lines := strings.Split(string(results), "\n")
 	rtest.Assert(t, len(lines) == 2, "expected one file found in repo (%v)", datafile)
 
-	results = testRunFind(t, false, env.gopts, "testfile*")
+	results = testRunFind(t, false, FindOptions{}, env.gopts, "testfile*")
 	lines = strings.Split(string(results), "\n")
 	rtest.Assert(t, len(lines) == 4, "expected three files found in repo (%v)", datafile)
 }
@@ -67,21 +65,69 @@ func TestFindJSON(t *testing.T) {
 
 	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
 	testRunCheck(t, env.gopts)
+	snapshot, _ := testRunSnapshots(t, env.gopts)
 
-	results := testRunFind(t, true, env.gopts, "unexistingfile")
+	results := testRunFind(t, true, FindOptions{}, env.gopts, "unexistingfile")
 	matches := []testMatches{}
 	rtest.OK(t, json.Unmarshal(results, &matches))
 	rtest.Assert(t, len(matches) == 0, "expected no match in repo (%v)", datafile)
 
-	results = testRunFind(t, true, env.gopts, "testfile")
+	results = testRunFind(t, true, FindOptions{}, env.gopts, "testfile")
 	rtest.OK(t, json.Unmarshal(results, &matches))
 	rtest.Assert(t, len(matches) == 1, "expected a single snapshot in repo (%v)", datafile)
 	rtest.Assert(t, len(matches[0].Matches) == 1, "expected a single file to match (%v)", datafile)
 	rtest.Assert(t, matches[0].Hits == 1, "expected hits to show 1 match (%v)", datafile)
 
-	results = testRunFind(t, true, env.gopts, "testfile*")
+	results = testRunFind(t, true, FindOptions{}, env.gopts, "testfile*")
 	rtest.OK(t, json.Unmarshal(results, &matches))
 	rtest.Assert(t, len(matches) == 1, "expected a single snapshot in repo (%v)", datafile)
 	rtest.Assert(t, len(matches[0].Matches) == 3, "expected 3 files to match (%v)", datafile)
 	rtest.Assert(t, matches[0].Hits == 3, "expected hits to show 3 matches (%v)", datafile)
+
+	results = testRunFind(t, true, FindOptions{TreeID: true}, env.gopts, snapshot.Tree.String())
+	rtest.OK(t, json.Unmarshal(results, &matches))
+	rtest.Assert(t, len(matches) == 1, "expected a single snapshot in repo (%v)", matches)
+	rtest.Assert(t, len(matches[0].Matches) == 3, "expected 3 files to match (%v)", matches[0].Matches)
+	rtest.Assert(t, matches[0].Hits == 3, "expected hits to show 3 matches (%v)", datafile)
+}
+
+func TestFindSorting(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	datafile := testSetupBackupData(t, env)
+	opts := BackupOptions{}
+
+	// first backup
+	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
+	sn1 := testListSnapshots(t, env.gopts, 1)[0]
+
+	// second backup
+	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
+	snapshots := testListSnapshots(t, env.gopts, 2)
+	// get id of new snapshot without depending on file order returned by filesystem
+	sn2 := snapshots[0]
+	if sn1.Equal(sn2) {
+		sn2 = snapshots[1]
+	}
+
+	// first restic find - with default FindOptions{}
+	results := testRunFind(t, true, FindOptions{}, env.gopts, "testfile")
+	lines := strings.Split(string(results), "\n")
+	rtest.Assert(t, len(lines) == 2, "expected two files found in repo (%v), found %d", datafile, len(lines))
+	matches := []testMatches{}
+	rtest.OK(t, json.Unmarshal(results, &matches))
+
+	// run second restic find with --reverse, sort oldest to newest
+	resultsReverse := testRunFind(t, true, FindOptions{Reverse: true}, env.gopts, "testfile")
+	lines = strings.Split(string(resultsReverse), "\n")
+	rtest.Assert(t, len(lines) == 2, "expected two files found in repo (%v), found %d", datafile, len(lines))
+	matchesReverse := []testMatches{}
+	rtest.OK(t, json.Unmarshal(resultsReverse, &matchesReverse))
+
+	// compare result sets
+	rtest.Assert(t, sn1.String() == matchesReverse[0].SnapshotID, "snapshot[0] must match old snapshot")
+	rtest.Assert(t, sn2.String() == matchesReverse[1].SnapshotID, "snapshot[1] must match new snapshot")
+	rtest.Assert(t, matches[0].SnapshotID == matchesReverse[1].SnapshotID, "matches should be sorted 1")
+	rtest.Assert(t, matches[1].SnapshotID == matchesReverse[0].SnapshotID, "matches should be sorted 2")
 }

cmd/restic/cmd_forget.go

@@ -8,16 +8,20 @@ import (
 	"strconv"
 
 	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/feature"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/ui/termstatus"
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 )
 
-var cmdForget = &cobra.Command{
-	Use:   "forget [flags] [snapshot ID] [...]",
-	Short: "Remove snapshots from the repository",
-	Long: `
+func newForgetCommand() *cobra.Command {
+	var opts ForgetOptions
+	var pruneOpts PruneOptions
+
+	cmd := &cobra.Command{
+		Use:   "forget [flags] [snapshot ID] [...]",
+		Short: "Remove snapshots from the repository",
+		Long: `
 The "forget" command removes snapshots according to a policy. All snapshots are
 first divided into groups according to "--group-by", and after that the policy
 specified by the "--keep-*" options is applied to each group individually.
@@ -37,22 +41,29 @@ EXIT STATUS
 
 Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
+Exit status is 3 if there was an error removing one or more snapshots.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	GroupID:           cmdGroupDefault,
-	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		term, cancel := setupTermstatus()
-		defer cancel()
-		return runForget(cmd.Context(), forgetOptions, forgetPruneOptions, globalOptions, term, args)
-	},
+		GroupID:           cmdGroupDefault,
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			term, cancel := setupTermstatus()
+			defer cancel()
+			return runForget(cmd.Context(), opts, pruneOpts, globalOptions, term, args)
+		},
+	}
+
+	opts.AddFlags(cmd.Flags())
+	pruneOpts.AddLimitedFlags(cmd.Flags())
+	return cmd
 }
 
 type ForgetPolicyCount int
 
 var ErrNegativePolicyCount = errors.New("negative values not allowed, use 'unlimited' instead")
+var ErrFailedToRemoveOneOrMoreSnapshots = errors.New("failed to remove one or more snapshots")
 
 func (c *ForgetPolicyCount) Set(s string) error {
 	switch s {
@@ -112,44 +123,38 @@ type ForgetOptions struct {
 	Prune   bool
 }
 
-var forgetOptions ForgetOptions
-var forgetPruneOptions PruneOptions
+func (opts *ForgetOptions) AddFlags(f *pflag.FlagSet) {
+	f.VarP(&opts.Last, "keep-last", "l", "keep the last `n` snapshots (use 'unlimited' to keep all snapshots)")
+	f.VarP(&opts.Hourly, "keep-hourly", "H", "keep the last `n` hourly snapshots (use 'unlimited' to keep all hourly snapshots)")
+	f.VarP(&opts.Daily, "keep-daily", "d", "keep the last `n` daily snapshots (use 'unlimited' to keep all daily snapshots)")
+	f.VarP(&opts.Weekly, "keep-weekly", "w", "keep the last `n` weekly snapshots (use 'unlimited' to keep all weekly snapshots)")
+	f.VarP(&opts.Monthly, "keep-monthly", "m", "keep the last `n` monthly snapshots (use 'unlimited' to keep all monthly snapshots)")
+	f.VarP(&opts.Yearly, "keep-yearly", "y", "keep the last `n` yearly snapshots (use 'unlimited' to keep all yearly snapshots)")
+	f.VarP(&opts.Within, "keep-within", "", "keep snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
+	f.VarP(&opts.WithinHourly, "keep-within-hourly", "", "keep hourly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
+	f.VarP(&opts.WithinDaily, "keep-within-daily", "", "keep daily snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
+	f.VarP(&opts.WithinWeekly, "keep-within-weekly", "", "keep weekly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
+	f.VarP(&opts.WithinMonthly, "keep-within-monthly", "", "keep monthly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
+	f.VarP(&opts.WithinYearly, "keep-within-yearly", "", "keep yearly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
+	f.Var(&opts.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)")
+	f.BoolVar(&opts.UnsafeAllowRemoveAll, "unsafe-allow-remove-all", false, "allow deleting all snapshots of a snapshot group")
 
-func init() {
-	cmdRoot.AddCommand(cmdForget)
-
-	f := cmdForget.Flags()
-	f.VarP(&forgetOptions.Last, "keep-last", "l", "keep the last `n` snapshots (use 'unlimited' to keep all snapshots)")
-	f.VarP(&forgetOptions.Hourly, "keep-hourly", "H", "keep the last `n` hourly snapshots (use 'unlimited' to keep all hourly snapshots)")
-	f.VarP(&forgetOptions.Daily, "keep-daily", "d", "keep the last `n` daily snapshots (use 'unlimited' to keep all daily snapshots)")
-	f.VarP(&forgetOptions.Weekly, "keep-weekly", "w", "keep the last `n` weekly snapshots (use 'unlimited' to keep all weekly snapshots)")
-	f.VarP(&forgetOptions.Monthly, "keep-monthly", "m", "keep the last `n` monthly snapshots (use 'unlimited' to keep all monthly snapshots)")
-	f.VarP(&forgetOptions.Yearly, "keep-yearly", "y", "keep the last `n` yearly snapshots (use 'unlimited' to keep all yearly snapshots)")
-	f.VarP(&forgetOptions.Within, "keep-within", "", "keep snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
-	f.VarP(&forgetOptions.WithinHourly, "keep-within-hourly", "", "keep hourly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
-	f.VarP(&forgetOptions.WithinDaily, "keep-within-daily", "", "keep daily snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
-	f.VarP(&forgetOptions.WithinWeekly, "keep-within-weekly", "", "keep weekly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
-	f.VarP(&forgetOptions.WithinMonthly, "keep-within-monthly", "", "keep monthly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
-	f.VarP(&forgetOptions.WithinYearly, "keep-within-yearly", "", "keep yearly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
-	f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)")
-	f.BoolVar(&forgetOptions.UnsafeAllowRemoveAll, "unsafe-allow-remove-all", false, "allow deleting all snapshots of a snapshot group")
-
-	initMultiSnapshotFilter(f, &forgetOptions.SnapshotFilter, false)
-	f.StringArrayVar(&forgetOptions.Hosts, "hostname", nil, "only consider snapshots with the given `hostname` (can be specified multiple times)")
+	f.StringArrayVar(&opts.Hosts, "hostname", nil, "only consider snapshots with the given `hostname` (can be specified multiple times)")
 	err := f.MarkDeprecated("hostname", "use --host")
 	if err != nil {
 		// MarkDeprecated only returns an error when the flag is not found
 		panic(err)
 	}
+	// must be defined after `--hostname` to not override the default value from the environment
+	initMultiSnapshotFilter(f, &opts.SnapshotFilter, false)
 
-	f.BoolVarP(&forgetOptions.Compact, "compact", "c", false, "use compact output format")
-	forgetOptions.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true}
-	f.VarP(&forgetOptions.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
-	f.BoolVarP(&forgetOptions.DryRun, "dry-run", "n", false, "do not delete anything, just print what would be done")
-	f.BoolVar(&forgetOptions.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed")
+	f.BoolVarP(&opts.Compact, "compact", "c", false, "use compact output format")
+	opts.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true}
+	f.VarP(&opts.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
+	f.BoolVarP(&opts.DryRun, "dry-run", "n", false, "do not delete anything, just print what would be done")
+	f.BoolVar(&opts.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed")
 
 	f.SortFlags = false
-	addPruneOptions(cmdForget, &forgetPruneOptions)
 }
 
 func verifyForgetOptions(opts *ForgetOptions) error {
@@ -271,7 +276,7 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
 
 			keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy)
 
-			if feature.Flag.Enabled(feature.SafeForgetKeepTags) && !policy.Empty() && len(keep) == 0 {
+			if !policy.Empty() && len(keep) == 0 {
 				return fmt.Errorf("refusing to delete last snapshot of snapshot group \"%v\"", key.String())
 			}
 			if len(keep) != 0 && !gopts.Quiet && !gopts.JSON {
@@ -302,12 +307,15 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
 		return ctx.Err()
 	}
 
+	// these are the snapshots that failed to be removed
+	failedSnIDs := restic.NewIDSet()
 	if len(removeSnIDs) > 0 {
 		if !opts.DryRun {
 			bar := printer.NewCounter("files deleted")
-			err := restic.ParallelRemove(ctx, repo, removeSnIDs, restic.SnapshotFile, func(id restic.ID, err error) error {
+			err := restic.ParallelRemove(ctx, repo, removeSnIDs, restic.WriteableSnapshotFile, func(id restic.ID, err error) error {
 				if err != nil {
 					printer.E("unable to remove %v/%v from the repository\n", restic.SnapshotFile, id)
+					failedSnIDs.Insert(id)
 				} else {
 					printer.VV("removed %v/%v\n", restic.SnapshotFile, id)
 				}
@@ -329,6 +337,10 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
 		}
 	}
 
+	if len(failedSnIDs) > 0 {
+		return ErrFailedToRemoveOneOrMoreSnapshots
+	}
+
 	if len(removeSnIDs) > 0 && opts.Prune {
 		if opts.DryRun {
 			printer.P("%d snapshots would be removed, running prune dry run\n", len(removeSnIDs))

cmd/restic/cmd_forget_test.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
+	"github.com/spf13/pflag"
 )
 
 func TestForgetPolicyValues(t *testing.T) {
@@ -92,3 +93,10 @@ func TestForgetOptionValues(t *testing.T) {
 		}
 	}
 }
+
+func TestForgetHostnameDefaulting(t *testing.T) {
+	t.Setenv("RESTIC_HOST", "testhost")
+	opts := ForgetOptions{}
+	opts.AddFlags(pflag.NewFlagSet("test", pflag.ContinueOnError))
+	rtest.Equals(t, []string{"testhost"}, opts.Hosts)
+}

cmd/restic/cmd_generate.go

@@ -1,17 +1,23 @@
 package main
 
 import (
+	"io"
+	"os"
 	"time"
 
 	"github.com/restic/restic/internal/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/cobra/doc"
+	"github.com/spf13/pflag"
 )
 
-var cmdGenerate = &cobra.Command{
-	Use:   "generate [flags]",
-	Short: "Generate manual pages and auto-completion files (bash, fish, zsh, powershell)",
-	Long: `
+func newGenerateCommand() *cobra.Command {
+	var opts generateOptions
+
+	cmd := &cobra.Command{
+		Use:   "generate [flags]",
+		Short: "Generate manual pages and auto-completion files (bash, fish, zsh, powershell)",
+		Long: `
 The "generate" command writes automatically generated files (like the man pages
 and the auto-completion files for bash, fish and zsh).
 
@@ -21,10 +27,13 @@ EXIT STATUS
 Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 `,
-	DisableAutoGenTag: true,
-	RunE: func(_ *cobra.Command, args []string) error {
-		return runGenerate(genOpts, args)
-	},
+		DisableAutoGenTag: true,
+		RunE: func(_ *cobra.Command, args []string) error {
+			return runGenerate(opts, args)
+		},
+	}
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 type generateOptions struct {
@@ -35,19 +44,15 @@ type generateOptions struct {
 	PowerShellCompletionFile string
 }
 
-var genOpts generateOptions
-
-func init() {
-	cmdRoot.AddCommand(cmdGenerate)
-	fs := cmdGenerate.Flags()
-	fs.StringVar(&genOpts.ManDir, "man", "", "write man pages to `directory`")
-	fs.StringVar(&genOpts.BashCompletionFile, "bash-completion", "", "write bash completion `file`")
-	fs.StringVar(&genOpts.FishCompletionFile, "fish-completion", "", "write fish completion `file`")
-	fs.StringVar(&genOpts.ZSHCompletionFile, "zsh-completion", "", "write zsh completion `file`")
-	fs.StringVar(&genOpts.PowerShellCompletionFile, "powershell-completion", "", "write powershell completion `file`")
+func (opts *generateOptions) AddFlags(f *pflag.FlagSet) {
+	f.StringVar(&opts.ManDir, "man", "", "write man pages to `directory`")
+	f.StringVar(&opts.BashCompletionFile, "bash-completion", "", "write bash completion `file` (`-` for stdout)")
+	f.StringVar(&opts.FishCompletionFile, "fish-completion", "", "write fish completion `file` (`-` for stdout)")
+	f.StringVar(&opts.ZSHCompletionFile, "zsh-completion", "", "write zsh completion `file` (`-` for stdout)")
+	f.StringVar(&opts.PowerShellCompletionFile, "powershell-completion", "", "write powershell completion `file` (`-` for stdout)")
 }
 
-func writeManpages(dir string) error {
+func writeManpages(root *cobra.Command, dir string) error {
 	// use a fixed date for the man pages so that generating them is deterministic
 	date, err := time.Parse("Jan 2006", "Jan 2017")
 	if err != nil {
@@ -62,35 +67,47 @@ func writeManpages(dir string) error {
 	}
 
 	Verbosef("writing man pages to directory %v\n", dir)
-	return doc.GenManTree(cmdRoot, header, dir)
+	return doc.GenManTree(root, header, dir)
 }
 
-func writeBashCompletion(file string) error {
+func writeCompletion(filename string, shell string, generate func(w io.Writer) error) (err error) {
 	if stdoutIsTerminal() {
-		Verbosef("writing bash completion file to %v\n", file)
+		Verbosef("writing %s completion file to %v\n", shell, filename)
 	}
-	return cmdRoot.GenBashCompletionFile(file)
+	var outWriter io.Writer
+	if filename != "-" {
+		var outFile *os.File
+		outFile, err = os.Create(filename)
+		if err != nil {
+			return
+		}
+		defer func() { err = outFile.Close() }()
+		outWriter = outFile
+	} else {
+		outWriter = globalOptions.stdout
+	}
+
+	err = generate(outWriter)
+	return
 }
 
-func writeFishCompletion(file string) error {
-	if stdoutIsTerminal() {
-		Verbosef("writing fish completion file to %v\n", file)
+func checkStdoutForSingleShell(opts generateOptions) error {
+	completionFileOpts := []string{
+		opts.BashCompletionFile,
+		opts.FishCompletionFile,
+		opts.ZSHCompletionFile,
+		opts.PowerShellCompletionFile,
 	}
-	return cmdRoot.GenFishCompletionFile(file, true)
-}
-
-func writeZSHCompletion(file string) error {
-	if stdoutIsTerminal() {
-		Verbosef("writing zsh completion file to %v\n", file)
+	seenIsStdout := false
+	for _, completionFileOpt := range completionFileOpts {
+		if completionFileOpt == "-" {
+			if seenIsStdout {
+				return errors.Fatal("the generate command can generate shell completions to stdout for single shell only")
+			}
+			seenIsStdout = true
+		}
 	}
-	return cmdRoot.GenZshCompletionFile(file)
-}
-
-func writePowerShellCompletion(file string) error {
-	if stdoutIsTerminal() {
-		Verbosef("writing powershell completion file to %v\n", file)
-	}
-	return cmdRoot.GenPowerShellCompletionFile(file)
+	return nil
 }
 
 func runGenerate(opts generateOptions, args []string) error {
@@ -98,36 +115,43 @@ func runGenerate(opts generateOptions, args []string) error {
 		return errors.Fatal("the generate command expects no arguments, only options - please see `restic help generate` for usage and flags")
 	}
 
+	cmdRoot := newRootCommand()
+
 	if opts.ManDir != "" {
-		err := writeManpages(opts.ManDir)
+		err := writeManpages(cmdRoot, opts.ManDir)
 		if err != nil {
 			return err
 		}
 	}
 
+	err := checkStdoutForSingleShell(opts)
+	if err != nil {
+		return err
+	}
+
 	if opts.BashCompletionFile != "" {
-		err := writeBashCompletion(opts.BashCompletionFile)
+		err := writeCompletion(opts.BashCompletionFile, "bash", cmdRoot.GenBashCompletion)
 		if err != nil {
 			return err
 		}
 	}
 
 	if opts.FishCompletionFile != "" {
-		err := writeFishCompletion(opts.FishCompletionFile)
+		err := writeCompletion(opts.FishCompletionFile, "fish", func(w io.Writer) error { return cmdRoot.GenFishCompletion(w, true) })
 		if err != nil {
 			return err
 		}
 	}
 
 	if opts.ZSHCompletionFile != "" {
-		err := writeZSHCompletion(opts.ZSHCompletionFile)
+		err := writeCompletion(opts.ZSHCompletionFile, "zsh", cmdRoot.GenZshCompletion)
 		if err != nil {
 			return err
 		}
 	}
 
 	if opts.PowerShellCompletionFile != "" {
-		err := writePowerShellCompletion(opts.PowerShellCompletionFile)
+		err := writeCompletion(opts.PowerShellCompletionFile, "powershell", cmdRoot.GenPowerShellCompletion)
 		if err != nil {
 			return err
 		}

cmd/restic/cmd_generate_integration_test.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func TestGenerateStdout(t *testing.T) {
+	testCases := []struct {
+		name string
+		opts generateOptions
+	}{
+		{"bash", generateOptions{BashCompletionFile: "-"}},
+		{"fish", generateOptions{FishCompletionFile: "-"}},
+		{"zsh", generateOptions{ZSHCompletionFile: "-"}},
+		{"powershell", generateOptions{PowerShellCompletionFile: "-"}},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			buf := bytes.NewBuffer(nil)
+			globalOptions.stdout = buf
+			err := runGenerate(tc.opts, []string{})
+			rtest.OK(t, err)
+			completionString := buf.String()
+			rtest.Assert(t, strings.Contains(completionString, "# "+tc.name+" completion for restic"), "has no expected completion header")
+		})
+	}
+
+	t.Run("Generate shell completions to stdout for two shells", func(t *testing.T) {
+		buf := bytes.NewBuffer(nil)
+		globalOptions.stdout = buf
+		opts := generateOptions{BashCompletionFile: "-", FishCompletionFile: "-"}
+		err := runGenerate(opts, []string{})
+		rtest.Assert(t, err != nil, "generate shell completions to stdout for two shells fails")
+	})
+}

cmd/restic/cmd_init.go

@@ -12,12 +12,16 @@ import (
 	"github.com/restic/restic/internal/restic"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 )
 
-var cmdInit = &cobra.Command{
-	Use:   "init",
-	Short: "Initialize a new repository",
-	Long: `
+func newInitCommand() *cobra.Command {
+	var opts InitOptions
+
+	cmd := &cobra.Command{
+		Use:   "init",
+		Short: "Initialize a new repository",
+		Long: `
 The "init" command initializes a new repository.
 
 EXIT STATUS
@@ -26,11 +30,14 @@ EXIT STATUS
 Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 `,
-	GroupID:           cmdGroupDefault,
-	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return runInit(cmd.Context(), initOptions, globalOptions, args)
-	},
+		GroupID:           cmdGroupDefault,
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runInit(cmd.Context(), opts, globalOptions, args)
+		},
+	}
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 // InitOptions bundles all options for the init command.
@@ -40,15 +47,10 @@ type InitOptions struct {
 	RepositoryVersion     string
 }
 
-var initOptions InitOptions
-
-func init() {
-	cmdRoot.AddCommand(cmdInit)
-
-	f := cmdInit.Flags()
-	initSecondaryRepoOptions(f, &initOptions.secondaryRepoOptions, "secondary", "to copy chunker parameters from")
-	f.BoolVar(&initOptions.CopyChunkerParameters, "copy-chunker-params", false, "copy chunker parameters from the secondary repository (useful with the copy command)")
-	f.StringVar(&initOptions.RepositoryVersion, "repository-version", "stable", "repository format version to use, allowed values are a format version, 'latest' and 'stable'")
+func (opts *InitOptions) AddFlags(f *pflag.FlagSet) {
+	opts.secondaryRepoOptions.AddFlags(f, "secondary", "to copy chunker parameters from")
+	f.BoolVar(&opts.CopyChunkerParameters, "copy-chunker-params", false, "copy chunker parameters from the secondary repository (useful with the copy command)")
+	f.StringVar(&opts.RepositoryVersion, "repository-version", "stable", "repository format version to use, allowed values are a format version, 'latest' and 'stable'")
 }
 
 func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []string) error {

cmd/restic/cmd_key.go

@@ -4,17 +4,23 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var cmdKey = &cobra.Command{
-	Use:   "key",
-	Short: "Manage keys (passwords)",
-	Long: `
+func newKeyCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "key",
+		Short: "Manage keys (passwords)",
+		Long: `
 The "key" command allows you to set multiple access keys or passwords
 per repository.
 	`,
-	DisableAutoGenTag: true,
-	GroupID:           cmdGroupDefault,
-}
+		DisableAutoGenTag: true,
+		GroupID:           cmdGroupDefault,
+	}
 
-func init() {
-	cmdRoot.AddCommand(cmdKey)
+	cmd.AddCommand(
+		newKeyAddCommand(),
+		newKeyListCommand(),
+		newKeyPasswdCommand(),
+		newKeyRemoveCommand(),
+	)
+	return cmd
 }

cmd/restic/cmd_key_add.go

@@ -10,10 +10,13 @@ import (
 	"github.com/spf13/pflag"
 )
 
-var cmdKeyAdd = &cobra.Command{
-	Use:   "add",
-	Short: "Add a new key (password) to the repository; returns the new key ID",
-	Long: `
+func newKeyAddCommand() *cobra.Command {
+	var opts KeyAddOptions
+
+	cmd := &cobra.Command{
+		Use:   "add",
+		Short: "Add a new key (password) to the repository; returns the new key ID",
+		Long: `
 The "add" sub-command creates a new key and validates the key. Returns the new key ID.
 
 EXIT STATUS
@@ -25,7 +28,14 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 	`,
-	DisableAutoGenTag: true,
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runKeyAdd(cmd.Context(), globalOptions, opts, args)
+		},
+	}
+
+	opts.Add(cmd.Flags())
+	return cmd
 }
 
 type KeyAddOptions struct {
@@ -42,16 +52,6 @@ func (opts *KeyAddOptions) Add(flags *pflag.FlagSet) {
 	flags.StringVarP(&opts.Hostname, "host", "", "", "the hostname for new key")
 }
 
-func init() {
-	cmdKey.AddCommand(cmdKeyAdd)
-
-	var keyAddOpts KeyAddOptions
-	keyAddOpts.Add(cmdKeyAdd.Flags())
-	cmdKeyAdd.RunE = func(cmd *cobra.Command, args []string) error {
-		return runKeyAdd(cmd.Context(), globalOptions, keyAddOpts, args)
-	}
-}
-
 func runKeyAdd(ctx context.Context, gopts GlobalOptions, opts KeyAddOptions, args []string) error {
 	if len(args) > 0 {
 		return fmt.Errorf("the key add command expects no arguments, only options - please see `restic help key add` for usage and flags")

cmd/restic/cmd_key_list.go

@@ -12,10 +12,11 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var cmdKeyList = &cobra.Command{
-	Use:   "list",
-	Short: "List keys (passwords)",
-	Long: `
+func newKeyListCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "list",
+		Short: "List keys (passwords)",
+		Long: `
 The "list" sub-command lists all the keys (passwords) associated with the repository.
 Returns the key ID, username, hostname, created time and if it's the current key being
 used to access the repository.
@@ -29,14 +30,12 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 	`,
-	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return runKeyList(cmd.Context(), globalOptions, args)
-	},
-}
-
-func init() {
-	cmdKey.AddCommand(cmdKeyList)
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runKeyList(cmd.Context(), globalOptions, args)
+		},
+	}
+	return cmd
 }
 
 func runKeyList(ctx context.Context, gopts GlobalOptions, args []string) error {

cmd/restic/cmd_key_passwd.go

@@ -7,12 +7,16 @@ import (
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 )
 
-var cmdKeyPasswd = &cobra.Command{
-	Use:   "passwd",
-	Short: "Change key (password); creates a new key ID and removes the old key ID, returns new key ID",
-	Long: `
+func newKeyPasswdCommand() *cobra.Command {
+	var opts KeyPasswdOptions
+
+	cmd := &cobra.Command{
+		Use:   "passwd",
+		Short: "Change key (password); creates a new key ID and removes the old key ID, returns new key ID",
+		Long: `
 The "passwd" sub-command creates a new key, validates the key and remove the old key ID.
 Returns the new key ID. 
 
@@ -25,21 +29,22 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 	`,
-	DisableAutoGenTag: true,
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runKeyPasswd(cmd.Context(), globalOptions, opts, args)
+		},
+	}
+
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 type KeyPasswdOptions struct {
 	KeyAddOptions
 }
 
-func init() {
-	cmdKey.AddCommand(cmdKeyPasswd)
-
-	var keyPasswdOpts KeyPasswdOptions
-	keyPasswdOpts.KeyAddOptions.Add(cmdKeyPasswd.Flags())
-	cmdKeyPasswd.RunE = func(cmd *cobra.Command, args []string) error {
-		return runKeyPasswd(cmd.Context(), globalOptions, keyPasswdOpts, args)
-	}
+func (opts *KeyPasswdOptions) AddFlags(flags *pflag.FlagSet) {
+	opts.KeyAddOptions.Add(flags)
 }
 
 func runKeyPasswd(ctx context.Context, gopts GlobalOptions, opts KeyPasswdOptions, args []string) error {

cmd/restic/cmd_key_remove.go

@@ -10,10 +10,11 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var cmdKeyRemove = &cobra.Command{
-	Use:   "remove [ID]",
-	Short: "Remove key ID (password) from the repository.",
-	Long: `
+func newKeyRemoveCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "remove [ID]",
+		Short: "Remove key ID (password) from the repository.",
+		Long: `
 The "remove" sub-command removes the selected key ID. The "remove" command does not allow
 removing the current key being used to access the repository. 
 
@@ -26,14 +27,12 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 	`,
-	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return runKeyRemove(cmd.Context(), globalOptions, args)
-	},
-}
-
-func init() {
-	cmdKey.AddCommand(cmdKeyRemove)
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runKeyRemove(cmd.Context(), globalOptions, args)
+		},
+	}
+	return cmd
 }
 
 func runKeyRemove(ctx context.Context, gopts GlobalOptions, args []string) error {

cmd/restic/cmd_list.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"strings"
 
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository/index"
@@ -10,10 +11,14 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var cmdList = &cobra.Command{
-	Use:   "list [flags] [blobs|packs|index|snapshots|keys|locks]",
-	Short: "List objects in the repository",
-	Long: `
+func newListCommand() *cobra.Command {
+	var listAllowedArgs = []string{"blobs", "packs", "index", "snapshots", "keys", "locks"}
+	var listAllowedArgsUseString = strings.Join(listAllowedArgs, "|")
+
+	cmd := &cobra.Command{
+		Use:   "list [flags] [" + listAllowedArgsUseString + "]",
+		Short: "List objects in the repository",
+		Long: `
 The "list" command allows listing objects in the repository based on type.
 
 EXIT STATUS
@@ -25,15 +30,15 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	DisableAutoGenTag: true,
-	GroupID:           cmdGroupDefault,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return runList(cmd.Context(), globalOptions, args)
-	},
-}
-
-func init() {
-	cmdRoot.AddCommand(cmdList)
+		DisableAutoGenTag: true,
+		GroupID:           cmdGroupDefault,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runList(cmd.Context(), globalOptions, args)
+		},
+		ValidArgs: listAllowedArgs,
+		Args:      cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs),
+	}
+	return cmd
 }
 
 func runList(ctx context.Context, gopts GlobalOptions, args []string) error {
@@ -60,7 +65,7 @@ func runList(ctx context.Context, gopts GlobalOptions, args []string) error {
 	case "locks":
 		t = restic.LockFile
 	case "blobs":
-		return index.ForAllIndexes(ctx, repo, repo, func(_ restic.ID, idx *index.Index, _ bool, err error) error {
+		return index.ForAllIndexes(ctx, repo, repo, func(_ restic.ID, idx *index.Index, err error) error {
 			if err != nil {
 				return err
 			}

cmd/restic/cmd_ls.go

@@ -1,15 +1,19 @@
 package main
 
 import (
+	"cmp"
 	"context"
 	"encoding/json"
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
+	"slices"
 	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/fs"
@@ -17,10 +21,13 @@ import (
 	"github.com/restic/restic/internal/walker"
 )
 
-var cmdLs = &cobra.Command{
-	Use:   "ls [flags] snapshotID [dir...]",
-	Short: "List files in a snapshot",
-	Long: `
+func newLsCommand() *cobra.Command {
+	var opts LsOptions
+
+	cmd := &cobra.Command{
+		Use:   "ls [flags] snapshotID [dir...]",
+		Short: "List files in a snapshot",
+		Long: `
 The "ls" command lists files and directories in a snapshot.
 
 The special snapshot ID "latest" can be used to list files and
@@ -36,6 +43,10 @@ will allow traversing into matching directories' subfolders.
 Any directory paths specified must be absolute (starting with
 a path separator); paths use the forward slash '/' as separator.
 
+File listings can be sorted by specifying --sort followed by one of the
+sort specifiers '(name|size|time=mtime|atime|ctime|extension)'.
+The sorting can be reversed by specifying --reverse.
+
 EXIT STATUS
 ===========
 
@@ -45,11 +56,14 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	DisableAutoGenTag: true,
-	GroupID:           cmdGroupDefault,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return runLs(cmd.Context(), lsOptions, globalOptions, args)
-	},
+		DisableAutoGenTag: true,
+		GroupID:           cmdGroupDefault,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runLs(cmd.Context(), opts, globalOptions, args)
+		},
+	}
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 // LsOptions collects all options for the ls command.
@@ -59,62 +73,55 @@ type LsOptions struct {
 	Recursive     bool
 	HumanReadable bool
 	Ncdu          bool
+	Sort          SortMode
+	Reverse       bool
 }
 
-var lsOptions LsOptions
-
-func init() {
-	cmdRoot.AddCommand(cmdLs)
-
-	flags := cmdLs.Flags()
-	initSingleSnapshotFilter(flags, &lsOptions.SnapshotFilter)
-	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
-	flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories")
-	flags.BoolVar(&lsOptions.HumanReadable, "human-readable", false, "print sizes in human readable format")
-	flags.BoolVar(&lsOptions.Ncdu, "ncdu", false, "output NCDU export format (pipe into 'ncdu -f -')")
+func (opts *LsOptions) AddFlags(f *pflag.FlagSet) {
+	initSingleSnapshotFilter(f, &opts.SnapshotFilter)
+	f.BoolVarP(&opts.ListLong, "long", "l", false, "use a long listing format showing size and mode")
+	f.BoolVar(&opts.Recursive, "recursive", false, "include files in subfolders of the listed directories")
+	f.BoolVar(&opts.HumanReadable, "human-readable", false, "print sizes in human readable format")
+	f.BoolVar(&opts.Ncdu, "ncdu", false, "output NCDU export format (pipe into 'ncdu -f -')")
+	f.VarP(&opts.Sort, "sort", "s", "sort output by (name|size|time=mtime|atime|ctime|extension)")
+	f.BoolVar(&opts.Reverse, "reverse", false, "reverse sorted output")
 }
 
 type lsPrinter interface {
-	Snapshot(sn *restic.Snapshot)
-	Node(path string, node *restic.Node, isPrefixDirectory bool)
-	LeaveDir(path string)
-	Close()
+	Snapshot(sn *restic.Snapshot) error
+	Node(path string, node *restic.Node, isPrefixDirectory bool) error
+	LeaveDir(path string) error
+	Close() error
 }
 
 type jsonLsPrinter struct {
 	enc *json.Encoder
 }
 
-func (p *jsonLsPrinter) Snapshot(sn *restic.Snapshot) {
+func (p *jsonLsPrinter) Snapshot(sn *restic.Snapshot) error {
 	type lsSnapshot struct {
 		*restic.Snapshot
 		ID          *restic.ID `json:"id"`
-		ShortID     string     `json:"short_id"`
+		ShortID     string     `json:"short_id"`     // deprecated
 		MessageType string     `json:"message_type"` // "snapshot"
 		StructType  string     `json:"struct_type"`  // "snapshot", deprecated
 	}
 
-	err := p.enc.Encode(lsSnapshot{
+	return p.enc.Encode(lsSnapshot{
 		Snapshot:    sn,
 		ID:          sn.ID(),
 		ShortID:     sn.ID().Str(),
 		MessageType: "snapshot",
 		StructType:  "snapshot",
 	})
-	if err != nil {
-		Warnf("JSON encode failed: %v\n", err)
-	}
 }
 
-// Print node in our custom JSON format, followed by a newline.
-func (p *jsonLsPrinter) Node(path string, node *restic.Node, isPrefixDirectory bool) {
+// Node formats node in our custom JSON format, followed by a newline.
+func (p *jsonLsPrinter) Node(path string, node *restic.Node, isPrefixDirectory bool) error {
 	if isPrefixDirectory {
-		return
-	}
-	err := lsNodeJSON(p.enc, path, node)
-	if err != nil {
-		Warnf("JSON encode failed: %v\n", err)
+		return nil
 	}
+	return lsNodeJSON(p.enc, path, node)
 }
 
 func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
@@ -137,7 +144,7 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
 		size uint64 // Target for Size pointer.
 	}{
 		Name:        node.Name,
-		Type:        node.Type,
+		Type:        string(node.Type),
 		Path:        path,
 		UID:         node.UID,
 		GID:         node.GID,
@@ -153,34 +160,35 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
 	}
 	// Always print size for regular files, even when empty,
 	// but never for other types.
-	if node.Type == "file" {
+	if node.Type == restic.NodeTypeFile {
 		n.Size = &n.size
 	}
 
 	return enc.Encode(n)
 }
 
-func (p *jsonLsPrinter) LeaveDir(_ string) {}
-func (p *jsonLsPrinter) Close()            {}
+func (p *jsonLsPrinter) LeaveDir(_ string) error { return nil }
+func (p *jsonLsPrinter) Close() error            { return nil }
 
 type ncduLsPrinter struct {
 	out   io.Writer
 	depth int
 }
 
-// lsSnapshotNcdu prints a restic snapshot in Ncdu save format.
+// Snapshot prints a restic snapshot in Ncdu save format.
 // It opens the JSON list. Nodes are added with lsNodeNcdu and the list is closed by lsCloseNcdu.
 // Format documentation: https://dev.yorhel.nl/ncdu/jsonfmt
-func (p *ncduLsPrinter) Snapshot(sn *restic.Snapshot) {
+func (p *ncduLsPrinter) Snapshot(sn *restic.Snapshot) error {
 	const NcduMajorVer = 1
 	const NcduMinorVer = 2
 
 	snapshotBytes, err := json.Marshal(sn)
 	if err != nil {
-		Warnf("JSON encode failed: %v\n", err)
+		return err
 	}
 	p.depth++
-	fmt.Fprintf(p.out, "[%d, %d, %s, [{\"name\":\"/\"}", NcduMajorVer, NcduMinorVer, string(snapshotBytes))
+	_, err = fmt.Fprintf(p.out, "[%d, %d, %s, [{\"name\":\"/\"}", NcduMajorVer, NcduMinorVer, string(snapshotBytes))
+	return err
 }
 
 func lsNcduNode(_ string, node *restic.Node) ([]byte, error) {
@@ -208,7 +216,7 @@ func lsNcduNode(_ string, node *restic.Node) ([]byte, error) {
 		Dev:    node.DeviceID,
 		Ino:    node.Inode,
 		NLink:  node.Links,
-		NotReg: node.Type != "dir" && node.Type != "file",
+		NotReg: node.Type != restic.NodeTypeDir && node.Type != restic.NodeTypeFile,
 		UID:    node.UID,
 		GID:    node.GID,
 		Mode:   uint16(node.Mode & os.ModePerm),
@@ -232,27 +240,30 @@ func lsNcduNode(_ string, node *restic.Node) ([]byte, error) {
 	return json.Marshal(outNode)
 }
 
-func (p *ncduLsPrinter) Node(path string, node *restic.Node, _ bool) {
+func (p *ncduLsPrinter) Node(path string, node *restic.Node, _ bool) error {
 	out, err := lsNcduNode(path, node)
 	if err != nil {
-		Warnf("JSON encode failed: %v\n", err)
+		return err
 	}
 
-	if node.Type == "dir" {
-		fmt.Fprintf(p.out, ",\n%s[\n%s%s", strings.Repeat("  ", p.depth), strings.Repeat("  ", p.depth+1), string(out))
+	if node.Type == restic.NodeTypeDir {
+		_, err = fmt.Fprintf(p.out, ",\n%s[\n%s%s", strings.Repeat("  ", p.depth), strings.Repeat("  ", p.depth+1), string(out))
 		p.depth++
 	} else {
-		fmt.Fprintf(p.out, ",\n%s%s", strings.Repeat("  ", p.depth), string(out))
+		_, err = fmt.Fprintf(p.out, ",\n%s%s", strings.Repeat("  ", p.depth), string(out))
 	}
+	return err
 }
 
-func (p *ncduLsPrinter) LeaveDir(_ string) {
+func (p *ncduLsPrinter) LeaveDir(_ string) error {
 	p.depth--
-	fmt.Fprintf(p.out, "\n%s]", strings.Repeat("  ", p.depth))
+	_, err := fmt.Fprintf(p.out, "\n%s]", strings.Repeat("  ", p.depth))
+	return err
 }
 
-func (p *ncduLsPrinter) Close() {
-	fmt.Fprint(p.out, "\n]\n]\n")
+func (p *ncduLsPrinter) Close() error {
+	_, err := fmt.Fprint(p.out, "\n]\n]\n")
+	return err
 }
 
 type textLsPrinter struct {
@@ -261,17 +272,29 @@ type textLsPrinter struct {
 	HumanReadable bool
 }
 
-func (p *textLsPrinter) Snapshot(sn *restic.Snapshot) {
+func (p *textLsPrinter) Snapshot(sn *restic.Snapshot) error {
 	Verbosef("%v filtered by %v:\n", sn, p.dirs)
+	return nil
 }
-func (p *textLsPrinter) Node(path string, node *restic.Node, isPrefixDirectory bool) {
+func (p *textLsPrinter) Node(path string, node *restic.Node, isPrefixDirectory bool) error {
 	if !isPrefixDirectory {
 		Printf("%s\n", formatNode(path, node, p.ListLong, p.HumanReadable))
 	}
+	return nil
 }
 
-func (p *textLsPrinter) LeaveDir(_ string) {}
-func (p *textLsPrinter) Close()            {}
+func (p *textLsPrinter) LeaveDir(_ string) error {
+	return nil
+}
+func (p *textLsPrinter) Close() error {
+	return nil
+}
+
+// for ls -l output sorting
+type toSortOutput struct {
+	nodepath string
+	node     *restic.Node
+}
 
 func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error {
 	if len(args) == 0 {
@@ -280,6 +303,12 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 	if opts.Ncdu && gopts.JSON {
 		return errors.Fatal("only either '--json' or '--ncdu' can be specified")
 	}
+	if opts.Sort != SortModeName && opts.Ncdu {
+		return errors.Fatal("--sort and --ncdu are mutually exclusive")
+	}
+	if opts.Reverse && opts.Ncdu {
+		return errors.Fatal("--reverse and --ncdu are mutually exclusive")
+	}
 
 	// extract any specific directories to walk
 	var dirs []string
@@ -359,6 +388,13 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 			HumanReadable: opts.HumanReadable,
 		}
 	}
+	if opts.Sort != SortModeName || opts.Reverse {
+		printer = &sortedPrinter{
+			printer:  printer,
+			sortMode: opts.Sort,
+			reverse:  opts.Reverse,
+		}
+	}
 
 	sn, subfolder, err := (&restic.SnapshotFilter{
 		Hosts: opts.Hosts,
@@ -374,7 +410,9 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		return err
 	}
 
-	printer.Snapshot(sn)
+	if err := printer.Snapshot(sn); err != nil {
+		return err
+	}
 
 	processNode := func(_ restic.ID, nodepath string, node *restic.Node, err error) error {
 		if err != nil {
@@ -387,7 +425,9 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		printedDir := false
 		if withinDir(nodepath) {
 			// if we're within a target path, print the node
-			printer.Node(nodepath, node, false)
+			if err := printer.Node(nodepath, node, false); err != nil {
+				return err
+			}
 			printedDir = true
 
 			// if recursive listing is requested, signal the walker that it
@@ -402,17 +442,19 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		if approachingMatchingTree(nodepath) {
 			// print node leading up to the target paths
 			if !printedDir {
-				printer.Node(nodepath, node, true)
+				return printer.Node(nodepath, node, true)
 			}
 			return nil
 		}
 
 		// otherwise, signal the walker to not walk recursively into any
 		// subdirs
-		if node.Type == "dir" {
+		if node.Type == restic.NodeTypeDir {
 			// immediately generate leaveDir if the directory is skipped
 			if printedDir {
-				printer.LeaveDir(nodepath)
+				if err := printer.LeaveDir(nodepath); err != nil {
+					return err
+				}
 			}
 			return walker.ErrSkipNode
 		}
@@ -421,11 +463,12 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 
 	err = walker.Walk(ctx, repo, *sn.Tree, walker.WalkVisitor{
 		ProcessNode: processNode,
-		LeaveDir: func(path string) {
+		LeaveDir: func(path string) error {
 			// the root path `/` has no corresponding node and is thus also skipped by processNode
 			if path != "/" {
-				printer.LeaveDir(path)
+				return printer.LeaveDir(path)
 			}
+			return nil
 		},
 	})
 
@@ -433,6 +476,147 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		return err
 	}
 
-	printer.Close()
+	return printer.Close()
+}
+
+type sortedPrinter struct {
+	printer   lsPrinter
+	collector []toSortOutput
+	sortMode  SortMode
+	reverse   bool
+}
+
+func (p *sortedPrinter) Snapshot(sn *restic.Snapshot) error {
+	return p.printer.Snapshot(sn)
+}
+func (p *sortedPrinter) Node(path string, node *restic.Node, isPrefixDirectory bool) error {
+	if !isPrefixDirectory {
+		p.collector = append(p.collector, toSortOutput{path, node})
+	}
 	return nil
 }
+
+func (p *sortedPrinter) LeaveDir(_ string) error {
+	return nil
+}
+func (p *sortedPrinter) Close() error {
+	var comparator func(a, b toSortOutput) int
+	switch p.sortMode {
+	case SortModeName:
+	case SortModeSize:
+		comparator = func(a, b toSortOutput) int {
+			return cmp.Or(
+				cmp.Compare(a.node.Size, b.node.Size),
+				cmp.Compare(a.nodepath, b.nodepath),
+			)
+		}
+	case SortModeMtime:
+		comparator = func(a, b toSortOutput) int {
+			return cmp.Or(
+				a.node.ModTime.Compare(b.node.ModTime),
+				cmp.Compare(a.nodepath, b.nodepath),
+			)
+		}
+	case SortModeAtime:
+		comparator = func(a, b toSortOutput) int {
+			return cmp.Or(
+				a.node.AccessTime.Compare(b.node.AccessTime),
+				cmp.Compare(a.nodepath, b.nodepath),
+			)
+		}
+	case SortModeCtime:
+		comparator = func(a, b toSortOutput) int {
+			return cmp.Or(
+				a.node.ChangeTime.Compare(b.node.ChangeTime),
+				cmp.Compare(a.nodepath, b.nodepath),
+			)
+		}
+	case SortModeExt:
+		// map name to extension
+		mapExt := make(map[string]string, len(p.collector))
+		for _, item := range p.collector {
+			ext := filepath.Ext(item.nodepath)
+			mapExt[item.nodepath] = ext
+		}
+
+		comparator = func(a, b toSortOutput) int {
+			return cmp.Or(
+				cmp.Compare(mapExt[a.nodepath], mapExt[b.nodepath]),
+				cmp.Compare(a.nodepath, b.nodepath),
+			)
+		}
+	}
+
+	if comparator != nil {
+		slices.SortStableFunc(p.collector, comparator)
+	}
+	if p.reverse {
+		slices.Reverse(p.collector)
+	}
+	for _, elem := range p.collector {
+		if err := p.printer.Node(elem.nodepath, elem.node, false); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// SortMode defines the allowed sorting modes
+type SortMode uint
+
+// Allowed sort modes
+const (
+	SortModeName SortMode = iota
+	SortModeSize
+	SortModeAtime
+	SortModeCtime
+	SortModeMtime
+	SortModeExt
+	SortModeInvalid
+)
+
+// Set implements the method needed for pflag command flag parsing.
+func (c *SortMode) Set(s string) error {
+	switch s {
+	case "name":
+		*c = SortModeName
+	case "size":
+		*c = SortModeSize
+	case "atime":
+		*c = SortModeAtime
+	case "ctime":
+		*c = SortModeCtime
+	case "mtime", "time":
+		*c = SortModeMtime
+	case "extension":
+		*c = SortModeExt
+	default:
+		*c = SortModeInvalid
+		return fmt.Errorf("invalid sort mode %q, must be one of (name|size|time=mtime|atime|ctime|extension)", s)
+	}
+
+	return nil
+}
+
+func (c *SortMode) String() string {
+	switch *c {
+	case SortModeName:
+		return "name"
+	case SortModeSize:
+		return "size"
+	case SortModeAtime:
+		return "atime"
+	case SortModeCtime:
+		return "ctime"
+	case SortModeMtime:
+		return "mtime"
+	case SortModeExt:
+		return "extension"
+	default:
+		return "invalid"
+	}
+}
+
+func (c *SortMode) Type() string {
+	return "mode"
+}

cmd/restic/cmd_ls_integration_test.go

@@ -1,11 +1,14 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 	"strings"
 	"testing"
 
+	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 )
 
@@ -49,3 +52,112 @@ func TestRunLsNcdu(t *testing.T) {
 		assertIsValidJSON(t, ncdu)
 	}
 }
+
+func TestRunLsSort(t *testing.T) {
+	rtest.Equals(t, SortMode(0), SortModeName, "unexpected default sort mode")
+
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{}
+	testRunBackup(t, env.testdata+"/0", []string{"for_cmd_ls"}, opts, env.gopts)
+
+	for _, test := range []struct {
+		mode     SortMode
+		expected []string
+	}{
+		{
+			SortModeSize,
+			[]string{
+				"/for_cmd_ls",
+				"/for_cmd_ls/file2.txt",
+				"/for_cmd_ls/file1.txt",
+				"/for_cmd_ls/python.py",
+				"",
+			},
+		},
+		{
+			SortModeExt,
+			[]string{
+				"/for_cmd_ls",
+				"/for_cmd_ls/python.py",
+				"/for_cmd_ls/file1.txt",
+				"/for_cmd_ls/file2.txt",
+				"",
+			},
+		},
+		{
+			SortModeName,
+			[]string{
+				"/for_cmd_ls",
+				"/for_cmd_ls/file1.txt",
+				"/for_cmd_ls/file2.txt",
+				"/for_cmd_ls/python.py",
+				"", // last empty line
+			},
+		},
+	} {
+		out := testRunLsWithOpts(t, env.gopts, LsOptions{Sort: test.mode}, []string{"latest"})
+		fileList := strings.Split(string(out), "\n")
+		rtest.Equals(t, test.expected, fileList, fmt.Sprintf("mismatch for mode %v", test.mode))
+	}
+}
+
+// JSON lines test
+func TestRunLsJson(t *testing.T) {
+	pathList := []string{
+		"/0",
+		"/0/for_cmd_ls",
+		"/0/for_cmd_ls/file1.txt",
+		"/0/for_cmd_ls/file2.txt",
+		"/0/for_cmd_ls/python.py",
+	}
+
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{}
+	testRunBackup(t, env.testdata, []string{"0/for_cmd_ls"}, opts, env.gopts)
+	snapshotIDs := testListSnapshots(t, env.gopts, 1)
+
+	env.gopts.Quiet = true
+	env.gopts.JSON = true
+	buf := testRunLsWithOpts(t, env.gopts, LsOptions{}, []string{"latest"})
+	byteLines := bytes.Split(buf, []byte{'\n'})
+
+	// partial copy of snapshot structure from cmd_ls
+	type lsSnapshot struct {
+		*restic.Snapshot
+		ID          *restic.ID `json:"id"`
+		ShortID     string     `json:"short_id"`     // deprecated
+		MessageType string     `json:"message_type"` // "snapshot"
+		StructType  string     `json:"struct_type"`  // "snapshot", deprecated
+	}
+
+	var snappy lsSnapshot
+	rtest.OK(t, json.Unmarshal(byteLines[0], &snappy))
+	rtest.Equals(t, snappy.ShortID, snapshotIDs[0].Str(), "expected snap IDs to be identical")
+
+	// partial copy of node structure from cmd_ls
+	type lsNode struct {
+		Name        string `json:"name"`
+		Type        string `json:"type"`
+		Path        string `json:"path"`
+		Permissions string `json:"permissions,omitempty"`
+		Inode       uint64 `json:"inode,omitempty"`
+		MessageType string `json:"message_type"` // "node"
+		StructType  string `json:"struct_type"`  // "node", deprecated
+	}
+
+	var testNode lsNode
+	for i, nodeLine := range byteLines[1:] {
+		if len(nodeLine) == 0 {
+			break
+		}
+
+		rtest.OK(t, json.Unmarshal(nodeLine, &testNode))
+		rtest.Equals(t, pathList[i], testNode.Path)
+	}
+}

cmd/restic/cmd_ls_test.go

@@ -23,7 +23,7 @@ var lsTestNodes = []lsTestNode{
 		path: "/bar/baz",
 		Node: restic.Node{
 			Name: "baz",
-			Type: "file",
+			Type: restic.NodeTypeFile,
 			Size: 12345,
 			UID:  10000000,
 			GID:  20000000,
@@ -39,7 +39,7 @@ var lsTestNodes = []lsTestNode{
 		path: "/foo/empty",
 		Node: restic.Node{
 			Name: "empty",
-			Type: "file",
+			Type: restic.NodeTypeFile,
 			Size: 0,
 			UID:  1001,
 			GID:  1001,
@@ -56,7 +56,7 @@ var lsTestNodes = []lsTestNode{
 		path: "/foo/link",
 		Node: restic.Node{
 			Name:       "link",
-			Type:       "symlink",
+			Type:       restic.NodeTypeSymlink,
 			Mode:       os.ModeSymlink | 0777,
 			LinkTarget: "not printed",
 		},
@@ -66,7 +66,7 @@ var lsTestNodes = []lsTestNode{
 		path: "/some/directory",
 		Node: restic.Node{
 			Name:       "directory",
-			Type:       "dir",
+			Type:       restic.NodeTypeDir,
 			Mode:       os.ModeDir | 0755,
 			ModTime:    time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC),
 			AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC),
@@ -79,7 +79,7 @@ var lsTestNodes = []lsTestNode{
 		path: "/some/sticky",
 		Node: restic.Node{
 			Name: "sticky",
-			Type: "dir",
+			Type: restic.NodeTypeDir,
 			Mode: os.ModeDir | 0755 | os.ModeSetuid | os.ModeSetgid | os.ModeSticky,
 		},
 	},
@@ -134,29 +134,29 @@ func TestLsNcdu(t *testing.T) {
 	}
 	modTime := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC)
 
-	printer.Snapshot(&restic.Snapshot{
+	rtest.OK(t, printer.Snapshot(&restic.Snapshot{
 		Hostname: "host",
 		Paths:    []string{"/example"},
-	})
-	printer.Node("/directory", &restic.Node{
-		Type:    "dir",
+	}))
+	rtest.OK(t, printer.Node("/directory", &restic.Node{
+		Type:    restic.NodeTypeDir,
 		Name:    "directory",
 		ModTime: modTime,
-	}, false)
-	printer.Node("/directory/data", &restic.Node{
-		Type:    "file",
+	}, false))
+	rtest.OK(t, printer.Node("/directory/data", &restic.Node{
+		Type:    restic.NodeTypeFile,
 		Name:    "data",
 		Size:    42,
 		ModTime: modTime,
-	}, false)
-	printer.LeaveDir("/directory")
-	printer.Node("/file", &restic.Node{
-		Type:    "file",
+	}, false))
+	rtest.OK(t, printer.LeaveDir("/directory"))
+	rtest.OK(t, printer.Node("/file", &restic.Node{
+		Type:    restic.NodeTypeFile,
 		Name:    "file",
 		Size:    12345,
 		ModTime: modTime,
-	}, false)
-	printer.Close()
+	}, false))
+	rtest.OK(t, printer.Close())
 
 	rtest.Equals(t, `[1, 2, {"time":"0001-01-01T00:00:00Z","tree":null,"paths":["/example"],"hostname":"host"}, [{"name":"/"},
   [

cmd/restic/cmd_migrate.go

@@ -9,12 +9,16 @@ import (
 	"github.com/restic/restic/internal/ui/termstatus"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 )
 
-var cmdMigrate = &cobra.Command{
-	Use:   "migrate [flags] [migration name] [...]",
-	Short: "Apply migrations",
-	Long: `
+func newMigrateCommand() *cobra.Command {
+	var opts MigrateOptions
+
+	cmd := &cobra.Command{
+		Use:   "migrate [flags] [migration name] [...]",
+		Short: "Apply migrations",
+		Long: `
 The "migrate" command checks which migrations can be applied for a repository
 and prints a list with available migration names. If one or more migration
 names are specified, these migrations are applied.
@@ -28,13 +32,17 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	DisableAutoGenTag: true,
-	GroupID:           cmdGroupDefault,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		term, cancel := setupTermstatus()
-		defer cancel()
-		return runMigrate(cmd.Context(), migrateOptions, globalOptions, args, term)
-	},
+		DisableAutoGenTag: true,
+		GroupID:           cmdGroupDefault,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			term, cancel := setupTermstatus()
+			defer cancel()
+			return runMigrate(cmd.Context(), opts, globalOptions, args, term)
+		},
+	}
+
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 // MigrateOptions bundles all options for the 'check' command.
@@ -42,12 +50,8 @@ type MigrateOptions struct {
 	Force bool
 }
 
-var migrateOptions MigrateOptions
-
-func init() {
-	cmdRoot.AddCommand(cmdMigrate)
-	f := cmdMigrate.Flags()
-	f.BoolVarP(&migrateOptions.Force, "force", "f", false, `apply a migration a second time`)
+func (opts *MigrateOptions) AddFlags(f *pflag.FlagSet) {
+	f.BoolVarP(&opts.Force, "force", "f", false, `apply a migration a second time`)
 }
 
 func checkMigrations(ctx context.Context, repo restic.Repository, printer progress.Printer) error {
@@ -105,7 +109,7 @@ func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptio
 					// the repository is already locked
 					checkGopts.NoLock = true
 
-					err = runCheck(ctx, checkOptions, checkGopts, []string{}, term)
+					_, err = runCheck(ctx, checkOptions, checkGopts, []string{}, term)
 					if err != nil {
 						return err
 					}

cmd/restic/cmd_mount.go

@@ -5,27 +5,35 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
 
-	resticfs "github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/fuse"
 
 	systemFuse "github.com/anacrolix/fuse"
 	"github.com/anacrolix/fuse/fs"
 )
 
-var cmdMount = &cobra.Command{
-	Use:   "mount [flags] mountpoint",
-	Short: "Mount the repository",
-	Long: `
+func registerMountCommand(cmdRoot *cobra.Command) {
+	cmdRoot.AddCommand(newMountCommand())
+}
+
+func newMountCommand() *cobra.Command {
+	var opts MountOptions
+
+	cmd := &cobra.Command{
+		Use:   "mount [flags] mountpoint",
+		Short: "Mount the repository",
+		Long: `
 The "mount" command mounts the repository via fuse to a directory. This is a
 read-only mount.
 
@@ -70,11 +78,15 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	DisableAutoGenTag: true,
-	GroupID:           cmdGroupDefault,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return runMount(cmd.Context(), mountOptions, globalOptions, args)
-	},
+		DisableAutoGenTag: true,
+		GroupID:           cmdGroupDefault,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runMount(cmd.Context(), opts, globalOptions, args)
+		},
+	}
+
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 // MountOptions collects all options for the mount command.
@@ -87,22 +99,17 @@ type MountOptions struct {
 	PathTemplates []string
 }
 
-var mountOptions MountOptions
+func (opts *MountOptions) AddFlags(f *pflag.FlagSet) {
+	f.BoolVar(&opts.OwnerRoot, "owner-root", false, "use 'root' as the owner of files and dirs")
+	f.BoolVar(&opts.AllowOther, "allow-other", false, "allow other users to access the data in the mounted directory")
+	f.BoolVar(&opts.NoDefaultPermissions, "no-default-permissions", false, "for 'allow-other', ignore Unix permissions and allow users to read all snapshot files")
 
-func init() {
-	cmdRoot.AddCommand(cmdMount)
+	initMultiSnapshotFilter(f, &opts.SnapshotFilter, true)
 
-	mountFlags := cmdMount.Flags()
-	mountFlags.BoolVar(&mountOptions.OwnerRoot, "owner-root", false, "use 'root' as the owner of files and dirs")
-	mountFlags.BoolVar(&mountOptions.AllowOther, "allow-other", false, "allow other users to access the data in the mounted directory")
-	mountFlags.BoolVar(&mountOptions.NoDefaultPermissions, "no-default-permissions", false, "for 'allow-other', ignore Unix permissions and allow users to read all snapshot files")
-
-	initMultiSnapshotFilter(mountFlags, &mountOptions.SnapshotFilter, true)
-
-	mountFlags.StringArrayVar(&mountOptions.PathTemplates, "path-template", nil, "set `template` for path names (can be specified multiple times)")
-	mountFlags.StringVar(&mountOptions.TimeTemplate, "snapshot-template", time.RFC3339, "set `template` to use for snapshot dirs")
-	mountFlags.StringVar(&mountOptions.TimeTemplate, "time-template", time.RFC3339, "set `template` to use for times")
-	_ = mountFlags.MarkDeprecated("snapshot-template", "use --time-template")
+	f.StringArrayVar(&opts.PathTemplates, "path-template", nil, "set `template` for path names (can be specified multiple times)")
+	f.StringVar(&opts.TimeTemplate, "snapshot-template", time.RFC3339, "set `template` to use for snapshot dirs")
+	f.StringVar(&opts.TimeTemplate, "time-template", time.RFC3339, "set `template` to use for times")
+	_ = f.MarkDeprecated("snapshot-template", "use --time-template")
 }
 
 func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args []string) error {
@@ -122,7 +129,7 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args
 
 	// Check the existence of the mount point at the earliest stage to
 	// prevent unnecessary computations while opening the repository.
-	if _, err := resticfs.Stat(mountpoint); errors.Is(err, os.ErrNotExist) {
+	if _, err := os.Stat(mountpoint); errors.Is(err, os.ErrNotExist) {
 		Verbosef("Mountpoint %s doesn't exist\n", mountpoint)
 		return err
 	}
@@ -142,9 +149,11 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args
 		return err
 	}
 
+	fuseMountName := fmt.Sprintf("restic:%s", repo.Config().ID[:10])
+
 	mountOptions := []systemFuse.MountOption{
 		systemFuse.ReadOnly(),
-		systemFuse.FSName("restic"),
+		systemFuse.FSName(fuseMountName),
 		systemFuse.MaxReadahead(128 * 1024),
 	}
 

cmd/restic/cmd_mount_disabled.go

@@ -0,0 +1,10 @@
+//go:build !darwin && !freebsd && !linux
+// +build !darwin,!freebsd,!linux
+
+package main
+
+import "github.com/spf13/cobra"
+
+func registerMountCommand(_ *cobra.Command) {
+	// Mount command not supported on these platforms
+}

cmd/restic/cmd_options.go

@@ -8,10 +8,11 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var optionsCmd = &cobra.Command{
-	Use:   "options",
-	Short: "Print list of extended options",
-	Long: `
+func newOptionsCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "options",
+		Short: "Print list of extended options",
+		Long: `
 The "options" command prints a list of extended options.
 
 EXIT STATUS
@@ -20,22 +21,20 @@ EXIT STATUS
 Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 `,
-	GroupID:           cmdGroupAdvanced,
-	DisableAutoGenTag: true,
-	Run: func(_ *cobra.Command, _ []string) {
-		fmt.Printf("All Extended Options:\n")
-		var maxLen int
-		for _, opt := range options.List() {
-			if l := len(opt.Namespace + "." + opt.Name); l > maxLen {
-				maxLen = l
+		GroupID:           cmdGroupAdvanced,
+		DisableAutoGenTag: true,
+		Run: func(_ *cobra.Command, _ []string) {
+			fmt.Printf("All Extended Options:\n")
+			var maxLen int
+			for _, opt := range options.List() {
+				if l := len(opt.Namespace + "." + opt.Name); l > maxLen {
+					maxLen = l
+				}
 			}
-		}
-		for _, opt := range options.List() {
-			fmt.Printf("  %*s  %s\n", -maxLen, opt.Namespace+"."+opt.Name, opt.Text)
-		}
-	},
-}
-
-func init() {
-	cmdRoot.AddCommand(optionsCmd)
+			for _, opt := range options.List() {
+				fmt.Printf("  %*s  %s\n", -maxLen, opt.Namespace+"."+opt.Name, opt.Text)
+			}
+		},
+	}
+	return cmd
 }

cmd/restic/cmd_prune.go

@@ -16,12 +16,16 @@ import (
 	"github.com/restic/restic/internal/ui/termstatus"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 )
 
-var cmdPrune = &cobra.Command{
-	Use:   "prune [flags]",
-	Short: "Remove unneeded data from the repository",
-	Long: `
+func newPruneCommand() *cobra.Command {
+	var opts PruneOptions
+
+	cmd := &cobra.Command{
+		Use:   "prune [flags]",
+		Short: "Remove unneeded data from the repository",
+		Long: `
 The "prune" command checks the repository and removes data that is not
 referenced and therefore not needed any more.
 
@@ -34,13 +38,17 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	GroupID:           cmdGroupDefault,
-	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, _ []string) error {
-		term, cancel := setupTermstatus()
-		defer cancel()
-		return runPrune(cmd.Context(), pruneOptions, globalOptions, term)
-	},
+		GroupID:           cmdGroupDefault,
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			term, cancel := setupTermstatus()
+			defer cancel()
+			return runPrune(cmd.Context(), opts, globalOptions, term)
+		},
+	}
+
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 // PruneOptions collects all options for the cleanup command.
@@ -59,25 +67,24 @@ type PruneOptions struct {
 	RepackCacheableOnly bool
 	RepackSmall         bool
 	RepackUncompressed  bool
+
+	SmallPackSize  string
+	SmallPackBytes uint64
 }
 
-var pruneOptions PruneOptions
-
-func init() {
-	cmdRoot.AddCommand(cmdPrune)
-	f := cmdPrune.Flags()
-	f.BoolVarP(&pruneOptions.DryRun, "dry-run", "n", false, "do not modify the repository, just print what would be done")
-	f.StringVarP(&pruneOptions.UnsafeNoSpaceRecovery, "unsafe-recover-no-free-space", "", "", "UNSAFE, READ THE DOCUMENTATION BEFORE USING! Try to recover a repository stuck with no free space. Do not use without trying out 'prune --max-repack-size 0' first.")
-	addPruneOptions(cmdPrune, &pruneOptions)
+func (opts *PruneOptions) AddFlags(f *pflag.FlagSet) {
+	opts.AddLimitedFlags(f)
+	f.BoolVarP(&opts.DryRun, "dry-run", "n", false, "do not modify the repository, just print what would be done")
+	f.StringVarP(&opts.UnsafeNoSpaceRecovery, "unsafe-recover-no-free-space", "", "", "UNSAFE, READ THE DOCUMENTATION BEFORE USING! Try to recover a repository stuck with no free space. Do not use without trying out 'prune --max-repack-size 0' first.")
 }
 
-func addPruneOptions(c *cobra.Command, pruneOptions *PruneOptions) {
-	f := c.Flags()
-	f.StringVar(&pruneOptions.MaxUnused, "max-unused", "5%", "tolerate given `limit` of unused data (absolute value in bytes with suffixes k/K, m/M, g/G, t/T, a value in % or the word 'unlimited')")
-	f.StringVar(&pruneOptions.MaxRepackSize, "max-repack-size", "", "maximum `size` to repack (allowed suffixes: k/K, m/M, g/G, t/T)")
-	f.BoolVar(&pruneOptions.RepackCacheableOnly, "repack-cacheable-only", false, "only repack packs which are cacheable")
-	f.BoolVar(&pruneOptions.RepackSmall, "repack-small", false, "repack pack files below 80% of target pack size")
-	f.BoolVar(&pruneOptions.RepackUncompressed, "repack-uncompressed", false, "repack all uncompressed data")
+func (opts *PruneOptions) AddLimitedFlags(f *pflag.FlagSet) {
+	f.StringVar(&opts.MaxUnused, "max-unused", "5%", "tolerate given `limit` of unused data (absolute value in bytes with suffixes k/K, m/M, g/G, t/T, a value in % or the word 'unlimited')")
+	f.StringVar(&opts.MaxRepackSize, "max-repack-size", "", "stop after repacking this much data in total (allowed suffixes for `size`: k/K, m/M, g/G, t/T)")
+	f.BoolVar(&opts.RepackCacheableOnly, "repack-cacheable-only", false, "only repack packs which are cacheable")
+	f.BoolVar(&opts.RepackSmall, "repack-small", false, "repack pack files below 80% of target pack size")
+	f.BoolVar(&opts.RepackUncompressed, "repack-uncompressed", false, "repack all uncompressed data")
+	f.StringVar(&opts.SmallPackSize, "repack-smaller-than", "", "pack `below-limit` packfiles (allowed suffixes: k/K, m/M)")
 }
 
 func verifyPruneOptions(opts *PruneOptions) error {
@@ -136,6 +143,15 @@ func verifyPruneOptions(opts *PruneOptions) error {
 		}
 	}
 
+	if opts.SmallPackSize != "" {
+		size, err := ui.ParseBytes(opts.SmallPackSize)
+		if err != nil {
+			return errors.Fatalf("invalid number of bytes %q for --repack-smaller-than: %v", opts.SmallPackSize, err)
+		}
+		opts.SmallPackBytes = uint64(size)
+		opts.RepackSmall = true
+	}
+
 	return nil
 }
 
@@ -149,7 +165,11 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, term
 		return errors.Fatal("disabled compression and `--repack-uncompressed` are mutually exclusive")
 	}
 
-	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
+	if gopts.NoLock && !opts.DryRun {
+		return errors.Fatal("--no-lock is only applicable in combination with --dry-run for prune command")
+	}
+
+	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, opts.DryRun && gopts.NoLock)
 	if err != nil {
 		return err
 	}
@@ -167,7 +187,7 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, term
 }
 
 func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet, term *termstatus.Terminal) error {
-	if repo.Cache == nil {
+	if repo.Cache() == nil {
 		Print("warning: running prune without a cache, this may be very slow!\n")
 	}
 
@@ -187,6 +207,7 @@ func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOption
 
 		MaxUnusedBytes: opts.maxUnusedBytes,
 		MaxRepackBytes: opts.MaxRepackBytes,
+		SmallPackBytes: opts.SmallPackBytes,
 
 		RepackCacheableOnly: opts.RepackCacheableOnly,
 		RepackSmall:         opts.RepackSmall,

cmd/restic/cmd_prune_integration_test.go

@@ -13,14 +13,25 @@ import (
 )
 
 func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) {
+	t.Helper()
+	rtest.OK(t, testRunPruneOutput(gopts, opts))
+}
+
+func testRunPruneMustFail(t testing.TB, gopts GlobalOptions, opts PruneOptions) {
+	t.Helper()
+	err := testRunPruneOutput(gopts, opts)
+	rtest.Assert(t, err != nil, "expected non nil error")
+}
+
+func testRunPruneOutput(gopts GlobalOptions, opts PruneOptions) error {
 	oldHook := gopts.backendTestHook
 	gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil }
 	defer func() {
 		gopts.backendTestHook = oldHook
 	}()
-	rtest.OK(t, withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
+	return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
 		return runPrune(context.TODO(), opts, gopts, term)
-	}))
+	})
 }
 
 func TestPrune(t *testing.T) {
@@ -112,7 +123,8 @@ func testPrune(t *testing.T, pruneOpts PruneOptions, checkOpts CheckOptions) {
 	createPrunableRepo(t, env)
 	testRunPrune(t, env.gopts, pruneOpts)
 	rtest.OK(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
-		return runCheck(context.TODO(), checkOpts, env.gopts, nil, term)
+		_, err := runCheck(context.TODO(), checkOpts, env.gopts, nil, term)
+		return err
 	}))
 }
 
@@ -220,7 +232,8 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o
 		testRunCheck(t, env.gopts)
 	} else {
 		rtest.Assert(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
-			return runCheck(context.TODO(), optionsCheck, env.gopts, nil, term)
+			_, err := runCheck(context.TODO(), optionsCheck, env.gopts, nil, term)
+			return err
 		}) != nil,
 			"check should have reported an error")
 	}
@@ -235,3 +248,20 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o
 			"prune should have reported an error")
 	}
 }
+
+func TestPruneRepackSmallerThanSmoke(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	// the implementation is already unit tested, so just check that
+	// the setting reaches its goal
+	createPrunableRepo(t, env)
+	testRunPrune(t, env.gopts, PruneOptions{
+		SmallPackSize: "4M",
+		MaxUnused:     "5%",
+	})
+	testRunPruneMustFail(t, env.gopts, PruneOptions{
+		SmallPackSize: "500M",
+		MaxUnused:     "5%",
+	})
+}

cmd/restic/cmd_recover.go

@@ -6,15 +6,19 @@ import (
 	"time"
 
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui/progress"
+	"github.com/restic/restic/internal/ui/termstatus"
 	"github.com/spf13/cobra"
 	"golang.org/x/sync/errgroup"
 )
 
-var cmdRecover = &cobra.Command{
-	Use:   "recover [flags]",
-	Short: "Recover data from the repository not referenced by snapshots",
-	Long: `
+func newRecoverCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "recover [flags]",
+		Short: "Recover data from the repository not referenced by snapshots",
+		Long: `
 The "recover" command builds a new snapshot from all directories it can find in
 the raw data of the repository which are not referenced in an existing snapshot.
 It can be used if, for example, a snapshot has been removed by accident with "forget".
@@ -28,36 +32,44 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	GroupID:           cmdGroupDefault,
-	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, _ []string) error {
-		return runRecover(cmd.Context(), globalOptions)
-	},
+		GroupID:           cmdGroupDefault,
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			term, cancel := setupTermstatus()
+			defer cancel()
+			return runRecover(cmd.Context(), globalOptions, term)
+		},
+	}
+	return cmd
 }
 
-func init() {
-	cmdRoot.AddCommand(cmdRecover)
-}
-
-func runRecover(ctx context.Context, gopts GlobalOptions) error {
+func runRecover(ctx context.Context, gopts GlobalOptions, term *termstatus.Terminal) error {
 	hostname, err := os.Hostname()
 	if err != nil {
 		return err
 	}
 
-	ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, false)
+	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
 	if err != nil {
 		return err
 	}
 	defer unlock()
 
+	printer := newTerminalProgressPrinter(gopts.verbosity, term)
+
 	snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
 	if err != nil {
 		return err
 	}
 
-	Verbosef("load index files\n")
-	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	printer.P("ensuring index is complete\n")
+	err = repository.RepairIndex(ctx, repo, repository.RepairIndexOptions{}, printer)
+	if err != nil {
+		return err
+	}
+
+	printer.P("load index files\n")
+	bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
 	if err = repo.LoadIndex(ctx, bar); err != nil {
 		return err
 	}
@@ -75,20 +87,20 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
 		return err
 	}
 
-	Verbosef("load %d trees\n", len(trees))
-	bar = newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
+	printer.P("load %d trees\n", len(trees))
+	bar = newTerminalProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded", term)
 	for id := range trees {
 		tree, err := restic.LoadTree(ctx, repo, id)
 		if ctx.Err() != nil {
 			return ctx.Err()
 		}
 		if err != nil {
-			Warnf("unable to load tree %v: %v\n", id.Str(), err)
+			printer.E("unable to load tree %v: %v\n", id.Str(), err)
 			continue
 		}
 
 		for _, node := range tree.Nodes {
-			if node.Type == "dir" && node.Subtree != nil {
+			if node.Type == restic.NodeTypeDir && node.Subtree != nil {
 				trees[*node.Subtree] = true
 			}
 		}
@@ -96,7 +108,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
 	}
 	bar.Done()
 
-	Verbosef("load snapshots\n")
+	printer.P("load snapshots\n")
 	err = restic.ForAllSnapshots(ctx, snapshotLister, repo, nil, func(_ restic.ID, sn *restic.Snapshot, _ error) error {
 		trees[*sn.Tree] = true
 		return nil
@@ -104,19 +116,19 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
 	if err != nil {
 		return err
 	}
-	Verbosef("done\n")
+	printer.P("done\n")
 
 	roots := restic.NewIDSet()
 	for id, seen := range trees {
 		if !seen {
-			Verboseff("found root tree %v\n", id.Str())
+			printer.V("found root tree %v\n", id.Str())
 			roots.Insert(id)
 		}
 	}
-	Printf("\nfound %d unreferenced roots\n", len(roots))
+	printer.S("\nfound %d unreferenced roots\n", len(roots))
 
 	if len(roots) == 0 {
-		Verbosef("no snapshot to write.\n")
+		printer.P("no snapshot to write.\n")
 		return nil
 	}
 
@@ -128,7 +140,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
 	for id := range roots {
 		var subtreeID = id
 		node := restic.Node{
-			Type:       "dir",
+			Type:       restic.NodeTypeDir,
 			Name:       id.Str(),
 			Mode:       0755,
 			Subtree:    &subtreeID,
@@ -164,11 +176,11 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
 		return err
 	}
 
-	return createSnapshot(ctx, "/recover", hostname, []string{"recovered"}, repo, &treeID)
+	return createSnapshot(ctx, printer, "/recover", hostname, []string{"recovered"}, repo, &treeID)
 
 }
 
-func createSnapshot(ctx context.Context, name, hostname string, tags []string, repo restic.SaverUnpacked, tree *restic.ID) error {
+func createSnapshot(ctx context.Context, printer progress.Printer, name, hostname string, tags []string, repo restic.SaverUnpacked[restic.WriteableFileType], tree *restic.ID) error {
 	sn, err := restic.NewSnapshot([]string{name}, tags, hostname, time.Now())
 	if err != nil {
 		return errors.Fatalf("unable to save snapshot: %v", err)
@@ -181,6 +193,6 @@ func createSnapshot(ctx context.Context, name, hostname string, tags []string, r
 		return errors.Fatalf("unable to save snapshot: %v", err)
 	}
 
-	Printf("saved new snapshot %v\n", id.Str())
+	printer.S("saved new snapshot %v\n", id.Str())
 	return nil
 }

cmd/restic/cmd_recover_integration_test.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"context"
+	"testing"
+
+	rtest "github.com/restic/restic/internal/test"
+	"github.com/restic/restic/internal/ui/termstatus"
+)
+
+func testRunRecover(t testing.TB, gopts GlobalOptions) {
+	rtest.OK(t, withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
+		return runRecover(context.TODO(), gopts, term)
+	}))
+}
+
+func TestRecover(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	// must list index more than once
+	env.gopts.backendTestHook = nil
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+
+	// create backup and forget it afterwards
+	testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts)
+	ids := testListSnapshots(t, env.gopts, 1)
+	sn := testLoadSnapshot(t, env.gopts, ids[0])
+	testRunForget(t, env.gopts, ForgetOptions{}, ids[0].String())
+	testListSnapshots(t, env.gopts, 0)
+
+	testRunRecover(t, env.gopts)
+	ids = testListSnapshots(t, env.gopts, 1)
+	testRunCheck(t, env.gopts)
+	// check that the root tree is included in the snapshot
+	rtest.OK(t, runCat(context.TODO(), env.gopts, []string{"tree", ids[0].String() + ":" + sn.Tree.Str()}))
+}

cmd/restic/cmd_repair.go

@@ -4,13 +4,18 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var cmdRepair = &cobra.Command{
-	Use:               "repair",
-	Short:             "Repair the repository",
-	GroupID:           cmdGroupDefault,
-	DisableAutoGenTag: true,
-}
+func newRepairCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:               "repair",
+		Short:             "Repair the repository",
+		GroupID:           cmdGroupDefault,
+		DisableAutoGenTag: true,
+	}
 
-func init() {
-	cmdRoot.AddCommand(cmdRepair)
+	cmd.AddCommand(
+		newRepairIndexCommand(),
+		newRepairPacksCommand(),
+		newRepairSnapshotsCommand(),
+	)
+	return cmd
 }

cmd/restic/cmd_repair_index.go

@@ -9,10 +9,13 @@ import (
 	"github.com/spf13/pflag"
 )
 
-var cmdRepairIndex = &cobra.Command{
-	Use:   "index [flags]",
-	Short: "Build a new index",
-	Long: `
+func newRepairIndexCommand() *cobra.Command {
+	var opts RepairIndexOptions
+
+	cmd := &cobra.Command{
+		Use:   "index [flags]",
+		Short: "Build a new index",
+		Long: `
 The "repair index" command creates a new index based on the pack files in the
 repository.
 
@@ -25,21 +28,16 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, _ []string) error {
-		term, cancel := setupTermstatus()
-		defer cancel()
-		return runRebuildIndex(cmd.Context(), repairIndexOptions, globalOptions, term)
-	},
-}
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			term, cancel := setupTermstatus()
+			defer cancel()
+			return runRebuildIndex(cmd.Context(), opts, globalOptions, term)
+		},
+	}
 
-var cmdRebuildIndex = &cobra.Command{
-	Use:               "rebuild-index [flags]",
-	Short:             cmdRepairIndex.Short,
-	Long:              cmdRepairIndex.Long,
-	Deprecated:        `Use "repair index" instead`,
-	DisableAutoGenTag: true,
-	RunE:              cmdRepairIndex.RunE,
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 // RepairIndexOptions collects all options for the repair index command.
@@ -47,16 +45,31 @@ type RepairIndexOptions struct {
 	ReadAllPacks bool
 }
 
-var repairIndexOptions RepairIndexOptions
+func (opts *RepairIndexOptions) AddFlags(f *pflag.FlagSet) {
+	f.BoolVar(&opts.ReadAllPacks, "read-all-packs", false, "read all pack files to generate new index from scratch")
+}
 
-func init() {
-	cmdRepair.AddCommand(cmdRepairIndex)
-	// add alias for old name
-	cmdRoot.AddCommand(cmdRebuildIndex)
+func newRebuildIndexCommand() *cobra.Command {
+	var opts RepairIndexOptions
 
-	for _, f := range []*pflag.FlagSet{cmdRepairIndex.Flags(), cmdRebuildIndex.Flags()} {
-		f.BoolVar(&repairIndexOptions.ReadAllPacks, "read-all-packs", false, "read all pack files to generate new index from scratch")
+	replacement := newRepairIndexCommand()
+	cmd := &cobra.Command{
+		Use:               "rebuild-index [flags]",
+		Short:             replacement.Short,
+		Long:              replacement.Long,
+		Deprecated:        `Use "repair index" instead`,
+		DisableAutoGenTag: true,
+		// must create a new instance of the run function as it captures opts
+		// by reference
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			term, cancel := setupTermstatus()
+			defer cancel()
+			return runRebuildIndex(cmd.Context(), opts, globalOptions, term)
+		},
 	}
+
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 func runRebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions, term *termstatus.Terminal) error {

cmd/restic/cmd_repair_index_integration_test.go

@@ -64,11 +64,11 @@ func TestRebuildIndex(t *testing.T) {
 }
 
 func TestRebuildIndexAlwaysFull(t *testing.T) {
-	indexFull := index.IndexFull
+	indexFull := index.Full
 	defer func() {
-		index.IndexFull = indexFull
+		index.Full = indexFull
 	}()
-	index.IndexFull = func(*index.Index) bool { return true }
+	index.Full = func(*index.Index) bool { return true }
 	testRebuildIndex(t, nil)
 }
 

cmd/restic/cmd_repair_packs.go

@@ -13,10 +13,11 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var cmdRepairPacks = &cobra.Command{
-	Use:   "packs [packIDs...]",
-	Short: "Salvage damaged pack files",
-	Long: `
+func newRepairPacksCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "packs [packIDs...]",
+		Short: "Salvage damaged pack files",
+		Long: `
 The "repair packs" command extracts intact blobs from the specified pack files, rebuilds
 the index to remove the damaged pack files and removes the pack files from the repository.
 
@@ -29,16 +30,14 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		term, cancel := setupTermstatus()
-		defer cancel()
-		return runRepairPacks(cmd.Context(), globalOptions, term, args)
-	},
-}
-
-func init() {
-	cmdRepair.AddCommand(cmdRepairPacks)
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			term, cancel := setupTermstatus()
+			defer cancel()
+			return runRepairPacks(cmd.Context(), globalOptions, term, args)
+		},
+	}
+	return cmd
 }
 
 func runRepairPacks(ctx context.Context, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {

cmd/restic/cmd_repair_snapshots.go

@@ -8,12 +8,16 @@ import (
 	"github.com/restic/restic/internal/walker"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 )
 
-var cmdRepairSnapshots = &cobra.Command{
-	Use:   "snapshots [flags] [snapshot ID] [...]",
-	Short: "Repair snapshots",
-	Long: `
+func newRepairSnapshotsCommand() *cobra.Command {
+	var opts RepairOptions
+
+	cmd := &cobra.Command{
+		Use:   "snapshots [flags] [snapshot ID] [...]",
+		Short: "Repair snapshots",
+		Long: `
 The "repair snapshots" command repairs broken snapshots. It scans the given
 snapshots and generates new ones with damaged directories and file contents
 removed. If the broken snapshots are deleted, a prune run will be able to
@@ -43,10 +47,14 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return runRepairSnapshots(cmd.Context(), globalOptions, repairSnapshotOptions, args)
-	},
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runRepairSnapshots(cmd.Context(), globalOptions, opts, args)
+		},
+	}
+
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 // RepairOptions collects all options for the repair command.
@@ -57,16 +65,11 @@ type RepairOptions struct {
 	restic.SnapshotFilter
 }
 
-var repairSnapshotOptions RepairOptions
+func (opts *RepairOptions) AddFlags(f *pflag.FlagSet) {
+	f.BoolVarP(&opts.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done")
+	f.BoolVarP(&opts.Forget, "forget", "", false, "remove original snapshots after creating new ones")
 
-func init() {
-	cmdRepair.AddCommand(cmdRepairSnapshots)
-	flags := cmdRepairSnapshots.Flags()
-
-	flags.BoolVarP(&repairSnapshotOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done")
-	flags.BoolVarP(&repairSnapshotOptions.Forget, "forget", "", false, "remove original snapshots after creating new ones")
-
-	initMultiSnapshotFilter(flags, &repairSnapshotOptions.SnapshotFilter, true)
+	initMultiSnapshotFilter(f, &opts.SnapshotFilter, true)
 }
 
 func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOptions, args []string) error {
@@ -92,12 +95,16 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt
 	// - files whose contents are not fully available  (-> file will be modified)
 	rewriter := walker.NewTreeRewriter(walker.RewriteOpts{
 		RewriteNode: func(node *restic.Node, path string) *restic.Node {
-			if node.Type != "file" {
+			if node.Type == restic.NodeTypeIrregular || node.Type == restic.NodeTypeInvalid {
+				Verbosef("  file %q: removed node with invalid type %q\n", path, node.Type)
+				return nil
+			}
+			if node.Type != restic.NodeTypeFile {
 				return node
 			}
 
 			ok := true
-			var newContent restic.IDs = restic.IDs{}
+			var newContent = restic.IDs{}
 			var newSize uint64
 			// check all contents and remove if not available
 			for _, id := range node.Content {
@@ -139,8 +146,9 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt
 	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) {
 		Verbosef("\n%v\n", sn)
 		changed, err := filterAndReplaceSnapshot(ctx, repo, sn,
-			func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) {
-				return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
+			func(ctx context.Context, sn *restic.Snapshot) (restic.ID, *restic.SnapshotSummary, error) {
+				id, err := rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
+				return id, nil, err
 			}, opts.DryRun, opts.Forget, nil, "repaired")
 		if err != nil {
 			return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err)