Add version for 0.13.0

Review, reword and polish unreleased changelog entries

.dockerignore

@@ -0,0 +1,12 @@
+# Folders
+.git/
+.github/
+changelog/
+doc/
+docker/
+helpers/
+
+# Files
+.gitignore
+.golangci.yml
+*.md

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,13 +1,7 @@
-
-
 <!--
 Thank you very much for contributing code or documentation to restic! Please
 fill out the following questions to make it easier for us to review your
 changes.
-
-You do not need to check all the boxes below all at once, feel free to take
-your time and add more commits. If you're done and ready for review, please
-check the last box.
 -->
 
 What does this PR change? What problem does it solve?
@@ -17,8 +11,8 @@ What does this PR change? What problem does it solve?
 Describe the changes and their purpose here, as detailed as needed.
 -->
 
-Was the change discussed in an issue or in the forum before?
-------------------------------------------------------------
+Was the change previously discussed in an issue or on the forum?
+----------------------------------------------------------------
 
 <!--
 Link issues and relevant forum posts here.
@@ -30,11 +24,17 @@ is closed automatically when this PR is merged.
 Checklist
 ---------
 
-- [ ] I have read the [Contribution Guidelines](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#providing-patches)
-- [ ] I have enabled [maintainer edits for this PR](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/allowing-changes-to-a-pull-request-branch-created-from-a-fork)
-- [ ] I have added tests for all changes in this PR
-- [ ] I have added documentation for the changes (in the manual)
-- [ ] There's a new file in `changelog/unreleased/` that describes the changes for our users (template [here](https://github.com/restic/restic/blob/master/changelog/TEMPLATE))
-- [ ] I have run `gofmt` on the code in all commits
-- [ ] All commit messages are formatted in the same style as [the other commits in the repo](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#git-commits)
-- [ ] I'm done, this Pull Request is ready for review
+<!--
+You do not need to check all the boxes below all at once. Feel free to take
+your time and add more commits. If you're done and ready for review, please
+check the last box. Enable a checkbox by replacing [ ] with [x].
+-->
+
+- [ ] I have read the [contribution guidelines](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#providing-patches).
+- [ ] I have [enabled maintainer edits](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/allowing-changes-to-a-pull-request-branch-created-from-a-fork).
+- [ ] I have added tests for all code changes.
+- [ ] I have added documentation for relevant changes (in the manual).
+- [ ] There's a new file in `changelog/unreleased/` that describes the changes for our users (see [template](https://github.com/restic/restic/blob/master/changelog/TEMPLATE)).
+- [ ] I have run `gofmt` on the code in all commits.
+- [ ] All commit messages are formatted in the same style as [the other commits in the repo](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#git-commits).
+- [ ] I'm done! This pull request is ready for review.

.github/workflows/tests.yml

@@ -8,6 +8,10 @@ on:
   # run tests for all pull requests
   pull_request:
 
+env:
+  latest_go: "1.18.x"
+  GO111MODULE: on
+
 jobs:
   test:
     strategy:
@@ -15,30 +19,47 @@ jobs:
         # list of jobs to run:
         include:
           - job_name: Windows
-            go: 1.15.x
+            go: 1.18.x
             os: windows-latest
+            install_verb: install
 
           - job_name: macOS
-            go: 1.15.x
+            go: 1.18.x
             os: macOS-latest
             test_fuse: false
+            install_verb: install
 
           - job_name: Linux
-            go: 1.15.x
+            go: 1.18.x
             os: ubuntu-latest
             test_cloud_backends: true
             test_fuse: true
             check_changelog: true
+            install_verb: install
+
+          - job_name: Linux
+            go: 1.17.x
+            os: ubuntu-latest
+            test_fuse: true
+            install_verb: install
+
+          - job_name: Linux
+            go: 1.16.x
+            os: ubuntu-latest
+            test_fuse: true
+            install_verb: get
+
+          - job_name: Linux
+            go: 1.15.x
+            os: ubuntu-latest
+            test_fuse: true
+            install_verb: get
 
           - job_name: Linux
             go: 1.14.x
             os: ubuntu-latest
             test_fuse: true
-
-          - job_name: Linux
-            go: 1.13.x
-            os: ubuntu-latest
-            test_fuse: true
+            install_verb: get
 
     name: ${{ matrix.job_name }} Go ${{ matrix.go }}
     runs-on: ${{ matrix.os }}
@@ -55,7 +76,7 @@ jobs:
       - name: Get programs (Linux/macOS)
         run: |
           echo "build Go tools"
-          go get github.com/restic/rest-server/...
+          go ${{ matrix.install_verb }} github.com/restic/rest-server/cmd/rest-server@latest
 
           echo "install minio server"
           mkdir $HOME/bin
@@ -87,7 +108,7 @@ jobs:
           $ProgressPreference = 'SilentlyContinue'
 
           echo "build Go tools"
-          go get github.com/restic/rest-server/...
+          go ${{ matrix.install_verb }} github.com/restic/rest-server/...
 
           echo "install minio server"
           mkdir $Env:USERPROFILE/bin
@@ -177,7 +198,7 @@ jobs:
       - name: Check changelog files with calens
         run: |
           echo "install calens"
-          go get github.com/restic/calens
+          go install github.com/restic/calens@latest
 
           echo "check changelog files"
           calens
@@ -190,18 +211,17 @@ jobs:
       matrix:
         # run cross-compile in two batches parallel so the overall tests run faster
         targets:
-          - "linux/386 linux/amd64 linux/arm linux/arm64 linux/ppc64le linux/mips linux/mipsle linux/mips64 linux/mips64le \
+          - "linux/386 linux/amd64 linux/arm linux/arm64 linux/ppc64le linux/mips linux/mipsle linux/mips64 linux/mips64le linux/s390x \
             openbsd/386 openbsd/amd64"
 
           - "freebsd/386 freebsd/amd64 freebsd/arm \
             aix/ppc64 \
-            darwin/amd64 \
+            darwin/amd64 darwin/arm64 \
             netbsd/386 netbsd/amd64 \
             windows/386 windows/amd64 \
             solaris/amd64"
 
     env:
-      go: 1.15.x
       GOPROXY: https://proxy.golang.org
 
     runs-on: ubuntu-latest
@@ -209,17 +229,17 @@ jobs:
     name: Cross Compile for ${{ matrix.targets }}
 
     steps:
-      - name: Set up Go ${{ env.go }}
+      - name: Set up Go ${{ env.latest_go }}
         uses: actions/setup-go@v2
         with:
-          go-version: ${{ env.go }}
-
-      - name: Check out code
-        uses: actions/checkout@v2
+          go-version: ${{ env.latest_go }}
 
       - name: Install gox
         run: |
-          go get github.com/mitchellh/gox
+          go install github.com/mitchellh/gox@latest
+
+      - name: Check out code
+        uses: actions/checkout@v2
 
       - name: Cross-compile with gox for ${{ matrix.targets }}
         env:
@@ -234,6 +254,11 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
+      - name: Set up Go ${{ env.latest_go }}
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.latest_go }}
+
       - name: Check out code
         uses: actions/checkout@v2
 
@@ -241,10 +266,11 @@ jobs:
         uses: golangci/golangci-lint-action@v2
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.36
+          version: v1.45
           # Optional: show only new issues if it's a pull request. The default value is `false`.
           only-new-issues: true
           args: --verbose --timeout 5m
+          skip-go-installation: true
 
         # only run golangci-lint for pull requests, otherwise ALL hints get
         # reported. We need to slowly address all issues until we can enable
@@ -256,3 +282,44 @@ jobs:
           echo "check if go.mod and go.sum are up to date"
           go mod tidy
           git diff --exit-code go.mod go.sum
+
+  docker:
+    name: docker
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          # list of Docker images to use as base name for tags
+          images: |
+            restic/restic
+          # generate Docker tags based on the following events/attributes
+          tags: |
+            type=schedule
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=sha
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          push: false
+          context: .
+          file: docker/Dockerfile
+          pull: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

CHANGELOG.md

@@ -1,3 +1,654 @@
+Changelog for restic 0.13.0 (2022-03-26)
+=======================================
+
+The following sections list the changes in restic 0.13.0 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #1106: Never lock repository for `list locks`
+ * Fix #2345: Make cache crash-resistant and usable by multiple concurrent processes
+ * Fix #2452: Improve error handling of repository locking
+ * Fix #2738: Don't print progress for `backup --json --quiet`
+ * Fix #3382: Make `check` command honor `RESTIC_CACHE_DIR` environment variable
+ * Fix #3518: Make `copy` command honor `--no-lock` for source repository
+ * Fix #3556: Fix hang with Backblaze B2 on SSL certificate authority error
+ * Fix #3601: Fix rclone backend prematurely exiting when receiving SIGINT on Windows
+ * Fix #3667: The `mount` command now reports symlinks sizes
+ * Fix #3488: `rebuild-index` failed if an index file was damaged
+ * Fix #3591: Fix handling of `prune --max-repack-size=0`
+ * Fix #3619: Avoid choosing parent snapshots newer than time of new snapshot
+ * Chg #3641: Ignore parent snapshot for `backup --stdin`
+ * Chg #3519: Require Go 1.14 or newer
+ * Enh #1542: Add `--dry-run`/`-n` option to `backup` command
+ * Enh #2202: Add upload checksum for Azure, GS, S3 and Swift backends
+ * Enh #233: Support negative include/exclude patterns
+ * Enh #2388: Add warning for S3 if partial credentials are provided
+ * Enh #2508: Support JSON output and quiet mode for the `diff` command
+ * Enh #2656: Add flag to disable TLS verification for self-signed certificates
+ * Enh #3003: Atomic uploads for the SFTP backend
+ * Enh #3127: Add xattr (extended attributes) support for Solaris
+ * Enh #3464: Skip lock creation on `forget` if `--no-lock` and `--dry-run`
+ * Enh #3490: Support random subset by size in `check --read-data-subset`
+ * Enh #3541: Improve handling of temporary B2 delete errors
+ * Enh #3542: Add file mode in symbolic notation to `ls --json`
+ * Enh #2594: Speed up the `restore --verify` command
+ * Enh #2816: The `backup` command no longer updates file access times on Linux
+ * Enh #2880: Make `recover` collect only unreferenced trees
+ * Enh #3429: Verify that new or modified keys are stored correctly
+ * Enh #3436: Improve local backend's resilience to (system) crashes
+ * Enh #3508: Cache blobs read by the `dump` command
+ * Enh #3511: Support configurable timeout for the rclone backend
+ * Enh #3593: Improve `copy` performance by parallelizing IO
+
+Details
+-------
+
+ * Bugfix #1106: Never lock repository for `list locks`
+
+   The `list locks` command previously locked to the repository by default. This had the problem
+   that it wouldn't work for an exclusively locked repository and that the command would also
+   display its own lock file which can be confusing.
+
+   Now, the `list locks` command never locks the repository.
+
+   https://github.com/restic/restic/issues/1106
+   https://github.com/restic/restic/pull/3665
+
+ * Bugfix #2345: Make cache crash-resistant and usable by multiple concurrent processes
+
+   The restic cache directory (`RESTIC_CACHE_DIR`) could end up in a broken state in the event of
+   restic (or the OS) crashing. This is now less likely to occur as files are downloaded to a
+   temporary location before being moved to their proper location.
+
+   This also allows multiple concurrent restic processes to operate on a single repository
+   without conflicts. Previously, concurrent operations could cause segfaults because the
+   processes saw each other's partially downloaded files.
+
+   https://github.com/restic/restic/issues/2345
+   https://github.com/restic/restic/pull/2838
+
+ * Bugfix #2452: Improve error handling of repository locking
+
+   Previously, when the lock refresh failed to delete the old lock file, it forgot about the newly
+   created one. Instead it continued trying to delete the old (usually no longer existing) lock
+   file and thus over time lots of lock files accumulated. This has now been fixed.
+
+   https://github.com/restic/restic/issues/2452
+   https://github.com/restic/restic/issues/2473
+   https://github.com/restic/restic/issues/2562
+   https://github.com/restic/restic/pull/3512
+
+ * Bugfix #2738: Don't print progress for `backup --json --quiet`
+
+   Unlike the text output, the `--json` output format still printed progress information even in
+   `--quiet` mode. This has now been fixed by always disabling the progress output in quiet mode.
+
+   https://github.com/restic/restic/issues/2738
+   https://github.com/restic/restic/pull/3264
+
+ * Bugfix #3382: Make `check` command honor `RESTIC_CACHE_DIR` environment variable
+
+   Previously, the `check` command didn't honor the `RESTIC_CACHE_DIR` environment variable,
+   which caused problems in certain system/usage configurations. This has now been fixed.
+
+   https://github.com/restic/restic/issues/3382
+   https://github.com/restic/restic/pull/3474
+
+ * Bugfix #3518: Make `copy` command honor `--no-lock` for source repository
+
+   The `copy` command previously did not respect the `--no-lock` option for the source
+   repository, causing failures with read-only storage backends. This has now been fixed such
+   that the option is now respected.
+
+   https://github.com/restic/restic/issues/3518
+   https://github.com/restic/restic/pull/3589
+
+ * Bugfix #3556: Fix hang with Backblaze B2 on SSL certificate authority error
+
+   Previously, if a request failed with an SSL unknown certificate authority error, the B2
+   backend retried indefinitely and restic would appear to hang.
+
+   This has now been fixed and restic instead fails with an error message.
+
+   https://github.com/restic/restic/issues/3556
+   https://github.com/restic/restic/issues/2355
+   https://github.com/restic/restic/pull/3571
+
+ * Bugfix #3601: Fix rclone backend prematurely exiting when receiving SIGINT on Windows
+
+   Previously, pressing Ctrl+C in a Windows console where restic was running with rclone as the
+   backend would cause rclone to exit prematurely due to getting a `SIGINT` signal at the same time
+   as restic. Restic would then wait for a long time for time with "unexpected EOF" and "rclone
+   stdio connection already closed" errors.
+
+   This has now been fixed by restic starting the rclone process detached from the console restic
+   runs in (similar to starting processes in a new process group on Linux), which enables restic to
+   gracefully clean up rclone (which now never gets the `SIGINT`).
+
+   https://github.com/restic/restic/issues/3601
+   https://github.com/restic/restic/pull/3602
+
+ * Bugfix #3667: The `mount` command now reports symlinks sizes
+
+   Symlinks used to have size zero in restic mountpoints, confusing some third-party tools. They
+   now have a size equal to the byte length of their target path, as required by POSIX.
+
+   https://github.com/restic/restic/issues/3667
+   https://github.com/restic/restic/pull/3668
+
+ * Bugfix #3488: `rebuild-index` failed if an index file was damaged
+
+   Previously, the `rebuild-index` command would fail with an error if an index file was damaged
+   or truncated. This has now been fixed.
+
+   On older restic versions, a (slow) workaround is to use `rebuild-index --read-all-packs` or
+   to manually delete the damaged index.
+
+   https://github.com/restic/restic/pull/3488
+
+ * Bugfix #3591: Fix handling of `prune --max-repack-size=0`
+
+   Restic ignored the `--max-repack-size` option when passing a value of 0. This has now been
+   fixed.
+
+   As a workaround, `--max-repack-size=1` can be used with older versions of restic.
+
+   https://github.com/restic/restic/pull/3591
+
+ * Bugfix #3619: Avoid choosing parent snapshots newer than time of new snapshot
+
+   The `backup` command, when a `--parent` was not provided, previously chose the most recent
+   matching snapshot as the parent snapshot. However, this didn't make sense when the user passed
+   `--time` to create a new snapshot older than the most recent snapshot.
+
+   Instead, `backup` now chooses the most recent snapshot which is not newer than the
+   snapshot-being-created's timestamp, to avoid any time travel.
+
+   https://github.com/restic/restic/pull/3619
+
+ * Change #3641: Ignore parent snapshot for `backup --stdin`
+
+   Restic uses a parent snapshot to speed up directory scanning when performing backups, but this
+   only wasted time and memory when the backup source is stdin (using the `--stdin` option of the
+   `backup` command), since no directory scanning is performed in this case.
+
+   Snapshots made with `backup --stdin` no longer have a parent snapshot, which allows restic to
+   skip some startup operations and saves a bit of resources.
+
+   The `--parent` option is still available for `backup --stdin`, but is now ignored.
+
+   https://github.com/restic/restic/issues/3641
+   https://github.com/restic/restic/pull/3645
+
+ * Change #3519: Require Go 1.14 or newer
+
+   Restic now requires Go 1.14 to build. This allows it to use new standard library features
+   instead of an external dependency.
+
+   https://github.com/restic/restic/issues/3519
+
+ * Enhancement #1542: Add `--dry-run`/`-n` option to `backup` command
+
+   Testing exclude filters and other configuration options was error prone as wrong filters
+   could cause files to be uploaded unintentionally. It was also not possible to estimate
+   beforehand how much data would be uploaded.
+
+   The `backup` command now has a `--dry-run`/`-n` option, which performs all the normal steps of
+   a backup without actually writing anything to the repository.
+
+   Passing -vv will log information about files that would be added, allowing for verification of
+   source and exclusion options before running the real backup.
+
+   https://github.com/restic/restic/issues/1542
+   https://github.com/restic/restic/pull/2308
+   https://github.com/restic/restic/pull/3210
+   https://github.com/restic/restic/pull/3300
+
+ * Enhancement #2202: Add upload checksum for Azure, GS, S3 and Swift backends
+
+   Previously only the B2 and partially the Swift backends verified the integrity of uploaded
+   (encrypted) files. The verification works by informing the backend about the expected hash of
+   the uploaded file. The backend then verifies the upload and thereby rules out any data
+   corruption during upload.
+
+   We have now added upload checksums for the Azure, GS, S3 and Swift backends, which besides
+   integrity checking for uploads also means that restic can now be used to store backups in S3
+   buckets which have Object Lock enabled.
+
+   https://github.com/restic/restic/issues/2202
+   https://github.com/restic/restic/issues/2700
+   https://github.com/restic/restic/issues/3023
+   https://github.com/restic/restic/pull/3246
+
+ * Enhancement #233: Support negative include/exclude patterns
+
+   If a pattern starts with an exclamation mark and it matches a file that was previously matched by
+   a regular pattern, the match is cancelled. Notably, this can be used with `--exclude-file` to
+   cancel the exclusion of some files.
+
+   It works similarly to `.gitignore`, with the same limitation; Once a directory is excluded, it
+   is not possible to include files inside the directory.
+
+   Example of use as an exclude pattern for the `backup` command:
+
+   $HOME/**/* !$HOME/Documents !$HOME/code !$HOME/.emacs.d !$HOME/games # [...]
+   node_modules *~ *.o *.lo *.pyc # [...] $HOME/code/linux/* !$HOME/code/linux/.git # [...]
+
+   https://github.com/restic/restic/issues/233
+   https://github.com/restic/restic/pull/2311
+
+ * Enhancement #2388: Add warning for S3 if partial credentials are provided
+
+   Previously restic did not notify about incomplete credentials when using the S3 backend,
+   instead just reporting access denied.
+
+   Restic now checks that both the AWS key ID and secret environment variables are set before
+   connecting to the remote server, and reports an error if not.
+
+   https://github.com/restic/restic/issues/2388
+   https://github.com/restic/restic/pull/3532
+
+ * Enhancement #2508: Support JSON output and quiet mode for the `diff` command
+
+   The `diff` command now supports outputting machine-readable output in JSON format. To enable
+   this, pass the `--json` option to the command. To only print the summary and suppress detailed
+   output, pass the `--quiet` option.
+
+   https://github.com/restic/restic/issues/2508
+   https://github.com/restic/restic/pull/3592
+
+ * Enhancement #2656: Add flag to disable TLS verification for self-signed certificates
+
+   There is now an `--insecure-tls` global option in restic, which disables TLS verification for
+   self-signed certificates in order to support some development workflows.
+
+   https://github.com/restic/restic/issues/2656
+   https://github.com/restic/restic/pull/2657
+
+ * Enhancement #3003: Atomic uploads for the SFTP backend
+
+   The SFTP backend did not upload files atomically. An interrupted upload could leave an
+   incomplete file behind which could prevent restic from accessing the repository. This has now
+   been fixed and uploads in the SFTP backend are done atomically.
+
+   https://github.com/restic/restic/issues/3003
+   https://github.com/restic/restic/pull/3524
+
+ * Enhancement #3127: Add xattr (extended attributes) support for Solaris
+
+   Restic now supports xattr for the Solaris operating system.
+
+   https://github.com/restic/restic/issues/3127
+   https://github.com/restic/restic/pull/3628
+
+ * Enhancement #3464: Skip lock creation on `forget` if `--no-lock` and `--dry-run`
+
+   Restic used to silently ignore the `--no-lock` option of the `forget` command.
+
+   It now skips creation of lock file in case both `--dry-run` and `--no-lock` are specified. If
+   `--no-lock` option is specified without `--dry-run`, restic prints a warning message to
+   stderr.
+
+   https://github.com/restic/restic/issues/3464
+   https://github.com/restic/restic/pull/3623
+
+ * Enhancement #3490: Support random subset by size in `check --read-data-subset`
+
+   The `--read-data-subset` option of the `check` command now supports a third way of specifying
+   the subset to check, namely `nS` where `n` is a size in bytes with suffix `S` as k/K, m/M, g/G or
+   t/T.
+
+   https://github.com/restic/restic/issues/3490
+   https://github.com/restic/restic/pull/3548
+
+ * Enhancement #3541: Improve handling of temporary B2 delete errors
+
+   Deleting files on B2 could sometimes fail temporarily, which required restic to retry the
+   delete operation. In some cases the file was deleted nevertheless, causing the retries and
+   ultimately the restic command to fail. This has now been fixed.
+
+   https://github.com/restic/restic/issues/3541
+   https://github.com/restic/restic/pull/3544
+
+ * Enhancement #3542: Add file mode in symbolic notation to `ls --json`
+
+   The `ls --json` command now provides the file mode in symbolic notation (using the
+   `permissions` key), aligned with `find --json`.
+
+   https://github.com/restic/restic/issues/3542
+   https://github.com/restic/restic/pull/3573
+   https://forum.restic.net/t/restic-ls-understanding-file-mode-with-json/4371
+
+ * Enhancement #2594: Speed up the `restore --verify` command
+
+   The `--verify` option lets the `restore` command verify the file content after it has restored
+   a snapshot. The performance of this operation has now been improved by up to a factor of two.
+
+   https://github.com/restic/restic/pull/2594
+
+ * Enhancement #2816: The `backup` command no longer updates file access times on Linux
+
+   When reading files during backup, restic used to cause the operating system to update the
+   files' access times. Note that this did not apply to filesystems with disabled file access
+   times.
+
+   Restic now instructs the operating system not to update the file access time, if the user
+   running restic is the file owner or has root permissions.
+
+   https://github.com/restic/restic/pull/2816
+
+ * Enhancement #2880: Make `recover` collect only unreferenced trees
+
+   Previously, the `recover` command used to generate a snapshot containing *all* root trees,
+   even those which were already referenced by a snapshot.
+
+   This has been improved such that it now only processes trees not already referenced by any
+   snapshot.
+
+   https://github.com/restic/restic/pull/2880
+
+ * Enhancement #3429: Verify that new or modified keys are stored correctly
+
+   When adding a new key or changing the password of a key, restic used to just create the new key (and
+   remove the old one, when changing the password). There was no verification that the new key was
+   stored correctly and works properly. As the repository cannot be decrypted without a valid key
+   file, this could in rare cases cause the repository to become inaccessible.
+
+   Restic now checks that new key files actually work before continuing. This can protect against
+   some (rare) cases of hardware or storage problems.
+
+   https://github.com/restic/restic/pull/3429
+
+ * Enhancement #3436: Improve local backend's resilience to (system) crashes
+
+   Restic now ensures that files stored using the `local` backend are created atomically (that
+   is, files are either stored completely or not at all). This ensures that no incomplete files are
+   left behind even if restic is terminated while writing a file.
+
+   In addition, restic now tries to ensure that the directory in the repository which contains a
+   newly uploaded file is also written to disk. This can prevent missing files if the system
+   crashes or the disk is not properly unmounted.
+
+   https://github.com/restic/restic/pull/3436
+
+ * Enhancement #3508: Cache blobs read by the `dump` command
+
+   When dumping a file using the `dump` command, restic did not cache blobs in any way, so even
+   consecutive runs of the same blob were loaded from the repository again and again, slowing down
+   the dump.
+
+   Now, the caching mechanism already used by the `fuse` command is also used by the `dump`
+   command. This makes dumping much faster, especially for sparse files.
+
+   https://github.com/restic/restic/pull/3508
+
+ * Enhancement #3511: Support configurable timeout for the rclone backend
+
+   A slow rclone backend could cause restic to time out while waiting for the repository to open.
+   Restic now offers an `-o rclone.timeout` option to make this timeout configurable.
+
+   https://github.com/restic/restic/issues/3511
+   https://github.com/restic/restic/pull/3514
+
+ * Enhancement #3593: Improve `copy` performance by parallelizing IO
+
+   Restic copy previously only used a single thread for copying blobs between repositories,
+   which resulted in limited performance when copying small blobs to/from a high latency backend
+   (i.e. any remote backend, especially b2).
+
+   Copying will now use 8 parallel threads to increase the throughput of the copy operation.
+
+   https://github.com/restic/restic/pull/3593
+
+
+Changelog for restic 0.12.1 (2021-08-03)
+=======================================
+
+The following sections list the changes in restic 0.12.1 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #2742: Improve error handling for rclone and REST backend over HTTP2
+ * Fix #3111: Fix terminal output redirection for PowerShell
+ * Fix #3214: Treat an empty password as a fatal error for repository init
+ * Fix #3267: `copy` failed to copy snapshots in rare cases
+ * Fix #3184: `backup --quiet` no longer prints status information
+ * Fix #3296: Fix crash of `check --read-data-subset=x%` run for an empty repository
+ * Fix #3302: Fix `fdopendir: not a directory` error for local backend
+ * Fix #3334: Print `created new cache` message only on a terminal
+ * Fix #3380: Fix crash of `backup --exclude='**'`
+ * Fix #3305: Fix possibly missing backup summary of JSON output in case of error
+ * Fix #3439: Correctly handle download errors during `restore`
+ * Chg #3247: Empty files now have size of 0 in `ls --json` output
+ * Enh #2780: Add release binaries for s390x architecture on Linux
+ * Enh #3293: Add `--repository-file2` option to `init` and `copy` command
+ * Enh #3312: Add auto-completion support for fish
+ * Enh #3336: SFTP backend now checks for disk space
+ * Enh #3377: Add release binaries for Apple Silicon
+ * Enh #3414: Add `--keep-within-hourly` option to restic forget
+ * Enh #3456: Support filtering and specifying untagged snapshots
+ * Enh #3167: Allow specifying limit of `snapshots` list
+ * Enh #3426: Optimize read performance of mount command
+ * Enh #3427: `find --pack` fallback to index if data file is missing
+
+Details
+-------
+
+ * Bugfix #2742: Improve error handling for rclone and REST backend over HTTP2
+
+   When retrieving data from the rclone / REST backend while also using HTTP2 restic did not detect
+   when no data was returned at all. This could cause for example the `check` command to report the
+   following error:
+
+   Pack ID does not match, want [...], got e3b0c442
+
+   This has been fixed by correctly detecting and retrying the incomplete download.
+
+   https://github.com/restic/restic/issues/2742
+   https://github.com/restic/restic/pull/3453
+   https://forum.restic.net/t/http2-stream-closed-connection-reset-context-canceled/3743/10
+
+ * Bugfix #3111: Fix terminal output redirection for PowerShell
+
+   When redirecting the output of restic using PowerShell on Windows, the output contained
+   terminal escape characters. This has been fixed by properly detecting the terminal type.
+
+   In addition, the mintty terminal now shows progress output for the backup command.
+
+   https://github.com/restic/restic/issues/3111
+   https://github.com/restic/restic/pull/3325
+
+ * Bugfix #3214: Treat an empty password as a fatal error for repository init
+
+   When attempting to initialize a new repository, if an empty password was supplied, the
+   repository would be created but the init command would return an error with a stack trace. Now,
+   if an empty password is provided, it is treated as a fatal error, and no repository is created.
+
+   https://github.com/restic/restic/issues/3214
+   https://github.com/restic/restic/pull/3283
+
+ * Bugfix #3267: `copy` failed to copy snapshots in rare cases
+
+   The `copy` command could in rare cases fail with the error message `SaveTree(...) returned
+   unexpected id ...`. This has been fixed.
+
+   On Linux/BSDs, the error could be caused by backing up symlinks with non-UTF-8 target paths.
+   Note that, due to limitations in the repository format, these are not stored properly and
+   should be avoided if possible.
+
+   https://github.com/restic/restic/issues/3267
+   https://github.com/restic/restic/pull/3310
+
+ * Bugfix #3184: `backup --quiet` no longer prints status information
+
+   A regression in the latest restic version caused the output of `backup --quiet` to contain
+   large amounts of backup progress information when run using an interactive terminal. This is
+   fixed now.
+
+   A workaround for this bug is to run restic as follows: `restic backup --quiet [..] | cat -`.
+
+   https://github.com/restic/restic/issues/3184
+   https://github.com/restic/restic/pull/3186
+
+ * Bugfix #3296: Fix crash of `check --read-data-subset=x%` run for an empty repository
+
+   The command `restic check --read-data-subset=x%` crashed when run for an empty repository.
+   This has been fixed.
+
+   https://github.com/restic/restic/issues/3296
+   https://github.com/restic/restic/pull/3309
+
+ * Bugfix #3302: Fix `fdopendir: not a directory` error for local backend
+
+   The `check`, `list packs`, `prune` and `rebuild-index` commands failed for the local backend
+   when the `data` folder in the repository contained files. This has been fixed.
+
+   https://github.com/restic/restic/issues/3302
+   https://github.com/restic/restic/pull/3308
+
+ * Bugfix #3334: Print `created new cache` message only on a terminal
+
+   The message `created new cache` was printed even when the output wasn't a terminal. That broke
+   piping `restic dump` output to tar or zip if cache directory didn't exist. The message is now
+   only printed on a terminal.
+
+   https://github.com/restic/restic/issues/3334
+   https://github.com/restic/restic/pull/3343
+
+ * Bugfix #3380: Fix crash of `backup --exclude='**'`
+
+   The exclude filter `**`, which excludes all files, caused restic to crash. This has been
+   corrected.
+
+   https://github.com/restic/restic/issues/3380
+   https://github.com/restic/restic/pull/3393
+
+ * Bugfix #3305: Fix possibly missing backup summary of JSON output in case of error
+
+   When using `--json` output it happened from time to time that the summary output was missing in
+   case an error occurred. This has been fixed.
+
+   https://github.com/restic/restic/pull/3305
+
+ * Bugfix #3439: Correctly handle download errors during `restore`
+
+   Due to a regression in restic 0.12.0, the `restore` command in some cases did not retry download
+   errors and only printed a warning. This has been fixed by retrying incomplete data downloads.
+
+   https://github.com/restic/restic/issues/3439
+   https://github.com/restic/restic/pull/3449
+
+ * Change #3247: Empty files now have size of 0 in `ls --json` output
+
+   The `ls --json` command used to omit the sizes of empty files in its output. It now reports a size
+   of zero explicitly for regular files, while omitting the size field for all other types.
+
+   https://github.com/restic/restic/issues/3247
+   https://github.com/restic/restic/pull/3257
+
+ * Enhancement #2780: Add release binaries for s390x architecture on Linux
+
+   We've added release binaries for Linux using the s390x architecture.
+
+   https://github.com/restic/restic/issues/2780
+   https://github.com/restic/restic/pull/3452
+
+ * Enhancement #3293: Add `--repository-file2` option to `init` and `copy` command
+
+   The `init` and `copy` command can now be used with the `--repository-file2` option or the
+   `$RESTIC_REPOSITORY_FILE2` environment variable. These to options are in addition to the
+   `--repo2` flag and allow you to read the destination repository from a file.
+
+   Using both `--repository-file` and `--repo2` options resulted in an error for the `copy` or
+   `init` command. The handling of this combination of options has been fixed. A workaround for
+   this issue is to only use `--repo` or `-r` and `--repo2` for `init` or `copy`.
+
+   https://github.com/restic/restic/issues/3293
+   https://github.com/restic/restic/pull/3294
+
+ * Enhancement #3312: Add auto-completion support for fish
+
+   The `generate` command now supports fish auto completion.
+
+   https://github.com/restic/restic/pull/3312
+
+ * Enhancement #3336: SFTP backend now checks for disk space
+
+   Backing up over SFTP previously spewed multiple generic "failure" messages when the remote
+   disk was full. It now checks for disk space before writing a file and fails immediately with a "no
+   space left on device" message.
+
+   https://github.com/restic/restic/issues/3336
+   https://github.com/restic/restic/pull/3345
+
+ * Enhancement #3377: Add release binaries for Apple Silicon
+
+   We've added release binaries for macOS on Apple Silicon (M1).
+
+   https://github.com/restic/restic/issues/3377
+   https://github.com/restic/restic/pull/3394
+
+ * Enhancement #3414: Add `--keep-within-hourly` option to restic forget
+
+   The `forget` command allowed keeping a given number of hourly backups or to keep all backups
+   within a given interval, but it was not possible to specify keeping hourly backups within a
+   given interval.
+
+   The new `--keep-within-hourly` option now offers this functionality. Similar options for
+   daily/weekly/monthly/yearly are also implemented, the new options are:
+
+   --keep-within-hourly <1y2m3d4h> --keep-within-daily <1y2m3d4h> --keep-within-weekly
+   <1y2m3d4h> --keep-within-monthly <1y2m3d4h> --keep-within-yearly <1y2m3d4h>
+
+   https://github.com/restic/restic/issues/3414
+   https://github.com/restic/restic/pull/3416
+   https://forum.restic.net/t/forget-policy/4014/11
+
+ * Enhancement #3456: Support filtering and specifying untagged snapshots
+
+   It was previously not possible to specify an empty tag with the `--tag` and `--keep-tag`
+   options. This has now been fixed, such that `--tag ''` and `--keep-tag ''` now matches
+   snapshots without tags. This allows e.g. the `snapshots` and `forget` commands to only
+   operate on untagged snapshots.
+
+   https://github.com/restic/restic/issues/3456
+   https://github.com/restic/restic/pull/3457
+
+ * Enhancement #3167: Allow specifying limit of `snapshots` list
+
+   The `--last` option allowed limiting the output of the `snapshots` command to the latest
+   snapshot for each host. The new `--latest n` option allows limiting the output to the latest `n`
+   snapshots.
+
+   This change deprecates the option `--last` in favour of `--latest 1`.
+
+   https://github.com/restic/restic/pull/3167
+
+ * Enhancement #3426: Optimize read performance of mount command
+
+   Reading large files in a mounted repository may be up to five times faster. This improvement
+   primarily applies to repositories stored at a backend that can be accessed with low latency,
+   like e.g. the local backend.
+
+   https://github.com/restic/restic/pull/3426
+
+ * Enhancement #3427: `find --pack` fallback to index if data file is missing
+
+   When investigating a repository with missing data files, it might be useful to determine
+   affected snapshots before running `rebuild-index`. Previously, `find --pack pack-id`
+   returned no data as it required accessing the data file. Now, if the necessary data is still
+   available in the repository index, it gets retrieved from there.
+
+   The command now also supports looking up multiple pack files in a single `find` run.
+
+   https://github.com/restic/restic/pull/3427
+   https://forum.restic.net/t/missing-packs-not-found/2600
+
+
 Changelog for restic 0.12.0 (2021-02-14)
 =======================================
 
@@ -2837,7 +3488,7 @@ Summary
  * Enh #1055: Create subdirs below `data/` for local/sftp backends
  * Enh #1067: Allow loading credentials for s3 from IAM
  * Enh #1073: Add `migrate` cmd to migrate from `s3legacy` to `default` layout
- * Enh #1081: Clarify semantic for `--tasg` for the `forget` command
+ * Enh #1081: Clarify semantic for `--tag` for the `forget` command
  * Enh #1080: Ignore chmod() errors on filesystems which do not support it
  * Enh #1082: Print stats on SIGINFO on Darwin and FreeBSD (ctrl+t)
 
@@ -2881,7 +3532,7 @@ Details
    https://github.com/restic/restic/issues/1073
    https://github.com/restic/restic/pull/1075
 
- * Enhancement #1081: Clarify semantic for `--tasg` for the `forget` command
+ * Enhancement #1081: Clarify semantic for `--tag` for the `forget` command
 
    https://github.com/restic/restic/issues/1081
    https://github.com/restic/restic/pull/1090

CONTRIBUTING.md

@@ -13,11 +13,10 @@ bug fixes are most welcome. However even "minor" details as fixing spelling
 errors, improving documentation or pointing out usability issues are a great
 help also.
 
-
 The restic project uses the GitHub infrastructure (see the
 [project page](https://github.com/restic/restic)) for all related discussions
 as well as the [forum](https://forum.restic.net/) and the `#restic` channel
-on [irc.freenode.net](https://kiwiirc.com/nextclient/irc.freenode.net/restic).
+on [irc.libera.chat](https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:6697/#restic).
 
 If you want to find an area that currently needs improving have a look at the
 open issues listed at the
@@ -67,7 +66,7 @@ Development Environment
 The repository contains the code written for restic in the directories
 `cmd/` and `internal/`.
 
-Restic requires Go version 1.13 or later for compiling. Clone the repo (without
+Restic requires Go version 1.14 or later for compiling. Clone the repo (without
 having `$GOPATH` set) and `cd` into the directory:
 
     $ unset GOPATH
@@ -124,7 +123,10 @@ down to the following steps:
     writing, ask yourself: If I were the user, what would I need to be aware
     of with this change?
 
- 8. Once your code looks good and passes all the tests, we'll merge it. Thanks
+ 8. Do not edit the man pages under `doc/man` or `doc/manual_rest.rst` -
+    these are autogenerated before new releases.
+
+ 9. Once your code looks good and passes all the tests, we'll merge it. Thanks
     a lot for your contribution!
 
 Please provide the patches for each bug or feature in a separate branch and

README.md

@@ -46,8 +46,8 @@ Therefore, restic supports the following backends for storing backups natively:
 
 - [Local directory](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#local)
 - [sftp server (via SSH)](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#sftp)
-- [HTTP REST server](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server) ([protocol](doc/100_references.rst#rest-backend), [rest-server](https://github.com/restic/rest-server))
-- [AWS S3](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) (either from Amazon or using the [Minio](https://minio.io) server)
+- [HTTP REST server](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server) ([protocol](https://restic.readthedocs.io/en/latest/100_references.html#rest-backend), [rest-server](https://github.com/restic/rest-server))
+- [Amazon S3](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) (either from Amazon or using the [Minio](https://minio.io) server)
 - [OpenStack Swift](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#openstack-swift)
 - [BackBlaze B2](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#backblaze-b2)
 - [Microsoft Azure Blob Storage](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#microsoft-azure-blob-storage)

VERSION

@@ -1 +1 @@
-0.12.0
+0.13.0

build.go

@@ -35,6 +35,7 @@
 // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+//go:build ignore_build_go
 // +build ignore_build_go
 
 package main
@@ -58,7 +59,7 @@ var config = Config{
 	Main:             "./cmd/restic",                           // package name for the main package
 	DefaultBuildTags: []string{"selfupdate"},                   // specify build tags which are always used
 	Tests:            []string{"./..."},                        // tests to run
-	MinVersion:       GoVersion{Major: 1, Minor: 11, Patch: 0}, // minimum Go version supported
+	MinVersion:       GoVersion{Major: 1, Minor: 14, Patch: 0}, // minimum Go version supported
 }
 
 // Config configures the build.
@@ -123,17 +124,8 @@ func printEnv(env []string) {
 
 // build runs "go build args..." with GOPATH set to gopath.
 func build(cwd string, env map[string]string, args ...string) error {
-	a := []string{"build"}
-
-	// try to remove all absolute paths from resulting binary
-	if goVersion.AtLeast(GoVersion{1, 13, 0}) {
-		// use the new flag introduced by Go 1.13
-		a = append(a, "-trimpath")
-	} else {
-		// otherwise try to trim as many paths as possible
-		a = append(a, "-asmflags", fmt.Sprintf("all=-trimpath=%s", cwd))
-		a = append(a, "-gcflags", fmt.Sprintf("all=-trimpath=%s", cwd))
-	}
+	// -trimpath removes all absolute paths from the binary.
+	a := []string{"build", "-trimpath"}
 
 	if enablePIE {
 		a = append(a, "-buildmode=pie")

changelog/0.12.1_2021-08-03/issue-2742

@@ -0,0 +1,13 @@
+Bugfix: Improve error handling for rclone and REST backend over HTTP2
+
+When retrieving data from the rclone / REST backend while also using HTTP2
+restic did not detect when no data was returned at all. This could cause
+for example the `check` command to report the following error:
+
+    Pack ID does not match, want [...], got e3b0c442
+
+This has been fixed by correctly detecting and retrying the incomplete download.
+
+https://github.com/restic/restic/issues/2742
+https://github.com/restic/restic/pull/3453
+https://forum.restic.net/t/http2-stream-closed-connection-reset-context-canceled/3743/10

changelog/0.12.1_2021-08-03/issue-2780

@@ -0,0 +1,6 @@
+Enhancement: Add release binaries for s390x architecture on Linux
+
+We've added release binaries for Linux using the s390x architecture.
+
+https://github.com/restic/restic/issues/2780
+https://github.com/restic/restic/pull/3452

changelog/0.12.1_2021-08-03/issue-3111

@@ -0,0 +1,11 @@
+Bugfix: Fix terminal output redirection for PowerShell
+
+When redirecting the output of restic using PowerShell on Windows, the
+output contained terminal escape characters. This has been fixed by
+properly detecting the terminal type.
+
+In addition, the mintty terminal now shows progress output for the backup
+command.
+
+https://github.com/restic/restic/issues/3111
+https://github.com/restic/restic/pull/3325

changelog/0.12.1_2021-08-03/issue-3214

@@ -0,0 +1,9 @@
+Bugfix: Treat an empty password as a fatal error for repository init
+
+When attempting to initialize a new repository, if an empty password was
+supplied, the repository would be created but the init command would return
+an error with a stack trace. Now, if an empty password is provided, it is
+treated as a fatal error, and no repository is created.
+
+https://github.com/restic/restic/issues/3214
+https://github.com/restic/restic/pull/3283

changelog/0.12.1_2021-08-03/issue-3247

@@ -0,0 +1,8 @@
+Change: Empty files now have size of 0 in `ls --json` output
+
+The `ls --json` command used to omit the sizes of empty files in its
+output. It now reports a size of zero explicitly for regular files,
+while omitting the size field for all other types.
+
+https://github.com/restic/restic/issues/3247
+https://github.com/restic/restic/pull/3257

changelog/0.12.1_2021-08-03/issue-3267

@@ -0,0 +1,11 @@
+Bugfix: `copy` failed to copy snapshots in rare cases
+
+The `copy` command could in rare cases fail with the error message `SaveTree(...)
+returned unexpected id ...`.  This has been fixed.
+
+On Linux/BSDs, the error could be caused by backing up symlinks with non-UTF-8
+target paths. Note that, due to limitations in the repository format, these are
+not stored properly and should be avoided if possible.
+
+https://github.com/restic/restic/issues/3267
+https://github.com/restic/restic/pull/3310

changelog/0.12.1_2021-08-03/issue-3284

@@ -0,0 +1,11 @@
+Bugfix: `backup --quiet` no longer prints status information
+
+A regression in the latest restic version caused the output of `backup --quiet`
+to contain large amounts of backup progress information when run using an
+interactive terminal. This is fixed now.
+
+A workaround for this bug is to run restic as follows:
+`restic backup --quiet [..] | cat -`.
+
+https://github.com/restic/restic/issues/3184
+https://github.com/restic/restic/pull/3186

changelog/0.12.1_2021-08-03/issue-3293

@@ -0,0 +1,14 @@
+Enhancement: Add `--repository-file2` option to `init` and `copy` command
+
+The `init` and `copy` command can now be used with the `--repository-file2`
+option or the `$RESTIC_REPOSITORY_FILE2` environment variable.
+These to options are in addition to the `--repo2` flag and allow you to read
+the destination repository from a file.
+
+Using both `--repository-file` and `--repo2` options resulted in an error for
+the `copy` or `init` command. The handling of this combination of options has
+been fixed. A workaround for this issue is to only use `--repo` or `-r` and
+`--repo2` for `init` or `copy`.
+
+https://github.com/restic/restic/issues/3293
+https://github.com/restic/restic/pull/3294

changelog/0.12.1_2021-08-03/issue-3296

@@ -0,0 +1,7 @@
+Bugfix: Fix crash of `check --read-data-subset=x%` run for an empty repository
+
+The command `restic check --read-data-subset=x%` crashed when run for an empty
+repository. This has been fixed.
+
+https://github.com/restic/restic/issues/3296
+https://github.com/restic/restic/pull/3309

changelog/0.12.1_2021-08-03/issue-3302

@@ -0,0 +1,8 @@
+Bugfix: Fix `fdopendir: not a directory` error for local backend
+
+The `check`, `list packs`, `prune` and `rebuild-index` commands failed
+for the local backend when the `data` folder in the repository contained
+files. This has been fixed.
+
+https://github.com/restic/restic/issues/3302
+https://github.com/restic/restic/pull/3308

changelog/0.12.1_2021-08-03/issue-3312

@@ -0,0 +1,5 @@
+Enhancement: Add auto-completion support for fish
+
+The `generate` command now supports fish auto completion.
+
+https://github.com/restic/restic/pull/3312

changelog/0.12.1_2021-08-03/issue-3334

@@ -0,0 +1,8 @@
+Bugfix: Print `created new cache` message only on a terminal
+
+The message `created new cache` was printed even when the output wasn't a
+terminal. That broke piping `restic dump` output to tar or zip if cache
+directory didn't exist. The message is now only printed on a terminal.
+
+https://github.com/restic/restic/issues/3334
+https://github.com/restic/restic/pull/3343

changelog/0.12.1_2021-08-03/issue-3336

@@ -0,0 +1,8 @@
+Enhancement: SFTP backend now checks for disk space
+
+Backing up over SFTP previously spewed multiple generic "failure" messages
+when the remote disk was full. It now checks for disk space before writing
+a file and fails immediately with a "no space left on device" message.
+
+https://github.com/restic/restic/issues/3336
+https://github.com/restic/restic/pull/3345

changelog/0.12.1_2021-08-03/issue-3377

@@ -0,0 +1,6 @@
+Enhancement: Add release binaries for Apple Silicon
+
+We've added release binaries for macOS on Apple Silicon (M1).
+
+https://github.com/restic/restic/issues/3377
+https://github.com/restic/restic/pull/3394

changelog/0.12.1_2021-08-03/issue-3380

@@ -0,0 +1,7 @@
+Bugfix: Fix crash of `backup --exclude='**'`
+
+The exclude filter `**`, which excludes all files, caused restic to crash. This
+has been corrected.
+
+https://github.com/restic/restic/issues/3380
+https://github.com/restic/restic/pull/3393

changelog/0.12.1_2021-08-03/issue-3414

@@ -0,0 +1,20 @@
+Enhancement: Add `--keep-within-hourly` option to restic forget
+
+The `forget` command allowed keeping a given number of hourly
+backups or to keep all backups within a given interval, but it
+was not possible to specify keeping hourly backups within a given
+interval.
+
+The new `--keep-within-hourly` option now offers this functionality.
+Similar options for daily/weekly/monthly/yearly are also implemented,
+the new options are:
+
+    --keep-within-hourly <1y2m3d4h>
+    --keep-within-daily <1y2m3d4h>
+    --keep-within-weekly <1y2m3d4h>
+    --keep-within-monthly <1y2m3d4h>
+    --keep-within-yearly <1y2m3d4h>
+
+https://github.com/restic/restic/issues/3414
+https://github.com/restic/restic/pull/3416
+https://forum.restic.net/t/forget-policy/4014/11

changelog/0.12.1_2021-08-03/issue-3456

@@ -0,0 +1,9 @@
+Enhancement: Support filtering and specifying untagged snapshots
+
+It was previously not possible to specify an empty tag with the `--tag` and
+`--keep-tag` options. This has now been fixed, such that `--tag ''` and
+`--keep-tag ''` now matches snapshots without tags. This allows e.g. the
+`snapshots` and `forget` commands to only operate on untagged snapshots.
+
+https://github.com/restic/restic/issues/3456
+https://github.com/restic/restic/pull/3457

changelog/0.12.1_2021-08-03/pull-3167

@@ -0,0 +1,9 @@
+Enhancement: Allow specifying limit of `snapshots` list
+
+The `--last` option allowed limiting the output of the `snapshots`
+command to the latest snapshot for each host. The new `--latest n`
+option allows limiting the output to the latest `n` snapshots.
+
+This change deprecates the option `--last` in favour of `--latest 1`.
+
+https://github.com/restic/restic/pull/3167

changelog/0.12.1_2021-08-03/pull-3305

@@ -0,0 +1,6 @@
+Bugfix: Fix possibly missing backup summary of JSON output in case of error
+
+When using `--json` output it happened from time to time that the summary
+output was missing in case an error occurred. This has been fixed.
+
+https://github.com/restic/restic/pull/3305

changelog/0.12.1_2021-08-03/pull-3426

@@ -0,0 +1,7 @@
+Enhancement: Optimize read performance of mount command
+
+Reading large files in a mounted repository may be up to five times faster.
+This improvement primarily applies to repositories stored at a backend that can
+be accessed with low latency, like e.g. the local backend.
+
+https://github.com/restic/restic/pull/3426

changelog/0.12.1_2021-08-03/pull-3427

@@ -0,0 +1,13 @@
+Enhancement: `find --pack` fallback to index if data file is missing
+
+When investigating a repository with missing data files, it might be useful to
+determine affected snapshots before running `rebuild-index`. Previously,
+`find --pack pack-id` returned no data as it required accessing the data file.
+Now, if the necessary data is still available in the repository index, it gets
+retrieved from there.
+
+The command now also supports looking up multiple pack files in a single `find`
+run.
+
+https://github.com/restic/restic/pull/3427
+https://forum.restic.net/t/missing-packs-not-found/2600

changelog/0.12.1_2021-08-03/pull-3449

@@ -0,0 +1,8 @@
+Bugfix: Correctly handle download errors during `restore`
+
+Due to a regression in restic 0.12.0, the `restore` command in some cases did
+not retry download errors and only printed a warning. This has been fixed by
+retrying incomplete data downloads.
+
+https://github.com/restic/restic/issues/3439
+https://github.com/restic/restic/pull/3449

changelog/0.13.0_2022-03-26/issue-1106

@@ -0,0 +1,10 @@
+Bugfix: Never lock repository for `list locks`
+
+The `list locks` command previously locked to the repository by default. This
+had the problem that it wouldn't work for an exclusively locked repository and
+that the command would also display its own lock file which can be confusing.
+
+Now, the `list locks` command never locks the repository.
+
+https://github.com/restic/restic/issues/1106
+https://github.com/restic/restic/pull/3665

changelog/0.13.0_2022-03-26/issue-1542

@@ -0,0 +1,16 @@
+Enhancement: Add `--dry-run`/`-n` option to `backup` command
+
+Testing exclude filters and other configuration options was error prone as
+wrong filters could cause files to be uploaded unintentionally. It was also
+not possible to estimate beforehand how much data would be uploaded.
+
+The `backup` command now has a `--dry-run`/`-n` option, which performs all the
+normal steps of a backup without actually writing anything to the repository.
+
+Passing -vv will log information about files that would be added, allowing for
+verification of source and exclusion options before running the real backup.
+
+https://github.com/restic/restic/issues/1542
+https://github.com/restic/restic/pull/2308
+https://github.com/restic/restic/pull/3210
+https://github.com/restic/restic/pull/3300

changelog/0.13.0_2022-03-26/issue-2202

@@ -0,0 +1,15 @@
+Enhancement: Add upload checksum for Azure, GS, S3 and Swift backends
+
+Previously only the B2 and partially the Swift backends verified the integrity
+of uploaded (encrypted) files. The verification works by informing the backend
+about the expected hash of the uploaded file. The backend then verifies the
+upload and thereby rules out any data corruption during upload.
+
+We have now added upload checksums for the Azure, GS, S3 and Swift backends,
+which besides integrity checking for uploads also means that restic can now be
+used to store backups in S3 buckets which have Object Lock enabled.
+
+https://github.com/restic/restic/issues/2202
+https://github.com/restic/restic/issues/2700
+https://github.com/restic/restic/issues/3023
+https://github.com/restic/restic/pull/3246

changelog/0.13.0_2022-03-26/issue-233

@@ -0,0 +1,29 @@
+Enhancement: Support negative include/exclude patterns
+
+If a pattern starts with an exclamation mark and it matches a file that was
+previously matched by a regular pattern, the match is cancelled. Notably,
+this can be used with `--exclude-file` to cancel the exclusion of some files.
+
+It works similarly to `.gitignore`, with the same limitation; Once a directory
+is excluded, it is not possible to include files inside the directory.
+
+Example of use as an exclude pattern for the `backup` command:
+
+    $HOME/**/*
+    !$HOME/Documents
+    !$HOME/code
+    !$HOME/.emacs.d
+    !$HOME/games
+    # [...]
+    node_modules
+    *~
+    *.o
+    *.lo
+    *.pyc
+    # [...]
+    $HOME/code/linux/*
+    !$HOME/code/linux/.git
+    # [...]
+
+https://github.com/restic/restic/issues/233
+https://github.com/restic/restic/pull/2311

changelog/0.13.0_2022-03-26/issue-2345

@@ -0,0 +1,13 @@
+Bugfix: Make cache crash-resistant and usable by multiple concurrent processes
+
+The restic cache directory (`RESTIC_CACHE_DIR`) could end up in a broken state
+in the event of restic (or the OS) crashing. This is now less likely to occur
+as files are downloaded to a temporary location before being moved to their
+proper location.
+
+This also allows multiple concurrent restic processes to operate on a single
+repository without conflicts. Previously, concurrent operations could cause
+segfaults because the processes saw each other's partially downloaded files.
+
+https://github.com/restic/restic/issues/2345
+https://github.com/restic/restic/pull/2838

changelog/0.13.0_2022-03-26/issue-2388

@@ -0,0 +1,10 @@
+Enhancement: Add warning for S3 if partial credentials are provided
+
+Previously restic did not notify about incomplete credentials when using the
+S3 backend, instead just reporting access denied.
+
+Restic now checks that both the AWS key ID and secret environment variables are
+set before connecting to the remote server, and reports an error if not.
+
+https://github.com/restic/restic/issues/2388
+https://github.com/restic/restic/pull/3532

changelog/0.13.0_2022-03-26/issue-2452

@@ -0,0 +1,11 @@
+Bugfix: Improve error handling of repository locking
+
+Previously, when the lock refresh failed to delete the old lock file, it forgot
+about the newly created one. Instead it continued trying to delete the old
+(usually no longer existing) lock file and thus over time lots of lock files
+accumulated. This has now been fixed.
+
+https://github.com/restic/restic/issues/2452
+https://github.com/restic/restic/issues/2473
+https://github.com/restic/restic/issues/2562
+https://github.com/restic/restic/pull/3512

changelog/0.13.0_2022-03-26/issue-2508

@@ -0,0 +1,8 @@
+Enhancement: Support JSON output and quiet mode for the `diff` command
+
+The `diff` command now supports outputting machine-readable output in JSON
+format. To enable this, pass the `--json` option to the command. To only print
+the summary and suppress detailed output, pass the `--quiet` option.
+
+https://github.com/restic/restic/issues/2508
+https://github.com/restic/restic/pull/3592

changelog/0.13.0_2022-03-26/issue-2656

@@ -0,0 +1,8 @@
+Enhancement: Add flag to disable TLS verification for self-signed certificates
+
+There is now an `--insecure-tls` global option in restic, which disables TLS
+verification for self-signed certificates in order to support some development
+workflows.
+
+https://github.com/restic/restic/issues/2656
+https://github.com/restic/restic/pull/2657

changelog/0.13.0_2022-03-26/issue-2738

@@ -0,0 +1,8 @@
+Bugfix: Don't print progress for `backup --json --quiet`
+
+Unlike the text output, the `--json` output format still printed progress
+information even in `--quiet` mode. This has now been fixed by always
+disabling the progress output in quiet mode.
+
+https://github.com/restic/restic/issues/2738
+https://github.com/restic/restic/pull/3264

changelog/0.13.0_2022-03-26/issue-3003

@@ -0,0 +1,9 @@
+Enhancement: Atomic uploads for the SFTP backend
+
+The SFTP backend did not upload files atomically. An interrupted upload could
+leave an incomplete file behind which could prevent restic from accessing the
+repository. This has now been fixed and uploads in the SFTP backend are done
+atomically.
+
+https://github.com/restic/restic/issues/3003
+https://github.com/restic/restic/pull/3524

changelog/0.13.0_2022-03-26/issue-3127

@@ -0,0 +1,6 @@
+Enhancement: Add xattr (extended attributes) support for Solaris
+
+Restic now supports xattr for the Solaris operating system.
+
+https://github.com/restic/restic/issues/3127
+https://github.com/restic/restic/pull/3628

changelog/0.13.0_2022-03-26/issue-3382

@@ -0,0 +1,8 @@
+Bugfix: Make `check` command honor `RESTIC_CACHE_DIR` environment variable
+
+Previously, the `check` command didn't honor the `RESTIC_CACHE_DIR` environment
+variable, which caused problems in certain system/usage configurations. This
+has now been fixed.
+
+https://github.com/restic/restic/issues/3382
+https://github.com/restic/restic/pull/3474

changelog/0.13.0_2022-03-26/issue-3464

@@ -0,0 +1,10 @@
+Enhancement: Skip lock creation on `forget` if `--no-lock` and `--dry-run`
+
+Restic used to silently ignore the `--no-lock` option of the `forget` command.
+
+It now skips creation of lock file in case both `--dry-run` and `--no-lock`
+are specified. If `--no-lock` option is specified without `--dry-run`, restic
+prints a warning message to stderr.
+
+https://github.com/restic/restic/issues/3464
+https://github.com/restic/restic/pull/3623

changelog/0.13.0_2022-03-26/issue-3490

@@ -0,0 +1,8 @@
+Enhancement: Support random subset by size in `check --read-data-subset`
+
+The `--read-data-subset` option of the `check` command now supports a third way
+of specifying the subset to check, namely `nS` where `n` is a size in bytes with
+suffix `S` as k/K, m/M, g/G or t/T.
+
+https://github.com/restic/restic/issues/3490
+https://github.com/restic/restic/pull/3548

changelog/0.13.0_2022-03-26/issue-3518

@@ -0,0 +1,8 @@
+Bugfix: Make `copy` command honor `--no-lock` for source repository
+
+The `copy` command previously did not respect the `--no-lock` option for the
+source repository, causing failures with read-only storage backends. This has
+now been fixed such that the option is now respected.
+
+https://github.com/restic/restic/issues/3518
+https://github.com/restic/restic/pull/3589

changelog/0.13.0_2022-03-26/issue-3541

@@ -0,0 +1,9 @@
+Enhancement: Improve handling of temporary B2 delete errors
+
+Deleting files on B2 could sometimes fail temporarily, which required restic to
+retry the delete operation. In some cases the file was deleted nevertheless,
+causing the retries and ultimately the restic command to fail. This has now been
+fixed.
+
+https://github.com/restic/restic/issues/3541
+https://github.com/restic/restic/pull/3544

changelog/0.13.0_2022-03-26/issue-3542

@@ -0,0 +1,8 @@
+Enhancement: Add file mode in symbolic notation to `ls --json`
+
+The `ls --json` command now provides the file mode in symbolic notation (using
+the `permissions` key), aligned with `find --json`.
+
+https://github.com/restic/restic/issues/3542
+https://github.com/restic/restic/pull/3573
+https://forum.restic.net/t/restic-ls-understanding-file-mode-with-json/4371

changelog/0.13.0_2022-03-26/issue-3556

@@ -0,0 +1,10 @@
+Bugfix: Fix hang with Backblaze B2 on SSL certificate authority error
+
+Previously, if a request failed with an SSL unknown certificate authority
+error, the B2 backend retried indefinitely and restic would appear to hang.
+
+This has now been fixed and restic instead fails with an error message.
+
+https://github.com/restic/restic/issues/3556
+https://github.com/restic/restic/issues/2355
+https://github.com/restic/restic/pull/3571

changelog/0.13.0_2022-03-26/issue-3601

@@ -0,0 +1,15 @@
+Bugfix: Fix rclone backend prematurely exiting when receiving SIGINT on Windows
+
+Previously, pressing Ctrl+C in a Windows console where restic was running with
+rclone as the backend would cause rclone to exit prematurely due to getting a
+`SIGINT` signal at the same time as restic. Restic would then wait for a long
+time for time with "unexpected EOF" and "rclone stdio connection already closed"
+errors.
+
+This has now been fixed by restic starting the rclone process detached from the
+console restic runs in (similar to starting processes in a new process group on
+Linux), which enables restic to gracefully clean up rclone (which now never gets
+the `SIGINT`).
+
+https://github.com/restic/restic/issues/3601
+https://github.com/restic/restic/pull/3602

changelog/0.13.0_2022-03-26/issue-3641

@@ -0,0 +1,14 @@
+Change: Ignore parent snapshot for `backup --stdin`
+
+Restic uses a parent snapshot to speed up directory scanning when performing
+backups, but this only wasted time and memory when the backup source is stdin
+(using the `--stdin` option of the `backup` command), since no directory scanning
+is performed in this case.
+
+Snapshots made with `backup --stdin` no longer have a parent snapshot, which allows
+restic to skip some startup operations and saves a bit of resources.
+
+The `--parent` option is still available for `backup --stdin`, but is now ignored.
+
+https://github.com/restic/restic/issues/3641
+https://github.com/restic/restic/pull/3645

changelog/0.13.0_2022-03-26/issue-3667

@@ -0,0 +1,8 @@
+Bugfix: The `mount` command now reports symlinks sizes
+
+Symlinks used to have size zero in restic mountpoints, confusing some
+third-party tools. They now have a size equal to the byte length of their
+target path, as required by POSIX.
+
+https://github.com/restic/restic/issues/3667
+https://github.com/restic/restic/pull/3668

changelog/0.13.0_2022-03-26/pull-2594

@@ -0,0 +1,7 @@
+Enhancement: Speed up the `restore --verify` command
+
+The `--verify` option lets the `restore` command verify the file content
+after it has restored a snapshot. The performance of this operation has
+now been improved by up to a factor of two.
+
+https://github.com/restic/restic/pull/2594

changelog/0.13.0_2022-03-26/pull-2816

@@ -0,0 +1,10 @@
+Enhancement: The `backup` command no longer updates file access times on Linux
+
+When reading files during backup, restic used to cause the operating system to
+update the files' access times. Note that this did not apply to filesystems with
+disabled file access times.
+
+Restic now instructs the operating system not to update the file access time,
+if the user running restic is the file owner or has root permissions.
+
+https://github.com/restic/restic/pull/2816

changelog/0.13.0_2022-03-26/pull-2880

@@ -0,0 +1,9 @@
+Enhancement: Make `recover` collect only unreferenced trees
+
+Previously, the `recover` command used to generate a snapshot containing *all*
+root trees, even those which were already referenced by a snapshot.
+
+This has been improved such that it now only processes trees not already
+referenced by any snapshot.
+
+https://github.com/restic/restic/pull/2880

changelog/0.13.0_2022-03-26/pull-3429

@@ -0,0 +1,12 @@
+Enhancement: Verify that new or modified keys are stored correctly
+
+When adding a new key or changing the password of a key, restic used to just
+create the new key (and remove the old one, when changing the password). There
+was no verification that the new key was stored correctly and works properly.
+As the repository cannot be decrypted without a valid key file, this could in
+rare cases cause the repository to become inaccessible.
+
+Restic now checks that new key files actually work before continuing. This
+can protect against some (rare) cases of hardware or storage problems.
+
+https://github.com/restic/restic/pull/3429

changelog/0.13.0_2022-03-26/pull-3436

@@ -0,0 +1,12 @@
+Enhancement: Improve local backend's resilience to (system) crashes
+
+Restic now ensures that files stored using the `local` backend are created
+atomically (that is, files are either stored completely or not at all). This
+ensures that no incomplete files are left behind even if restic is terminated
+while writing a file.
+
+In addition, restic now tries to ensure that the directory in the repository
+which contains a newly uploaded file is also written to disk. This can prevent
+missing files if the system crashes or the disk is not properly unmounted.
+
+https://github.com/restic/restic/pull/3436

changelog/0.13.0_2022-03-26/pull-3488

@@ -0,0 +1,9 @@
+Bugfix: `rebuild-index` failed if an index file was damaged
+
+Previously, the `rebuild-index` command would fail with an error if an index
+file was damaged or truncated. This has now been fixed.
+
+On older restic versions, a (slow) workaround is to use
+`rebuild-index --read-all-packs` or to manually delete the damaged index.
+
+https://github.com/restic/restic/pull/3488

changelog/0.13.0_2022-03-26/pull-3508

@@ -0,0 +1,10 @@
+Enhancement: Cache blobs read by the `dump` command
+
+When dumping a file using the `dump` command, restic did not cache blobs in any
+way, so even consecutive runs of the same blob were loaded from the repository
+again and again, slowing down the dump.
+
+Now, the caching mechanism already used by the `fuse` command is also used by
+the `dump` command. This makes dumping much faster, especially for sparse files.
+
+https://github.com/restic/restic/pull/3508

changelog/0.13.0_2022-03-26/pull-3514

@@ -0,0 +1,8 @@
+Enhancement: Support configurable timeout for the rclone backend
+
+A slow rclone backend could cause restic to time out while waiting for the
+repository to open. Restic now offers an `-o rclone.timeout` option to make
+this timeout configurable.
+
+https://github.com/restic/restic/issues/3511
+https://github.com/restic/restic/pull/3514

changelog/0.13.0_2022-03-26/pull-3519

@@ -0,0 +1,6 @@
+Change: Require Go 1.14 or newer
+
+Restic now requires Go 1.14 to build. This allows it to use new
+standard library features instead of an external dependency.
+
+https://github.com/restic/restic/issues/3519

changelog/0.13.0_2022-03-26/pull-3591

@@ -0,0 +1,8 @@
+Bugfix: Fix handling of `prune --max-repack-size=0`
+
+Restic ignored the `--max-repack-size` option when passing a value of 0. This
+has now been fixed.
+
+As a workaround, `--max-repack-size=1` can be used with older versions of restic.
+
+https://github.com/restic/restic/pull/3591

changelog/0.13.0_2022-03-26/pull-3593

@@ -0,0 +1,10 @@
+Enhancement: Improve `copy` performance by parallelizing IO
+
+Restic copy previously only used a single thread for copying blobs between
+repositories, which resulted in limited performance when copying small blobs
+to/from a high latency backend (i.e. any remote backend, especially b2).
+
+Copying will now use 8 parallel threads to increase the throughput of the copy
+operation.
+
+https://github.com/restic/restic/pull/3593

changelog/0.13.0_2022-03-26/pull-3619

@@ -0,0 +1,11 @@
+Bugfix: Avoid choosing parent snapshots newer than time of new snapshot
+
+The `backup` command, when a `--parent` was not provided, previously chose the
+most recent matching snapshot as the parent snapshot. However, this didn't make
+sense when the user passed `--time` to create a new snapshot older than the most
+recent snapshot.
+
+Instead, `backup` now chooses the most recent snapshot which is not newer than
+the snapshot-being-created's timestamp, to avoid any time travel.
+
+https://github.com/restic/restic/pull/3619

changelog/0.7.1_2017-07-22/issue-1081

@@ -1,4 +1,4 @@
-Enhancement: Clarify semantic for `--tasg` for the `forget` command
+Enhancement: Clarify semantic for `--tag` for the `forget` command
 
 https://github.com/restic/restic/issues/1081
 https://github.com/restic/restic/pull/1090

cmd/restic/cleanup.go

@@ -58,7 +58,7 @@ func RunCleanupHandlers() {
 func CleanupHandler(c <-chan os.Signal) {
 	for s := range c {
 		debug.Log("signal %v received, cleaning up", s)
-		Warnf("%ssignal %v received, cleaning up\n", ClearLine(), s)
+		Warnf("%ssignal %v received, cleaning up\n", clearLine(0), s)
 
 		code := 0
 

cmd/restic/cmd_backup.go

@@ -24,8 +24,7 @@ import (
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/textfile"
-	"github.com/restic/restic/internal/ui"
-	"github.com/restic/restic/internal/ui/json"
+	"github.com/restic/restic/internal/ui/backup"
 	"github.com/restic/restic/internal/ui/termstatus"
 )
 
@@ -60,11 +59,11 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
 		t.Go(func() error { term.Run(t.Context(globalOptions.ctx)); return nil })
 
 		err := runBackup(backupOptions, globalOptions, term, args)
-		if err != nil {
-			return err
-		}
 		t.Kill(nil)
-		return t.Wait()
+		if werr := t.Wait(); werr != nil {
+			panic(fmt.Sprintf("term.Run() returned err: %v", err))
+		}
+		return err
 	},
 }
 
@@ -92,24 +91,25 @@ type BackupOptions struct {
 	IgnoreInode             bool
 	IgnoreCtime             bool
 	UseFsSnapshot           bool
+	DryRun                  bool
 }
 
 var backupOptions BackupOptions
 
 // ErrInvalidSourceData is used to report an incomplete backup
-var ErrInvalidSourceData = errors.New("failed to read all source data during backup")
+var ErrInvalidSourceData = errors.New("at least one source file could not be read")
 
 func init() {
 	cmdRoot.AddCommand(cmdBackup)
 
 	f := cmdBackup.Flags()
-	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: last snapshot in the repo that has the same target files/directories)")
+	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: last snapshot in the repo that has the same target files/directories, and is not newer than the snapshot time)")
 	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
 	f.StringArrayVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
 	f.StringArrayVar(&backupOptions.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
 	f.StringArrayVar(&backupOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
 	f.StringArrayVar(&backupOptions.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns")
-	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems")
+	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems, don't cross filesystem boundaries and subvolumes")
 	f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes `filename[:header]`, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)")
 	f.BoolVar(&backupOptions.ExcludeCaches, "exclude-caches", false, `excludes cache directories that are marked with a CACHEDIR.TAG file. See https://bford.info/cachedir/ for the Cache Directory Tagging Standard`)
 	f.StringVar(&backupOptions.ExcludeLargerThan, "exclude-larger-than", "", "max `size` of the files to be backed up (allowed suffixes: k/K, m/M, g/G, t/T)")
@@ -132,6 +132,7 @@ func init() {
 	f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories")
 	f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number changes when checking for modified files")
 	f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files")
+	f.BoolVarP(&backupOptions.DryRun, "dry-run", "n", false, "do not upload or write any data, just show what would be done")
 	if runtime.GOOS == "windows" {
 		f.BoolVar(&backupOptions.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)")
 	}
@@ -471,7 +472,7 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 
 // parent returns the ID of the parent snapshot. If there is none, nil is
 // returned.
-func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string) (parentID *restic.ID, err error) {
+func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string, timeStampLimit time.Time) (parentID *restic.ID, err error) {
 	// Force using a parent
 	if !opts.Force && opts.Parent != "" {
 		id, err := restic.FindSnapshot(ctx, repo, opts.Parent)
@@ -484,7 +485,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
 
 	// Find last snapshot to set it as parent, if not already set
 	if !opts.Force && parentID == nil {
-		id, err := restic.FindLatestSnapshot(ctx, repo, targets, []restic.TagList{}, []string{opts.Host})
+		id, err := restic.FindLatestSnapshot(ctx, repo, targets, []restic.TagList{}, []string{opts.Host}, &timeStampLimit)
 		if err == nil {
 			parentID = &id
 		} else if err != restic.ErrNoSnapshotFound {
@@ -525,33 +526,17 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		return err
 	}
 
-	type ArchiveProgressReporter interface {
-		CompleteItem(item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration)
-		StartFile(filename string)
-		CompleteBlob(filename string, bytes uint64)
-		ScannerError(item string, fi os.FileInfo, err error) error
-		ReportTotal(item string, s archiver.ScanStats)
-		SetMinUpdatePause(d time.Duration)
-		Run(ctx context.Context) error
-		Error(item string, fi os.FileInfo, err error) error
-		Finish(snapshotID restic.ID)
-
-		// ui.StdioWrapper
-		Stdout() io.WriteCloser
-		Stderr() io.WriteCloser
-
-		// ui.Message
-		E(msg string, args ...interface{})
-		P(msg string, args ...interface{})
-		V(msg string, args ...interface{})
-		VV(msg string, args ...interface{})
-	}
-
-	var p ArchiveProgressReporter
+	var progressPrinter backup.ProgressPrinter
 	if gopts.JSON {
-		p = json.NewBackup(term, gopts.verbosity)
+		progressPrinter = backup.NewJSONProgress(term, gopts.verbosity)
 	} else {
-		p = ui.NewBackup(term, gopts.verbosity)
+		progressPrinter = backup.NewTextProgress(term, gopts.verbosity)
+	}
+	progressReporter := backup.NewProgress(progressPrinter)
+
+	if opts.DryRun {
+		repo.SetDryRun()
+		progressReporter.SetDryRun()
 	}
 
 	// use the terminal for stdout/stderr
@@ -559,14 +544,14 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	defer func() {
 		gopts.stdout, gopts.stderr = prevStdout, prevStderr
 	}()
-	gopts.stdout, gopts.stderr = p.Stdout(), p.Stderr()
+	gopts.stdout, gopts.stderr = progressPrinter.Stdout(), progressPrinter.Stderr()
 
-	p.SetMinUpdatePause(calculateProgressInterval())
+	progressReporter.SetMinUpdatePause(calculateProgressInterval(!gopts.Quiet, gopts.JSON))
 
-	t.Go(func() error { return p.Run(t.Context(gopts.ctx)) })
+	t.Go(func() error { return progressReporter.Run(t.Context(gopts.ctx)) })
 
 	if !gopts.JSON {
-		p.V("lock repository")
+		progressPrinter.V("lock repository")
 	}
 	lock, err := lockRepo(gopts.ctx, repo)
 	defer unlockRepo(lock)
@@ -587,23 +572,26 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}
 
 	if !gopts.JSON {
-		p.V("load index files")
+		progressPrinter.V("load index files")
 	}
 	err = repo.LoadIndex(gopts.ctx)
 	if err != nil {
 		return err
 	}
 
-	parentSnapshotID, err := findParentSnapshot(gopts.ctx, repo, opts, targets)
-	if err != nil {
-		return err
-	}
+	var parentSnapshotID *restic.ID
+	if !opts.Stdin {
+		parentSnapshotID, err = findParentSnapshot(gopts.ctx, repo, opts, targets, timeStamp)
+		if err != nil {
+			return err
+		}
 
-	if !gopts.JSON {
-		if parentSnapshotID != nil {
-			p.P("using parent snapshot %v\n", parentSnapshotID.Str())
-		} else {
-			p.P("no parent snapshot found, will read all files\n")
+		if !gopts.JSON {
+			if parentSnapshotID != nil {
+				progressPrinter.P("using parent snapshot %v\n", parentSnapshotID.Str())
+			} else {
+				progressPrinter.P("no parent snapshot found, will read all files\n")
+			}
 		}
 	}
 
@@ -632,12 +620,12 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		}
 
 		errorHandler := func(item string, err error) error {
-			return p.Error(item, nil, err)
+			return progressReporter.Error(item, nil, err)
 		}
 
 		messageHandler := func(msg string, args ...interface{}) {
 			if !gopts.JSON {
-				p.P(msg, args...)
+				progressPrinter.P(msg, args...)
 			}
 		}
 
@@ -647,7 +635,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}
 	if opts.Stdin {
 		if !gopts.JSON {
-			p.V("read data from stdin")
+			progressPrinter.V("read data from stdin")
 		}
 		filename := path.Join("/", opts.StdinFilename)
 		targetFS = &fs.Reader{
@@ -662,11 +650,11 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	sc := archiver.NewScanner(targetFS)
 	sc.SelectByName = selectByNameFilter
 	sc.Select = selectFilter
-	sc.Error = p.ScannerError
-	sc.Result = p.ReportTotal
+	sc.Error = progressReporter.ScannerError
+	sc.Result = progressReporter.ReportTotal
 
 	if !gopts.JSON {
-		p.V("start scan on %v", targets)
+		progressPrinter.V("start scan on %v", targets)
 	}
 	t.Go(func() error { return sc.Scan(t.Context(gopts.ctx), targets) })
 
@@ -677,11 +665,11 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	success := true
 	arch.Error = func(item string, fi os.FileInfo, err error) error {
 		success = false
-		return p.Error(item, fi, err)
+		return progressReporter.Error(item, fi, err)
 	}
-	arch.CompleteItem = p.CompleteItem
-	arch.StartFile = p.StartFile
-	arch.CompleteBlob = p.CompleteBlob
+	arch.CompleteItem = progressReporter.CompleteItem
+	arch.StartFile = progressReporter.StartFile
+	arch.CompleteBlob = progressReporter.CompleteBlob
 
 	if opts.IgnoreInode {
 		// --ignore-inode implies --ignore-ctime: on FUSE, the ctime is not
@@ -705,28 +693,30 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}
 
 	if !gopts.JSON {
-		p.V("start backup on %v", targets)
+		progressPrinter.V("start backup on %v", targets)
 	}
 	_, id, err := arch.Snapshot(gopts.ctx, targets, snapshotOpts)
-	if err != nil {
-		return errors.Fatalf("unable to save snapshot: %v", err)
-	}
 
 	// cleanly shutdown all running goroutines
 	t.Kill(nil)
 
 	// let's see if one returned an error
-	err = t.Wait()
+	werr := t.Wait()
+
+	// return original error
+	if err != nil {
+		return errors.Fatalf("unable to save snapshot: %v", err)
+	}
 
 	// Report finished execution
-	p.Finish(id)
-	if !gopts.JSON {
-		p.P("snapshot %s saved\n", id.Str())
+	progressReporter.Finish(id)
+	if !gopts.JSON && !opts.DryRun {
+		progressPrinter.P("snapshot %s saved\n", id.Str())
 	}
 	if !success {
 		return ErrInvalidSourceData
 	}
 
 	// Return error if any
-	return err
+	return werr
 }

cmd/restic/cmd_cache.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
 	"time"
 
 	"github.com/restic/restic/internal/cache"
@@ -140,8 +141,13 @@ func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
 			size = fmt.Sprintf("%11s", formatBytes(uint64(bytes)))
 		}
 
+		name := entry.Name()
+		if !strings.HasPrefix(name, "restic-check-cache-") {
+			name = name[:10]
+		}
+
 		tab.AddRow(data{
-			entry.Name()[:10],
+			name,
 			fmt.Sprintf("%d days ago", uint(time.Since(entry.ModTime()).Hours()/24)),
 			old,
 			size,

cmd/restic/cmd_cat.go

@@ -69,7 +69,6 @@ func runCat(gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	// handle all types that don't need an index
 	switch tpe {
 	case "config":
 		buf, err := json.MarshalIndent(repo.Config(), "", "  ")
@@ -142,15 +141,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 
 		Println(string(buf))
 		return nil
-	}
 
-	// load index, handle all the other types
-	err = repo.LoadIndex(gopts.ctx)
-	if err != nil {
-		return err
-	}
-
-	switch tpe {
 	case "pack":
 		h := restic.Handle{Type: restic.PackFile, Name: id.String()}
 		buf, err := backend.LoadAll(gopts.ctx, nil, repo.Backend(), h)
@@ -167,6 +158,11 @@ func runCat(gopts GlobalOptions, args []string) error {
 		return err
 
 	case "blob":
+		err = repo.LoadIndex(gopts.ctx)
+		if err != nil {
+			return err
+		}
+
 		for _, t := range []restic.BlobType{restic.DataBlob, restic.TreeBlob} {
 			bh := restic.BlobHandle{ID: id, Type: t}
 			if !repo.Index().Has(bh) {

cmd/restic/cmd_check.go

@@ -5,10 +5,12 @@ import (
 	"math/rand"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/spf13/cobra"
 
+	"github.com/restic/restic/internal/cache"
 	"github.com/restic/restic/internal/checker"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/fs"
@@ -54,7 +56,7 @@ func init() {
 
 	f := cmdCheck.Flags()
 	f.BoolVar(&checkOptions.ReadData, "read-data", false, "read all data blobs")
-	f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read a `subset` of data packs, specified as 'n/t' for specific subset or either 'x%' or 'x.y%' for random subset")
+	f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read a `subset` of data packs, specified as 'n/t' for specific part, or either 'x%' or 'x.y%' or a size in bytes with suffixes k/K, m/M, g/G, t/T for a random subset")
 	f.BoolVar(&checkOptions.CheckUnused, "check-unused", false, "find unused blobs")
 	f.BoolVar(&checkOptions.WithCache, "with-cache", false, "use the cache")
 }
@@ -65,7 +67,7 @@ func checkFlags(opts CheckOptions) error {
 	}
 	if opts.ReadDataSubset != "" {
 		dataSubset, err := stringToIntSlice(opts.ReadDataSubset)
-		argumentError := errors.Fatal("check flag --read-data-subset must have two positive integer values or a percentage, e.g. --read-data-subset=1/2 or --read-data-subset=2.5%%")
+		argumentError := errors.Fatal("check flag --read-data-subset has invalid value, please see documentation")
 		if err == nil {
 			if len(dataSubset) != 2 {
 				return argumentError
@@ -76,7 +78,7 @@ func checkFlags(opts CheckOptions) error {
 			if dataSubset[1] > totalBucketsMax {
 				return errors.Fatalf("check flag --read-data-subset=n/t t must be at most %d", totalBucketsMax)
 			}
-		} else {
+		} else if strings.HasSuffix(opts.ReadDataSubset, "%") {
 			percentage, err := parsePercentage(opts.ReadDataSubset)
 			if err != nil {
 				return argumentError
@@ -84,8 +86,19 @@ func checkFlags(opts CheckOptions) error {
 
 			if percentage <= 0.0 || percentage > 100.0 {
 				return errors.Fatal(
-					"check flag --read-data-subset=n% n must be above 0.0% and at most 100.0%")
+					"check flag --read-data-subset=x% x must be above 0.0% and at most 100.0%")
 			}
+
+		} else {
+			fileSize, err := parseSizeStr(opts.ReadDataSubset)
+			if err != nil {
+				return argumentError
+			}
+			if fileSize <= 0.0 {
+				return errors.Fatal(
+					"check flag --read-data-subset=n n must be above 0")
+			}
+
 		}
 	}
 
@@ -146,6 +159,9 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func())
 	}
 
 	cachedir := gopts.CacheDir
+	if cachedir == "" {
+		cachedir = cache.EnvDir()
+	}
 
 	// use a cache in a temporary directory
 	tempdir, err := ioutil.TempDir(cachedir, "restic-check-cache-")
@@ -241,7 +257,11 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 
 	Verbosef("check snapshots, trees and blobs\n")
 	errChan = make(chan error)
+	var wg sync.WaitGroup
+
+	wg.Add(1)
 	go func() {
+		defer wg.Done()
 		bar := newProgressMax(!gopts.Quiet, 0, "snapshots")
 		defer bar.Done()
 		chkr.Structure(gopts.ctx, bar, errChan)
@@ -259,6 +279,11 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
+	// Wait for the progress bar to be complete before printing more below.
+	// Must happen after `errChan` is read from in the above loop to avoid
+	// deadlocking in the case of errors.
+	wg.Wait()
+
 	if opts.CheckUnused {
 		for _, id := range chkr.UnusedBlobs(gopts.ctx) {
 			Verbosef("unused blob %v\n", id)
@@ -294,10 +319,27 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 			packs = selectPacksByBucket(chkr.GetPacks(), bucket, totalBuckets)
 			packCount := uint64(len(packs))
 			Verbosef("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, packCount, chkr.CountPacks(), totalBuckets)
+		} else if strings.HasSuffix(opts.ReadDataSubset, "%") {
+			percentage, err := parsePercentage(opts.ReadDataSubset)
+			if err == nil {
+				packs = selectRandomPacksByPercentage(chkr.GetPacks(), percentage)
+				Verbosef("read %.1f%% of data packs\n", percentage)
+			}
 		} else {
-			percentage, _ := parsePercentage(opts.ReadDataSubset)
-			packs = selectRandomPacksByPercentage(chkr.GetPacks(), percentage)
-			Verbosef("read %.1f%% of data packs\n", percentage)
+			repoSize := int64(0)
+			allPacks := chkr.GetPacks()
+			for _, size := range allPacks {
+				repoSize += size
+			}
+			if repoSize == 0 {
+				return errors.Fatal("Cannot read from a repository having size 0")
+			}
+			subsetSize, _ := parseSizeStr(opts.ReadDataSubset)
+			if subsetSize > repoSize {
+				subsetSize = repoSize
+			}
+			packs = selectRandomPacksByFileSize(chkr.GetPacks(), subsetSize, repoSize)
+			Verbosef("read %d bytes of data packs\n", subsetSize)
 		}
 		if packs == nil {
 			return errors.Fatal("internal error: failed to select packs to check")
@@ -331,7 +373,7 @@ func selectPacksByBucket(allPacks map[restic.ID]int64, bucket, totalBuckets uint
 func selectRandomPacksByPercentage(allPacks map[restic.ID]int64, percentage float64) map[restic.ID]int64 {
 	packCount := len(allPacks)
 	packsToCheck := int(float64(packCount) * (percentage / 100.0))
-	if packsToCheck < 1 {
+	if packCount > 0 && packsToCheck < 1 {
 		packsToCheck = 1
 	}
 	timeNs := time.Now().UnixNano()
@@ -349,6 +391,11 @@ func selectRandomPacksByPercentage(allPacks map[restic.ID]int64, percentage floa
 		id := keys[idx[i]]
 		packs[id] = allPacks[id]
 	}
-
+	return packs
+}
+
+func selectRandomPacksByFileSize(allPacks map[restic.ID]int64, subsetSize int64, repoSize int64) map[restic.ID]int64 {
+	subsetPercentage := (float64(subsetSize) / float64(repoSize)) * 100.0
+	packs := selectRandomPacksByPercentage(allPacks, subsetPercentage)
 	return packs
 }

cmd/restic/cmd_check_test.go

@@ -122,3 +122,44 @@ func TestSelectRandomPacksByPercentage(t *testing.T) {
 		rtest.Assert(t, ok, "Expected input and output to be equal")
 	}
 }
+
+func TestSelectNoRandomPacksByPercentage(t *testing.T) {
+	// that the a repository without pack files works
+	var testPacks = make(map[restic.ID]int64)
+	selectedPacks := selectRandomPacksByPercentage(testPacks, 10.0)
+	rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs")
+}
+
+func TestSelectRandomPacksByFileSize(t *testing.T) {
+	var testPacks = make(map[restic.ID]int64)
+	for i := 1; i <= 10; i++ {
+		id := restic.NewRandomID()
+		// ensure unique ids
+		id[0] = byte(i)
+		testPacks[id] = 0
+	}
+
+	selectedPacks := selectRandomPacksByFileSize(testPacks, 10, 500)
+	rtest.Assert(t, len(selectedPacks) == 1, "Expected 1 selected packs")
+
+	selectedPacks = selectRandomPacksByFileSize(testPacks, 10240, 51200)
+	rtest.Assert(t, len(selectedPacks) == 2, "Expected 2 selected packs")
+	for pack := range selectedPacks {
+		_, ok := testPacks[pack]
+		rtest.Assert(t, ok, "Unexpected selection")
+	}
+
+	selectedPacks = selectRandomPacksByFileSize(testPacks, 500, 500)
+	rtest.Assert(t, len(selectedPacks) == 10, "Expected 10 selected packs")
+	for pack := range selectedPacks {
+		_, ok := testPacks[pack]
+		rtest.Assert(t, ok, "Unexpected item in selection")
+	}
+}
+
+func TestSelectNoRandomPacksByFileSize(t *testing.T) {
+	// that the a repository without pack files works
+	var testPacks = make(map[restic.ID]int64)
+	selectedPacks := selectRandomPacksByFileSize(testPacks, 10, 500)
+	rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs")
+}

cmd/restic/cmd_copy.go

@@ -73,10 +73,12 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	srcLock, err := lockRepo(ctx, srcRepo)
-	defer unlockRepo(srcLock)
-	if err != nil {
-		return err
+	if !gopts.NoLock {
+		srcLock, err := lockRepo(ctx, srcRepo)
+		defer unlockRepo(srcLock)
+		if err != nil {
+			return err
+		}
 	}
 
 	dstLock, err := lockRepo(ctx, dstRepo)
@@ -174,9 +176,12 @@ func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool {
 	return true
 }
 
+const numCopyWorkers = 8
+
 func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository,
 	visitedTrees restic.IDSet, rootTreeID restic.ID) error {
 
+	idChan := make(chan restic.ID)
 	wg, ctx := errgroup.WithContext(ctx)
 
 	treeStream := restic.StreamTrees(ctx, wg, srcRepo, restic.IDs{rootTreeID}, func(treeID restic.ID) bool {
@@ -186,9 +191,9 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep
 	}, nil)
 
 	wg.Go(func() error {
+		defer close(idChan)
 		// reused buffer
 		var buf []byte
-
 		for tree := range treeStream {
 			if tree.Error != nil {
 				return fmt.Errorf("LoadTree(%v) returned error %v", tree.ID.Str(), tree.Error)
@@ -196,42 +201,57 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep
 
 			// Do we already have this tree blob?
 			if !dstRepo.Index().Has(restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob}) {
-				newTreeID, err := dstRepo.SaveTree(ctx, tree.Tree)
+				// copy raw tree bytes to avoid problems if the serialization changes
+				var err error
+				buf, err = srcRepo.LoadBlob(ctx, restic.TreeBlob, tree.ID, buf)
 				if err != nil {
-					return fmt.Errorf("SaveTree(%v) returned error %v", tree.ID.Str(), err)
+					return fmt.Errorf("LoadBlob(%v) for tree returned error %v", tree.ID, err)
 				}
-				// Assurance only.
-				if newTreeID != tree.ID {
-					return fmt.Errorf("SaveTree(%v) returned unexpected id %s", tree.ID.Str(), newTreeID.Str())
+
+				_, _, err = dstRepo.SaveBlob(ctx, restic.TreeBlob, buf, tree.ID, false)
+				if err != nil {
+					return fmt.Errorf("SaveBlob(%v) for tree returned error %v", tree.ID.Str(), err)
 				}
 			}
 
-			// TODO: parallelize blob down/upload
-
 			for _, entry := range tree.Nodes {
 				// Recursion into directories is handled by StreamTrees
 				// Copy the blobs for this file.
 				for _, blobID := range entry.Content {
-					// Do we already have this data blob?
-					if dstRepo.Index().Has(restic.BlobHandle{ID: blobID, Type: restic.DataBlob}) {
-						continue
-					}
-					debug.Log("Copying blob %s\n", blobID.Str())
-					var err error
-					buf, err = srcRepo.LoadBlob(ctx, restic.DataBlob, blobID, buf)
-					if err != nil {
-						return fmt.Errorf("LoadBlob(%v) returned error %v", blobID, err)
-					}
-
-					_, _, err = dstRepo.SaveBlob(ctx, restic.DataBlob, buf, blobID, false)
-					if err != nil {
-						return fmt.Errorf("SaveBlob(%v) returned error %v", blobID, err)
+					select {
+					case idChan <- blobID:
+					case <-ctx.Done():
+						return ctx.Err()
 					}
 				}
 			}
-
 		}
 		return nil
 	})
+
+	for i := 0; i < numCopyWorkers; i++ {
+		wg.Go(func() error {
+			// reused buffer
+			var buf []byte
+			for blobID := range idChan {
+				// Do we already have this data blob?
+				if dstRepo.Index().Has(restic.BlobHandle{ID: blobID, Type: restic.DataBlob}) {
+					continue
+				}
+				debug.Log("Copying blob %s\n", blobID.Str())
+				var err error
+				buf, err = srcRepo.LoadBlob(ctx, restic.DataBlob, blobID, buf)
+				if err != nil {
+					return fmt.Errorf("LoadBlob(%v) returned error %v", blobID, err)
+				}
+
+				_, _, err = dstRepo.SaveBlob(ctx, restic.DataBlob, buf, blobID, false)
+				if err != nil {
+					return fmt.Errorf("SaveBlob(%v) returned error %v", blobID, err)
+				}
+			}
+			return nil
+		})
+	}
 	return wg.Wait()
 }

cmd/restic/cmd_debug.go

@@ -4,12 +4,21 @@ package main
 
 import (
 	"context"
+	"crypto/aes"
+	"crypto/cipher"
 	"encoding/json"
 	"fmt"
 	"io"
+	"os"
+	"runtime"
+	"sort"
+	"time"
 
 	"github.com/spf13/cobra"
+	"golang.org/x/sync/errgroup"
 
+	"github.com/restic/restic/internal/backend"
+	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/pack"
 	"github.com/restic/restic/internal/repository"
@@ -39,9 +48,17 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 	},
 }
 
+var tryRepair bool
+var repairByte bool
+var extractPack bool
+
 func init() {
 	cmdRoot.AddCommand(cmdDebug)
 	cmdDebug.AddCommand(cmdDebugDump)
+	cmdDebug.AddCommand(cmdDebugExamine)
+	cmdDebugExamine.Flags().BoolVar(&extractPack, "extract-pack", false, "write blobs to the current directory")
+	cmdDebugExamine.Flags().BoolVar(&tryRepair, "try-repair", false, "try to repair broken blobs with single bit flips")
+	cmdDebugExamine.Flags().BoolVar(&repairByte, "repair-byte", false, "try to repair broken blobs by trying bytes")
 }
 
 func prettyPrintJSON(wr io.Writer, item interface{}) error {
@@ -165,3 +182,367 @@ func runDebugDump(gopts GlobalOptions, args []string) error {
 		return errors.Fatalf("no such type %q", tpe)
 	}
 }
+
+var cmdDebugExamine = &cobra.Command{
+	Use:               "examine pack-ID...",
+	Short:             "Examine a pack file",
+	DisableAutoGenTag: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runDebugExamine(globalOptions, args)
+	},
+}
+
+func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, bytewise bool) []byte {
+	if bytewise {
+		Printf("        trying to repair blob by finding a broken byte\n")
+	} else {
+		Printf("        trying to repair blob with single bit flip\n")
+	}
+
+	ch := make(chan int)
+	var wg errgroup.Group
+	done := make(chan struct{})
+	var fixed []byte
+	var found bool
+
+	workers := runtime.GOMAXPROCS(0)
+	Printf("         spinning up %d worker functions\n", runtime.GOMAXPROCS(0))
+	for i := 0; i < workers; i++ {
+		wg.Go(func() error {
+			// make a local copy of the buffer
+			buf := make([]byte, len(input))
+			copy(buf, input)
+
+			testFlip := func(idx int, pattern byte) bool {
+				// flip bits
+				buf[idx] ^= pattern
+
+				nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
+				plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil)
+				if err == nil {
+					Printf("\n")
+					Printf("        blob could be repaired by XORing byte %v with 0x%02x\n", idx, pattern)
+					Printf("        hash is %v\n", restic.Hash(plaintext))
+					close(done)
+					found = true
+					fixed = plaintext
+					return true
+				}
+
+				// flip bits back
+				buf[idx] ^= pattern
+				return false
+			}
+
+			for i := range ch {
+				if bytewise {
+					for j := 0; j < 255; j++ {
+						if testFlip(i, byte(j)) {
+							return nil
+						}
+					}
+				} else {
+					for j := 0; j < 7; j++ {
+						// flip each bit once
+						if testFlip(i, (1 << uint(j))) {
+							return nil
+						}
+					}
+				}
+			}
+			return nil
+		})
+	}
+
+	wg.Go(func() error {
+		defer close(ch)
+
+		start := time.Now()
+		info := time.Now()
+		for i := range input {
+			select {
+			case ch <- i:
+			case <-done:
+				Printf("     done after %v\n", time.Since(start))
+				return nil
+			}
+
+			if time.Since(info) > time.Second {
+				secs := time.Since(start).Seconds()
+				gps := float64(i) / secs
+				remaining := len(input) - i
+				eta := time.Duration(float64(remaining)/gps) * time.Second
+
+				Printf("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v",
+					i, len(input), float32(i)/float32(len(input))*100, gps, eta)
+				info = time.Now()
+			}
+		}
+		return nil
+	})
+	err := wg.Wait()
+	if err != nil {
+		panic("all go rountines can only return nil")
+	}
+
+	if !found {
+		Printf("\n        blob could not be repaired\n")
+	}
+	return fixed
+}
+
+func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte {
+	// strip signature at the end
+	l := len(buf)
+	nonce, ct := buf[:16], buf[16:l-16]
+	out := make([]byte, len(ct))
+
+	c, err := aes.NewCipher(k.EncryptionKey[:])
+	if err != nil {
+		panic(fmt.Sprintf("unable to create cipher: %v", err))
+	}
+	e := cipher.NewCTR(c, nonce)
+	e.XORKeyStream(out, ct)
+
+	return out
+}
+
+func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list []restic.Blob) error {
+	be := repo.Backend()
+	h := restic.Handle{
+		Name: pack.String(),
+		Type: restic.PackFile,
+	}
+	for _, blob := range list {
+		Printf("      loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length)
+		buf := make([]byte, blob.Length)
+		err := be.Load(ctx, h, int(blob.Length), int64(blob.Offset), func(rd io.Reader) error {
+			n, err := io.ReadFull(rd, buf)
+			if err != nil {
+				return fmt.Errorf("read error after %d bytes: %v", n, err)
+			}
+			return nil
+		})
+		if err != nil {
+			Warnf("error read: %v\n", err)
+			continue
+		}
+
+		key := repo.Key()
+
+		nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
+		plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
+		if err != nil {
+			Warnf("error decrypting blob: %v\n", err)
+			var plain []byte
+			if tryRepair || repairByte {
+				plain = tryRepairWithBitflip(ctx, key, buf, repairByte)
+			}
+			var prefix string
+			if plain != nil {
+				id := restic.Hash(plain)
+				if !id.Equal(blob.ID) {
+					Printf("         repaired blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plain), id, blob.ID)
+					prefix = "repaired-wrong-hash-"
+				} else {
+					Printf("         successfully repaired blob (length %v), hash is %v, ID matches\n", len(plain), id)
+					prefix = "repaired-"
+				}
+			} else {
+				plain = decryptUnsigned(ctx, key, buf)
+				prefix = "damaged-"
+			}
+			err = storePlainBlob(blob.ID, prefix, plain)
+			if err != nil {
+				return err
+			}
+			continue
+		}
+
+		id := restic.Hash(plaintext)
+		var prefix string
+		if !id.Equal(blob.ID) {
+			Printf("         successfully decrypted blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plaintext), id, blob.ID)
+			prefix = "wrong-hash-"
+		} else {
+			Printf("         successfully decrypted blob (length %v), hash is %v, ID matches\n", len(plaintext), id)
+			prefix = "correct-"
+		}
+		if extractPack {
+			err = storePlainBlob(id, prefix, plaintext)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func storePlainBlob(id restic.ID, prefix string, plain []byte) error {
+	filename := fmt.Sprintf("%s%s.bin", prefix, id)
+	f, err := os.Create(filename)
+	if err != nil {
+		return err
+	}
+
+	_, err = f.Write(plain)
+	if err != nil {
+		_ = f.Close()
+		return err
+	}
+
+	err = f.Close()
+	if err != nil {
+		return err
+	}
+
+	Printf("decrypt of blob %v stored at %v\n", id, filename)
+	return nil
+}
+
+func runDebugExamine(gopts GlobalOptions, args []string) error {
+	ids := make([]restic.ID, 0)
+	for _, name := range args {
+		id, err := restic.ParseID(name)
+		if err != nil {
+			Warnf("error: %v\n", err)
+			continue
+		}
+		ids = append(ids, id)
+	}
+
+	if len(ids) == 0 {
+		return errors.Fatal("no pack files to examine")
+	}
+
+	repo, err := OpenRepository(gopts)
+	if err != nil {
+		return err
+	}
+
+	if !gopts.NoLock {
+		lock, err := lockRepo(gopts.ctx, repo)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	}
+
+	err = repo.LoadIndex(gopts.ctx)
+	if err != nil {
+		return err
+	}
+
+	for _, id := range ids {
+		err := examinePack(gopts.ctx, repo, id)
+		if err != nil {
+			Warnf("error: %v\n", err)
+		}
+		if err == context.Canceled {
+			break
+		}
+	}
+	return nil
+}
+
+func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) error {
+	Printf("examine %v\n", id)
+
+	h := restic.Handle{
+		Type: restic.PackFile,
+		Name: id.String(),
+	}
+	fi, err := repo.Backend().Stat(ctx, h)
+	if err != nil {
+		return err
+	}
+	Printf("  file size is %v\n", fi.Size)
+
+	buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h)
+	if err != nil {
+		return err
+	}
+	gotID := restic.Hash(buf)
+	if !id.Equal(gotID) {
+		Printf("  wanted hash %v, got %v\n", id, gotID)
+	} else {
+		Printf("  hash for file content matches\n")
+	}
+
+	Printf("  ========================================\n")
+	Printf("  looking for info in the indexes\n")
+
+	blobsLoaded := false
+	// examine all data the indexes have for the pack file
+	for _, idx := range repo.Index().(*repository.MasterIndex).All() {
+		idxIDs, err := idx.IDs()
+		if err != nil {
+			idxIDs = restic.IDs{}
+		}
+
+		blobs := idx.ListPack(id)
+		if len(blobs) == 0 {
+			continue
+		}
+
+		Printf("    index %v:\n", idxIDs)
+
+		// convert list of blobs to []restic.Blob
+		var list []restic.Blob
+		for _, b := range blobs {
+			list = append(list, b.Blob)
+		}
+		checkPackSize(list, fi.Size)
+
+		err = loadBlobs(ctx, repo, id, list)
+		if err != nil {
+			Warnf("error: %v\n", err)
+		} else {
+			blobsLoaded = true
+		}
+	}
+
+	Printf("  ========================================\n")
+	Printf("  inspect the pack itself\n")
+
+	blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), fi.Size)
+	if err != nil {
+		return fmt.Errorf("pack %v: %v", id.Str(), err)
+	}
+	checkPackSize(blobs, fi.Size)
+
+	if !blobsLoaded {
+		return loadBlobs(ctx, repo, id, blobs)
+	}
+	return nil
+}
+
+func checkPackSize(blobs []restic.Blob, fileSize int64) {
+	// track current size and offset
+	var size, offset uint64
+
+	sort.Slice(blobs, func(i, j int) bool {
+		return blobs[i].Offset < blobs[j].Offset
+	})
+
+	for _, pb := range blobs {
+		Printf("      %v blob %v, offset %-6d, raw length %-6d\n", pb.Type, pb.ID, pb.Offset, pb.Length)
+		if offset != uint64(pb.Offset) {
+			Printf("      hole in file, want offset %v, got %v\n", offset, pb.Offset)
+		}
+		offset += uint64(pb.Length)
+		size += uint64(pb.Length)
+	}
+
+	// compute header size, per blob: 1 byte type, 4 byte length, 32 byte id
+	size += uint64(restic.CiphertextLength(len(blobs) * (1 + 4 + 32)))
+	// length in uint32 little endian
+	size += 4
+
+	if uint64(fileSize) != size {
+		Printf("      file sizes do not match: computed %v from index, file size is %v\n", size, fileSize)
+	} else {
+		Printf("      file sizes match\n")
+	}
+}

cmd/restic/cmd_diff.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"path"
 	"reflect"
 	"sort"
@@ -62,15 +63,29 @@ func loadSnapshot(ctx context.Context, repo *repository.Repository, desc string)
 
 // Comparer collects all things needed to compare two snapshots.
 type Comparer struct {
-	repo restic.Repository
-	opts DiffOptions
+	repo        restic.Repository
+	opts        DiffOptions
+	printChange func(change *Change)
+}
+
+type Change struct {
+	MessageType string `json:"message_type"` // "change"
+	Path        string `json:"path"`
+	Modifier    string `json:"modifier"`
+}
+
+func NewChange(path string, mode string) *Change {
+	return &Change{MessageType: "change", Path: path, Modifier: mode}
 }
 
 // DiffStat collects stats for all types of items.
 type DiffStat struct {
-	Files, Dirs, Others  int
-	DataBlobs, TreeBlobs int
-	Bytes                uint64
+	Files     int    `json:"files"`
+	Dirs      int    `json:"dirs"`
+	Others    int    `json:"others"`
+	DataBlobs int    `json:"data_blobs"`
+	TreeBlobs int    `json:"tree_blobs"`
+	Bytes     uint64 `json:"bytes"`
 }
 
 // Add adds stats information for node to s.
@@ -113,21 +128,14 @@ func addBlobs(bs restic.BlobSet, node *restic.Node) {
 	}
 }
 
-// DiffStats collects the differences between two snapshots.
-type DiffStats struct {
-	ChangedFiles                         int
-	Added                                DiffStat
-	Removed                              DiffStat
-	BlobsBefore, BlobsAfter, BlobsCommon restic.BlobSet
-}
-
-// NewDiffStats creates new stats for a diff run.
-func NewDiffStats() *DiffStats {
-	return &DiffStats{
-		BlobsBefore: restic.NewBlobSet(),
-		BlobsAfter:  restic.NewBlobSet(),
-		BlobsCommon: restic.NewBlobSet(),
-	}
+type DiffStatsContainer struct {
+	MessageType                          string         `json:"message_type"` // "statistics"
+	SourceSnapshot                       string         `json:"source_snapshot"`
+	TargetSnapshot                       string         `json:"target_snapshot"`
+	ChangedFiles                         int            `json:"changed_files"`
+	Added                                DiffStat       `json:"added"`
+	Removed                              DiffStat       `json:"removed"`
+	BlobsBefore, BlobsAfter, BlobsCommon restic.BlobSet `json:"-"`
 }
 
 // updateBlobs updates the blob counters in the stats struct.
@@ -162,7 +170,7 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b
 		if node.Type == "dir" {
 			name += "/"
 		}
-		Printf("%-5s%v\n", mode, name)
+		c.printChange(NewChange(name, "+"))
 		stats.Add(node)
 		addBlobs(blobs, node)
 
@@ -221,7 +229,7 @@ func uniqueNodeNames(tree1, tree2 *restic.Tree) (tree1Nodes, tree2Nodes map[stri
 	return tree1Nodes, tree2Nodes, uniqueNames
 }
 
-func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string, id1, id2 restic.ID) error {
+func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, prefix string, id1, id2 restic.ID) error {
 	debug.Log("diffing %v to %v", id1, id2)
 	tree1, err := c.repo.LoadTree(ctx, id1)
 	if err != nil {
@@ -265,7 +273,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string
 			}
 
 			if mod != "" {
-				Printf("%-5s%v\n", mod, name)
+				c.printChange(NewChange(name, mod))
 			}
 
 			if node1.Type == "dir" && node2.Type == "dir" {
@@ -284,7 +292,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string
 			if node1.Type == "dir" {
 				prefix += "/"
 			}
-			Printf("%-5s%v\n", "-", prefix)
+			c.printChange(NewChange(prefix, "-"))
 			stats.Removed.Add(node1)
 
 			if node1.Type == "dir" {
@@ -298,7 +306,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string
 			if node2.Type == "dir" {
 				prefix += "/"
 			}
-			Printf("%-5s%v\n", "+", prefix)
+			c.printChange(NewChange(prefix, "+"))
 			stats.Added.Add(node2)
 
 			if node2.Type == "dir" {
@@ -348,7 +356,9 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
+	if !gopts.JSON {
+		Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
+	}
 
 	if sn1.Tree == nil {
 		return errors.Errorf("snapshot %v has nil tree", sn1.ID().Str())
@@ -361,9 +371,33 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 	c := &Comparer{
 		repo: repo,
 		opts: diffOptions,
+		printChange: func(change *Change) {
+			Printf("%-5s%v\n", change.Modifier, change.Path)
+		},
 	}
 
-	stats := NewDiffStats()
+	if gopts.JSON {
+		enc := json.NewEncoder(gopts.stdout)
+		c.printChange = func(change *Change) {
+			err := enc.Encode(change)
+			if err != nil {
+				Warnf("JSON encode failed: %v\n", err)
+			}
+		}
+	}
+
+	if gopts.Quiet {
+		c.printChange = func(change *Change) {}
+	}
+
+	stats := &DiffStatsContainer{
+		MessageType:    "statistics",
+		SourceSnapshot: args[0],
+		TargetSnapshot: args[1],
+		BlobsBefore:    restic.NewBlobSet(),
+		BlobsAfter:     restic.NewBlobSet(),
+		BlobsCommon:    restic.NewBlobSet(),
+	}
 	stats.BlobsBefore.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn1.Tree})
 	stats.BlobsAfter.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn2.Tree})
 
@@ -376,14 +410,21 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 	updateBlobs(repo, stats.BlobsBefore.Sub(both).Sub(stats.BlobsCommon), &stats.Removed)
 	updateBlobs(repo, stats.BlobsAfter.Sub(both).Sub(stats.BlobsCommon), &stats.Added)
 
-	Printf("\n")
-	Printf("Files:       %5d new, %5d removed, %5d changed\n", stats.Added.Files, stats.Removed.Files, stats.ChangedFiles)
-	Printf("Dirs:        %5d new, %5d removed\n", stats.Added.Dirs, stats.Removed.Dirs)
-	Printf("Others:      %5d new, %5d removed\n", stats.Added.Others, stats.Removed.Others)
-	Printf("Data Blobs:  %5d new, %5d removed\n", stats.Added.DataBlobs, stats.Removed.DataBlobs)
-	Printf("Tree Blobs:  %5d new, %5d removed\n", stats.Added.TreeBlobs, stats.Removed.TreeBlobs)
-	Printf("  Added:   %-5s\n", formatBytes(uint64(stats.Added.Bytes)))
-	Printf("  Removed: %-5s\n", formatBytes(uint64(stats.Removed.Bytes)))
+	if gopts.JSON {
+		err := json.NewEncoder(gopts.stdout).Encode(stats)
+		if err != nil {
+			Warnf("JSON encode failed: %v\n", err)
+		}
+	} else {
+		Printf("\n")
+		Printf("Files:       %5d new, %5d removed, %5d changed\n", stats.Added.Files, stats.Removed.Files, stats.ChangedFiles)
+		Printf("Dirs:        %5d new, %5d removed\n", stats.Added.Dirs, stats.Removed.Dirs)
+		Printf("Others:      %5d new, %5d removed\n", stats.Added.Others, stats.Removed.Others)
+		Printf("Data Blobs:  %5d new, %5d removed\n", stats.Added.DataBlobs, stats.Removed.DataBlobs)
+		Printf("Tree Blobs:  %5d new, %5d removed\n", stats.Added.TreeBlobs, stats.Removed.TreeBlobs)
+		Printf("  Added:   %-5s\n", formatBytes(uint64(stats.Added.Bytes)))
+		Printf("  Removed: %-5s\n", formatBytes(uint64(stats.Removed.Bytes)))
+	}
 
 	return nil
 }

cmd/restic/cmd_dump.go

@@ -67,41 +67,31 @@ func splitPath(p string) []string {
 	return append(s, f)
 }
 
-func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string, writeDump dump.WriteDump) error {
-	if tree == nil {
-		return fmt.Errorf("called with a nil tree")
-	}
-	if repo == nil {
-		return fmt.Errorf("called with a nil repository")
-	}
-	l := len(pathComponents)
-	if l == 0 {
-		return fmt.Errorf("empty path components")
-	}
-
+func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string, d *dump.Dumper) error {
 	// If we print / we need to assume that there are multiple nodes at that
 	// level in the tree.
 	if pathComponents[0] == "" {
 		if err := checkStdoutArchive(); err != nil {
 			return err
 		}
-		return writeDump(ctx, repo, tree, "/", os.Stdout)
+		return d.DumpTree(ctx, tree, "/")
 	}
 
 	item := filepath.Join(prefix, pathComponents[0])
+	l := len(pathComponents)
 	for _, node := range tree.Nodes {
 		// If dumping something in the highest level it will just take the
 		// first item it finds and dump that according to the switch case below.
 		if node.Name == pathComponents[0] {
 			switch {
 			case l == 1 && dump.IsFile(node):
-				return dump.GetNodeData(ctx, os.Stdout, repo, node)
+				return d.WriteNode(ctx, node)
 			case l > 1 && dump.IsDir(node):
 				subtree, err := repo.LoadTree(ctx, *node.Subtree)
 				if err != nil {
 					return errors.Wrapf(err, "cannot load subtree for %q", item)
 				}
-				return printFromTree(ctx, subtree, repo, item, pathComponents[1:], writeDump)
+				return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d)
 			case dump.IsDir(node):
 				if err := checkStdoutArchive(); err != nil {
 					return err
@@ -110,7 +100,7 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
 				if err != nil {
 					return err
 				}
-				return writeDump(ctx, repo, subtree, item, os.Stdout)
+				return d.DumpTree(ctx, subtree, item)
 			case l > 1:
 				return fmt.Errorf("%q should be a dir, but is a %q", item, node.Type)
 			case !dump.IsFile(node):
@@ -128,12 +118,8 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 		return errors.Fatal("no file and no snapshot ID specified")
 	}
 
-	var wd dump.WriteDump
 	switch opts.Archive {
-	case "tar":
-		wd = dump.WriteTar
-	case "zip":
-		wd = dump.WriteZip
+	case "tar", "zip":
 	default:
 		return fmt.Errorf("unknown archive format %q", opts.Archive)
 	}
@@ -166,7 +152,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 	var id restic.ID
 
 	if snapshotIDString == "latest" {
-		id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Hosts)
+		id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Hosts, nil)
 		if err != nil {
 			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Hosts:%v", err, opts.Paths, opts.Hosts)
 		}
@@ -187,7 +173,8 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 		Exitf(2, "loading tree for snapshot %q failed: %v", snapshotIDString, err)
 	}
 
-	err = printFromTree(ctx, tree, repo, "/", splittedPath, wd)
+	d := dump.New(opts.Archive, repo, os.Stdout)
+	err = printFromTree(ctx, tree, repo, "/", splittedPath, d)
 	if err != nil {
 		Exitf(2, "cannot dump file: %v", err)
 	}

cmd/restic/cmd_find.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"sort"
 	"strings"
 	"time"
 
@@ -40,8 +41,6 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 	},
 }
 
-const shortStr = 8 // Length of short IDs: 4 bytes as hex strings
-
 // FindOptions bundles all options for the find command.
 type FindOptions struct {
 	Oldest             string
@@ -268,7 +267,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 		if err != nil {
 			debug.Log("Error loading tree %v: %v", parentTreeID, err)
 
-			Printf("Unable to load tree %s\n ... which belongs to snapshot %s.\n", parentTreeID, sn.ID())
+			Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID())
 
 			return false, walker.ErrSkipNode
 		}
@@ -352,7 +351,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 		if err != nil {
 			debug.Log("Error loading tree %v: %v", parentTreeID, err)
 
-			Printf("Unable to load tree %s\n ... which belongs to snapshot %s.\n", parentTreeID, sn.ID())
+			Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID())
 
 			return false, walker.ErrSkipNode
 		}
@@ -386,12 +385,12 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 				idStr := id.String()
 				if _, ok := f.blobIDs[idStr]; !ok {
 					// Look for short ID form
-					if _, ok := f.blobIDs[idStr[:shortStr]]; !ok {
+					if _, ok := f.blobIDs[id.Str()]; !ok {
 						continue
 					}
 					// Replace the short ID with the long one
 					f.blobIDs[idStr] = struct{}{}
-					delete(f.blobIDs, idStr[:shortStr])
+					delete(f.blobIDs, id.Str())
 				}
 				f.out.PrintObject("blob", idStr, nodepath, parentTreeID.String(), sn)
 			}
@@ -401,6 +400,8 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 	})
 }
 
+var errAllPacksFound = errors.New("all packs found")
+
 // packsToBlobs converts the list of pack IDs to a list of blob IDs that
 // belong to those packs.
 func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
@@ -412,20 +413,18 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
 		f.blobIDs = make(map[string]struct{})
 	}
 
-	allPacksFound := false
-	packsFound := 0
-
 	debug.Log("Looking for packs...")
 	err := f.repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
-		if allPacksFound {
-			return nil
-		}
 		idStr := id.String()
 		if _, ok := packIDs[idStr]; !ok {
 			// Look for short ID form
-			if _, ok := packIDs[idStr[:shortStr]]; !ok {
+			if _, ok := packIDs[id.Str()]; !ok {
 				return nil
 			}
+			delete(packIDs, id.Str())
+		} else {
+			// forget found id
+			delete(packIDs, idStr)
 		}
 		debug.Log("Found pack %s", idStr)
 		blobs, _, err := f.repo.ListPack(ctx, id, size)
@@ -436,25 +435,75 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
 			f.blobIDs[b.ID.String()] = struct{}{}
 		}
 		// Stop searching when all packs have been found
-		packsFound++
-		if packsFound >= len(packIDs) {
-			allPacksFound = true
+		if len(packIDs) == 0 {
+			return errAllPacksFound
 		}
 		return nil
 	})
 
-	if err != nil {
+	if err != nil && err != errAllPacksFound {
 		return err
 	}
 
-	if !allPacksFound {
-		return errors.Fatal("unable to find all specified pack(s)")
+	if err != errAllPacksFound {
+		// try to resolve unknown pack ids from the index
+		packIDs = f.indexPacksToBlobs(ctx, packIDs)
+	}
+
+	if len(packIDs) > 0 {
+		list := make([]string, 0, len(packIDs))
+		for h := range packIDs {
+			list = append(list, h)
+		}
+
+		sort.Strings(list)
+		return errors.Fatalf("unable to find pack(s): %v", list)
 	}
 
 	debug.Log("%d blobs found", len(f.blobIDs))
 	return nil
 }
 
+func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struct{}) map[string]struct{} {
+	wctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	// remember which packs were found in the index
+	indexPackIDs := make(map[string]struct{})
+	for pb := range f.repo.Index().Each(wctx) {
+		idStr := pb.PackID.String()
+		// keep entry in packIDs as Each() returns individual index entries
+		matchingID := false
+		if _, ok := packIDs[idStr]; ok {
+			matchingID = true
+		} else {
+			if _, ok := packIDs[pb.PackID.Str()]; ok {
+				// expand id
+				delete(packIDs, pb.PackID.Str())
+				packIDs[idStr] = struct{}{}
+				matchingID = true
+			}
+		}
+		if matchingID {
+			f.blobIDs[pb.ID.String()] = struct{}{}
+			indexPackIDs[idStr] = struct{}{}
+		}
+	}
+
+	for id := range indexPackIDs {
+		delete(packIDs, id)
+	}
+
+	if len(indexPackIDs) > 0 {
+		list := make([]string, 0, len(indexPackIDs))
+		for h := range indexPackIDs {
+			list = append(list, h)
+		}
+		Warnf("some pack files are missing from the repository, getting their blobs from the repository index: %v\n\n", list)
+	}
+	return packIDs
+}
+
 func (f *Finder) findObjectPack(ctx context.Context, id string, t restic.BlobType) {
 	idx := f.repo.Index()
 
@@ -563,7 +612,7 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	if opts.PackID {
-		err := f.packsToBlobs(ctx, []string{f.pat.pattern[0]}) // TODO: support multiple packs
+		err := f.packsToBlobs(ctx, f.pat.pattern)
 		if err != nil {
 			return err
 		}

cmd/restic/cmd_forget.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"io"
 
+	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
 	"github.com/spf13/cobra"
 )
@@ -15,8 +16,9 @@ var cmdForget = &cobra.Command{
 	Long: `
 The "forget" command removes snapshots according to a policy. Please note that
 this command really only deletes the snapshot object in the repository, which
-is a reference to data stored there. In order to remove this (now unreferenced)
-data after 'forget' was run successfully, see the 'prune' command.
+is a reference to data stored there. In order to remove the unreferenced data
+after "forget" was run successfully, see the "prune" command. Please also read
+the documentation for "forget" to learn about important security considerations.
 
 EXIT STATUS
 ===========
@@ -31,14 +33,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 
 // ForgetOptions collects all options for the forget command.
 type ForgetOptions struct {
-	Last     int
-	Hourly   int
-	Daily    int
-	Weekly   int
-	Monthly  int
-	Yearly   int
-	Within   restic.Duration
-	KeepTags restic.TagLists
+	Last          int
+	Hourly        int
+	Daily         int
+	Weekly        int
+	Monthly       int
+	Yearly        int
+	Within        restic.Duration
+	WithinHourly  restic.Duration
+	WithinDaily   restic.Duration
+	WithinWeekly  restic.Duration
+	WithinMonthly restic.Duration
+	WithinYearly  restic.Duration
+	KeepTags      restic.TagLists
 
 	Hosts   []string
 	Tags    restic.TagLists
@@ -64,6 +71,11 @@ func init() {
 	f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots")
 	f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots")
 	f.VarP(&forgetOptions.Within, "keep-within", "", "keep snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
+	f.VarP(&forgetOptions.WithinHourly, "keep-within-hourly", "", "keep hourly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
+	f.VarP(&forgetOptions.WithinDaily, "keep-within-daily", "", "keep daily snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
+	f.VarP(&forgetOptions.WithinWeekly, "keep-within-weekly", "", "keep weekly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
+	f.VarP(&forgetOptions.WithinMonthly, "keep-within-monthly", "", "keep monthly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
+	f.VarP(&forgetOptions.WithinYearly, "keep-within-yearly", "", "keep yearly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
 
 	f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)")
 	f.StringArrayVar(&forgetOptions.Hosts, "host", nil, "only consider snapshots with the given `host` (can be specified multiple times)")
@@ -98,10 +110,16 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	lock, err := lockRepoExclusive(gopts.ctx, repo)
-	defer unlockRepo(lock)
-	if err != nil {
-		return err
+	if gopts.NoLock && !opts.DryRun {
+		return errors.Fatal("--no-lock is only applicable in combination with --dry-run for forget command")
+	}
+
+	if !opts.DryRun || !gopts.NoLock {
+		lock, err := lockRepoExclusive(gopts.ctx, repo)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
 	}
 
 	ctx, cancel := context.WithCancel(gopts.ctx)
@@ -128,14 +146,19 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		}
 
 		policy := restic.ExpirePolicy{
-			Last:    opts.Last,
-			Hourly:  opts.Hourly,
-			Daily:   opts.Daily,
-			Weekly:  opts.Weekly,
-			Monthly: opts.Monthly,
-			Yearly:  opts.Yearly,
-			Within:  opts.Within,
-			Tags:    opts.KeepTags,
+			Last:          opts.Last,
+			Hourly:        opts.Hourly,
+			Daily:         opts.Daily,
+			Weekly:        opts.Weekly,
+			Monthly:       opts.Monthly,
+			Yearly:        opts.Yearly,
+			Within:        opts.Within,
+			WithinHourly:  opts.WithinHourly,
+			WithinDaily:   opts.WithinDaily,
+			WithinWeekly:  opts.WithinWeekly,
+			WithinMonthly: opts.WithinMonthly,
+			WithinYearly:  opts.WithinYearly,
+			Tags:          opts.KeepTags,
 		}
 
 		if policy.Empty() && len(args) == 0 {

cmd/restic/cmd_generate.go

@@ -10,10 +10,10 @@ import (
 
 var cmdGenerate = &cobra.Command{
 	Use:   "generate [flags]",
-	Short: "Generate manual pages and auto-completion files (bash, zsh)",
+	Short: "Generate manual pages and auto-completion files (bash, fish, zsh)",
 	Long: `
 The "generate" command writes automatically generated files (like the man pages
-and the auto-completion files for bash and zsh).
+and the auto-completion files for bash, fish and zsh).
 
 EXIT STATUS
 ===========
@@ -27,6 +27,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 type generateOptions struct {
 	ManDir             string
 	BashCompletionFile string
+	FishCompletionFile string
 	ZSHCompletionFile  string
 }
 
@@ -37,6 +38,7 @@ func init() {
 	fs := cmdGenerate.Flags()
 	fs.StringVar(&genOpts.ManDir, "man", "", "write man pages to `directory`")
 	fs.StringVar(&genOpts.BashCompletionFile, "bash-completion", "", "write bash completion `file`")
+	fs.StringVar(&genOpts.FishCompletionFile, "fish-completion", "", "write fish completion `file`")
 	fs.StringVar(&genOpts.ZSHCompletionFile, "zsh-completion", "", "write zsh completion `file`")
 }
 
@@ -63,6 +65,11 @@ func writeBashCompletion(file string) error {
 	return cmdRoot.GenBashCompletionFile(file)
 }
 
+func writeFishCompletion(file string) error {
+	Verbosef("writing fish completion file to %v\n", file)
+	return cmdRoot.GenFishCompletionFile(file, true)
+}
+
 func writeZSHCompletion(file string) error {
 	Verbosef("writing zsh completion file to %v\n", file)
 	return cmdRoot.GenZshCompletionFile(file)
@@ -83,6 +90,13 @@ func runGenerate(cmd *cobra.Command, args []string) error {
 		}
 	}
 
+	if genOpts.FishCompletionFile != "" {
+		err := writeFishCompletion(genOpts.FishCompletionFile)
+		if err != nil {
+			return err
+		}
+	}
+
 	if genOpts.ZSHCompletionFile != "" {
 		err := writeZSHCompletion(genOpts.ZSHCompletionFile)
 		if err != nil {

cmd/restic/cmd_init.go

@@ -53,11 +53,6 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	be, err := create(repo, gopts.extended)
-	if err != nil {
-		return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
-	}
-
 	gopts.password, err = ReadPasswordTwice(gopts,
 		"enter password for new repository: ",
 		"enter password again: ")
@@ -65,6 +60,11 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
+	be, err := create(repo, gopts.extended)
+	if err != nil {
+		return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
+	}
+
 	s := repository.New(be)
 
 	err = s.Init(gopts.ctx, gopts.password, chunkerPolynomial)

cmd/restic/cmd_key.go

@@ -131,6 +131,11 @@ func addKey(gopts GlobalOptions, repo *repository.Repository) error {
 		return errors.Fatalf("creating new key failed: %v\n", err)
 	}
 
+	err = switchToNewKeyAndRemoveIfBroken(gopts.ctx, repo, id, pw)
+	if err != nil {
+		return err
+	}
+
 	Verbosef("saved new key as %s\n", id)
 
 	return nil
@@ -161,8 +166,14 @@ func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
 	if err != nil {
 		return errors.Fatalf("creating new key failed: %v\n", err)
 	}
+	oldID := repo.KeyName()
 
-	h := restic.Handle{Type: restic.KeyFile, Name: repo.KeyName()}
+	err = switchToNewKeyAndRemoveIfBroken(gopts.ctx, repo, id, pw)
+	if err != nil {
+		return err
+	}
+
+	h := restic.Handle{Type: restic.KeyFile, Name: oldID}
 	err = repo.Backend().Remove(gopts.ctx, h)
 	if err != nil {
 		return err
@@ -173,6 +184,19 @@ func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
 	return nil
 }
 
+func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repository, key *repository.Key, pw string) error {
+	// Verify new key to make sure it really works. A broken key can render the
+	// whole repository inaccessible
+	err := repo.SearchKey(ctx, pw, 0, key.Name())
+	if err != nil {
+		// the key is invalid, try to remove it
+		h := restic.Handle{Type: restic.KeyFile, Name: key.Name()}
+		_ = repo.Backend().Remove(ctx, h)
+		return errors.Fatalf("failed to access repository with new key: %v", err)
+	}
+	return nil
+}
+
 func runKey(gopts GlobalOptions, args []string) error {
 	if len(args) < 1 || (args[0] == "remove" && len(args) != 2) || (args[0] != "remove" && len(args) != 1) {
 		return errors.Fatal("wrong number of arguments")

cmd/restic/cmd_list.go

@@ -39,7 +39,7 @@ func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error {
 		return err
 	}
 
-	if !opts.NoLock {
+	if !opts.NoLock && args[0] != "locks" {
 		lock, err := lockRepo(opts.ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {

cmd/restic/cmd_ls.go

@@ -61,9 +61,9 @@ func init() {
 
 	flags := cmdLs.Flags()
 	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
-	flags.StringArrayVarP(&lsOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)")
-	flags.Var(&lsOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot ID is given")
-	flags.StringArrayVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
+	flags.StringArrayVarP(&lsOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when snapshot ID \"latest\" is given (can be specified multiple times)")
+	flags.Var(&lsOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when snapshot ID \"latest\" is given (can be specified multiple times)")
+	flags.StringArrayVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when snapshot ID \"latest\" is given (can be specified multiple times)")
 	flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories")
 }
 
@@ -74,23 +74,49 @@ type lsSnapshot struct {
 	StructType string     `json:"struct_type"` // "snapshot"
 }
 
-type lsNode struct {
-	Name       string      `json:"name"`
-	Type       string      `json:"type"`
-	Path       string      `json:"path"`
-	UID        uint32      `json:"uid"`
-	GID        uint32      `json:"gid"`
-	Size       uint64      `json:"size,omitempty"`
-	Mode       os.FileMode `json:"mode,omitempty"`
-	ModTime    time.Time   `json:"mtime,omitempty"`
-	AccessTime time.Time   `json:"atime,omitempty"`
-	ChangeTime time.Time   `json:"ctime,omitempty"`
-	StructType string      `json:"struct_type"` // "node"
+// Print node in our custom JSON format, followed by a newline.
+func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
+	n := &struct {
+		Name        string      `json:"name"`
+		Type        string      `json:"type"`
+		Path        string      `json:"path"`
+		UID         uint32      `json:"uid"`
+		GID         uint32      `json:"gid"`
+		Size        *uint64     `json:"size,omitempty"`
+		Mode        os.FileMode `json:"mode,omitempty"`
+		Permissions string      `json:"permissions,omitempty"`
+		ModTime     time.Time   `json:"mtime,omitempty"`
+		AccessTime  time.Time   `json:"atime,omitempty"`
+		ChangeTime  time.Time   `json:"ctime,omitempty"`
+		StructType  string      `json:"struct_type"` // "node"
+
+		size uint64 // Target for Size pointer.
+	}{
+		Name:        node.Name,
+		Type:        node.Type,
+		Path:        path,
+		UID:         node.UID,
+		GID:         node.GID,
+		size:        node.Size,
+		Mode:        node.Mode,
+		Permissions: node.Mode.String(),
+		ModTime:     node.ModTime,
+		AccessTime:  node.AccessTime,
+		ChangeTime:  node.ChangeTime,
+		StructType:  "node",
+	}
+	// Always print size for regular files, even when empty,
+	// but never for other types.
+	if node.Type == "file" {
+		n.Size = &n.size
+	}
+
+	return enc.Encode(n)
 }
 
 func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 	if len(args) == 0 {
-		return errors.Fatal("no snapshot ID specified")
+		return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'")
 	}
 
 	// extract any specific directories to walk
@@ -159,7 +185,7 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 		enc := json.NewEncoder(gopts.stdout)
 
 		printSnapshot = func(sn *restic.Snapshot) {
-			err = enc.Encode(lsSnapshot{
+			err := enc.Encode(lsSnapshot{
 				Snapshot:   sn,
 				ID:         sn.ID(),
 				ShortID:    sn.ID().Str(),
@@ -171,19 +197,7 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 		}
 
 		printNode = func(path string, node *restic.Node) {
-			err = enc.Encode(lsNode{
-				Name:       node.Name,
-				Type:       node.Type,
-				Path:       path,
-				UID:        node.UID,
-				GID:        node.GID,
-				Size:       node.Size,
-				Mode:       node.Mode,
-				ModTime:    node.ModTime,
-				AccessTime: node.AccessTime,
-				ChangeTime: node.ChangeTime,
-				StructType: "node",
-			})
+			err := lsNodeJSON(enc, path, node)
 			if err != nil {
 				Warnf("JSON encode failed: %v\n", err)
 			}

cmd/restic/cmd_ls_test.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/restic/restic/internal/restic"
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func TestLsNodeJSON(t *testing.T) {
+	for _, c := range []struct {
+		path string
+		restic.Node
+		expect string
+	}{
+		// Mode is omitted when zero.
+		// Permissions, by convention is "-" per mode bit
+		{
+			path: "/bar/baz",
+			Node: restic.Node{
+				Name: "baz",
+				Type: "file",
+				Size: 12345,
+				UID:  10000000,
+				GID:  20000000,
+
+				User:  "nobody",
+				Group: "nobodies",
+				Links: 1,
+			},
+			expect: `{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
+		},
+
+		// Even empty files get an explicit size.
+		{
+			path: "/foo/empty",
+			Node: restic.Node{
+				Name: "empty",
+				Type: "file",
+				Size: 0,
+				UID:  1001,
+				GID:  1001,
+
+				User:  "not printed",
+				Group: "not printed",
+				Links: 0xF00,
+			},
+			expect: `{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
+		},
+
+		// Non-regular files do not get a size.
+		// Mode is printed in decimal, including the type bits.
+		{
+			path: "/foo/link",
+			Node: restic.Node{
+				Name:       "link",
+				Type:       "symlink",
+				Mode:       os.ModeSymlink | 0777,
+				LinkTarget: "not printed",
+			},
+			expect: `{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
+		},
+
+		{
+			path: "/some/directory",
+			Node: restic.Node{
+				Name:       "directory",
+				Type:       "dir",
+				Mode:       os.ModeDir | 0755,
+				ModTime:    time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC),
+				AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC),
+				ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC),
+			},
+			expect: `{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`,
+		},
+	} {
+		buf := new(bytes.Buffer)
+		enc := json.NewEncoder(buf)
+		err := lsNodeJSON(enc, c.path, &c.Node)
+		rtest.OK(t, err)
+		rtest.Equals(t, c.expect+"\n", buf.String())
+
+		// Sanity check: output must be valid JSON.
+		var v interface{}
+		err = json.NewDecoder(buf).Decode(&v)
+		rtest.OK(t, err)
+	}
+}

cmd/restic/cmd_mount.go

@@ -122,6 +122,7 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
 	mountOptions := []systemFuse.MountOption{
 		systemFuse.ReadOnly(),
 		systemFuse.FSName("restic"),
+		systemFuse.MaxReadahead(128 * 1024),
 	}
 
 	if opts.AllowOther {
@@ -161,7 +162,8 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
 	root := fuse.NewRoot(repo, cfg)
 
 	Printf("Now serving the repository at %s\n", mountpoint)
-	Printf("When finished, quit with Ctrl-c or umount the mountpoint.\n")
+	Printf("Use another terminal or tool to browse the contents of this folder.\n")
+	Printf("When finished, quit with Ctrl-c here or umount the mountpoint.\n")
 
 	debug.Log("serving mount at %v", mountpoint)
 	err = fs.Serve(c, root)

cmd/restic/cmd_prune.go

@@ -66,6 +66,7 @@ func addPruneOptions(c *cobra.Command) {
 }
 
 func verifyPruneOptions(opts *PruneOptions) error {
+	opts.MaxRepackBytes = math.MaxUint64
 	if len(opts.MaxRepackSize) > 0 {
 		size, err := parseSizeStr(opts.MaxRepackSize)
 		if err != nil {
@@ -119,18 +120,6 @@ func verifyPruneOptions(opts *PruneOptions) error {
 	return nil
 }
 
-func shortenStatus(maxLength int, s string) string {
-	if len(s) <= maxLength {
-		return s
-	}
-
-	if maxLength < 3 {
-		return s[:maxLength]
-	}
-
-	return s[:maxLength-3] + "..."
-}
-
 func runPrune(opts PruneOptions, gopts GlobalOptions) error {
 	err := verifyPruneOptions(&opts)
 	if err != nil {
@@ -249,7 +238,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 			"Integrity check failed: Data seems to be missing.\n"+
 			"Will not start prune to prevent (additional) data loss!\n"+
 			"Please report this error (along with the output of the 'prune' run) at\n"+
-			"https://github.com/restic/restic/issues/new/choose", usedBlobs)
+			"https://github.com/restic/restic/issues/new/choose\n", usedBlobs)
 		return errorIndexIncomplete
 	}
 
@@ -324,7 +313,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 			// Pack size does not fit and pack is needed => error
 			// If the pack is not needed, this is no error, the pack can
 			// and will be simply removed, see below.
-			Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic rebuild-index'.",
+			Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic rebuild-index'.\n",
 				id.Str(), p.unusedSize+p.usedSize, packSize)
 			return errorSizeNotMatching
 		}
@@ -430,11 +419,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 
 	for _, p := range repackCandidates {
 		reachedUnusedSizeAfter := (stats.size.unused-stats.size.remove-stats.size.repackrm < maxUnusedSizeAfter)
-
-		reachedRepackSize := false
-		if opts.MaxRepackBytes > 0 {
-			reachedRepackSize = stats.size.repack+p.unusedSize+p.usedSize > opts.MaxRepackBytes
-		}
+		reachedRepackSize := stats.size.repack+p.unusedSize+p.usedSize >= opts.MaxRepackBytes
 
 		switch {
 		case reachedRepackSize:
@@ -459,26 +444,26 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 		stats.size.repackrm += stats.size.duplicate
 	}
 
-	Verboseff("\nused:        %10d blobs / %s\n", stats.blobs.used, formatBytes(stats.size.used))
+	Verboseff("\nused:         %10d blobs / %s\n", stats.blobs.used, formatBytes(stats.size.used))
 	if stats.blobs.duplicate > 0 {
-		Verboseff("duplicates:  %10d blobs / %s\n", stats.blobs.duplicate, formatBytes(stats.size.duplicate))
+		Verboseff("duplicates:   %10d blobs / %s\n", stats.blobs.duplicate, formatBytes(stats.size.duplicate))
 	}
-	Verboseff("unused:      %10d blobs / %s\n", stats.blobs.unused, formatBytes(stats.size.unused))
+	Verboseff("unused:       %10d blobs / %s\n", stats.blobs.unused, formatBytes(stats.size.unused))
 	if stats.size.unref > 0 {
-		Verboseff("unreferenced:                   %s\n", formatBytes(stats.size.unref))
+		Verboseff("unreferenced:                    %s\n", formatBytes(stats.size.unref))
 	}
 	totalBlobs := stats.blobs.used + stats.blobs.unused + stats.blobs.duplicate
 	totalSize := stats.size.used + stats.size.duplicate + stats.size.unused + stats.size.unref
 	unusedSize := stats.size.duplicate + stats.size.unused
-	Verboseff("total:       %10d blobs / %s\n", totalBlobs, formatBytes(totalSize))
+	Verboseff("total:        %10d blobs / %s\n", totalBlobs, formatBytes(totalSize))
 	Verboseff("unused size: %s of total size\n", formatPercent(unusedSize, totalSize))
 
-	Verbosef("\nto repack:   %10d blobs / %s\n", stats.blobs.repack, formatBytes(stats.size.repack))
-	Verbosef("this removes %10d blobs / %s\n", stats.blobs.repackrm, formatBytes(stats.size.repackrm))
-	Verbosef("to delete:   %10d blobs / %s\n", stats.blobs.remove, formatBytes(stats.size.remove+stats.size.unref))
+	Verbosef("\nto repack:    %10d blobs / %s\n", stats.blobs.repack, formatBytes(stats.size.repack))
+	Verbosef("this removes: %10d blobs / %s\n", stats.blobs.repackrm, formatBytes(stats.size.repackrm))
+	Verbosef("to delete:    %10d blobs / %s\n", stats.blobs.remove, formatBytes(stats.size.remove+stats.size.unref))
 	totalPruneSize := stats.size.remove + stats.size.repackrm + stats.size.unref
-	Verbosef("total prune: %10d blobs / %s\n", stats.blobs.remove+stats.blobs.repackrm, formatBytes(totalPruneSize))
-	Verbosef("remaining:   %10d blobs / %s\n", totalBlobs-(stats.blobs.remove+stats.blobs.repackrm), formatBytes(totalSize-totalPruneSize))
+	Verbosef("total prune:  %10d blobs / %s\n", stats.blobs.remove+stats.blobs.repackrm, formatBytes(totalPruneSize))
+	Verbosef("remaining:    %10d blobs / %s\n", totalBlobs-(stats.blobs.remove+stats.blobs.repackrm), formatBytes(totalSize-totalPruneSize))
 	unusedAfter := unusedSize - stats.size.remove - stats.size.repackrm
 	Verbosef("unused size after prune: %s (%s of remaining size)\n",
 		formatBytes(unusedAfter), formatPercent(unusedAfter, totalSize-totalPruneSize))
@@ -487,11 +472,11 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 	Verboseff("partly used packs:  %10d\n", stats.packs.partlyUsed)
 	Verboseff("unused packs:       %10d\n\n", stats.packs.unused)
 
-	Verboseff("to keep:   %10d packs\n", stats.packs.keep)
-	Verboseff("to repack: %10d packs\n", len(repackPacks))
-	Verboseff("to delete: %10d packs\n", len(removePacks))
+	Verboseff("to keep:      %10d packs\n", stats.packs.keep)
+	Verboseff("to repack:    %10d packs\n", len(repackPacks))
+	Verboseff("to delete:    %10d packs\n", len(removePacks))
 	if len(removePacksFirst) > 0 {
-		Verboseff("to delete: %10d unreferenced packs\n\n", len(removePacksFirst))
+		Verboseff("to delete:    %10d unreferenced packs\n\n", len(removePacksFirst))
 	}
 
 	if opts.DryRun {

cmd/restic/cmd_rebuild_index.go

@@ -73,7 +73,27 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor
 		}
 	} else {
 		Verbosef("loading indexes...\n")
-		err := repo.LoadIndex(gopts.ctx)
+		mi := repository.NewMasterIndex()
+		err := repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
+			if err != nil {
+				Warnf("removing invalid index %v: %v\n", id, err)
+				obsoleteIndexes = append(obsoleteIndexes, id)
+				return nil
+			}
+
+			mi.Insert(idx)
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+
+		err = mi.MergeFinalIndexes()
+		if err != nil {
+			return err
+		}
+
+		err = repo.SetIndex(mi)
 		if err != nil {
 			return err
 		}

cmd/restic/cmd_recover.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"os"
 	"time"
 
@@ -11,11 +12,11 @@ import (
 
 var cmdRecover = &cobra.Command{
 	Use:   "recover [flags]",
-	Short: "Recover data from the repository",
+	Short: "Recover data from the repository not referenced by snapshots",
 	Long: `
 The "recover" command builds a new snapshot from all directories it can find in
-the raw data of the repository. It can be used if, for example, a snapshot has
-been removed by accident with "forget".
+the raw data of the repository which are not referenced in an existing snapshot.
+It can be used if, for example, a snapshot has been removed by accident with "forget".
 
 EXIT STATUS
 ===========
@@ -59,24 +60,14 @@ func runRecover(gopts GlobalOptions) error {
 	trees := make(map[restic.ID]bool)
 
 	for blob := range repo.Index().Each(gopts.ctx) {
-		if blob.Blob.Type != restic.TreeBlob {
-			continue
+		if blob.Type == restic.TreeBlob {
+			trees[blob.Blob.ID] = false
 		}
-		trees[blob.Blob.ID] = false
 	}
 
-	cur := 0
-	max := len(trees)
-	Verbosef("load %d trees\n\n", len(trees))
-
+	Verbosef("load %d trees\n", len(trees))
+	bar := newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
 	for id := range trees {
-		cur++
-		Verbosef("\rtree (%v/%v)", cur, max)
-
-		if !trees[id] {
-			trees[id] = false
-		}
-
 		tree, err := repo.LoadTree(gopts.ctx, id)
 		if err != nil {
 			Warnf("unable to load tree %v: %v\n", id.Str(), err)
@@ -84,28 +75,39 @@ func runRecover(gopts GlobalOptions) error {
 		}
 
 		for _, node := range tree.Nodes {
-			if node.Type != "dir" || node.Subtree == nil {
-				continue
+			if node.Type == "dir" && node.Subtree != nil {
+				trees[*node.Subtree] = true
 			}
-
-			subtree := *node.Subtree
-			trees[subtree] = true
 		}
+		bar.Add(1)
 	}
-	Verbosef("\ndone\n")
+	bar.Done()
+
+	Verbosef("load snapshots\n")
+	err = restic.ForAllSnapshots(gopts.ctx, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error {
+		trees[*sn.Tree] = true
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	Verbosef("done\n")
 
 	roots := restic.NewIDSet()
 	for id, seen := range trees {
-		if seen {
-			continue
+		if !seen {
+			Verboseff("found root tree %v\n", id.Str())
+			roots.Insert(id)
 		}
+	}
+	Printf("\nfound %d unreferenced roots\n", len(roots))
 
-		roots.Insert(id)
+	if len(roots) == 0 {
+		Verbosef("no snapshot to write.\n")
+		return nil
 	}
 
-	Verbosef("found %d roots\n", len(roots))
-
-	tree := restic.NewTree()
+	tree := restic.NewTree(len(roots))
 	for id := range roots {
 		var subtreeID = id
 		node := restic.Node{
@@ -117,7 +119,7 @@ func runRecover(gopts GlobalOptions) error {
 			ModTime:    time.Now(),
 			ChangeTime: time.Now(),
 		}
-		err = tree.Insert(&node)
+		err := tree.Insert(&node)
 		if err != nil {
 			return err
 		}
@@ -133,19 +135,23 @@ func runRecover(gopts GlobalOptions) error {
 		return errors.Fatalf("unable to save blobs to the repo: %v", err)
 	}
 
-	sn, err := restic.NewSnapshot([]string{"/recover"}, []string{}, hostname, time.Now())
+	return createSnapshot(gopts.ctx, "/recover", hostname, []string{"recovered"}, repo, &treeID)
+
+}
+
+func createSnapshot(ctx context.Context, name, hostname string, tags []string, repo restic.Repository, tree *restic.ID) error {
+	sn, err := restic.NewSnapshot([]string{name}, tags, hostname, time.Now())
 	if err != nil {
 		return errors.Fatalf("unable to save snapshot: %v", err)
 	}
 
-	sn.Tree = &treeID
+	sn.Tree = tree
 
-	id, err := repo.SaveJSONUnpacked(gopts.ctx, restic.SnapshotFile, sn)
+	id, err := repo.SaveJSONUnpacked(ctx, restic.SnapshotFile, sn)
 	if err != nil {
 		return errors.Fatalf("unable to save snapshot: %v", err)
 	}
 
 	Printf("saved new snapshot %v\n", id.Str())
-
 	return nil
 }

cmd/restic/cmd_restore.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"strings"
+	"time"
 
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
@@ -117,7 +118,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 	var id restic.ID
 
 	if snapshotIDString == "latest" {
-		id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Hosts)
+		id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Hosts, nil)
 		if err != nil {
 			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Hosts:%v", err, opts.Paths, opts.Hosts)
 		}
@@ -202,6 +203,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 	if opts.Verify {
 		Verbosef("verifying files in %s\n", opts.Target)
 		var count int
+		t0 := time.Now()
 		count, err = res.VerifyFiles(ctx, opts.Target)
 		if err != nil {
 			return err
@@ -209,7 +211,8 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 		if totalErrors > 0 {
 			return errors.Fatalf("There were %d errors\n", totalErrors)
 		}
-		Verbosef("finished verifying %d files in %s\n", count, opts.Target)
+		Verbosef("finished verifying %d files in %s (took %s)\n", count, opts.Target,
+			time.Since(t0).Round(time.Millisecond))
 	}
 
 	return nil

cmd/restic/cmd_self_update.go

@@ -71,7 +71,7 @@ func runSelfUpdate(opts SelfUpdateOptions, gopts GlobalOptions, args []string) e
 		}
 	}
 
-	Printf("writing restic to %v\n", opts.Output)
+	Verbosef("writing restic to %v\n", opts.Output)
 
 	v, err := selfupdate.DownloadLatestStableRelease(gopts.ctx, opts.Output, version, Verbosef)
 	if err != nil {

cmd/restic/cmd_snapshots.go

@@ -36,7 +36,8 @@ type SnapshotOptions struct {
 	Tags    restic.TagLists
 	Paths   []string
 	Compact bool
-	Last    bool
+	Last    bool // This option should be removed in favour of Latest.
+	Latest  int
 	GroupBy string
 }
 
@@ -51,6 +52,12 @@ func init() {
 	f.StringArrayVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)")
 	f.BoolVarP(&snapshotOptions.Compact, "compact", "c", false, "use compact output format")
 	f.BoolVar(&snapshotOptions.Last, "last", false, "only show the last snapshot for each host and path")
+	err := f.MarkDeprecated("last", "use --latest 1")
+	if err != nil {
+		// MarkDeprecated only returns an error when the flag is not found
+		panic(err)
+	}
+	f.IntVar(&snapshotOptions.Latest, "latest", 0, "only show the last `n` snapshots for each host and path")
 	f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "string for grouping snapshots by host,paths,tags")
 }
 
@@ -82,7 +89,11 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 
 	for k, list := range snapshotGroups {
 		if opts.Last {
-			list = FilterLastSnapshots(list)
+			// This branch should be removed in the same time
+			// that --last.
+			list = FilterLastestSnapshots(list, 1)
+		} else if opts.Latest > 0 {
+			list = FilterLastestSnapshots(list, opts.Latest)
 		}
 		sort.Sort(sort.Reverse(list))
 		snapshotGroups[k] = list
@@ -125,21 +136,22 @@ func newFilterLastSnapshotsKey(sn *restic.Snapshot) filterLastSnapshotsKey {
 	return filterLastSnapshotsKey{sn.Hostname, strings.Join(paths, "|")}
 }
 
-// FilterLastSnapshots filters a list of snapshots to only return the last
-// entry for each hostname and path. If the snapshot contains multiple paths,
-// they will be joined and treated as one item.
-func FilterLastSnapshots(list restic.Snapshots) restic.Snapshots {
+// FilterLastestSnapshots filters a list of snapshots to only return
+// the limit last entries for each hostname and path. If the snapshot
+// contains multiple paths, they will be joined and treated as one
+// item.
+func FilterLastestSnapshots(list restic.Snapshots, limit int) restic.Snapshots {
 	// Sort the snapshots so that the newer ones are listed first
 	sort.SliceStable(list, func(i, j int) bool {
 		return list[i].Time.After(list[j].Time)
 	})
 
 	var results restic.Snapshots
-	seen := make(map[filterLastSnapshotsKey]bool)
+	seen := make(map[filterLastSnapshotsKey]int)
 	for _, sn := range list {
 		key := newFilterLastSnapshotsKey(sn)
-		if !seen[key] {
-			seen[key] = true
+		if seen[key] < limit {
+			seen[key]++
 			results = append(results, sn)
 		}
 	}

cmd/restic/delete.go

@@ -25,16 +25,21 @@ const numDeleteWorkers = 8
 func deleteFiles(gopts GlobalOptions, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
 	totalCount := len(fileList)
 	fileChan := make(chan restic.ID)
-	go func() {
+	wg, ctx := errgroup.WithContext(gopts.ctx)
+	wg.Go(func() error {
+		defer close(fileChan)
 		for id := range fileList {
-			fileChan <- id
+			select {
+			case fileChan <- id:
+			case <-ctx.Done():
+				return ctx.Err()
+			}
 		}
-		close(fileChan)
-	}()
+		return nil
+	})
 
 	bar := newProgressMax(!gopts.JSON && !gopts.Quiet, uint64(totalCount), "files deleted")
 	defer bar.Done()
-	wg, ctx := errgroup.WithContext(gopts.ctx)
 	for i := 0; i < numDeleteWorkers; i++ {
 		wg.Go(func() error {
 			for id := range fileChan {

cmd/restic/find.go

@@ -23,7 +23,7 @@ func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, hos
 			for _, s := range snapshotIDs {
 				if s == "latest" {
 					usedFilter = true
-					id, err = restic.FindLatestSnapshot(ctx, repo, paths, tags, hosts)
+					id, err = restic.FindLatestSnapshot(ctx, repo, paths, tags, hosts, nil)
 					if err != nil {
 						Warnf("Ignoring %q, no snapshot matched given filter (Paths:%v Tags:%v Hosts:%v)\n", s, paths, tags, hosts)
 						continue
@@ -31,7 +31,7 @@ func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, hos
 				} else {
 					id, err = restic.FindSnapshot(ctx, repo, s)
 					if err != nil {
-						Warnf("Ignoring %q, it is not a snapshot id\n", s)
+						Warnf("Ignoring %q: %v\n", s, err)
 						continue
 					}
 				}

cmd/restic/global.go

@@ -31,6 +31,7 @@ import (
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/textfile"
+	"github.com/restic/restic/internal/ui/termstatus"
 
 	"github.com/restic/restic/internal/errors"
 
@@ -39,7 +40,7 @@ import (
 	"golang.org/x/crypto/ssh/terminal"
 )
 
-var version = "0.12.0"
+var version = "0.13.0"
 
 // TimeFormat is the format used for all timestamps printed by restic.
 const TimeFormat = "2006-01-02 15:04:05"
@@ -60,6 +61,7 @@ type GlobalOptions struct {
 	CacheDir        string
 	NoCache         bool
 	CACerts         []string
+	InsecureTLS     bool
 	TLSClientCert   string
 	CleanupCache    bool
 
@@ -96,6 +98,8 @@ func init() {
 	var cancel context.CancelFunc
 	globalOptions.ctx, cancel = context.WithCancel(context.Background())
 	AddCleanupHandler(func() error {
+		// Must be called before the unlock cleanup handler to ensure that the latter is
+		// not blocked due to limited number of backend connections, see #1434
 		cancel()
 		return nil
 	})
@@ -114,10 +118,13 @@ func init() {
 	f.BoolVar(&globalOptions.NoCache, "no-cache", false, "do not use a local cache")
 	f.StringSliceVar(&globalOptions.CACerts, "cacert", nil, "`file` to load root certificates from (default: use system certificates)")
 	f.StringVar(&globalOptions.TLSClientCert, "tls-client-cert", "", "path to a `file` containing PEM encoded TLS client certificate and private key")
+	f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repo (insecure)")
 	f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories")
 	f.IntVar(&globalOptions.LimitUploadKb, "limit-upload", 0, "limits uploads to a maximum rate in KiB/s. (default: unlimited)")
 	f.IntVar(&globalOptions.LimitDownloadKb, "limit-download", 0, "limits downloads to a maximum rate in KiB/s. (default: unlimited)")
 	f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)")
+	// Use our "generate" command instead of the cobra provided "completion" command
+	cmdRoot.CompletionOptions.DisableDefaultCmd = true
 
 	restoreTerminal()
 }
@@ -142,7 +149,13 @@ func stdinIsTerminal() bool {
 }
 
 func stdoutIsTerminal() bool {
-	return terminal.IsTerminal(int(os.Stdout.Fd()))
+	// mintty on windows can use pipes which behave like a posix terminal,
+	// but which are not a terminal handle
+	return terminal.IsTerminal(int(os.Stdout.Fd())) || stdoutCanUpdateStatus()
+}
+
+func stdoutCanUpdateStatus() bool {
+	return termstatus.CanUpdateStatus(os.Stdout.Fd())
 }
 
 func stdoutTerminalWidth() int {
@@ -159,7 +172,7 @@ func stdoutTerminalWidth() int {
 // program execution must revert changes to the terminal configuration itself.
 // The terminal configuration is only restored while reading a password.
 func restoreTerminal() {
-	if !stdoutIsTerminal() {
+	if !terminal.IsTerminal(int(os.Stdout.Fd())) {
 		return
 	}
 
@@ -188,16 +201,21 @@ func restoreTerminal() {
 }
 
 // ClearLine creates a platform dependent string to clear the current
-// line, so it can be overwritten. ANSI sequences are not supported on
-// current windows cmd shell.
-func ClearLine() string {
-	if runtime.GOOS == "windows" {
-		if w := stdoutTerminalWidth(); w > 0 {
-			return strings.Repeat(" ", w-1) + "\r"
-		}
-		return ""
+// line, so it can be overwritten.
+//
+// w should be the terminal width, or 0 to let clearLine figure it out.
+func clearLine(w int) string {
+	if runtime.GOOS != "windows" {
+		return "\x1b[2K"
 	}
-	return "\x1b[2K"
+
+	// ANSI sequences are not supported on Windows cmd shell.
+	if w <= 0 {
+		if w = stdoutTerminalWidth(); w <= 0 {
+			return ""
+		}
+	}
+	return strings.Repeat(" ", w-1) + "\r"
 }
 
 // Printf writes the message to the configured stdout stream.
@@ -238,31 +256,6 @@ func Verboseff(format string, args ...interface{}) {
 	}
 }
 
-// PrintProgress wraps fmt.Printf to handle the difference in writing progress
-// information to terminals and non-terminal stdout
-func PrintProgress(format string, args ...interface{}) {
-	var (
-		message         string
-		carriageControl string
-	)
-	message = fmt.Sprintf(format, args...)
-
-	if !(strings.HasSuffix(message, "\r") || strings.HasSuffix(message, "\n")) {
-		if stdoutIsTerminal() {
-			carriageControl = "\r"
-		} else {
-			carriageControl = "\n"
-		}
-		message = fmt.Sprintf("%s%s", message, carriageControl)
-	}
-
-	if stdoutIsTerminal() {
-		message = fmt.Sprintf("%s%s", ClearLine(), message)
-	}
-
-	fmt.Print(message)
-}
-
 // Warnf writes the message to the configured stderr stream.
 func Warnf(format string, args ...interface{}) {
 	_, err := fmt.Fprintf(globalOptions.stderr, format, args...)
@@ -364,7 +357,7 @@ func ReadPassword(opts GlobalOptions, prompt string) (string, error) {
 	}
 
 	if len(password) == 0 {
-		return "", errors.New("an empty password is not a password")
+		return "", errors.Fatal("an empty password is not a password")
 	}
 
 	return password, nil
@@ -492,7 +485,7 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
 		return s, nil
 	}
 
-	if c.Created && !opts.JSON {
+	if c.Created && !opts.JSON && stdoutIsTerminal() {
 		Verbosef("created new cache in %v\n", c.Base)
 	}
 
@@ -511,8 +504,9 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
 
 	// cleanup old cache dirs if instructed to do so
 	if opts.CleanupCache {
-		Printf("removing %d old cache dirs from %v\n", len(oldCacheDirs), c.Base)
-
+		if stdoutIsTerminal() && !opts.JSON {
+			Verbosef("removing %d old cache dirs from %v\n", len(oldCacheDirs), c.Base)
+		}
 		for _, item := range oldCacheDirs {
 			dir := filepath.Join(c.Base, item.Name())
 			err = fs.RemoveAll(dir)
@@ -563,6 +557,12 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
 			cfg.Secret = os.Getenv("AWS_SECRET_ACCESS_KEY")
 		}
 
+		if cfg.KeyID == "" && cfg.Secret != "" {
+			return nil, errors.Fatalf("unable to open S3 backend: Key ID ($AWS_ACCESS_KEY_ID) is empty")
+		} else if cfg.KeyID != "" && cfg.Secret == "" {
+			return nil, errors.Fatalf("unable to open S3 backend: Secret ($AWS_SECRET_ACCESS_KEY) is empty")
+		}
+
 		if cfg.Region == "" {
 			cfg.Region = os.Getenv("AWS_DEFAULT_REGION")
 		}
@@ -682,6 +682,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
 	tropts := backend.TransportOptions{
 		RootCertFilenames:        globalOptions.CACerts,
 		TLSClientCertKeyFilename: globalOptions.TLSClientCert,
+		InsecureTLS:              globalOptions.InsecureTLS,
 	}
 	rt, err := backend.Transport(tropts)
 	if err != nil {
@@ -704,7 +705,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
 	case "azure":
 		be, err = azure.Open(cfg.(azure.Config), rt)
 	case "swift":
-		be, err = swift.Open(cfg.(swift.Config), rt)
+		be, err = swift.Open(globalOptions.ctx, cfg.(swift.Config), rt)
 	case "b2":
 		be, err = b2.Open(globalOptions.ctx, cfg.(b2.Config), rt)
 	case "rest":
@@ -762,6 +763,7 @@ func create(s string, opts options.Options) (restic.Backend, error) {
 	tropts := backend.TransportOptions{
 		RootCertFilenames:        globalOptions.CACerts,
 		TLSClientCertKeyFilename: globalOptions.TLSClientCert,
+		InsecureTLS:              globalOptions.InsecureTLS,
 	}
 	rt, err := backend.Transport(tropts)
 	if err != nil {
@@ -780,7 +782,7 @@ func create(s string, opts options.Options) (restic.Backend, error) {
 	case "azure":
 		return azure.Create(cfg.(azure.Config), rt)
 	case "swift":
-		return swift.Open(cfg.(swift.Config), rt)
+		return swift.Open(globalOptions.ctx, cfg.(swift.Config), rt)
 	case "b2":
 		return b2.Create(globalOptions.ctx, cfg.(b2.Config), rt)
 	case "rest":

cmd/restic/integration_helpers_test.go

@@ -67,7 +67,7 @@ func isSymlink(fi os.FileInfo) bool {
 
 func sameModTime(fi1, fi2 os.FileInfo) bool {
 	switch runtime.GOOS {
-	case "darwin", "freebsd", "openbsd", "netbsd":
+	case "darwin", "freebsd", "openbsd", "netbsd", "solaris":
 		if isSymlink(fi1) && isSymlink(fi2) {
 			return true
 		}

cmd/restic/integration_test.go

@@ -15,6 +15,7 @@ import (
 	"regexp"
 	"runtime"
 	"strings"
+	"sync"
 	"syscall"
 	"testing"
 	"time"
@@ -158,8 +159,11 @@ func testRunDiffOutput(gopts GlobalOptions, firstSnapshotID string, secondSnapsh
 	buf := bytes.NewBuffer(nil)
 
 	globalOptions.stdout = buf
+	oldStdout := gopts.stdout
+	gopts.stdout = buf
 	defer func() {
 		globalOptions.stdout = os.Stdout
+		gopts.stdout = oldStdout
 	}()
 
 	opts := DiffOptions{
@@ -345,6 +349,57 @@ func testBackup(t *testing.T, useFsSnapshot bool) {
 	testRunCheck(t, env.gopts)
 }
 
+func TestDryRunBackup(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{}
+	dryOpts := BackupOptions{DryRun: true}
+
+	// dry run before first backup
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
+	snapshotIDs := testRunList(t, "snapshots", env.gopts)
+	rtest.Assert(t, len(snapshotIDs) == 0,
+		"expected no snapshot, got %v", snapshotIDs)
+	packIDs := testRunList(t, "packs", env.gopts)
+	rtest.Assert(t, len(packIDs) == 0,
+		"expected no data, got %v", snapshotIDs)
+	indexIDs := testRunList(t, "index", env.gopts)
+	rtest.Assert(t, len(indexIDs) == 0,
+		"expected no index, got %v", snapshotIDs)
+
+	// first backup
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	snapshotIDs = testRunList(t, "snapshots", env.gopts)
+	packIDs = testRunList(t, "packs", env.gopts)
+	indexIDs = testRunList(t, "index", env.gopts)
+
+	// dry run between backups
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
+	snapshotIDsAfter := testRunList(t, "snapshots", env.gopts)
+	rtest.Equals(t, snapshotIDs, snapshotIDsAfter)
+	dataIDsAfter := testRunList(t, "packs", env.gopts)
+	rtest.Equals(t, packIDs, dataIDsAfter)
+	indexIDsAfter := testRunList(t, "index", env.gopts)
+	rtest.Equals(t, indexIDs, indexIDsAfter)
+
+	// second backup, implicit incremental
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	snapshotIDs = testRunList(t, "snapshots", env.gopts)
+	packIDs = testRunList(t, "packs", env.gopts)
+	indexIDs = testRunList(t, "index", env.gopts)
+
+	// another dry run
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
+	snapshotIDsAfter = testRunList(t, "snapshots", env.gopts)
+	rtest.Equals(t, snapshotIDs, snapshotIDsAfter)
+	dataIDsAfter = testRunList(t, "packs", env.gopts)
+	rtest.Equals(t, packIDs, dataIDsAfter)
+	indexIDsAfter = testRunList(t, "index", env.gopts)
+	rtest.Equals(t, indexIDs, indexIDsAfter)
+}
+
 func TestBackupNonExistingFile(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
@@ -374,10 +429,11 @@ func removePacksExcept(gopts GlobalOptions, t *testing.T, keep restic.IDSet, rem
 
 	// Get all tree packs
 	rtest.OK(t, r.LoadIndex(gopts.ctx))
+
 	treePacks := restic.NewIDSet()
-	for _, idx := range r.Index().(*repository.MasterIndex).All() {
-		for _, id := range idx.TreePacks() {
-			treePacks.Insert(id)
+	for pb := range r.Index().Each(context.TODO()) {
+		if pb.Type == restic.TreeBlob {
+			treePacks.Insert(pb.PackID)
 		}
 	}
 
@@ -436,11 +492,10 @@ func TestBackupTreeLoadError(t *testing.T) {
 	r, err := OpenRepository(env.gopts)
 	rtest.OK(t, err)
 	rtest.OK(t, r.LoadIndex(env.gopts.ctx))
-	// collect tree packs of subdirectory
-	subTreePacks := restic.NewIDSet()
-	for _, idx := range r.Index().(*repository.MasterIndex).All() {
-		for _, id := range idx.TreePacks() {
-			subTreePacks.Insert(id)
+	treePacks := restic.NewIDSet()
+	for pb := range r.Index().Each(context.TODO()) {
+		if pb.Type == restic.TreeBlob {
+			treePacks.Insert(pb.PackID)
 		}
 	}
 
@@ -448,7 +503,7 @@ func TestBackupTreeLoadError(t *testing.T) {
 	testRunCheck(t, env.gopts)
 
 	// delete the subdirectory pack first
-	for id := range subTreePacks {
+	for id := range treePacks {
 		rtest.OK(t, r.Backend().Remove(env.gopts.ctx, restic.Handle{Type: restic.PackFile, Name: id.String()}))
 	}
 	testRunRebuildIndex(t, env.gopts)
@@ -799,6 +854,25 @@ func TestCopyIncremental(t *testing.T) {
 		len(copiedSnapshotIDs), len(snapshotIDs))
 }
 
+func TestCopyUnstableJSON(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+	env2, cleanup2 := withTestEnvironment(t)
+	defer cleanup2()
+
+	// contains a symlink created using `ln -s '../i/'$'\355\246\361''d/samba' broken-symlink`
+	datafile := filepath.Join("testdata", "copy-unstable-json.tar.gz")
+	rtest.SetupTarTestFixture(t, env.base, datafile)
+
+	testRunInit(t, env2.gopts)
+	testRunCopy(t, env.gopts, env2.gopts)
+	testRunCheck(t, env2.gopts)
+
+	copiedSnapshotIDs := testRunList(t, "snapshots", env2.gopts)
+	rtest.Assert(t, 1 == len(copiedSnapshotIDs), "still expected %v snapshot, found %v",
+		1, len(copiedSnapshotIDs))
+}
+
 func TestInitCopyChunkerParams(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
@@ -1014,6 +1088,41 @@ func TestKeyAddRemove(t *testing.T) {
 	testRunKeyAddNewKeyUserHost(t, env.gopts)
 }
 
+type emptySaveBackend struct {
+	restic.Backend
+}
+
+func (b *emptySaveBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
+	return b.Backend.Save(ctx, h, restic.NewByteReader([]byte{}, nil))
+}
+
+func TestKeyProblems(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testRunInit(t, env.gopts)
+	env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
+		return &emptySaveBackend{r}, nil
+	}
+
+	testKeyNewPassword = "geheim2"
+	defer func() {
+		testKeyNewPassword = ""
+	}()
+
+	err := runKey(env.gopts, []string{"passwd"})
+	t.Log(err)
+	rtest.Assert(t, err != nil, "expected passwd change to fail")
+
+	err = runKey(env.gopts, []string{"add"})
+	t.Log(err)
+	rtest.Assert(t, err != nil, "expected key adding to fail")
+
+	t.Logf("testing access with initial password %q\n", env.gopts.password)
+	rtest.OK(t, runKey(env.gopts, []string{"list"}))
+	testRunCheck(t, env.gopts)
+}
+
 func testFileSize(filename string, size int64) error {
 	fi, err := os.Stat(filename)
 	if err != nil {
@@ -1311,7 +1420,7 @@ func TestFindJSON(t *testing.T) {
 	rtest.Assert(t, matches[0].Hits == 3, "expected hits to show 3 matches (%v)", datafile)
 }
 
-func TestRebuildIndex(t *testing.T) {
+func testRebuildIndex(t *testing.T, backendTestHook backendWrapper) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
 
@@ -1331,8 +1440,10 @@ func TestRebuildIndex(t *testing.T) {
 		t.Fatalf("did not find hint for rebuild-index command")
 	}
 
+	env.gopts.backendTestHook = backendTestHook
 	testRunRebuildIndex(t, env.gopts)
 
+	env.gopts.backendTestHook = nil
 	out, err = testRunCheckOutput(env.gopts)
 	if len(out) != 0 {
 		t.Fatalf("expected no output from the checker, got: %v", out)
@@ -1343,9 +1454,57 @@ func TestRebuildIndex(t *testing.T) {
 	}
 }
 
+func TestRebuildIndex(t *testing.T) {
+	testRebuildIndex(t, nil)
+}
+
 func TestRebuildIndexAlwaysFull(t *testing.T) {
+	indexFull := repository.IndexFull
+	defer func() {
+		repository.IndexFull = indexFull
+	}()
 	repository.IndexFull = func(*repository.Index) bool { return true }
-	TestRebuildIndex(t)
+	testRebuildIndex(t, nil)
+}
+
+// indexErrorBackend modifies the first index after reading.
+type indexErrorBackend struct {
+	restic.Backend
+	lock     sync.Mutex
+	hasErred bool
+}
+
+func (b *indexErrorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
+	return b.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error {
+		// protect hasErred
+		b.lock.Lock()
+		defer b.lock.Unlock()
+		if !b.hasErred && h.Type == restic.IndexFile {
+			b.hasErred = true
+			return consumer(errorReadCloser{rd})
+		}
+		return consumer(rd)
+	})
+}
+
+type errorReadCloser struct {
+	io.Reader
+}
+
+func (erd errorReadCloser) Read(p []byte) (int, error) {
+	n, err := erd.Reader.Read(p)
+	if n > 0 {
+		p[0] ^= 1
+	}
+	return n, err
+}
+
+func TestRebuildIndexDamage(t *testing.T) {
+	testRebuildIndex(t, func(r restic.Backend) (restic.Backend, error) {
+		return &indexErrorBackend{
+			Backend: r,
+		}, nil
+	})
 }
 
 type appendOnlyBackend struct {
@@ -1588,7 +1747,7 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o
 
 // a listOnceBackend only allows listing once per filetype
 // listing filetypes more than once may cause problems with eventually consistent
-// backends (like e.g. AWS S3) as the second listing may be inconsistent to what
+// backends (like e.g. Amazon S3) as the second listing may be inconsistent to what
 // is expected by the first listing + some operations.
 type listOnceBackend struct {
 	restic.Backend
@@ -1816,10 +1975,8 @@ var diffOutputRegexPatterns = []string{
 	"Removed: +2[0-9]{2}\\.[0-9]{3} KiB",
 }
 
-func TestDiff(t *testing.T) {
+func setupDiffRepo(t *testing.T) (*testEnvironment, func(), string, string) {
 	env, cleanup := withTestEnvironment(t)
-	defer cleanup()
-
 	testRunInit(t, env.gopts)
 
 	datadir := filepath.Join(env.base, "testdata")
@@ -1855,19 +2012,82 @@ func TestDiff(t *testing.T) {
 	testRunBackup(t, "", []string{datadir}, opts, env.gopts)
 	_, secondSnapshotID := lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts))
 
+	return env, cleanup, firstSnapshotID, secondSnapshotID
+}
+
+func TestDiff(t *testing.T) {
+	env, cleanup, firstSnapshotID, secondSnapshotID := setupDiffRepo(t)
+	defer cleanup()
+
+	// quiet suppresses the diff output except for the summary
+	env.gopts.Quiet = false
 	_, err := testRunDiffOutput(env.gopts, "", secondSnapshotID)
 	rtest.Assert(t, err != nil, "expected error on invalid snapshot id")
 
 	out, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
-	if err != nil {
-		t.Fatalf("expected no error from diff for test repository, got %v", err)
-	}
+	rtest.OK(t, err)
 
 	for _, pattern := range diffOutputRegexPatterns {
 		r, err := regexp.Compile(pattern)
 		rtest.Assert(t, err == nil, "failed to compile regexp %v", pattern)
 		rtest.Assert(t, r.MatchString(out), "expected pattern %v in output, got\n%v", pattern, out)
 	}
+
+	// check quiet output
+	env.gopts.Quiet = true
+	outQuiet, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
+	rtest.OK(t, err)
+
+	rtest.Assert(t, len(outQuiet) < len(out), "expected shorter output on quiet mode %v vs. %v", len(outQuiet), len(out))
+}
+
+type typeSniffer struct {
+	MessageType string `json:"message_type"`
+}
+
+func TestDiffJSON(t *testing.T) {
+	env, cleanup, firstSnapshotID, secondSnapshotID := setupDiffRepo(t)
+	defer cleanup()
+
+	// quiet suppresses the diff output except for the summary
+	env.gopts.Quiet = false
+	env.gopts.JSON = true
+	out, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
+	rtest.OK(t, err)
+
+	var stat DiffStatsContainer
+	var changes int
+
+	scanner := bufio.NewScanner(strings.NewReader(out))
+	for scanner.Scan() {
+		line := scanner.Text()
+		var sniffer typeSniffer
+		rtest.OK(t, json.Unmarshal([]byte(line), &sniffer))
+		switch sniffer.MessageType {
+		case "change":
+			changes++
+		case "statistics":
+			rtest.OK(t, json.Unmarshal([]byte(line), &stat))
+		default:
+			t.Fatalf("unexpected message type %v", sniffer.MessageType)
+		}
+	}
+	rtest.Equals(t, 9, changes)
+	rtest.Assert(t, stat.Added.Files == 2 && stat.Added.Dirs == 3 && stat.Added.DataBlobs == 2 &&
+		stat.Removed.Files == 1 && stat.Removed.Dirs == 2 && stat.Removed.DataBlobs == 1 &&
+		stat.ChangedFiles == 1, "unexpected statistics")
+
+	// check quiet output
+	env.gopts.Quiet = true
+	outQuiet, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
+	rtest.OK(t, err)
+
+	stat = DiffStatsContainer{}
+	rtest.OK(t, json.Unmarshal([]byte(outQuiet), &stat))
+	rtest.Assert(t, stat.Added.Files == 2 && stat.Added.Dirs == 3 && stat.Added.DataBlobs == 2 &&
+		stat.Removed.Files == 1 && stat.Removed.Dirs == 2 && stat.Removed.DataBlobs == 1 &&
+		stat.ChangedFiles == 1, "unexpected statistics")
+	rtest.Assert(t, stat.SourceSnapshot == firstSnapshotID && stat.TargetSnapshot == secondSnapshotID, "unexpected snapshot ids")
 }
 
 type writeToOnly struct {

cmd/restic/lock.go

@@ -16,6 +16,7 @@ var globalLocks struct {
 	cancelRefresh chan struct{}
 	refreshWG     sync.WaitGroup
 	sync.Mutex
+	sync.Once
 }
 
 func lockRepo(ctx context.Context, repo *repository.Repository) (*restic.Lock, error) {
@@ -27,6 +28,12 @@ func lockRepoExclusive(ctx context.Context, repo *repository.Repository) (*resti
 }
 
 func lockRepository(ctx context.Context, repo *repository.Repository, exclusive bool) (*restic.Lock, error) {
+	// make sure that a repository is unlocked properly and after cancel() was
+	// called by the cleanup handler in global.go
+	globalLocks.Do(func() {
+		AddCleanupHandler(unlockAll)
+	})
+
 	lockFn := restic.NewLock
 	if exclusive {
 		lockFn = restic.NewExclusiveLock
@@ -128,7 +135,3 @@ func unlockAll() error {
 
 	return nil
 }
-
-func init() {
-	AddCleanupHandler(unlockAll)
-}

cmd/restic/progress.go

@@ -4,15 +4,17 @@ import (
 	"fmt"
 	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/restic/restic/internal/ui/progress"
+	"github.com/restic/restic/internal/ui/termstatus"
 )
 
 // calculateProgressInterval returns the interval configured via RESTIC_PROGRESS_FPS
 // or if unset returns an interval for 60fps on interactive terminals and 0 (=disabled)
-// for non-interactive terminals
-func calculateProgressInterval() time.Duration {
+// for non-interactive terminals or when run using the --quiet flag
+func calculateProgressInterval(show bool, json bool) time.Duration {
 	interval := time.Second / 60
 	fps, err := strconv.ParseFloat(os.Getenv("RESTIC_PROGRESS_FPS"), 64)
 	if err == nil && fps > 0 {
@@ -20,7 +22,7 @@ func calculateProgressInterval() time.Duration {
 			fps = 60
 		}
 		interval = time.Duration(float64(time.Second) / fps)
-	} else if !stdoutIsTerminal() {
+	} else if !json && !stdoutCanUpdateStatus() || !show {
 		interval = 0
 	}
 	return interval
@@ -31,7 +33,8 @@ func newProgressMax(show bool, max uint64, description string) *progress.Counter
 	if !show {
 		return nil
 	}
-	interval := calculateProgressInterval()
+	interval := calculateProgressInterval(show, false)
+	canUpdateStatus := stdoutCanUpdateStatus()
 
 	return progress.New(interval, max, func(v uint64, max uint64, d time.Duration, final bool) {
 		var status string
@@ -42,13 +45,36 @@ func newProgressMax(show bool, max uint64, description string) *progress.Counter
 				formatDuration(d), formatPercent(v, max), v, max, description)
 		}
 
-		if w := stdoutTerminalWidth(); w > 0 {
-			status = shortenStatus(w, status)
-		}
-
-		PrintProgress("%s", status)
+		printProgress(status, canUpdateStatus)
 		if final {
 			fmt.Print("\n")
 		}
 	})
 }
+
+func printProgress(status string, canUpdateStatus bool) {
+	w := stdoutTerminalWidth()
+	if w > 0 {
+		if w < 3 {
+			status = termstatus.Truncate(status, w)
+		} else {
+			status = termstatus.Truncate(status, w-3) + "..."
+		}
+	}
+
+	var carriageControl, clear string
+
+	if canUpdateStatus {
+		clear = clearLine(w)
+	}
+
+	if !(strings.HasSuffix(status, "\r") || strings.HasSuffix(status, "\n")) {
+		if canUpdateStatus {
+			carriageControl = "\r"
+		} else {
+			carriageControl = "\n"
+		}
+	}
+
+	_, _ = os.Stdout.Write([]byte(clear + status + carriageControl))
+}

cmd/restic/secondary_repo.go

@@ -9,6 +9,7 @@ import (
 
 type secondaryRepoOptions struct {
 	Repo            string
+	RepositoryFile  string
 	password        string
 	PasswordFile    string
 	PasswordCommand string
@@ -17,18 +18,25 @@ type secondaryRepoOptions struct {
 
 func initSecondaryRepoOptions(f *pflag.FlagSet, opts *secondaryRepoOptions, repoPrefix string, repoUsage string) {
 	f.StringVarP(&opts.Repo, "repo2", "", os.Getenv("RESTIC_REPOSITORY2"), repoPrefix+" `repository` "+repoUsage+" (default: $RESTIC_REPOSITORY2)")
+	f.StringVarP(&opts.RepositoryFile, "repository-file2", "", os.Getenv("RESTIC_REPOSITORY_FILE2"), "`file` from which to read the "+repoPrefix+" repository location "+repoUsage+" (default: $RESTIC_REPOSITORY_FILE2)")
 	f.StringVarP(&opts.PasswordFile, "password-file2", "", os.Getenv("RESTIC_PASSWORD_FILE2"), "`file` to read the "+repoPrefix+" repository password from (default: $RESTIC_PASSWORD_FILE2)")
 	f.StringVarP(&opts.KeyHint, "key-hint2", "", os.Getenv("RESTIC_KEY_HINT2"), "key ID of key to try decrypting the "+repoPrefix+" repository first (default: $RESTIC_KEY_HINT2)")
 	f.StringVarP(&opts.PasswordCommand, "password-command2", "", os.Getenv("RESTIC_PASSWORD_COMMAND2"), "shell `command` to obtain the "+repoPrefix+" repository password from (default: $RESTIC_PASSWORD_COMMAND2)")
 }
 
 func fillSecondaryGlobalOpts(opts secondaryRepoOptions, gopts GlobalOptions, repoPrefix string) (GlobalOptions, error) {
-	if opts.Repo == "" {
-		return GlobalOptions{}, errors.Fatal("Please specify a " + repoPrefix + " repository location (--repo2)")
+	if opts.Repo == "" && opts.RepositoryFile == "" {
+		return GlobalOptions{}, errors.Fatal("Please specify a " + repoPrefix + " repository location (--repo2 or --repository-file2)")
 	}
+
+	if opts.Repo != "" && opts.RepositoryFile != "" {
+		return GlobalOptions{}, errors.Fatal("Options --repo2 and --repository-file2 are mutually exclusive, please specify only one")
+	}
+
 	var err error
 	dstGopts := gopts
 	dstGopts.Repo = opts.Repo
+	dstGopts.RepositoryFile = opts.RepositoryFile
 	dstGopts.PasswordFile = opts.PasswordFile
 	dstGopts.PasswordCommand = opts.PasswordCommand
 	dstGopts.KeyHint = opts.KeyHint

cmd/restic/secondary_repo_test.go

@@ -0,0 +1,132 @@
+package main
+
+import (
+	"io/ioutil"
+	"path/filepath"
+	"testing"
+
+	rtest "github.com/restic/restic/internal/test"
+)
+
+//TestFillSecondaryGlobalOpts tests valid and invalid data on fillSecondaryGlobalOpts-function
+func TestFillSecondaryGlobalOpts(t *testing.T) {
+	//secondaryRepoTestCase defines a struct for test cases
+	type secondaryRepoTestCase struct {
+		Opts     secondaryRepoOptions
+		DstGOpts GlobalOptions
+	}
+
+	//validSecondaryRepoTestCases is a list with test cases that must pass
+	var validSecondaryRepoTestCases = []secondaryRepoTestCase{
+		{
+			// Test if Repo and Password are parsed correctly.
+			Opts: secondaryRepoOptions{
+				Repo:     "backupDst",
+				password: "secretDst",
+			},
+			DstGOpts: GlobalOptions{
+				Repo:     "backupDst",
+				password: "secretDst",
+			},
+		},
+		{
+			// Test if RepositoryFile and PasswordFile are parsed correctly.
+			Opts: secondaryRepoOptions{
+				RepositoryFile: "backupDst",
+				PasswordFile:   "passwordFileDst",
+			},
+			DstGOpts: GlobalOptions{
+				RepositoryFile: "backupDst",
+				password:       "secretDst",
+				PasswordFile:   "passwordFileDst",
+			},
+		},
+		{
+			// Test if RepositoryFile and PasswordCommand are parsed correctly.
+			Opts: secondaryRepoOptions{
+				RepositoryFile:  "backupDst",
+				PasswordCommand: "echo secretDst",
+			},
+			DstGOpts: GlobalOptions{
+				RepositoryFile:  "backupDst",
+				password:        "secretDst",
+				PasswordCommand: "echo secretDst",
+			},
+		},
+	}
+
+	//invalidSecondaryRepoTestCases is a list with test cases that must fail
+	var invalidSecondaryRepoTestCases = []secondaryRepoTestCase{
+		{
+			// Test must fail on no repo given.
+			Opts: secondaryRepoOptions{},
+		},
+		{
+			// Test must fail as Repo and RepositoryFile are both given
+			Opts: secondaryRepoOptions{
+				Repo:           "backupDst",
+				RepositoryFile: "backupDst",
+			},
+		},
+		{
+			// Test must fail as PasswordFile and PasswordCommand are both given
+			Opts: secondaryRepoOptions{
+				Repo:            "backupDst",
+				PasswordFile:    "passwordFileDst",
+				PasswordCommand: "notEmpty",
+			},
+		},
+		{
+			// Test must fail as PasswordFile does not exist
+			Opts: secondaryRepoOptions{
+				Repo:         "backupDst",
+				PasswordFile: "NonExistingFile",
+			},
+		},
+		{
+			// Test must fail as PasswordCommand does not exist
+			Opts: secondaryRepoOptions{
+				Repo:            "backupDst",
+				PasswordCommand: "notEmpty",
+			},
+		},
+		{
+			// Test must fail as no password is given.
+			Opts: secondaryRepoOptions{
+				Repo: "backupDst",
+			},
+		},
+	}
+
+	//gOpts defines the Global options used in the secondary repository tests
+	var gOpts = GlobalOptions{
+		Repo:           "backupSrc",
+		RepositoryFile: "backupSrc",
+		password:       "secretSrc",
+		PasswordFile:   "passwordFileSrc",
+	}
+
+	//Create temp dir to create password file.
+	dir, cleanup := rtest.TempDir(t)
+	defer cleanup()
+
+	cleanup = rtest.Chdir(t, dir)
+	defer cleanup()
+
+	//Create temporary password file
+	err := ioutil.WriteFile(filepath.Join(dir, "passwordFileDst"), []byte("secretDst"), 0666)
+	rtest.OK(t, err)
+
+	// Test all valid cases
+	for _, testCase := range validSecondaryRepoTestCases {
+		DstGOpts, err := fillSecondaryGlobalOpts(testCase.Opts, gOpts, "destination")
+		rtest.OK(t, err)
+		rtest.Equals(t, DstGOpts, testCase.DstGOpts)
+	}
+
+	// Test all invalid cases
+	for _, testCase := range invalidSecondaryRepoTestCases {
+		_, err := fillSecondaryGlobalOpts(testCase.Opts, gOpts, "destination")
+		rtest.Assert(t, err != nil, "Expected error, but function did not return an error")
+	}
+}