report result of delete operation after failed upload

The sanity check shouldn't replace the error message if there is already
one.

.dockerignore

@@ -1,12 +0,0 @@
-# Folders
-.git/
-.github/
-changelog/
-doc/
-docker/
-helpers/
-
-# Files
-.gitignore
-.golangci.yml
-*.md

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,7 +1,13 @@
+
+
 <!--
 Thank you very much for contributing code or documentation to restic! Please
 fill out the following questions to make it easier for us to review your
 changes.
+
+You do not need to check all the boxes below all at once, feel free to take
+your time and add more commits. If you're done and ready for review, please
+check the last box.
 -->
 
 What does this PR change? What problem does it solve?
@@ -11,8 +17,8 @@ What does this PR change? What problem does it solve?
 Describe the changes and their purpose here, as detailed as needed.
 -->
 
-Was the change previously discussed in an issue or on the forum?
-----------------------------------------------------------------
+Was the change discussed in an issue or in the forum before?
+------------------------------------------------------------
 
 <!--
 Link issues and relevant forum posts here.
@@ -24,17 +30,11 @@ is closed automatically when this PR is merged.
 Checklist
 ---------
 
-<!--
-You do not need to check all the boxes below all at once. Feel free to take
-your time and add more commits. If you're done and ready for review, please
-check the last box. Enable a checkbox by replacing [ ] with [x].
--->
-
-- [ ] I have read the [contribution guidelines](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#providing-patches).
-- [ ] I have [enabled maintainer edits](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/allowing-changes-to-a-pull-request-branch-created-from-a-fork).
-- [ ] I have added tests for all code changes.
-- [ ] I have added documentation for relevant changes (in the manual).
-- [ ] There's a new file in `changelog/unreleased/` that describes the changes for our users (see [template](https://github.com/restic/restic/blob/master/changelog/TEMPLATE)).
-- [ ] I have run `gofmt` on the code in all commits.
-- [ ] All commit messages are formatted in the same style as [the other commits in the repo](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#git-commits).
-- [ ] I'm done! This pull request is ready for review.
+- [ ] I have read the [Contribution Guidelines](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#providing-patches)
+- [ ] I have enabled [maintainer edits for this PR](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/allowing-changes-to-a-pull-request-branch-created-from-a-fork)
+- [ ] I have added tests for all changes in this PR
+- [ ] I have added documentation for the changes (in the manual)
+- [ ] There's a new file in `changelog/unreleased/` that describes the changes for our users (template [here](https://github.com/restic/restic/blob/master/changelog/TEMPLATE))
+- [ ] I have run `gofmt` on the code in all commits
+- [ ] All commit messages are formatted in the same style as [the other commits in the repo](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#git-commits)
+- [ ] I'm done, this Pull Request is ready for review

.github/workflows/tests.yml

@@ -8,10 +8,6 @@ on:
   # run tests for all pull requests
   pull_request:
 
-env:
-  latest_go: "1.18.x"
-  GO111MODULE: on
-
 jobs:
   test:
     strategy:
@@ -19,47 +15,30 @@ jobs:
         # list of jobs to run:
         include:
           - job_name: Windows
-            go: 1.18.x
+            go: 1.15.x
             os: windows-latest
-            install_verb: install
 
           - job_name: macOS
-            go: 1.18.x
+            go: 1.15.x
             os: macOS-latest
             test_fuse: false
-            install_verb: install
-
-          - job_name: Linux
-            go: 1.18.x
-            os: ubuntu-latest
-            test_cloud_backends: true
-            test_fuse: true
-            check_changelog: true
-            install_verb: install
-
-          - job_name: Linux
-            go: 1.17.x
-            os: ubuntu-latest
-            test_fuse: true
-            install_verb: install
-
-          - job_name: Linux
-            go: 1.16.x
-            os: ubuntu-latest
-            test_fuse: true
-            install_verb: get
 
           - job_name: Linux
             go: 1.15.x
             os: ubuntu-latest
+            test_cloud_backends: true
             test_fuse: true
-            install_verb: get
+            check_changelog: true
 
           - job_name: Linux
             go: 1.14.x
             os: ubuntu-latest
             test_fuse: true
-            install_verb: get
+
+          - job_name: Linux
+            go: 1.13.x
+            os: ubuntu-latest
+            test_fuse: true
 
     name: ${{ matrix.job_name }} Go ${{ matrix.go }}
     runs-on: ${{ matrix.os }}
@@ -76,7 +55,7 @@ jobs:
       - name: Get programs (Linux/macOS)
         run: |
           echo "build Go tools"
-          go ${{ matrix.install_verb }} github.com/restic/rest-server/cmd/rest-server@latest
+          go get github.com/restic/rest-server/...
 
           echo "install minio server"
           mkdir $HOME/bin
@@ -108,7 +87,7 @@ jobs:
           $ProgressPreference = 'SilentlyContinue'
 
           echo "build Go tools"
-          go ${{ matrix.install_verb }} github.com/restic/rest-server/...
+          go get github.com/restic/rest-server/...
 
           echo "install minio server"
           mkdir $Env:USERPROFILE/bin
@@ -198,7 +177,7 @@ jobs:
       - name: Check changelog files with calens
         run: |
           echo "install calens"
-          go install github.com/restic/calens@latest
+          go get github.com/restic/calens
 
           echo "check changelog files"
           calens
@@ -211,17 +190,18 @@ jobs:
       matrix:
         # run cross-compile in two batches parallel so the overall tests run faster
         targets:
-          - "linux/386 linux/amd64 linux/arm linux/arm64 linux/ppc64le linux/mips linux/mipsle linux/mips64 linux/mips64le linux/s390x \
+          - "linux/386 linux/amd64 linux/arm linux/arm64 linux/ppc64le linux/mips linux/mipsle linux/mips64 linux/mips64le \
             openbsd/386 openbsd/amd64"
 
           - "freebsd/386 freebsd/amd64 freebsd/arm \
             aix/ppc64 \
-            darwin/amd64 darwin/arm64 \
+            darwin/amd64 \
             netbsd/386 netbsd/amd64 \
             windows/386 windows/amd64 \
             solaris/amd64"
 
     env:
+      go: 1.15.x
       GOPROXY: https://proxy.golang.org
 
     runs-on: ubuntu-latest
@@ -229,18 +209,18 @@ jobs:
     name: Cross Compile for ${{ matrix.targets }}
 
     steps:
-      - name: Set up Go ${{ env.latest_go }}
+      - name: Set up Go ${{ env.go }}
         uses: actions/setup-go@v2
         with:
-          go-version: ${{ env.latest_go }}
-
-      - name: Install gox
-        run: |
-          go install github.com/mitchellh/gox@latest
+          go-version: ${{ env.go }}
 
       - name: Check out code
         uses: actions/checkout@v2
 
+      - name: Install gox
+        run: |
+          go get github.com/mitchellh/gox
+
       - name: Cross-compile with gox for ${{ matrix.targets }}
         env:
           GOFLAGS: "-trimpath"
@@ -254,11 +234,6 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go ${{ env.latest_go }}
-        uses: actions/setup-go@v2
-        with:
-          go-version: ${{ env.latest_go }}
-
       - name: Check out code
         uses: actions/checkout@v2
 
@@ -266,11 +241,10 @@ jobs:
         uses: golangci/golangci-lint-action@v2
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.45
+          version: v1.36
           # Optional: show only new issues if it's a pull request. The default value is `false`.
           only-new-issues: true
           args: --verbose --timeout 5m
-          skip-go-installation: true
 
         # only run golangci-lint for pull requests, otherwise ALL hints get
         # reported. We need to slowly address all issues until we can enable
@@ -282,44 +256,3 @@ jobs:
           echo "check if go.mod and go.sum are up to date"
           go mod tidy
           git diff --exit-code go.mod go.sum
-
-  docker:
-    name: docker
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v3
-        with:
-          # list of Docker images to use as base name for tags
-          images: |
-            restic/restic
-          # generate Docker tags based on the following events/attributes
-          tags: |
-            type=schedule
-            type=ref,event=branch
-            type=ref,event=pr
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=semver,pattern={{major}}
-            type=sha
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Build and push
-        id: docker_build
-        uses: docker/build-push-action@v2
-        with:
-          push: false
-          context: .
-          file: docker/Dockerfile
-          pull: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}

CHANGELOG.md

@@ -1,654 +1,3 @@
-Changelog for restic 0.13.0 (2022-03-26)
-=======================================
-
-The following sections list the changes in restic 0.13.0 relevant to
-restic users. The changes are ordered by importance.
-
-Summary
--------
-
- * Fix #1106: Never lock repository for `list locks`
- * Fix #2345: Make cache crash-resistant and usable by multiple concurrent processes
- * Fix #2452: Improve error handling of repository locking
- * Fix #2738: Don't print progress for `backup --json --quiet`
- * Fix #3382: Make `check` command honor `RESTIC_CACHE_DIR` environment variable
- * Fix #3518: Make `copy` command honor `--no-lock` for source repository
- * Fix #3556: Fix hang with Backblaze B2 on SSL certificate authority error
- * Fix #3601: Fix rclone backend prematurely exiting when receiving SIGINT on Windows
- * Fix #3667: The `mount` command now reports symlinks sizes
- * Fix #3488: `rebuild-index` failed if an index file was damaged
- * Fix #3591: Fix handling of `prune --max-repack-size=0`
- * Fix #3619: Avoid choosing parent snapshots newer than time of new snapshot
- * Chg #3641: Ignore parent snapshot for `backup --stdin`
- * Chg #3519: Require Go 1.14 or newer
- * Enh #1542: Add `--dry-run`/`-n` option to `backup` command
- * Enh #2202: Add upload checksum for Azure, GS, S3 and Swift backends
- * Enh #233: Support negative include/exclude patterns
- * Enh #2388: Add warning for S3 if partial credentials are provided
- * Enh #2508: Support JSON output and quiet mode for the `diff` command
- * Enh #2656: Add flag to disable TLS verification for self-signed certificates
- * Enh #3003: Atomic uploads for the SFTP backend
- * Enh #3127: Add xattr (extended attributes) support for Solaris
- * Enh #3464: Skip lock creation on `forget` if `--no-lock` and `--dry-run`
- * Enh #3490: Support random subset by size in `check --read-data-subset`
- * Enh #3541: Improve handling of temporary B2 delete errors
- * Enh #3542: Add file mode in symbolic notation to `ls --json`
- * Enh #2594: Speed up the `restore --verify` command
- * Enh #2816: The `backup` command no longer updates file access times on Linux
- * Enh #2880: Make `recover` collect only unreferenced trees
- * Enh #3429: Verify that new or modified keys are stored correctly
- * Enh #3436: Improve local backend's resilience to (system) crashes
- * Enh #3508: Cache blobs read by the `dump` command
- * Enh #3511: Support configurable timeout for the rclone backend
- * Enh #3593: Improve `copy` performance by parallelizing IO
-
-Details
--------
-
- * Bugfix #1106: Never lock repository for `list locks`
-
-   The `list locks` command previously locked to the repository by default. This had the problem
-   that it wouldn't work for an exclusively locked repository and that the command would also
-   display its own lock file which can be confusing.
-
-   Now, the `list locks` command never locks the repository.
-
-   https://github.com/restic/restic/issues/1106
-   https://github.com/restic/restic/pull/3665
-
- * Bugfix #2345: Make cache crash-resistant and usable by multiple concurrent processes
-
-   The restic cache directory (`RESTIC_CACHE_DIR`) could end up in a broken state in the event of
-   restic (or the OS) crashing. This is now less likely to occur as files are downloaded to a
-   temporary location before being moved to their proper location.
-
-   This also allows multiple concurrent restic processes to operate on a single repository
-   without conflicts. Previously, concurrent operations could cause segfaults because the
-   processes saw each other's partially downloaded files.
-
-   https://github.com/restic/restic/issues/2345
-   https://github.com/restic/restic/pull/2838
-
- * Bugfix #2452: Improve error handling of repository locking
-
-   Previously, when the lock refresh failed to delete the old lock file, it forgot about the newly
-   created one. Instead it continued trying to delete the old (usually no longer existing) lock
-   file and thus over time lots of lock files accumulated. This has now been fixed.
-
-   https://github.com/restic/restic/issues/2452
-   https://github.com/restic/restic/issues/2473
-   https://github.com/restic/restic/issues/2562
-   https://github.com/restic/restic/pull/3512
-
- * Bugfix #2738: Don't print progress for `backup --json --quiet`
-
-   Unlike the text output, the `--json` output format still printed progress information even in
-   `--quiet` mode. This has now been fixed by always disabling the progress output in quiet mode.
-
-   https://github.com/restic/restic/issues/2738
-   https://github.com/restic/restic/pull/3264
-
- * Bugfix #3382: Make `check` command honor `RESTIC_CACHE_DIR` environment variable
-
-   Previously, the `check` command didn't honor the `RESTIC_CACHE_DIR` environment variable,
-   which caused problems in certain system/usage configurations. This has now been fixed.
-
-   https://github.com/restic/restic/issues/3382
-   https://github.com/restic/restic/pull/3474
-
- * Bugfix #3518: Make `copy` command honor `--no-lock` for source repository
-
-   The `copy` command previously did not respect the `--no-lock` option for the source
-   repository, causing failures with read-only storage backends. This has now been fixed such
-   that the option is now respected.
-
-   https://github.com/restic/restic/issues/3518
-   https://github.com/restic/restic/pull/3589
-
- * Bugfix #3556: Fix hang with Backblaze B2 on SSL certificate authority error
-
-   Previously, if a request failed with an SSL unknown certificate authority error, the B2
-   backend retried indefinitely and restic would appear to hang.
-
-   This has now been fixed and restic instead fails with an error message.
-
-   https://github.com/restic/restic/issues/3556
-   https://github.com/restic/restic/issues/2355
-   https://github.com/restic/restic/pull/3571
-
- * Bugfix #3601: Fix rclone backend prematurely exiting when receiving SIGINT on Windows
-
-   Previously, pressing Ctrl+C in a Windows console where restic was running with rclone as the
-   backend would cause rclone to exit prematurely due to getting a `SIGINT` signal at the same time
-   as restic. Restic would then wait for a long time for time with "unexpected EOF" and "rclone
-   stdio connection already closed" errors.
-
-   This has now been fixed by restic starting the rclone process detached from the console restic
-   runs in (similar to starting processes in a new process group on Linux), which enables restic to
-   gracefully clean up rclone (which now never gets the `SIGINT`).
-
-   https://github.com/restic/restic/issues/3601
-   https://github.com/restic/restic/pull/3602
-
- * Bugfix #3667: The `mount` command now reports symlinks sizes
-
-   Symlinks used to have size zero in restic mountpoints, confusing some third-party tools. They
-   now have a size equal to the byte length of their target path, as required by POSIX.
-
-   https://github.com/restic/restic/issues/3667
-   https://github.com/restic/restic/pull/3668
-
- * Bugfix #3488: `rebuild-index` failed if an index file was damaged
-
-   Previously, the `rebuild-index` command would fail with an error if an index file was damaged
-   or truncated. This has now been fixed.
-
-   On older restic versions, a (slow) workaround is to use `rebuild-index --read-all-packs` or
-   to manually delete the damaged index.
-
-   https://github.com/restic/restic/pull/3488
-
- * Bugfix #3591: Fix handling of `prune --max-repack-size=0`
-
-   Restic ignored the `--max-repack-size` option when passing a value of 0. This has now been
-   fixed.
-
-   As a workaround, `--max-repack-size=1` can be used with older versions of restic.
-
-   https://github.com/restic/restic/pull/3591
-
- * Bugfix #3619: Avoid choosing parent snapshots newer than time of new snapshot
-
-   The `backup` command, when a `--parent` was not provided, previously chose the most recent
-   matching snapshot as the parent snapshot. However, this didn't make sense when the user passed
-   `--time` to create a new snapshot older than the most recent snapshot.
-
-   Instead, `backup` now chooses the most recent snapshot which is not newer than the
-   snapshot-being-created's timestamp, to avoid any time travel.
-
-   https://github.com/restic/restic/pull/3619
-
- * Change #3641: Ignore parent snapshot for `backup --stdin`
-
-   Restic uses a parent snapshot to speed up directory scanning when performing backups, but this
-   only wasted time and memory when the backup source is stdin (using the `--stdin` option of the
-   `backup` command), since no directory scanning is performed in this case.
-
-   Snapshots made with `backup --stdin` no longer have a parent snapshot, which allows restic to
-   skip some startup operations and saves a bit of resources.
-
-   The `--parent` option is still available for `backup --stdin`, but is now ignored.
-
-   https://github.com/restic/restic/issues/3641
-   https://github.com/restic/restic/pull/3645
-
- * Change #3519: Require Go 1.14 or newer
-
-   Restic now requires Go 1.14 to build. This allows it to use new standard library features
-   instead of an external dependency.
-
-   https://github.com/restic/restic/issues/3519
-
- * Enhancement #1542: Add `--dry-run`/`-n` option to `backup` command
-
-   Testing exclude filters and other configuration options was error prone as wrong filters
-   could cause files to be uploaded unintentionally. It was also not possible to estimate
-   beforehand how much data would be uploaded.
-
-   The `backup` command now has a `--dry-run`/`-n` option, which performs all the normal steps of
-   a backup without actually writing anything to the repository.
-
-   Passing -vv will log information about files that would be added, allowing for verification of
-   source and exclusion options before running the real backup.
-
-   https://github.com/restic/restic/issues/1542
-   https://github.com/restic/restic/pull/2308
-   https://github.com/restic/restic/pull/3210
-   https://github.com/restic/restic/pull/3300
-
- * Enhancement #2202: Add upload checksum for Azure, GS, S3 and Swift backends
-
-   Previously only the B2 and partially the Swift backends verified the integrity of uploaded
-   (encrypted) files. The verification works by informing the backend about the expected hash of
-   the uploaded file. The backend then verifies the upload and thereby rules out any data
-   corruption during upload.
-
-   We have now added upload checksums for the Azure, GS, S3 and Swift backends, which besides
-   integrity checking for uploads also means that restic can now be used to store backups in S3
-   buckets which have Object Lock enabled.
-
-   https://github.com/restic/restic/issues/2202
-   https://github.com/restic/restic/issues/2700
-   https://github.com/restic/restic/issues/3023
-   https://github.com/restic/restic/pull/3246
-
- * Enhancement #233: Support negative include/exclude patterns
-
-   If a pattern starts with an exclamation mark and it matches a file that was previously matched by
-   a regular pattern, the match is cancelled. Notably, this can be used with `--exclude-file` to
-   cancel the exclusion of some files.
-
-   It works similarly to `.gitignore`, with the same limitation; Once a directory is excluded, it
-   is not possible to include files inside the directory.
-
-   Example of use as an exclude pattern for the `backup` command:
-
-   $HOME/**/* !$HOME/Documents !$HOME/code !$HOME/.emacs.d !$HOME/games # [...]
-   node_modules *~ *.o *.lo *.pyc # [...] $HOME/code/linux/* !$HOME/code/linux/.git # [...]
-
-   https://github.com/restic/restic/issues/233
-   https://github.com/restic/restic/pull/2311
-
- * Enhancement #2388: Add warning for S3 if partial credentials are provided
-
-   Previously restic did not notify about incomplete credentials when using the S3 backend,
-   instead just reporting access denied.
-
-   Restic now checks that both the AWS key ID and secret environment variables are set before
-   connecting to the remote server, and reports an error if not.
-
-   https://github.com/restic/restic/issues/2388
-   https://github.com/restic/restic/pull/3532
-
- * Enhancement #2508: Support JSON output and quiet mode for the `diff` command
-
-   The `diff` command now supports outputting machine-readable output in JSON format. To enable
-   this, pass the `--json` option to the command. To only print the summary and suppress detailed
-   output, pass the `--quiet` option.
-
-   https://github.com/restic/restic/issues/2508
-   https://github.com/restic/restic/pull/3592
-
- * Enhancement #2656: Add flag to disable TLS verification for self-signed certificates
-
-   There is now an `--insecure-tls` global option in restic, which disables TLS verification for
-   self-signed certificates in order to support some development workflows.
-
-   https://github.com/restic/restic/issues/2656
-   https://github.com/restic/restic/pull/2657
-
- * Enhancement #3003: Atomic uploads for the SFTP backend
-
-   The SFTP backend did not upload files atomically. An interrupted upload could leave an
-   incomplete file behind which could prevent restic from accessing the repository. This has now
-   been fixed and uploads in the SFTP backend are done atomically.
-
-   https://github.com/restic/restic/issues/3003
-   https://github.com/restic/restic/pull/3524
-
- * Enhancement #3127: Add xattr (extended attributes) support for Solaris
-
-   Restic now supports xattr for the Solaris operating system.
-
-   https://github.com/restic/restic/issues/3127
-   https://github.com/restic/restic/pull/3628
-
- * Enhancement #3464: Skip lock creation on `forget` if `--no-lock` and `--dry-run`
-
-   Restic used to silently ignore the `--no-lock` option of the `forget` command.
-
-   It now skips creation of lock file in case both `--dry-run` and `--no-lock` are specified. If
-   `--no-lock` option is specified without `--dry-run`, restic prints a warning message to
-   stderr.
-
-   https://github.com/restic/restic/issues/3464
-   https://github.com/restic/restic/pull/3623
-
- * Enhancement #3490: Support random subset by size in `check --read-data-subset`
-
-   The `--read-data-subset` option of the `check` command now supports a third way of specifying
-   the subset to check, namely `nS` where `n` is a size in bytes with suffix `S` as k/K, m/M, g/G or
-   t/T.
-
-   https://github.com/restic/restic/issues/3490
-   https://github.com/restic/restic/pull/3548
-
- * Enhancement #3541: Improve handling of temporary B2 delete errors
-
-   Deleting files on B2 could sometimes fail temporarily, which required restic to retry the
-   delete operation. In some cases the file was deleted nevertheless, causing the retries and
-   ultimately the restic command to fail. This has now been fixed.
-
-   https://github.com/restic/restic/issues/3541
-   https://github.com/restic/restic/pull/3544
-
- * Enhancement #3542: Add file mode in symbolic notation to `ls --json`
-
-   The `ls --json` command now provides the file mode in symbolic notation (using the
-   `permissions` key), aligned with `find --json`.
-
-   https://github.com/restic/restic/issues/3542
-   https://github.com/restic/restic/pull/3573
-   https://forum.restic.net/t/restic-ls-understanding-file-mode-with-json/4371
-
- * Enhancement #2594: Speed up the `restore --verify` command
-
-   The `--verify` option lets the `restore` command verify the file content after it has restored
-   a snapshot. The performance of this operation has now been improved by up to a factor of two.
-
-   https://github.com/restic/restic/pull/2594
-
- * Enhancement #2816: The `backup` command no longer updates file access times on Linux
-
-   When reading files during backup, restic used to cause the operating system to update the
-   files' access times. Note that this did not apply to filesystems with disabled file access
-   times.
-
-   Restic now instructs the operating system not to update the file access time, if the user
-   running restic is the file owner or has root permissions.
-
-   https://github.com/restic/restic/pull/2816
-
- * Enhancement #2880: Make `recover` collect only unreferenced trees
-
-   Previously, the `recover` command used to generate a snapshot containing *all* root trees,
-   even those which were already referenced by a snapshot.
-
-   This has been improved such that it now only processes trees not already referenced by any
-   snapshot.
-
-   https://github.com/restic/restic/pull/2880
-
- * Enhancement #3429: Verify that new or modified keys are stored correctly
-
-   When adding a new key or changing the password of a key, restic used to just create the new key (and
-   remove the old one, when changing the password). There was no verification that the new key was
-   stored correctly and works properly. As the repository cannot be decrypted without a valid key
-   file, this could in rare cases cause the repository to become inaccessible.
-
-   Restic now checks that new key files actually work before continuing. This can protect against
-   some (rare) cases of hardware or storage problems.
-
-   https://github.com/restic/restic/pull/3429
-
- * Enhancement #3436: Improve local backend's resilience to (system) crashes
-
-   Restic now ensures that files stored using the `local` backend are created atomically (that
-   is, files are either stored completely or not at all). This ensures that no incomplete files are
-   left behind even if restic is terminated while writing a file.
-
-   In addition, restic now tries to ensure that the directory in the repository which contains a
-   newly uploaded file is also written to disk. This can prevent missing files if the system
-   crashes or the disk is not properly unmounted.
-
-   https://github.com/restic/restic/pull/3436
-
- * Enhancement #3508: Cache blobs read by the `dump` command
-
-   When dumping a file using the `dump` command, restic did not cache blobs in any way, so even
-   consecutive runs of the same blob were loaded from the repository again and again, slowing down
-   the dump.
-
-   Now, the caching mechanism already used by the `fuse` command is also used by the `dump`
-   command. This makes dumping much faster, especially for sparse files.
-
-   https://github.com/restic/restic/pull/3508
-
- * Enhancement #3511: Support configurable timeout for the rclone backend
-
-   A slow rclone backend could cause restic to time out while waiting for the repository to open.
-   Restic now offers an `-o rclone.timeout` option to make this timeout configurable.
-
-   https://github.com/restic/restic/issues/3511
-   https://github.com/restic/restic/pull/3514
-
- * Enhancement #3593: Improve `copy` performance by parallelizing IO
-
-   Restic copy previously only used a single thread for copying blobs between repositories,
-   which resulted in limited performance when copying small blobs to/from a high latency backend
-   (i.e. any remote backend, especially b2).
-
-   Copying will now use 8 parallel threads to increase the throughput of the copy operation.
-
-   https://github.com/restic/restic/pull/3593
-
-
-Changelog for restic 0.12.1 (2021-08-03)
-=======================================
-
-The following sections list the changes in restic 0.12.1 relevant to
-restic users. The changes are ordered by importance.
-
-Summary
--------
-
- * Fix #2742: Improve error handling for rclone and REST backend over HTTP2
- * Fix #3111: Fix terminal output redirection for PowerShell
- * Fix #3214: Treat an empty password as a fatal error for repository init
- * Fix #3267: `copy` failed to copy snapshots in rare cases
- * Fix #3184: `backup --quiet` no longer prints status information
- * Fix #3296: Fix crash of `check --read-data-subset=x%` run for an empty repository
- * Fix #3302: Fix `fdopendir: not a directory` error for local backend
- * Fix #3334: Print `created new cache` message only on a terminal
- * Fix #3380: Fix crash of `backup --exclude='**'`
- * Fix #3305: Fix possibly missing backup summary of JSON output in case of error
- * Fix #3439: Correctly handle download errors during `restore`
- * Chg #3247: Empty files now have size of 0 in `ls --json` output
- * Enh #2780: Add release binaries for s390x architecture on Linux
- * Enh #3293: Add `--repository-file2` option to `init` and `copy` command
- * Enh #3312: Add auto-completion support for fish
- * Enh #3336: SFTP backend now checks for disk space
- * Enh #3377: Add release binaries for Apple Silicon
- * Enh #3414: Add `--keep-within-hourly` option to restic forget
- * Enh #3456: Support filtering and specifying untagged snapshots
- * Enh #3167: Allow specifying limit of `snapshots` list
- * Enh #3426: Optimize read performance of mount command
- * Enh #3427: `find --pack` fallback to index if data file is missing
-
-Details
--------
-
- * Bugfix #2742: Improve error handling for rclone and REST backend over HTTP2
-
-   When retrieving data from the rclone / REST backend while also using HTTP2 restic did not detect
-   when no data was returned at all. This could cause for example the `check` command to report the
-   following error:
-
-   Pack ID does not match, want [...], got e3b0c442
-
-   This has been fixed by correctly detecting and retrying the incomplete download.
-
-   https://github.com/restic/restic/issues/2742
-   https://github.com/restic/restic/pull/3453
-   https://forum.restic.net/t/http2-stream-closed-connection-reset-context-canceled/3743/10
-
- * Bugfix #3111: Fix terminal output redirection for PowerShell
-
-   When redirecting the output of restic using PowerShell on Windows, the output contained
-   terminal escape characters. This has been fixed by properly detecting the terminal type.
-
-   In addition, the mintty terminal now shows progress output for the backup command.
-
-   https://github.com/restic/restic/issues/3111
-   https://github.com/restic/restic/pull/3325
-
- * Bugfix #3214: Treat an empty password as a fatal error for repository init
-
-   When attempting to initialize a new repository, if an empty password was supplied, the
-   repository would be created but the init command would return an error with a stack trace. Now,
-   if an empty password is provided, it is treated as a fatal error, and no repository is created.
-
-   https://github.com/restic/restic/issues/3214
-   https://github.com/restic/restic/pull/3283
-
- * Bugfix #3267: `copy` failed to copy snapshots in rare cases
-
-   The `copy` command could in rare cases fail with the error message `SaveTree(...) returned
-   unexpected id ...`. This has been fixed.
-
-   On Linux/BSDs, the error could be caused by backing up symlinks with non-UTF-8 target paths.
-   Note that, due to limitations in the repository format, these are not stored properly and
-   should be avoided if possible.
-
-   https://github.com/restic/restic/issues/3267
-   https://github.com/restic/restic/pull/3310
-
- * Bugfix #3184: `backup --quiet` no longer prints status information
-
-   A regression in the latest restic version caused the output of `backup --quiet` to contain
-   large amounts of backup progress information when run using an interactive terminal. This is
-   fixed now.
-
-   A workaround for this bug is to run restic as follows: `restic backup --quiet [..] | cat -`.
-
-   https://github.com/restic/restic/issues/3184
-   https://github.com/restic/restic/pull/3186
-
- * Bugfix #3296: Fix crash of `check --read-data-subset=x%` run for an empty repository
-
-   The command `restic check --read-data-subset=x%` crashed when run for an empty repository.
-   This has been fixed.
-
-   https://github.com/restic/restic/issues/3296
-   https://github.com/restic/restic/pull/3309
-
- * Bugfix #3302: Fix `fdopendir: not a directory` error for local backend
-
-   The `check`, `list packs`, `prune` and `rebuild-index` commands failed for the local backend
-   when the `data` folder in the repository contained files. This has been fixed.
-
-   https://github.com/restic/restic/issues/3302
-   https://github.com/restic/restic/pull/3308
-
- * Bugfix #3334: Print `created new cache` message only on a terminal
-
-   The message `created new cache` was printed even when the output wasn't a terminal. That broke
-   piping `restic dump` output to tar or zip if cache directory didn't exist. The message is now
-   only printed on a terminal.
-
-   https://github.com/restic/restic/issues/3334
-   https://github.com/restic/restic/pull/3343
-
- * Bugfix #3380: Fix crash of `backup --exclude='**'`
-
-   The exclude filter `**`, which excludes all files, caused restic to crash. This has been
-   corrected.
-
-   https://github.com/restic/restic/issues/3380
-   https://github.com/restic/restic/pull/3393
-
- * Bugfix #3305: Fix possibly missing backup summary of JSON output in case of error
-
-   When using `--json` output it happened from time to time that the summary output was missing in
-   case an error occurred. This has been fixed.
-
-   https://github.com/restic/restic/pull/3305
-
- * Bugfix #3439: Correctly handle download errors during `restore`
-
-   Due to a regression in restic 0.12.0, the `restore` command in some cases did not retry download
-   errors and only printed a warning. This has been fixed by retrying incomplete data downloads.
-
-   https://github.com/restic/restic/issues/3439
-   https://github.com/restic/restic/pull/3449
-
- * Change #3247: Empty files now have size of 0 in `ls --json` output
-
-   The `ls --json` command used to omit the sizes of empty files in its output. It now reports a size
-   of zero explicitly for regular files, while omitting the size field for all other types.
-
-   https://github.com/restic/restic/issues/3247
-   https://github.com/restic/restic/pull/3257
-
- * Enhancement #2780: Add release binaries for s390x architecture on Linux
-
-   We've added release binaries for Linux using the s390x architecture.
-
-   https://github.com/restic/restic/issues/2780
-   https://github.com/restic/restic/pull/3452
-
- * Enhancement #3293: Add `--repository-file2` option to `init` and `copy` command
-
-   The `init` and `copy` command can now be used with the `--repository-file2` option or the
-   `$RESTIC_REPOSITORY_FILE2` environment variable. These to options are in addition to the
-   `--repo2` flag and allow you to read the destination repository from a file.
-
-   Using both `--repository-file` and `--repo2` options resulted in an error for the `copy` or
-   `init` command. The handling of this combination of options has been fixed. A workaround for
-   this issue is to only use `--repo` or `-r` and `--repo2` for `init` or `copy`.
-
-   https://github.com/restic/restic/issues/3293
-   https://github.com/restic/restic/pull/3294
-
- * Enhancement #3312: Add auto-completion support for fish
-
-   The `generate` command now supports fish auto completion.
-
-   https://github.com/restic/restic/pull/3312
-
- * Enhancement #3336: SFTP backend now checks for disk space
-
-   Backing up over SFTP previously spewed multiple generic "failure" messages when the remote
-   disk was full. It now checks for disk space before writing a file and fails immediately with a "no
-   space left on device" message.
-
-   https://github.com/restic/restic/issues/3336
-   https://github.com/restic/restic/pull/3345
-
- * Enhancement #3377: Add release binaries for Apple Silicon
-
-   We've added release binaries for macOS on Apple Silicon (M1).
-
-   https://github.com/restic/restic/issues/3377
-   https://github.com/restic/restic/pull/3394
-
- * Enhancement #3414: Add `--keep-within-hourly` option to restic forget
-
-   The `forget` command allowed keeping a given number of hourly backups or to keep all backups
-   within a given interval, but it was not possible to specify keeping hourly backups within a
-   given interval.
-
-   The new `--keep-within-hourly` option now offers this functionality. Similar options for
-   daily/weekly/monthly/yearly are also implemented, the new options are:
-
-   --keep-within-hourly <1y2m3d4h> --keep-within-daily <1y2m3d4h> --keep-within-weekly
-   <1y2m3d4h> --keep-within-monthly <1y2m3d4h> --keep-within-yearly <1y2m3d4h>
-
-   https://github.com/restic/restic/issues/3414
-   https://github.com/restic/restic/pull/3416
-   https://forum.restic.net/t/forget-policy/4014/11
-
- * Enhancement #3456: Support filtering and specifying untagged snapshots
-
-   It was previously not possible to specify an empty tag with the `--tag` and `--keep-tag`
-   options. This has now been fixed, such that `--tag ''` and `--keep-tag ''` now matches
-   snapshots without tags. This allows e.g. the `snapshots` and `forget` commands to only
-   operate on untagged snapshots.
-
-   https://github.com/restic/restic/issues/3456
-   https://github.com/restic/restic/pull/3457
-
- * Enhancement #3167: Allow specifying limit of `snapshots` list
-
-   The `--last` option allowed limiting the output of the `snapshots` command to the latest
-   snapshot for each host. The new `--latest n` option allows limiting the output to the latest `n`
-   snapshots.
-
-   This change deprecates the option `--last` in favour of `--latest 1`.
-
-   https://github.com/restic/restic/pull/3167
-
- * Enhancement #3426: Optimize read performance of mount command
-
-   Reading large files in a mounted repository may be up to five times faster. This improvement
-   primarily applies to repositories stored at a backend that can be accessed with low latency,
-   like e.g. the local backend.
-
-   https://github.com/restic/restic/pull/3426
-
- * Enhancement #3427: `find --pack` fallback to index if data file is missing
-
-   When investigating a repository with missing data files, it might be useful to determine
-   affected snapshots before running `rebuild-index`. Previously, `find --pack pack-id`
-   returned no data as it required accessing the data file. Now, if the necessary data is still
-   available in the repository index, it gets retrieved from there.
-
-   The command now also supports looking up multiple pack files in a single `find` run.
-
-   https://github.com/restic/restic/pull/3427
-   https://forum.restic.net/t/missing-packs-not-found/2600
-
-
 Changelog for restic 0.12.0 (2021-02-14)
 =======================================
 
@@ -3488,7 +2837,7 @@ Summary
  * Enh #1055: Create subdirs below `data/` for local/sftp backends
  * Enh #1067: Allow loading credentials for s3 from IAM
  * Enh #1073: Add `migrate` cmd to migrate from `s3legacy` to `default` layout
- * Enh #1081: Clarify semantic for `--tag` for the `forget` command
+ * Enh #1081: Clarify semantic for `--tasg` for the `forget` command
  * Enh #1080: Ignore chmod() errors on filesystems which do not support it
  * Enh #1082: Print stats on SIGINFO on Darwin and FreeBSD (ctrl+t)
 
@@ -3532,7 +2881,7 @@ Details
    https://github.com/restic/restic/issues/1073
    https://github.com/restic/restic/pull/1075
 
- * Enhancement #1081: Clarify semantic for `--tag` for the `forget` command
+ * Enhancement #1081: Clarify semantic for `--tasg` for the `forget` command
 
    https://github.com/restic/restic/issues/1081
    https://github.com/restic/restic/pull/1090

CONTRIBUTING.md

@@ -13,10 +13,11 @@ bug fixes are most welcome. However even "minor" details as fixing spelling
 errors, improving documentation or pointing out usability issues are a great
 help also.
 
+
 The restic project uses the GitHub infrastructure (see the
 [project page](https://github.com/restic/restic)) for all related discussions
 as well as the [forum](https://forum.restic.net/) and the `#restic` channel
-on [irc.libera.chat](https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:6697/#restic).
+on [irc.freenode.net](https://kiwiirc.com/nextclient/irc.freenode.net/restic).
 
 If you want to find an area that currently needs improving have a look at the
 open issues listed at the
@@ -66,7 +67,7 @@ Development Environment
 The repository contains the code written for restic in the directories
 `cmd/` and `internal/`.
 
-Restic requires Go version 1.14 or later for compiling. Clone the repo (without
+Restic requires Go version 1.13 or later for compiling. Clone the repo (without
 having `$GOPATH` set) and `cd` into the directory:
 
     $ unset GOPATH
@@ -123,10 +124,7 @@ down to the following steps:
     writing, ask yourself: If I were the user, what would I need to be aware
     of with this change?
 
- 8. Do not edit the man pages under `doc/man` or `doc/manual_rest.rst` -
-    these are autogenerated before new releases.
-
- 9. Once your code looks good and passes all the tests, we'll merge it. Thanks
+ 8. Once your code looks good and passes all the tests, we'll merge it. Thanks
     a lot for your contribution!
 
 Please provide the patches for each bug or feature in a separate branch and

README.md

@@ -46,8 +46,8 @@ Therefore, restic supports the following backends for storing backups natively:
 
 - [Local directory](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#local)
 - [sftp server (via SSH)](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#sftp)
-- [HTTP REST server](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server) ([protocol](https://restic.readthedocs.io/en/latest/100_references.html#rest-backend), [rest-server](https://github.com/restic/rest-server))
-- [Amazon S3](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) (either from Amazon or using the [Minio](https://minio.io) server)
+- [HTTP REST server](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server) ([protocol](doc/100_references.rst#rest-backend), [rest-server](https://github.com/restic/rest-server))
+- [AWS S3](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) (either from Amazon or using the [Minio](https://minio.io) server)
 - [OpenStack Swift](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#openstack-swift)
 - [BackBlaze B2](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#backblaze-b2)
 - [Microsoft Azure Blob Storage](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#microsoft-azure-blob-storage)

VERSION

@@ -1 +1 @@
-0.13.0
+0.12.0

build.go

@@ -35,7 +35,6 @@
 // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-//go:build ignore_build_go
 // +build ignore_build_go
 
 package main
@@ -59,7 +58,7 @@ var config = Config{
 	Main:             "./cmd/restic",                           // package name for the main package
 	DefaultBuildTags: []string{"selfupdate"},                   // specify build tags which are always used
 	Tests:            []string{"./..."},                        // tests to run
-	MinVersion:       GoVersion{Major: 1, Minor: 14, Patch: 0}, // minimum Go version supported
+	MinVersion:       GoVersion{Major: 1, Minor: 11, Patch: 0}, // minimum Go version supported
 }
 
 // Config configures the build.
@@ -124,8 +123,17 @@ func printEnv(env []string) {
 
 // build runs "go build args..." with GOPATH set to gopath.
 func build(cwd string, env map[string]string, args ...string) error {
-	// -trimpath removes all absolute paths from the binary.
-	a := []string{"build", "-trimpath"}
+	a := []string{"build"}
+
+	// try to remove all absolute paths from resulting binary
+	if goVersion.AtLeast(GoVersion{1, 13, 0}) {
+		// use the new flag introduced by Go 1.13
+		a = append(a, "-trimpath")
+	} else {
+		// otherwise try to trim as many paths as possible
+		a = append(a, "-asmflags", fmt.Sprintf("all=-trimpath=%s", cwd))
+		a = append(a, "-gcflags", fmt.Sprintf("all=-trimpath=%s", cwd))
+	}
 
 	if enablePIE {
 		a = append(a, "-buildmode=pie")

changelog/0.12.1_2021-08-03/issue-2742

@@ -1,13 +0,0 @@
-Bugfix: Improve error handling for rclone and REST backend over HTTP2
-
-When retrieving data from the rclone / REST backend while also using HTTP2
-restic did not detect when no data was returned at all. This could cause
-for example the `check` command to report the following error:
-
-    Pack ID does not match, want [...], got e3b0c442
-
-This has been fixed by correctly detecting and retrying the incomplete download.
-
-https://github.com/restic/restic/issues/2742
-https://github.com/restic/restic/pull/3453
-https://forum.restic.net/t/http2-stream-closed-connection-reset-context-canceled/3743/10

changelog/0.12.1_2021-08-03/issue-2780

@@ -1,6 +0,0 @@
-Enhancement: Add release binaries for s390x architecture on Linux
-
-We've added release binaries for Linux using the s390x architecture.
-
-https://github.com/restic/restic/issues/2780
-https://github.com/restic/restic/pull/3452

changelog/0.12.1_2021-08-03/issue-3111

@@ -1,11 +0,0 @@
-Bugfix: Fix terminal output redirection for PowerShell
-
-When redirecting the output of restic using PowerShell on Windows, the
-output contained terminal escape characters. This has been fixed by
-properly detecting the terminal type.
-
-In addition, the mintty terminal now shows progress output for the backup
-command.
-
-https://github.com/restic/restic/issues/3111
-https://github.com/restic/restic/pull/3325

changelog/0.12.1_2021-08-03/issue-3214

@@ -1,9 +0,0 @@
-Bugfix: Treat an empty password as a fatal error for repository init
-
-When attempting to initialize a new repository, if an empty password was
-supplied, the repository would be created but the init command would return
-an error with a stack trace. Now, if an empty password is provided, it is
-treated as a fatal error, and no repository is created.
-
-https://github.com/restic/restic/issues/3214
-https://github.com/restic/restic/pull/3283

changelog/0.12.1_2021-08-03/issue-3247

@@ -1,8 +0,0 @@
-Change: Empty files now have size of 0 in `ls --json` output
-
-The `ls --json` command used to omit the sizes of empty files in its
-output. It now reports a size of zero explicitly for regular files,
-while omitting the size field for all other types.
-
-https://github.com/restic/restic/issues/3247
-https://github.com/restic/restic/pull/3257

changelog/0.12.1_2021-08-03/issue-3267

@@ -1,11 +0,0 @@
-Bugfix: `copy` failed to copy snapshots in rare cases
-
-The `copy` command could in rare cases fail with the error message `SaveTree(...)
-returned unexpected id ...`.  This has been fixed.
-
-On Linux/BSDs, the error could be caused by backing up symlinks with non-UTF-8
-target paths. Note that, due to limitations in the repository format, these are
-not stored properly and should be avoided if possible.
-
-https://github.com/restic/restic/issues/3267
-https://github.com/restic/restic/pull/3310

changelog/0.12.1_2021-08-03/issue-3284

@@ -1,11 +0,0 @@
-Bugfix: `backup --quiet` no longer prints status information
-
-A regression in the latest restic version caused the output of `backup --quiet`
-to contain large amounts of backup progress information when run using an
-interactive terminal. This is fixed now.
-
-A workaround for this bug is to run restic as follows:
-`restic backup --quiet [..] | cat -`.
-
-https://github.com/restic/restic/issues/3184
-https://github.com/restic/restic/pull/3186

changelog/0.12.1_2021-08-03/issue-3293

@@ -1,14 +0,0 @@
-Enhancement: Add `--repository-file2` option to `init` and `copy` command
-
-The `init` and `copy` command can now be used with the `--repository-file2`
-option or the `$RESTIC_REPOSITORY_FILE2` environment variable.
-These to options are in addition to the `--repo2` flag and allow you to read
-the destination repository from a file.
-
-Using both `--repository-file` and `--repo2` options resulted in an error for
-the `copy` or `init` command. The handling of this combination of options has
-been fixed. A workaround for this issue is to only use `--repo` or `-r` and
-`--repo2` for `init` or `copy`.
-
-https://github.com/restic/restic/issues/3293
-https://github.com/restic/restic/pull/3294

changelog/0.12.1_2021-08-03/issue-3296

@@ -1,7 +0,0 @@
-Bugfix: Fix crash of `check --read-data-subset=x%` run for an empty repository
-
-The command `restic check --read-data-subset=x%` crashed when run for an empty
-repository. This has been fixed.
-
-https://github.com/restic/restic/issues/3296
-https://github.com/restic/restic/pull/3309

changelog/0.12.1_2021-08-03/issue-3302

@@ -1,8 +0,0 @@
-Bugfix: Fix `fdopendir: not a directory` error for local backend
-
-The `check`, `list packs`, `prune` and `rebuild-index` commands failed
-for the local backend when the `data` folder in the repository contained
-files. This has been fixed.
-
-https://github.com/restic/restic/issues/3302
-https://github.com/restic/restic/pull/3308

changelog/0.12.1_2021-08-03/issue-3312

@@ -1,5 +0,0 @@
-Enhancement: Add auto-completion support for fish
-
-The `generate` command now supports fish auto completion.
-
-https://github.com/restic/restic/pull/3312

changelog/0.12.1_2021-08-03/issue-3334

@@ -1,8 +0,0 @@
-Bugfix: Print `created new cache` message only on a terminal
-
-The message `created new cache` was printed even when the output wasn't a
-terminal. That broke piping `restic dump` output to tar or zip if cache
-directory didn't exist. The message is now only printed on a terminal.
-
-https://github.com/restic/restic/issues/3334
-https://github.com/restic/restic/pull/3343

changelog/0.12.1_2021-08-03/issue-3336

@@ -1,8 +0,0 @@
-Enhancement: SFTP backend now checks for disk space
-
-Backing up over SFTP previously spewed multiple generic "failure" messages
-when the remote disk was full. It now checks for disk space before writing
-a file and fails immediately with a "no space left on device" message.
-
-https://github.com/restic/restic/issues/3336
-https://github.com/restic/restic/pull/3345

changelog/0.12.1_2021-08-03/issue-3377

@@ -1,6 +0,0 @@
-Enhancement: Add release binaries for Apple Silicon
-
-We've added release binaries for macOS on Apple Silicon (M1).
-
-https://github.com/restic/restic/issues/3377
-https://github.com/restic/restic/pull/3394

changelog/0.12.1_2021-08-03/issue-3380

@@ -1,7 +0,0 @@
-Bugfix: Fix crash of `backup --exclude='**'`
-
-The exclude filter `**`, which excludes all files, caused restic to crash. This
-has been corrected.
-
-https://github.com/restic/restic/issues/3380
-https://github.com/restic/restic/pull/3393

changelog/0.12.1_2021-08-03/issue-3414

@@ -1,20 +0,0 @@
-Enhancement: Add `--keep-within-hourly` option to restic forget
-
-The `forget` command allowed keeping a given number of hourly
-backups or to keep all backups within a given interval, but it
-was not possible to specify keeping hourly backups within a given
-interval.
-
-The new `--keep-within-hourly` option now offers this functionality.
-Similar options for daily/weekly/monthly/yearly are also implemented,
-the new options are:
-
-    --keep-within-hourly <1y2m3d4h>
-    --keep-within-daily <1y2m3d4h>
-    --keep-within-weekly <1y2m3d4h>
-    --keep-within-monthly <1y2m3d4h>
-    --keep-within-yearly <1y2m3d4h>
-
-https://github.com/restic/restic/issues/3414
-https://github.com/restic/restic/pull/3416
-https://forum.restic.net/t/forget-policy/4014/11

changelog/0.12.1_2021-08-03/issue-3456

@@ -1,9 +0,0 @@
-Enhancement: Support filtering and specifying untagged snapshots
-
-It was previously not possible to specify an empty tag with the `--tag` and
-`--keep-tag` options. This has now been fixed, such that `--tag ''` and
-`--keep-tag ''` now matches snapshots without tags. This allows e.g. the
-`snapshots` and `forget` commands to only operate on untagged snapshots.
-
-https://github.com/restic/restic/issues/3456
-https://github.com/restic/restic/pull/3457

changelog/0.12.1_2021-08-03/pull-3167

@@ -1,9 +0,0 @@
-Enhancement: Allow specifying limit of `snapshots` list
-
-The `--last` option allowed limiting the output of the `snapshots`
-command to the latest snapshot for each host. The new `--latest n`
-option allows limiting the output to the latest `n` snapshots.
-
-This change deprecates the option `--last` in favour of `--latest 1`.
-
-https://github.com/restic/restic/pull/3167

changelog/0.12.1_2021-08-03/pull-3305

@@ -1,6 +0,0 @@
-Bugfix: Fix possibly missing backup summary of JSON output in case of error
-
-When using `--json` output it happened from time to time that the summary
-output was missing in case an error occurred. This has been fixed.
-
-https://github.com/restic/restic/pull/3305

changelog/0.12.1_2021-08-03/pull-3426

@@ -1,7 +0,0 @@
-Enhancement: Optimize read performance of mount command
-
-Reading large files in a mounted repository may be up to five times faster.
-This improvement primarily applies to repositories stored at a backend that can
-be accessed with low latency, like e.g. the local backend.
-
-https://github.com/restic/restic/pull/3426

changelog/0.12.1_2021-08-03/pull-3427

@@ -1,13 +0,0 @@
-Enhancement: `find --pack` fallback to index if data file is missing
-
-When investigating a repository with missing data files, it might be useful to
-determine affected snapshots before running `rebuild-index`. Previously,
-`find --pack pack-id` returned no data as it required accessing the data file.
-Now, if the necessary data is still available in the repository index, it gets
-retrieved from there.
-
-The command now also supports looking up multiple pack files in a single `find`
-run.
-
-https://github.com/restic/restic/pull/3427
-https://forum.restic.net/t/missing-packs-not-found/2600

changelog/0.12.1_2021-08-03/pull-3449

@@ -1,8 +0,0 @@
-Bugfix: Correctly handle download errors during `restore`
-
-Due to a regression in restic 0.12.0, the `restore` command in some cases did
-not retry download errors and only printed a warning. This has been fixed by
-retrying incomplete data downloads.
-
-https://github.com/restic/restic/issues/3439
-https://github.com/restic/restic/pull/3449

changelog/0.13.0_2022-03-26/issue-1106

@@ -1,10 +0,0 @@
-Bugfix: Never lock repository for `list locks`
-
-The `list locks` command previously locked to the repository by default. This
-had the problem that it wouldn't work for an exclusively locked repository and
-that the command would also display its own lock file which can be confusing.
-
-Now, the `list locks` command never locks the repository.
-
-https://github.com/restic/restic/issues/1106
-https://github.com/restic/restic/pull/3665

changelog/0.13.0_2022-03-26/issue-1542

@@ -1,16 +0,0 @@
-Enhancement: Add `--dry-run`/`-n` option to `backup` command
-
-Testing exclude filters and other configuration options was error prone as
-wrong filters could cause files to be uploaded unintentionally. It was also
-not possible to estimate beforehand how much data would be uploaded.
-
-The `backup` command now has a `--dry-run`/`-n` option, which performs all the
-normal steps of a backup without actually writing anything to the repository.
-
-Passing -vv will log information about files that would be added, allowing for
-verification of source and exclusion options before running the real backup.
-
-https://github.com/restic/restic/issues/1542
-https://github.com/restic/restic/pull/2308
-https://github.com/restic/restic/pull/3210
-https://github.com/restic/restic/pull/3300

changelog/0.13.0_2022-03-26/issue-2202

@@ -1,15 +0,0 @@
-Enhancement: Add upload checksum for Azure, GS, S3 and Swift backends
-
-Previously only the B2 and partially the Swift backends verified the integrity
-of uploaded (encrypted) files. The verification works by informing the backend
-about the expected hash of the uploaded file. The backend then verifies the
-upload and thereby rules out any data corruption during upload.
-
-We have now added upload checksums for the Azure, GS, S3 and Swift backends,
-which besides integrity checking for uploads also means that restic can now be
-used to store backups in S3 buckets which have Object Lock enabled.
-
-https://github.com/restic/restic/issues/2202
-https://github.com/restic/restic/issues/2700
-https://github.com/restic/restic/issues/3023
-https://github.com/restic/restic/pull/3246

changelog/0.13.0_2022-03-26/issue-233

@@ -1,29 +0,0 @@
-Enhancement: Support negative include/exclude patterns
-
-If a pattern starts with an exclamation mark and it matches a file that was
-previously matched by a regular pattern, the match is cancelled. Notably,
-this can be used with `--exclude-file` to cancel the exclusion of some files.
-
-It works similarly to `.gitignore`, with the same limitation; Once a directory
-is excluded, it is not possible to include files inside the directory.
-
-Example of use as an exclude pattern for the `backup` command:
-
-    $HOME/**/*
-    !$HOME/Documents
-    !$HOME/code
-    !$HOME/.emacs.d
-    !$HOME/games
-    # [...]
-    node_modules
-    *~
-    *.o
-    *.lo
-    *.pyc
-    # [...]
-    $HOME/code/linux/*
-    !$HOME/code/linux/.git
-    # [...]
-
-https://github.com/restic/restic/issues/233
-https://github.com/restic/restic/pull/2311

changelog/0.13.0_2022-03-26/issue-2345

@@ -1,13 +0,0 @@
-Bugfix: Make cache crash-resistant and usable by multiple concurrent processes
-
-The restic cache directory (`RESTIC_CACHE_DIR`) could end up in a broken state
-in the event of restic (or the OS) crashing. This is now less likely to occur
-as files are downloaded to a temporary location before being moved to their
-proper location.
-
-This also allows multiple concurrent restic processes to operate on a single
-repository without conflicts. Previously, concurrent operations could cause
-segfaults because the processes saw each other's partially downloaded files.
-
-https://github.com/restic/restic/issues/2345
-https://github.com/restic/restic/pull/2838

changelog/0.13.0_2022-03-26/issue-2388

@@ -1,10 +0,0 @@
-Enhancement: Add warning for S3 if partial credentials are provided
-
-Previously restic did not notify about incomplete credentials when using the
-S3 backend, instead just reporting access denied.
-
-Restic now checks that both the AWS key ID and secret environment variables are
-set before connecting to the remote server, and reports an error if not.
-
-https://github.com/restic/restic/issues/2388
-https://github.com/restic/restic/pull/3532

changelog/0.13.0_2022-03-26/issue-2452

@@ -1,11 +0,0 @@
-Bugfix: Improve error handling of repository locking
-
-Previously, when the lock refresh failed to delete the old lock file, it forgot
-about the newly created one. Instead it continued trying to delete the old
-(usually no longer existing) lock file and thus over time lots of lock files
-accumulated. This has now been fixed.
-
-https://github.com/restic/restic/issues/2452
-https://github.com/restic/restic/issues/2473
-https://github.com/restic/restic/issues/2562
-https://github.com/restic/restic/pull/3512

changelog/0.13.0_2022-03-26/issue-2508

@@ -1,8 +0,0 @@
-Enhancement: Support JSON output and quiet mode for the `diff` command
-
-The `diff` command now supports outputting machine-readable output in JSON
-format. To enable this, pass the `--json` option to the command. To only print
-the summary and suppress detailed output, pass the `--quiet` option.
-
-https://github.com/restic/restic/issues/2508
-https://github.com/restic/restic/pull/3592

changelog/0.13.0_2022-03-26/issue-2656

@@ -1,8 +0,0 @@
-Enhancement: Add flag to disable TLS verification for self-signed certificates
-
-There is now an `--insecure-tls` global option in restic, which disables TLS
-verification for self-signed certificates in order to support some development
-workflows.
-
-https://github.com/restic/restic/issues/2656
-https://github.com/restic/restic/pull/2657

changelog/0.13.0_2022-03-26/issue-2738

@@ -1,8 +0,0 @@
-Bugfix: Don't print progress for `backup --json --quiet`
-
-Unlike the text output, the `--json` output format still printed progress
-information even in `--quiet` mode. This has now been fixed by always
-disabling the progress output in quiet mode.
-
-https://github.com/restic/restic/issues/2738
-https://github.com/restic/restic/pull/3264

changelog/0.13.0_2022-03-26/issue-3003

@@ -1,9 +0,0 @@
-Enhancement: Atomic uploads for the SFTP backend
-
-The SFTP backend did not upload files atomically. An interrupted upload could
-leave an incomplete file behind which could prevent restic from accessing the
-repository. This has now been fixed and uploads in the SFTP backend are done
-atomically.
-
-https://github.com/restic/restic/issues/3003
-https://github.com/restic/restic/pull/3524

changelog/0.13.0_2022-03-26/issue-3127

@@ -1,6 +0,0 @@
-Enhancement: Add xattr (extended attributes) support for Solaris
-
-Restic now supports xattr for the Solaris operating system.
-
-https://github.com/restic/restic/issues/3127
-https://github.com/restic/restic/pull/3628

changelog/0.13.0_2022-03-26/issue-3382

@@ -1,8 +0,0 @@
-Bugfix: Make `check` command honor `RESTIC_CACHE_DIR` environment variable
-
-Previously, the `check` command didn't honor the `RESTIC_CACHE_DIR` environment
-variable, which caused problems in certain system/usage configurations. This
-has now been fixed.
-
-https://github.com/restic/restic/issues/3382
-https://github.com/restic/restic/pull/3474

changelog/0.13.0_2022-03-26/issue-3464

@@ -1,10 +0,0 @@
-Enhancement: Skip lock creation on `forget` if `--no-lock` and `--dry-run`
-
-Restic used to silently ignore the `--no-lock` option of the `forget` command.
-
-It now skips creation of lock file in case both `--dry-run` and `--no-lock`
-are specified. If `--no-lock` option is specified without `--dry-run`, restic
-prints a warning message to stderr.
-
-https://github.com/restic/restic/issues/3464
-https://github.com/restic/restic/pull/3623

changelog/0.13.0_2022-03-26/issue-3490

@@ -1,8 +0,0 @@
-Enhancement: Support random subset by size in `check --read-data-subset`
-
-The `--read-data-subset` option of the `check` command now supports a third way
-of specifying the subset to check, namely `nS` where `n` is a size in bytes with
-suffix `S` as k/K, m/M, g/G or t/T.
-
-https://github.com/restic/restic/issues/3490
-https://github.com/restic/restic/pull/3548

changelog/0.13.0_2022-03-26/issue-3518

@@ -1,8 +0,0 @@
-Bugfix: Make `copy` command honor `--no-lock` for source repository
-
-The `copy` command previously did not respect the `--no-lock` option for the
-source repository, causing failures with read-only storage backends. This has
-now been fixed such that the option is now respected.
-
-https://github.com/restic/restic/issues/3518
-https://github.com/restic/restic/pull/3589

changelog/0.13.0_2022-03-26/issue-3541

@@ -1,9 +0,0 @@
-Enhancement: Improve handling of temporary B2 delete errors
-
-Deleting files on B2 could sometimes fail temporarily, which required restic to
-retry the delete operation. In some cases the file was deleted nevertheless,
-causing the retries and ultimately the restic command to fail. This has now been
-fixed.
-
-https://github.com/restic/restic/issues/3541
-https://github.com/restic/restic/pull/3544

changelog/0.13.0_2022-03-26/issue-3542

@@ -1,8 +0,0 @@
-Enhancement: Add file mode in symbolic notation to `ls --json`
-
-The `ls --json` command now provides the file mode in symbolic notation (using
-the `permissions` key), aligned with `find --json`.
-
-https://github.com/restic/restic/issues/3542
-https://github.com/restic/restic/pull/3573
-https://forum.restic.net/t/restic-ls-understanding-file-mode-with-json/4371

changelog/0.13.0_2022-03-26/issue-3556

@@ -1,10 +0,0 @@
-Bugfix: Fix hang with Backblaze B2 on SSL certificate authority error
-
-Previously, if a request failed with an SSL unknown certificate authority
-error, the B2 backend retried indefinitely and restic would appear to hang.
-
-This has now been fixed and restic instead fails with an error message.
-
-https://github.com/restic/restic/issues/3556
-https://github.com/restic/restic/issues/2355
-https://github.com/restic/restic/pull/3571

changelog/0.13.0_2022-03-26/issue-3601

@@ -1,15 +0,0 @@
-Bugfix: Fix rclone backend prematurely exiting when receiving SIGINT on Windows
-
-Previously, pressing Ctrl+C in a Windows console where restic was running with
-rclone as the backend would cause rclone to exit prematurely due to getting a
-`SIGINT` signal at the same time as restic. Restic would then wait for a long
-time for time with "unexpected EOF" and "rclone stdio connection already closed"
-errors.
-
-This has now been fixed by restic starting the rclone process detached from the
-console restic runs in (similar to starting processes in a new process group on
-Linux), which enables restic to gracefully clean up rclone (which now never gets
-the `SIGINT`).
-
-https://github.com/restic/restic/issues/3601
-https://github.com/restic/restic/pull/3602

changelog/0.13.0_2022-03-26/issue-3641

@@ -1,14 +0,0 @@
-Change: Ignore parent snapshot for `backup --stdin`
-
-Restic uses a parent snapshot to speed up directory scanning when performing
-backups, but this only wasted time and memory when the backup source is stdin
-(using the `--stdin` option of the `backup` command), since no directory scanning
-is performed in this case.
-
-Snapshots made with `backup --stdin` no longer have a parent snapshot, which allows
-restic to skip some startup operations and saves a bit of resources.
-
-The `--parent` option is still available for `backup --stdin`, but is now ignored.
-
-https://github.com/restic/restic/issues/3641
-https://github.com/restic/restic/pull/3645

changelog/0.13.0_2022-03-26/issue-3667

@@ -1,8 +0,0 @@
-Bugfix: The `mount` command now reports symlinks sizes
-
-Symlinks used to have size zero in restic mountpoints, confusing some
-third-party tools. They now have a size equal to the byte length of their
-target path, as required by POSIX.
-
-https://github.com/restic/restic/issues/3667
-https://github.com/restic/restic/pull/3668

changelog/0.13.0_2022-03-26/pull-2594

@@ -1,7 +0,0 @@
-Enhancement: Speed up the `restore --verify` command
-
-The `--verify` option lets the `restore` command verify the file content
-after it has restored a snapshot. The performance of this operation has
-now been improved by up to a factor of two.
-
-https://github.com/restic/restic/pull/2594

changelog/0.13.0_2022-03-26/pull-2816

@@ -1,10 +0,0 @@
-Enhancement: The `backup` command no longer updates file access times on Linux
-
-When reading files during backup, restic used to cause the operating system to
-update the files' access times. Note that this did not apply to filesystems with
-disabled file access times.
-
-Restic now instructs the operating system not to update the file access time,
-if the user running restic is the file owner or has root permissions.
-
-https://github.com/restic/restic/pull/2816

changelog/0.13.0_2022-03-26/pull-2880

@@ -1,9 +0,0 @@
-Enhancement: Make `recover` collect only unreferenced trees
-
-Previously, the `recover` command used to generate a snapshot containing *all*
-root trees, even those which were already referenced by a snapshot.
-
-This has been improved such that it now only processes trees not already
-referenced by any snapshot.
-
-https://github.com/restic/restic/pull/2880

changelog/0.13.0_2022-03-26/pull-3429

@@ -1,12 +0,0 @@
-Enhancement: Verify that new or modified keys are stored correctly
-
-When adding a new key or changing the password of a key, restic used to just
-create the new key (and remove the old one, when changing the password). There
-was no verification that the new key was stored correctly and works properly.
-As the repository cannot be decrypted without a valid key file, this could in
-rare cases cause the repository to become inaccessible.
-
-Restic now checks that new key files actually work before continuing. This
-can protect against some (rare) cases of hardware or storage problems.
-
-https://github.com/restic/restic/pull/3429

changelog/0.13.0_2022-03-26/pull-3436

@@ -1,12 +0,0 @@
-Enhancement: Improve local backend's resilience to (system) crashes
-
-Restic now ensures that files stored using the `local` backend are created
-atomically (that is, files are either stored completely or not at all). This
-ensures that no incomplete files are left behind even if restic is terminated
-while writing a file.
-
-In addition, restic now tries to ensure that the directory in the repository
-which contains a newly uploaded file is also written to disk. This can prevent
-missing files if the system crashes or the disk is not properly unmounted.
-
-https://github.com/restic/restic/pull/3436

changelog/0.13.0_2022-03-26/pull-3488

@@ -1,9 +0,0 @@
-Bugfix: `rebuild-index` failed if an index file was damaged
-
-Previously, the `rebuild-index` command would fail with an error if an index
-file was damaged or truncated. This has now been fixed.
-
-On older restic versions, a (slow) workaround is to use
-`rebuild-index --read-all-packs` or to manually delete the damaged index.
-
-https://github.com/restic/restic/pull/3488

changelog/0.13.0_2022-03-26/pull-3508

@@ -1,10 +0,0 @@
-Enhancement: Cache blobs read by the `dump` command
-
-When dumping a file using the `dump` command, restic did not cache blobs in any
-way, so even consecutive runs of the same blob were loaded from the repository
-again and again, slowing down the dump.
-
-Now, the caching mechanism already used by the `fuse` command is also used by
-the `dump` command. This makes dumping much faster, especially for sparse files.
-
-https://github.com/restic/restic/pull/3508

changelog/0.13.0_2022-03-26/pull-3514

@@ -1,8 +0,0 @@
-Enhancement: Support configurable timeout for the rclone backend
-
-A slow rclone backend could cause restic to time out while waiting for the
-repository to open. Restic now offers an `-o rclone.timeout` option to make
-this timeout configurable.
-
-https://github.com/restic/restic/issues/3511
-https://github.com/restic/restic/pull/3514

changelog/0.13.0_2022-03-26/pull-3519

@@ -1,6 +0,0 @@
-Change: Require Go 1.14 or newer
-
-Restic now requires Go 1.14 to build. This allows it to use new
-standard library features instead of an external dependency.
-
-https://github.com/restic/restic/issues/3519

changelog/0.13.0_2022-03-26/pull-3591

@@ -1,8 +0,0 @@
-Bugfix: Fix handling of `prune --max-repack-size=0`
-
-Restic ignored the `--max-repack-size` option when passing a value of 0. This
-has now been fixed.
-
-As a workaround, `--max-repack-size=1` can be used with older versions of restic.
-
-https://github.com/restic/restic/pull/3591

changelog/0.13.0_2022-03-26/pull-3593

@@ -1,10 +0,0 @@
-Enhancement: Improve `copy` performance by parallelizing IO
-
-Restic copy previously only used a single thread for copying blobs between
-repositories, which resulted in limited performance when copying small blobs
-to/from a high latency backend (i.e. any remote backend, especially b2).
-
-Copying will now use 8 parallel threads to increase the throughput of the copy
-operation.
-
-https://github.com/restic/restic/pull/3593

changelog/0.13.0_2022-03-26/pull-3619

@@ -1,11 +0,0 @@
-Bugfix: Avoid choosing parent snapshots newer than time of new snapshot
-
-The `backup` command, when a `--parent` was not provided, previously chose the
-most recent matching snapshot as the parent snapshot. However, this didn't make
-sense when the user passed `--time` to create a new snapshot older than the most
-recent snapshot.
-
-Instead, `backup` now chooses the most recent snapshot which is not newer than
-the snapshot-being-created's timestamp, to avoid any time travel.
-
-https://github.com/restic/restic/pull/3619

changelog/0.7.1_2017-07-22/issue-1081

@@ -1,4 +1,4 @@
-Enhancement: Clarify semantic for `--tag` for the `forget` command
+Enhancement: Clarify semantic for `--tasg` for the `forget` command
 
 https://github.com/restic/restic/issues/1081
 https://github.com/restic/restic/pull/1090

cmd/restic/cleanup.go

@@ -58,7 +58,7 @@ func RunCleanupHandlers() {
 func CleanupHandler(c <-chan os.Signal) {
 	for s := range c {
 		debug.Log("signal %v received, cleaning up", s)
-		Warnf("%ssignal %v received, cleaning up\n", clearLine(0), s)
+		Warnf("%ssignal %v received, cleaning up\n", ClearLine(), s)
 
 		code := 0
 

cmd/restic/cmd_backup.go

@@ -24,7 +24,8 @@ import (
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/textfile"
-	"github.com/restic/restic/internal/ui/backup"
+	"github.com/restic/restic/internal/ui"
+	"github.com/restic/restic/internal/ui/json"
 	"github.com/restic/restic/internal/ui/termstatus"
 )
 
@@ -59,11 +60,11 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
 		t.Go(func() error { term.Run(t.Context(globalOptions.ctx)); return nil })
 
 		err := runBackup(backupOptions, globalOptions, term, args)
-		t.Kill(nil)
-		if werr := t.Wait(); werr != nil {
-			panic(fmt.Sprintf("term.Run() returned err: %v", err))
+		if err != nil {
+			return err
 		}
-		return err
+		t.Kill(nil)
+		return t.Wait()
 	},
 }
 
@@ -91,25 +92,24 @@ type BackupOptions struct {
 	IgnoreInode             bool
 	IgnoreCtime             bool
 	UseFsSnapshot           bool
-	DryRun                  bool
 }
 
 var backupOptions BackupOptions
 
 // ErrInvalidSourceData is used to report an incomplete backup
-var ErrInvalidSourceData = errors.New("at least one source file could not be read")
+var ErrInvalidSourceData = errors.New("failed to read all source data during backup")
 
 func init() {
 	cmdRoot.AddCommand(cmdBackup)
 
 	f := cmdBackup.Flags()
-	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: last snapshot in the repo that has the same target files/directories, and is not newer than the snapshot time)")
+	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: last snapshot in the repo that has the same target files/directories)")
 	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
 	f.StringArrayVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
 	f.StringArrayVar(&backupOptions.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
 	f.StringArrayVar(&backupOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
 	f.StringArrayVar(&backupOptions.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns")
-	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems, don't cross filesystem boundaries and subvolumes")
+	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems")
 	f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes `filename[:header]`, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)")
 	f.BoolVar(&backupOptions.ExcludeCaches, "exclude-caches", false, `excludes cache directories that are marked with a CACHEDIR.TAG file. See https://bford.info/cachedir/ for the Cache Directory Tagging Standard`)
 	f.StringVar(&backupOptions.ExcludeLargerThan, "exclude-larger-than", "", "max `size` of the files to be backed up (allowed suffixes: k/K, m/M, g/G, t/T)")
@@ -132,7 +132,6 @@ func init() {
 	f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories")
 	f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number changes when checking for modified files")
 	f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files")
-	f.BoolVarP(&backupOptions.DryRun, "dry-run", "n", false, "do not upload or write any data, just show what would be done")
 	if runtime.GOOS == "windows" {
 		f.BoolVar(&backupOptions.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)")
 	}
@@ -472,7 +471,7 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 
 // parent returns the ID of the parent snapshot. If there is none, nil is
 // returned.
-func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string, timeStampLimit time.Time) (parentID *restic.ID, err error) {
+func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string) (parentID *restic.ID, err error) {
 	// Force using a parent
 	if !opts.Force && opts.Parent != "" {
 		id, err := restic.FindSnapshot(ctx, repo, opts.Parent)
@@ -485,7 +484,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
 
 	// Find last snapshot to set it as parent, if not already set
 	if !opts.Force && parentID == nil {
-		id, err := restic.FindLatestSnapshot(ctx, repo, targets, []restic.TagList{}, []string{opts.Host}, &timeStampLimit)
+		id, err := restic.FindLatestSnapshot(ctx, repo, targets, []restic.TagList{}, []string{opts.Host})
 		if err == nil {
 			parentID = &id
 		} else if err != restic.ErrNoSnapshotFound {
@@ -526,17 +525,33 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		return err
 	}
 
-	var progressPrinter backup.ProgressPrinter
-	if gopts.JSON {
-		progressPrinter = backup.NewJSONProgress(term, gopts.verbosity)
-	} else {
-		progressPrinter = backup.NewTextProgress(term, gopts.verbosity)
-	}
-	progressReporter := backup.NewProgress(progressPrinter)
+	type ArchiveProgressReporter interface {
+		CompleteItem(item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration)
+		StartFile(filename string)
+		CompleteBlob(filename string, bytes uint64)
+		ScannerError(item string, fi os.FileInfo, err error) error
+		ReportTotal(item string, s archiver.ScanStats)
+		SetMinUpdatePause(d time.Duration)
+		Run(ctx context.Context) error
+		Error(item string, fi os.FileInfo, err error) error
+		Finish(snapshotID restic.ID)
 
-	if opts.DryRun {
-		repo.SetDryRun()
-		progressReporter.SetDryRun()
+		// ui.StdioWrapper
+		Stdout() io.WriteCloser
+		Stderr() io.WriteCloser
+
+		// ui.Message
+		E(msg string, args ...interface{})
+		P(msg string, args ...interface{})
+		V(msg string, args ...interface{})
+		VV(msg string, args ...interface{})
+	}
+
+	var p ArchiveProgressReporter
+	if gopts.JSON {
+		p = json.NewBackup(term, gopts.verbosity)
+	} else {
+		p = ui.NewBackup(term, gopts.verbosity)
 	}
 
 	// use the terminal for stdout/stderr
@@ -544,14 +559,14 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	defer func() {
 		gopts.stdout, gopts.stderr = prevStdout, prevStderr
 	}()
-	gopts.stdout, gopts.stderr = progressPrinter.Stdout(), progressPrinter.Stderr()
+	gopts.stdout, gopts.stderr = p.Stdout(), p.Stderr()
 
-	progressReporter.SetMinUpdatePause(calculateProgressInterval(!gopts.Quiet, gopts.JSON))
+	p.SetMinUpdatePause(calculateProgressInterval())
 
-	t.Go(func() error { return progressReporter.Run(t.Context(gopts.ctx)) })
+	t.Go(func() error { return p.Run(t.Context(gopts.ctx)) })
 
 	if !gopts.JSON {
-		progressPrinter.V("lock repository")
+		p.V("lock repository")
 	}
 	lock, err := lockRepo(gopts.ctx, repo)
 	defer unlockRepo(lock)
@@ -572,26 +587,23 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}
 
 	if !gopts.JSON {
-		progressPrinter.V("load index files")
+		p.V("load index files")
 	}
 	err = repo.LoadIndex(gopts.ctx)
 	if err != nil {
 		return err
 	}
 
-	var parentSnapshotID *restic.ID
-	if !opts.Stdin {
-		parentSnapshotID, err = findParentSnapshot(gopts.ctx, repo, opts, targets, timeStamp)
-		if err != nil {
-			return err
-		}
+	parentSnapshotID, err := findParentSnapshot(gopts.ctx, repo, opts, targets)
+	if err != nil {
+		return err
+	}
 
-		if !gopts.JSON {
-			if parentSnapshotID != nil {
-				progressPrinter.P("using parent snapshot %v\n", parentSnapshotID.Str())
-			} else {
-				progressPrinter.P("no parent snapshot found, will read all files\n")
-			}
+	if !gopts.JSON {
+		if parentSnapshotID != nil {
+			p.P("using parent snapshot %v\n", parentSnapshotID.Str())
+		} else {
+			p.P("no parent snapshot found, will read all files\n")
 		}
 	}
 
@@ -620,12 +632,12 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		}
 
 		errorHandler := func(item string, err error) error {
-			return progressReporter.Error(item, nil, err)
+			return p.Error(item, nil, err)
 		}
 
 		messageHandler := func(msg string, args ...interface{}) {
 			if !gopts.JSON {
-				progressPrinter.P(msg, args...)
+				p.P(msg, args...)
 			}
 		}
 
@@ -635,7 +647,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}
 	if opts.Stdin {
 		if !gopts.JSON {
-			progressPrinter.V("read data from stdin")
+			p.V("read data from stdin")
 		}
 		filename := path.Join("/", opts.StdinFilename)
 		targetFS = &fs.Reader{
@@ -650,11 +662,11 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	sc := archiver.NewScanner(targetFS)
 	sc.SelectByName = selectByNameFilter
 	sc.Select = selectFilter
-	sc.Error = progressReporter.ScannerError
-	sc.Result = progressReporter.ReportTotal
+	sc.Error = p.ScannerError
+	sc.Result = p.ReportTotal
 
 	if !gopts.JSON {
-		progressPrinter.V("start scan on %v", targets)
+		p.V("start scan on %v", targets)
 	}
 	t.Go(func() error { return sc.Scan(t.Context(gopts.ctx), targets) })
 
@@ -665,11 +677,11 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	success := true
 	arch.Error = func(item string, fi os.FileInfo, err error) error {
 		success = false
-		return progressReporter.Error(item, fi, err)
+		return p.Error(item, fi, err)
 	}
-	arch.CompleteItem = progressReporter.CompleteItem
-	arch.StartFile = progressReporter.StartFile
-	arch.CompleteBlob = progressReporter.CompleteBlob
+	arch.CompleteItem = p.CompleteItem
+	arch.StartFile = p.StartFile
+	arch.CompleteBlob = p.CompleteBlob
 
 	if opts.IgnoreInode {
 		// --ignore-inode implies --ignore-ctime: on FUSE, the ctime is not
@@ -693,30 +705,28 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}
 
 	if !gopts.JSON {
-		progressPrinter.V("start backup on %v", targets)
+		p.V("start backup on %v", targets)
 	}
 	_, id, err := arch.Snapshot(gopts.ctx, targets, snapshotOpts)
+	if err != nil {
+		return errors.Fatalf("unable to save snapshot: %v", err)
+	}
 
 	// cleanly shutdown all running goroutines
 	t.Kill(nil)
 
 	// let's see if one returned an error
-	werr := t.Wait()
-
-	// return original error
-	if err != nil {
-		return errors.Fatalf("unable to save snapshot: %v", err)
-	}
+	err = t.Wait()
 
 	// Report finished execution
-	progressReporter.Finish(id)
-	if !gopts.JSON && !opts.DryRun {
-		progressPrinter.P("snapshot %s saved\n", id.Str())
+	p.Finish(id)
+	if !gopts.JSON {
+		p.P("snapshot %s saved\n", id.Str())
 	}
 	if !success {
 		return ErrInvalidSourceData
 	}
 
 	// Return error if any
-	return werr
+	return err
 }

cmd/restic/cmd_cache.go

@@ -5,7 +5,6 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
-	"strings"
 	"time"
 
 	"github.com/restic/restic/internal/cache"
@@ -141,13 +140,8 @@ func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
 			size = fmt.Sprintf("%11s", formatBytes(uint64(bytes)))
 		}
 
-		name := entry.Name()
-		if !strings.HasPrefix(name, "restic-check-cache-") {
-			name = name[:10]
-		}
-
 		tab.AddRow(data{
-			name,
+			entry.Name()[:10],
 			fmt.Sprintf("%d days ago", uint(time.Since(entry.ModTime()).Hours()/24)),
 			old,
 			size,

cmd/restic/cmd_cat.go

@@ -69,6 +69,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 		}
 	}
 
+	// handle all types that don't need an index
 	switch tpe {
 	case "config":
 		buf, err := json.MarshalIndent(repo.Config(), "", "  ")
@@ -141,7 +142,15 @@ func runCat(gopts GlobalOptions, args []string) error {
 
 		Println(string(buf))
 		return nil
+	}
 
+	// load index, handle all the other types
+	err = repo.LoadIndex(gopts.ctx)
+	if err != nil {
+		return err
+	}
+
+	switch tpe {
 	case "pack":
 		h := restic.Handle{Type: restic.PackFile, Name: id.String()}
 		buf, err := backend.LoadAll(gopts.ctx, nil, repo.Backend(), h)
@@ -158,11 +167,6 @@ func runCat(gopts GlobalOptions, args []string) error {
 		return err
 
 	case "blob":
-		err = repo.LoadIndex(gopts.ctx)
-		if err != nil {
-			return err
-		}
-
 		for _, t := range []restic.BlobType{restic.DataBlob, restic.TreeBlob} {
 			bh := restic.BlobHandle{ID: id, Type: t}
 			if !repo.Index().Has(bh) {

cmd/restic/cmd_check.go

@@ -5,12 +5,10 @@ import (
 	"math/rand"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/spf13/cobra"
 
-	"github.com/restic/restic/internal/cache"
 	"github.com/restic/restic/internal/checker"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/fs"
@@ -56,7 +54,7 @@ func init() {
 
 	f := cmdCheck.Flags()
 	f.BoolVar(&checkOptions.ReadData, "read-data", false, "read all data blobs")
-	f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read a `subset` of data packs, specified as 'n/t' for specific part, or either 'x%' or 'x.y%' or a size in bytes with suffixes k/K, m/M, g/G, t/T for a random subset")
+	f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read a `subset` of data packs, specified as 'n/t' for specific subset or either 'x%' or 'x.y%' for random subset")
 	f.BoolVar(&checkOptions.CheckUnused, "check-unused", false, "find unused blobs")
 	f.BoolVar(&checkOptions.WithCache, "with-cache", false, "use the cache")
 }
@@ -67,7 +65,7 @@ func checkFlags(opts CheckOptions) error {
 	}
 	if opts.ReadDataSubset != "" {
 		dataSubset, err := stringToIntSlice(opts.ReadDataSubset)
-		argumentError := errors.Fatal("check flag --read-data-subset has invalid value, please see documentation")
+		argumentError := errors.Fatal("check flag --read-data-subset must have two positive integer values or a percentage, e.g. --read-data-subset=1/2 or --read-data-subset=2.5%%")
 		if err == nil {
 			if len(dataSubset) != 2 {
 				return argumentError
@@ -78,7 +76,7 @@ func checkFlags(opts CheckOptions) error {
 			if dataSubset[1] > totalBucketsMax {
 				return errors.Fatalf("check flag --read-data-subset=n/t t must be at most %d", totalBucketsMax)
 			}
-		} else if strings.HasSuffix(opts.ReadDataSubset, "%") {
+		} else {
 			percentage, err := parsePercentage(opts.ReadDataSubset)
 			if err != nil {
 				return argumentError
@@ -86,19 +84,8 @@ func checkFlags(opts CheckOptions) error {
 
 			if percentage <= 0.0 || percentage > 100.0 {
 				return errors.Fatal(
-					"check flag --read-data-subset=x% x must be above 0.0% and at most 100.0%")
+					"check flag --read-data-subset=n% n must be above 0.0% and at most 100.0%")
 			}
-
-		} else {
-			fileSize, err := parseSizeStr(opts.ReadDataSubset)
-			if err != nil {
-				return argumentError
-			}
-			if fileSize <= 0.0 {
-				return errors.Fatal(
-					"check flag --read-data-subset=n n must be above 0")
-			}
-
 		}
 	}
 
@@ -159,9 +146,6 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func())
 	}
 
 	cachedir := gopts.CacheDir
-	if cachedir == "" {
-		cachedir = cache.EnvDir()
-	}
 
 	// use a cache in a temporary directory
 	tempdir, err := ioutil.TempDir(cachedir, "restic-check-cache-")
@@ -257,11 +241,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 
 	Verbosef("check snapshots, trees and blobs\n")
 	errChan = make(chan error)
-	var wg sync.WaitGroup
-
-	wg.Add(1)
 	go func() {
-		defer wg.Done()
 		bar := newProgressMax(!gopts.Quiet, 0, "snapshots")
 		defer bar.Done()
 		chkr.Structure(gopts.ctx, bar, errChan)
@@ -279,11 +259,6 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	// Wait for the progress bar to be complete before printing more below.
-	// Must happen after `errChan` is read from in the above loop to avoid
-	// deadlocking in the case of errors.
-	wg.Wait()
-
 	if opts.CheckUnused {
 		for _, id := range chkr.UnusedBlobs(gopts.ctx) {
 			Verbosef("unused blob %v\n", id)
@@ -319,27 +294,10 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 			packs = selectPacksByBucket(chkr.GetPacks(), bucket, totalBuckets)
 			packCount := uint64(len(packs))
 			Verbosef("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, packCount, chkr.CountPacks(), totalBuckets)
-		} else if strings.HasSuffix(opts.ReadDataSubset, "%") {
-			percentage, err := parsePercentage(opts.ReadDataSubset)
-			if err == nil {
-				packs = selectRandomPacksByPercentage(chkr.GetPacks(), percentage)
-				Verbosef("read %.1f%% of data packs\n", percentage)
-			}
 		} else {
-			repoSize := int64(0)
-			allPacks := chkr.GetPacks()
-			for _, size := range allPacks {
-				repoSize += size
-			}
-			if repoSize == 0 {
-				return errors.Fatal("Cannot read from a repository having size 0")
-			}
-			subsetSize, _ := parseSizeStr(opts.ReadDataSubset)
-			if subsetSize > repoSize {
-				subsetSize = repoSize
-			}
-			packs = selectRandomPacksByFileSize(chkr.GetPacks(), subsetSize, repoSize)
-			Verbosef("read %d bytes of data packs\n", subsetSize)
+			percentage, _ := parsePercentage(opts.ReadDataSubset)
+			packs = selectRandomPacksByPercentage(chkr.GetPacks(), percentage)
+			Verbosef("read %.1f%% of data packs\n", percentage)
 		}
 		if packs == nil {
 			return errors.Fatal("internal error: failed to select packs to check")
@@ -373,7 +331,7 @@ func selectPacksByBucket(allPacks map[restic.ID]int64, bucket, totalBuckets uint
 func selectRandomPacksByPercentage(allPacks map[restic.ID]int64, percentage float64) map[restic.ID]int64 {
 	packCount := len(allPacks)
 	packsToCheck := int(float64(packCount) * (percentage / 100.0))
-	if packCount > 0 && packsToCheck < 1 {
+	if packsToCheck < 1 {
 		packsToCheck = 1
 	}
 	timeNs := time.Now().UnixNano()
@@ -391,11 +349,6 @@ func selectRandomPacksByPercentage(allPacks map[restic.ID]int64, percentage floa
 		id := keys[idx[i]]
 		packs[id] = allPacks[id]
 	}
-	return packs
-}
 
-func selectRandomPacksByFileSize(allPacks map[restic.ID]int64, subsetSize int64, repoSize int64) map[restic.ID]int64 {
-	subsetPercentage := (float64(subsetSize) / float64(repoSize)) * 100.0
-	packs := selectRandomPacksByPercentage(allPacks, subsetPercentage)
 	return packs
 }

cmd/restic/cmd_check_test.go

@@ -122,44 +122,3 @@ func TestSelectRandomPacksByPercentage(t *testing.T) {
 		rtest.Assert(t, ok, "Expected input and output to be equal")
 	}
 }
-
-func TestSelectNoRandomPacksByPercentage(t *testing.T) {
-	// that the a repository without pack files works
-	var testPacks = make(map[restic.ID]int64)
-	selectedPacks := selectRandomPacksByPercentage(testPacks, 10.0)
-	rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs")
-}
-
-func TestSelectRandomPacksByFileSize(t *testing.T) {
-	var testPacks = make(map[restic.ID]int64)
-	for i := 1; i <= 10; i++ {
-		id := restic.NewRandomID()
-		// ensure unique ids
-		id[0] = byte(i)
-		testPacks[id] = 0
-	}
-
-	selectedPacks := selectRandomPacksByFileSize(testPacks, 10, 500)
-	rtest.Assert(t, len(selectedPacks) == 1, "Expected 1 selected packs")
-
-	selectedPacks = selectRandomPacksByFileSize(testPacks, 10240, 51200)
-	rtest.Assert(t, len(selectedPacks) == 2, "Expected 2 selected packs")
-	for pack := range selectedPacks {
-		_, ok := testPacks[pack]
-		rtest.Assert(t, ok, "Unexpected selection")
-	}
-
-	selectedPacks = selectRandomPacksByFileSize(testPacks, 500, 500)
-	rtest.Assert(t, len(selectedPacks) == 10, "Expected 10 selected packs")
-	for pack := range selectedPacks {
-		_, ok := testPacks[pack]
-		rtest.Assert(t, ok, "Unexpected item in selection")
-	}
-}
-
-func TestSelectNoRandomPacksByFileSize(t *testing.T) {
-	// that the a repository without pack files works
-	var testPacks = make(map[restic.ID]int64)
-	selectedPacks := selectRandomPacksByFileSize(testPacks, 10, 500)
-	rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs")
-}

cmd/restic/cmd_copy.go

@@ -73,12 +73,10 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	if !gopts.NoLock {
-		srcLock, err := lockRepo(ctx, srcRepo)
-		defer unlockRepo(srcLock)
-		if err != nil {
-			return err
-		}
+	srcLock, err := lockRepo(ctx, srcRepo)
+	defer unlockRepo(srcLock)
+	if err != nil {
+		return err
 	}
 
 	dstLock, err := lockRepo(ctx, dstRepo)
@@ -176,12 +174,9 @@ func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool {
 	return true
 }
 
-const numCopyWorkers = 8
-
 func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository,
 	visitedTrees restic.IDSet, rootTreeID restic.ID) error {
 
-	idChan := make(chan restic.ID)
 	wg, ctx := errgroup.WithContext(ctx)
 
 	treeStream := restic.StreamTrees(ctx, wg, srcRepo, restic.IDs{rootTreeID}, func(treeID restic.ID) bool {
@@ -191,9 +186,9 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep
 	}, nil)
 
 	wg.Go(func() error {
-		defer close(idChan)
 		// reused buffer
 		var buf []byte
+
 		for tree := range treeStream {
 			if tree.Error != nil {
 				return fmt.Errorf("LoadTree(%v) returned error %v", tree.ID.Str(), tree.Error)
@@ -201,57 +196,42 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep
 
 			// Do we already have this tree blob?
 			if !dstRepo.Index().Has(restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob}) {
-				// copy raw tree bytes to avoid problems if the serialization changes
-				var err error
-				buf, err = srcRepo.LoadBlob(ctx, restic.TreeBlob, tree.ID, buf)
+				newTreeID, err := dstRepo.SaveTree(ctx, tree.Tree)
 				if err != nil {
-					return fmt.Errorf("LoadBlob(%v) for tree returned error %v", tree.ID, err)
+					return fmt.Errorf("SaveTree(%v) returned error %v", tree.ID.Str(), err)
 				}
-
-				_, _, err = dstRepo.SaveBlob(ctx, restic.TreeBlob, buf, tree.ID, false)
-				if err != nil {
-					return fmt.Errorf("SaveBlob(%v) for tree returned error %v", tree.ID.Str(), err)
+				// Assurance only.
+				if newTreeID != tree.ID {
+					return fmt.Errorf("SaveTree(%v) returned unexpected id %s", tree.ID.Str(), newTreeID.Str())
 				}
 			}
 
+			// TODO: parallelize blob down/upload
+
 			for _, entry := range tree.Nodes {
 				// Recursion into directories is handled by StreamTrees
 				// Copy the blobs for this file.
 				for _, blobID := range entry.Content {
-					select {
-					case idChan <- blobID:
-					case <-ctx.Done():
-						return ctx.Err()
+					// Do we already have this data blob?
+					if dstRepo.Index().Has(restic.BlobHandle{ID: blobID, Type: restic.DataBlob}) {
+						continue
+					}
+					debug.Log("Copying blob %s\n", blobID.Str())
+					var err error
+					buf, err = srcRepo.LoadBlob(ctx, restic.DataBlob, blobID, buf)
+					if err != nil {
+						return fmt.Errorf("LoadBlob(%v) returned error %v", blobID, err)
+					}
+
+					_, _, err = dstRepo.SaveBlob(ctx, restic.DataBlob, buf, blobID, false)
+					if err != nil {
+						return fmt.Errorf("SaveBlob(%v) returned error %v", blobID, err)
 					}
 				}
 			}
+
 		}
 		return nil
 	})
-
-	for i := 0; i < numCopyWorkers; i++ {
-		wg.Go(func() error {
-			// reused buffer
-			var buf []byte
-			for blobID := range idChan {
-				// Do we already have this data blob?
-				if dstRepo.Index().Has(restic.BlobHandle{ID: blobID, Type: restic.DataBlob}) {
-					continue
-				}
-				debug.Log("Copying blob %s\n", blobID.Str())
-				var err error
-				buf, err = srcRepo.LoadBlob(ctx, restic.DataBlob, blobID, buf)
-				if err != nil {
-					return fmt.Errorf("LoadBlob(%v) returned error %v", blobID, err)
-				}
-
-				_, _, err = dstRepo.SaveBlob(ctx, restic.DataBlob, buf, blobID, false)
-				if err != nil {
-					return fmt.Errorf("SaveBlob(%v) returned error %v", blobID, err)
-				}
-			}
-			return nil
-		})
-	}
 	return wg.Wait()
 }

cmd/restic/cmd_debug.go

@@ -4,21 +4,12 @@ package main
 
 import (
 	"context"
-	"crypto/aes"
-	"crypto/cipher"
 	"encoding/json"
 	"fmt"
 	"io"
-	"os"
-	"runtime"
-	"sort"
-	"time"
 
 	"github.com/spf13/cobra"
-	"golang.org/x/sync/errgroup"
 
-	"github.com/restic/restic/internal/backend"
-	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/pack"
 	"github.com/restic/restic/internal/repository"
@@ -48,17 +39,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 	},
 }
 
-var tryRepair bool
-var repairByte bool
-var extractPack bool
-
 func init() {
 	cmdRoot.AddCommand(cmdDebug)
 	cmdDebug.AddCommand(cmdDebugDump)
-	cmdDebug.AddCommand(cmdDebugExamine)
-	cmdDebugExamine.Flags().BoolVar(&extractPack, "extract-pack", false, "write blobs to the current directory")
-	cmdDebugExamine.Flags().BoolVar(&tryRepair, "try-repair", false, "try to repair broken blobs with single bit flips")
-	cmdDebugExamine.Flags().BoolVar(&repairByte, "repair-byte", false, "try to repair broken blobs by trying bytes")
 }
 
 func prettyPrintJSON(wr io.Writer, item interface{}) error {
@@ -182,367 +165,3 @@ func runDebugDump(gopts GlobalOptions, args []string) error {
 		return errors.Fatalf("no such type %q", tpe)
 	}
 }
-
-var cmdDebugExamine = &cobra.Command{
-	Use:               "examine pack-ID...",
-	Short:             "Examine a pack file",
-	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return runDebugExamine(globalOptions, args)
-	},
-}
-
-func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, bytewise bool) []byte {
-	if bytewise {
-		Printf("        trying to repair blob by finding a broken byte\n")
-	} else {
-		Printf("        trying to repair blob with single bit flip\n")
-	}
-
-	ch := make(chan int)
-	var wg errgroup.Group
-	done := make(chan struct{})
-	var fixed []byte
-	var found bool
-
-	workers := runtime.GOMAXPROCS(0)
-	Printf("         spinning up %d worker functions\n", runtime.GOMAXPROCS(0))
-	for i := 0; i < workers; i++ {
-		wg.Go(func() error {
-			// make a local copy of the buffer
-			buf := make([]byte, len(input))
-			copy(buf, input)
-
-			testFlip := func(idx int, pattern byte) bool {
-				// flip bits
-				buf[idx] ^= pattern
-
-				nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
-				plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil)
-				if err == nil {
-					Printf("\n")
-					Printf("        blob could be repaired by XORing byte %v with 0x%02x\n", idx, pattern)
-					Printf("        hash is %v\n", restic.Hash(plaintext))
-					close(done)
-					found = true
-					fixed = plaintext
-					return true
-				}
-
-				// flip bits back
-				buf[idx] ^= pattern
-				return false
-			}
-
-			for i := range ch {
-				if bytewise {
-					for j := 0; j < 255; j++ {
-						if testFlip(i, byte(j)) {
-							return nil
-						}
-					}
-				} else {
-					for j := 0; j < 7; j++ {
-						// flip each bit once
-						if testFlip(i, (1 << uint(j))) {
-							return nil
-						}
-					}
-				}
-			}
-			return nil
-		})
-	}
-
-	wg.Go(func() error {
-		defer close(ch)
-
-		start := time.Now()
-		info := time.Now()
-		for i := range input {
-			select {
-			case ch <- i:
-			case <-done:
-				Printf("     done after %v\n", time.Since(start))
-				return nil
-			}
-
-			if time.Since(info) > time.Second {
-				secs := time.Since(start).Seconds()
-				gps := float64(i) / secs
-				remaining := len(input) - i
-				eta := time.Duration(float64(remaining)/gps) * time.Second
-
-				Printf("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v",
-					i, len(input), float32(i)/float32(len(input))*100, gps, eta)
-				info = time.Now()
-			}
-		}
-		return nil
-	})
-	err := wg.Wait()
-	if err != nil {
-		panic("all go rountines can only return nil")
-	}
-
-	if !found {
-		Printf("\n        blob could not be repaired\n")
-	}
-	return fixed
-}
-
-func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte {
-	// strip signature at the end
-	l := len(buf)
-	nonce, ct := buf[:16], buf[16:l-16]
-	out := make([]byte, len(ct))
-
-	c, err := aes.NewCipher(k.EncryptionKey[:])
-	if err != nil {
-		panic(fmt.Sprintf("unable to create cipher: %v", err))
-	}
-	e := cipher.NewCTR(c, nonce)
-	e.XORKeyStream(out, ct)
-
-	return out
-}
-
-func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list []restic.Blob) error {
-	be := repo.Backend()
-	h := restic.Handle{
-		Name: pack.String(),
-		Type: restic.PackFile,
-	}
-	for _, blob := range list {
-		Printf("      loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length)
-		buf := make([]byte, blob.Length)
-		err := be.Load(ctx, h, int(blob.Length), int64(blob.Offset), func(rd io.Reader) error {
-			n, err := io.ReadFull(rd, buf)
-			if err != nil {
-				return fmt.Errorf("read error after %d bytes: %v", n, err)
-			}
-			return nil
-		})
-		if err != nil {
-			Warnf("error read: %v\n", err)
-			continue
-		}
-
-		key := repo.Key()
-
-		nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
-		plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
-		if err != nil {
-			Warnf("error decrypting blob: %v\n", err)
-			var plain []byte
-			if tryRepair || repairByte {
-				plain = tryRepairWithBitflip(ctx, key, buf, repairByte)
-			}
-			var prefix string
-			if plain != nil {
-				id := restic.Hash(plain)
-				if !id.Equal(blob.ID) {
-					Printf("         repaired blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plain), id, blob.ID)
-					prefix = "repaired-wrong-hash-"
-				} else {
-					Printf("         successfully repaired blob (length %v), hash is %v, ID matches\n", len(plain), id)
-					prefix = "repaired-"
-				}
-			} else {
-				plain = decryptUnsigned(ctx, key, buf)
-				prefix = "damaged-"
-			}
-			err = storePlainBlob(blob.ID, prefix, plain)
-			if err != nil {
-				return err
-			}
-			continue
-		}
-
-		id := restic.Hash(plaintext)
-		var prefix string
-		if !id.Equal(blob.ID) {
-			Printf("         successfully decrypted blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plaintext), id, blob.ID)
-			prefix = "wrong-hash-"
-		} else {
-			Printf("         successfully decrypted blob (length %v), hash is %v, ID matches\n", len(plaintext), id)
-			prefix = "correct-"
-		}
-		if extractPack {
-			err = storePlainBlob(id, prefix, plaintext)
-			if err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-func storePlainBlob(id restic.ID, prefix string, plain []byte) error {
-	filename := fmt.Sprintf("%s%s.bin", prefix, id)
-	f, err := os.Create(filename)
-	if err != nil {
-		return err
-	}
-
-	_, err = f.Write(plain)
-	if err != nil {
-		_ = f.Close()
-		return err
-	}
-
-	err = f.Close()
-	if err != nil {
-		return err
-	}
-
-	Printf("decrypt of blob %v stored at %v\n", id, filename)
-	return nil
-}
-
-func runDebugExamine(gopts GlobalOptions, args []string) error {
-	ids := make([]restic.ID, 0)
-	for _, name := range args {
-		id, err := restic.ParseID(name)
-		if err != nil {
-			Warnf("error: %v\n", err)
-			continue
-		}
-		ids = append(ids, id)
-	}
-
-	if len(ids) == 0 {
-		return errors.Fatal("no pack files to examine")
-	}
-
-	repo, err := OpenRepository(gopts)
-	if err != nil {
-		return err
-	}
-
-	if !gopts.NoLock {
-		lock, err := lockRepo(gopts.ctx, repo)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	}
-
-	err = repo.LoadIndex(gopts.ctx)
-	if err != nil {
-		return err
-	}
-
-	for _, id := range ids {
-		err := examinePack(gopts.ctx, repo, id)
-		if err != nil {
-			Warnf("error: %v\n", err)
-		}
-		if err == context.Canceled {
-			break
-		}
-	}
-	return nil
-}
-
-func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) error {
-	Printf("examine %v\n", id)
-
-	h := restic.Handle{
-		Type: restic.PackFile,
-		Name: id.String(),
-	}
-	fi, err := repo.Backend().Stat(ctx, h)
-	if err != nil {
-		return err
-	}
-	Printf("  file size is %v\n", fi.Size)
-
-	buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h)
-	if err != nil {
-		return err
-	}
-	gotID := restic.Hash(buf)
-	if !id.Equal(gotID) {
-		Printf("  wanted hash %v, got %v\n", id, gotID)
-	} else {
-		Printf("  hash for file content matches\n")
-	}
-
-	Printf("  ========================================\n")
-	Printf("  looking for info in the indexes\n")
-
-	blobsLoaded := false
-	// examine all data the indexes have for the pack file
-	for _, idx := range repo.Index().(*repository.MasterIndex).All() {
-		idxIDs, err := idx.IDs()
-		if err != nil {
-			idxIDs = restic.IDs{}
-		}
-
-		blobs := idx.ListPack(id)
-		if len(blobs) == 0 {
-			continue
-		}
-
-		Printf("    index %v:\n", idxIDs)
-
-		// convert list of blobs to []restic.Blob
-		var list []restic.Blob
-		for _, b := range blobs {
-			list = append(list, b.Blob)
-		}
-		checkPackSize(list, fi.Size)
-
-		err = loadBlobs(ctx, repo, id, list)
-		if err != nil {
-			Warnf("error: %v\n", err)
-		} else {
-			blobsLoaded = true
-		}
-	}
-
-	Printf("  ========================================\n")
-	Printf("  inspect the pack itself\n")
-
-	blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), fi.Size)
-	if err != nil {
-		return fmt.Errorf("pack %v: %v", id.Str(), err)
-	}
-	checkPackSize(blobs, fi.Size)
-
-	if !blobsLoaded {
-		return loadBlobs(ctx, repo, id, blobs)
-	}
-	return nil
-}
-
-func checkPackSize(blobs []restic.Blob, fileSize int64) {
-	// track current size and offset
-	var size, offset uint64
-
-	sort.Slice(blobs, func(i, j int) bool {
-		return blobs[i].Offset < blobs[j].Offset
-	})
-
-	for _, pb := range blobs {
-		Printf("      %v blob %v, offset %-6d, raw length %-6d\n", pb.Type, pb.ID, pb.Offset, pb.Length)
-		if offset != uint64(pb.Offset) {
-			Printf("      hole in file, want offset %v, got %v\n", offset, pb.Offset)
-		}
-		offset += uint64(pb.Length)
-		size += uint64(pb.Length)
-	}
-
-	// compute header size, per blob: 1 byte type, 4 byte length, 32 byte id
-	size += uint64(restic.CiphertextLength(len(blobs) * (1 + 4 + 32)))
-	// length in uint32 little endian
-	size += 4
-
-	if uint64(fileSize) != size {
-		Printf("      file sizes do not match: computed %v from index, file size is %v\n", size, fileSize)
-	} else {
-		Printf("      file sizes match\n")
-	}
-}

cmd/restic/cmd_diff.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"encoding/json"
 	"path"
 	"reflect"
 	"sort"
@@ -63,29 +62,15 @@ func loadSnapshot(ctx context.Context, repo *repository.Repository, desc string)
 
 // Comparer collects all things needed to compare two snapshots.
 type Comparer struct {
-	repo        restic.Repository
-	opts        DiffOptions
-	printChange func(change *Change)
-}
-
-type Change struct {
-	MessageType string `json:"message_type"` // "change"
-	Path        string `json:"path"`
-	Modifier    string `json:"modifier"`
-}
-
-func NewChange(path string, mode string) *Change {
-	return &Change{MessageType: "change", Path: path, Modifier: mode}
+	repo restic.Repository
+	opts DiffOptions
 }
 
 // DiffStat collects stats for all types of items.
 type DiffStat struct {
-	Files     int    `json:"files"`
-	Dirs      int    `json:"dirs"`
-	Others    int    `json:"others"`
-	DataBlobs int    `json:"data_blobs"`
-	TreeBlobs int    `json:"tree_blobs"`
-	Bytes     uint64 `json:"bytes"`
+	Files, Dirs, Others  int
+	DataBlobs, TreeBlobs int
+	Bytes                uint64
 }
 
 // Add adds stats information for node to s.
@@ -128,14 +113,21 @@ func addBlobs(bs restic.BlobSet, node *restic.Node) {
 	}
 }
 
-type DiffStatsContainer struct {
-	MessageType                          string         `json:"message_type"` // "statistics"
-	SourceSnapshot                       string         `json:"source_snapshot"`
-	TargetSnapshot                       string         `json:"target_snapshot"`
-	ChangedFiles                         int            `json:"changed_files"`
-	Added                                DiffStat       `json:"added"`
-	Removed                              DiffStat       `json:"removed"`
-	BlobsBefore, BlobsAfter, BlobsCommon restic.BlobSet `json:"-"`
+// DiffStats collects the differences between two snapshots.
+type DiffStats struct {
+	ChangedFiles                         int
+	Added                                DiffStat
+	Removed                              DiffStat
+	BlobsBefore, BlobsAfter, BlobsCommon restic.BlobSet
+}
+
+// NewDiffStats creates new stats for a diff run.
+func NewDiffStats() *DiffStats {
+	return &DiffStats{
+		BlobsBefore: restic.NewBlobSet(),
+		BlobsAfter:  restic.NewBlobSet(),
+		BlobsCommon: restic.NewBlobSet(),
+	}
 }
 
 // updateBlobs updates the blob counters in the stats struct.
@@ -170,7 +162,7 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b
 		if node.Type == "dir" {
 			name += "/"
 		}
-		c.printChange(NewChange(name, "+"))
+		Printf("%-5s%v\n", mode, name)
 		stats.Add(node)
 		addBlobs(blobs, node)
 
@@ -229,7 +221,7 @@ func uniqueNodeNames(tree1, tree2 *restic.Tree) (tree1Nodes, tree2Nodes map[stri
 	return tree1Nodes, tree2Nodes, uniqueNames
 }
 
-func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, prefix string, id1, id2 restic.ID) error {
+func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string, id1, id2 restic.ID) error {
 	debug.Log("diffing %v to %v", id1, id2)
 	tree1, err := c.repo.LoadTree(ctx, id1)
 	if err != nil {
@@ -273,7 +265,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
 			}
 
 			if mod != "" {
-				c.printChange(NewChange(name, mod))
+				Printf("%-5s%v\n", mod, name)
 			}
 
 			if node1.Type == "dir" && node2.Type == "dir" {
@@ -292,7 +284,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
 			if node1.Type == "dir" {
 				prefix += "/"
 			}
-			c.printChange(NewChange(prefix, "-"))
+			Printf("%-5s%v\n", "-", prefix)
 			stats.Removed.Add(node1)
 
 			if node1.Type == "dir" {
@@ -306,7 +298,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
 			if node2.Type == "dir" {
 				prefix += "/"
 			}
-			c.printChange(NewChange(prefix, "+"))
+			Printf("%-5s%v\n", "+", prefix)
 			stats.Added.Add(node2)
 
 			if node2.Type == "dir" {
@@ -356,9 +348,7 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	if !gopts.JSON {
-		Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
-	}
+	Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
 
 	if sn1.Tree == nil {
 		return errors.Errorf("snapshot %v has nil tree", sn1.ID().Str())
@@ -371,33 +361,9 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 	c := &Comparer{
 		repo: repo,
 		opts: diffOptions,
-		printChange: func(change *Change) {
-			Printf("%-5s%v\n", change.Modifier, change.Path)
-		},
 	}
 
-	if gopts.JSON {
-		enc := json.NewEncoder(gopts.stdout)
-		c.printChange = func(change *Change) {
-			err := enc.Encode(change)
-			if err != nil {
-				Warnf("JSON encode failed: %v\n", err)
-			}
-		}
-	}
-
-	if gopts.Quiet {
-		c.printChange = func(change *Change) {}
-	}
-
-	stats := &DiffStatsContainer{
-		MessageType:    "statistics",
-		SourceSnapshot: args[0],
-		TargetSnapshot: args[1],
-		BlobsBefore:    restic.NewBlobSet(),
-		BlobsAfter:     restic.NewBlobSet(),
-		BlobsCommon:    restic.NewBlobSet(),
-	}
+	stats := NewDiffStats()
 	stats.BlobsBefore.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn1.Tree})
 	stats.BlobsAfter.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn2.Tree})
 
@@ -410,21 +376,14 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 	updateBlobs(repo, stats.BlobsBefore.Sub(both).Sub(stats.BlobsCommon), &stats.Removed)
 	updateBlobs(repo, stats.BlobsAfter.Sub(both).Sub(stats.BlobsCommon), &stats.Added)
 
-	if gopts.JSON {
-		err := json.NewEncoder(gopts.stdout).Encode(stats)
-		if err != nil {
-			Warnf("JSON encode failed: %v\n", err)
-		}
-	} else {
-		Printf("\n")
-		Printf("Files:       %5d new, %5d removed, %5d changed\n", stats.Added.Files, stats.Removed.Files, stats.ChangedFiles)
-		Printf("Dirs:        %5d new, %5d removed\n", stats.Added.Dirs, stats.Removed.Dirs)
-		Printf("Others:      %5d new, %5d removed\n", stats.Added.Others, stats.Removed.Others)
-		Printf("Data Blobs:  %5d new, %5d removed\n", stats.Added.DataBlobs, stats.Removed.DataBlobs)
-		Printf("Tree Blobs:  %5d new, %5d removed\n", stats.Added.TreeBlobs, stats.Removed.TreeBlobs)
-		Printf("  Added:   %-5s\n", formatBytes(uint64(stats.Added.Bytes)))
-		Printf("  Removed: %-5s\n", formatBytes(uint64(stats.Removed.Bytes)))
-	}
+	Printf("\n")
+	Printf("Files:       %5d new, %5d removed, %5d changed\n", stats.Added.Files, stats.Removed.Files, stats.ChangedFiles)
+	Printf("Dirs:        %5d new, %5d removed\n", stats.Added.Dirs, stats.Removed.Dirs)
+	Printf("Others:      %5d new, %5d removed\n", stats.Added.Others, stats.Removed.Others)
+	Printf("Data Blobs:  %5d new, %5d removed\n", stats.Added.DataBlobs, stats.Removed.DataBlobs)
+	Printf("Tree Blobs:  %5d new, %5d removed\n", stats.Added.TreeBlobs, stats.Removed.TreeBlobs)
+	Printf("  Added:   %-5s\n", formatBytes(uint64(stats.Added.Bytes)))
+	Printf("  Removed: %-5s\n", formatBytes(uint64(stats.Removed.Bytes)))
 
 	return nil
 }

cmd/restic/cmd_dump.go

@@ -67,31 +67,41 @@ func splitPath(p string) []string {
 	return append(s, f)
 }
 
-func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string, d *dump.Dumper) error {
+func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string, writeDump dump.WriteDump) error {
+	if tree == nil {
+		return fmt.Errorf("called with a nil tree")
+	}
+	if repo == nil {
+		return fmt.Errorf("called with a nil repository")
+	}
+	l := len(pathComponents)
+	if l == 0 {
+		return fmt.Errorf("empty path components")
+	}
+
 	// If we print / we need to assume that there are multiple nodes at that
 	// level in the tree.
 	if pathComponents[0] == "" {
 		if err := checkStdoutArchive(); err != nil {
 			return err
 		}
-		return d.DumpTree(ctx, tree, "/")
+		return writeDump(ctx, repo, tree, "/", os.Stdout)
 	}
 
 	item := filepath.Join(prefix, pathComponents[0])
-	l := len(pathComponents)
 	for _, node := range tree.Nodes {
 		// If dumping something in the highest level it will just take the
 		// first item it finds and dump that according to the switch case below.
 		if node.Name == pathComponents[0] {
 			switch {
 			case l == 1 && dump.IsFile(node):
-				return d.WriteNode(ctx, node)
+				return dump.GetNodeData(ctx, os.Stdout, repo, node)
 			case l > 1 && dump.IsDir(node):
 				subtree, err := repo.LoadTree(ctx, *node.Subtree)
 				if err != nil {
 					return errors.Wrapf(err, "cannot load subtree for %q", item)
 				}
-				return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d)
+				return printFromTree(ctx, subtree, repo, item, pathComponents[1:], writeDump)
 			case dump.IsDir(node):
 				if err := checkStdoutArchive(); err != nil {
 					return err
@@ -100,7 +110,7 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
 				if err != nil {
 					return err
 				}
-				return d.DumpTree(ctx, subtree, item)
+				return writeDump(ctx, repo, subtree, item, os.Stdout)
 			case l > 1:
 				return fmt.Errorf("%q should be a dir, but is a %q", item, node.Type)
 			case !dump.IsFile(node):
@@ -118,8 +128,12 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 		return errors.Fatal("no file and no snapshot ID specified")
 	}
 
+	var wd dump.WriteDump
 	switch opts.Archive {
-	case "tar", "zip":
+	case "tar":
+		wd = dump.WriteTar
+	case "zip":
+		wd = dump.WriteZip
 	default:
 		return fmt.Errorf("unknown archive format %q", opts.Archive)
 	}
@@ -152,7 +166,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 	var id restic.ID
 
 	if snapshotIDString == "latest" {
-		id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Hosts, nil)
+		id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Hosts)
 		if err != nil {
 			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Hosts:%v", err, opts.Paths, opts.Hosts)
 		}
@@ -173,8 +187,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 		Exitf(2, "loading tree for snapshot %q failed: %v", snapshotIDString, err)
 	}
 
-	d := dump.New(opts.Archive, repo, os.Stdout)
-	err = printFromTree(ctx, tree, repo, "/", splittedPath, d)
+	err = printFromTree(ctx, tree, repo, "/", splittedPath, wd)
 	if err != nil {
 		Exitf(2, "cannot dump file: %v", err)
 	}

cmd/restic/cmd_find.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"encoding/json"
-	"sort"
 	"strings"
 	"time"
 
@@ -41,6 +40,8 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 	},
 }
 
+const shortStr = 8 // Length of short IDs: 4 bytes as hex strings
+
 // FindOptions bundles all options for the find command.
 type FindOptions struct {
 	Oldest             string
@@ -267,7 +268,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 		if err != nil {
 			debug.Log("Error loading tree %v: %v", parentTreeID, err)
 
-			Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID())
+			Printf("Unable to load tree %s\n ... which belongs to snapshot %s.\n", parentTreeID, sn.ID())
 
 			return false, walker.ErrSkipNode
 		}
@@ -351,7 +352,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 		if err != nil {
 			debug.Log("Error loading tree %v: %v", parentTreeID, err)
 
-			Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID())
+			Printf("Unable to load tree %s\n ... which belongs to snapshot %s.\n", parentTreeID, sn.ID())
 
 			return false, walker.ErrSkipNode
 		}
@@ -385,12 +386,12 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 				idStr := id.String()
 				if _, ok := f.blobIDs[idStr]; !ok {
 					// Look for short ID form
-					if _, ok := f.blobIDs[id.Str()]; !ok {
+					if _, ok := f.blobIDs[idStr[:shortStr]]; !ok {
 						continue
 					}
 					// Replace the short ID with the long one
 					f.blobIDs[idStr] = struct{}{}
-					delete(f.blobIDs, id.Str())
+					delete(f.blobIDs, idStr[:shortStr])
 				}
 				f.out.PrintObject("blob", idStr, nodepath, parentTreeID.String(), sn)
 			}
@@ -400,8 +401,6 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 	})
 }
 
-var errAllPacksFound = errors.New("all packs found")
-
 // packsToBlobs converts the list of pack IDs to a list of blob IDs that
 // belong to those packs.
 func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
@@ -413,18 +412,20 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
 		f.blobIDs = make(map[string]struct{})
 	}
 
+	allPacksFound := false
+	packsFound := 0
+
 	debug.Log("Looking for packs...")
 	err := f.repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
+		if allPacksFound {
+			return nil
+		}
 		idStr := id.String()
 		if _, ok := packIDs[idStr]; !ok {
 			// Look for short ID form
-			if _, ok := packIDs[id.Str()]; !ok {
+			if _, ok := packIDs[idStr[:shortStr]]; !ok {
 				return nil
 			}
-			delete(packIDs, id.Str())
-		} else {
-			// forget found id
-			delete(packIDs, idStr)
 		}
 		debug.Log("Found pack %s", idStr)
 		blobs, _, err := f.repo.ListPack(ctx, id, size)
@@ -435,75 +436,25 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
 			f.blobIDs[b.ID.String()] = struct{}{}
 		}
 		// Stop searching when all packs have been found
-		if len(packIDs) == 0 {
-			return errAllPacksFound
+		packsFound++
+		if packsFound >= len(packIDs) {
+			allPacksFound = true
 		}
 		return nil
 	})
 
-	if err != nil && err != errAllPacksFound {
+	if err != nil {
 		return err
 	}
 
-	if err != errAllPacksFound {
-		// try to resolve unknown pack ids from the index
-		packIDs = f.indexPacksToBlobs(ctx, packIDs)
-	}
-
-	if len(packIDs) > 0 {
-		list := make([]string, 0, len(packIDs))
-		for h := range packIDs {
-			list = append(list, h)
-		}
-
-		sort.Strings(list)
-		return errors.Fatalf("unable to find pack(s): %v", list)
+	if !allPacksFound {
+		return errors.Fatal("unable to find all specified pack(s)")
 	}
 
 	debug.Log("%d blobs found", len(f.blobIDs))
 	return nil
 }
 
-func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struct{}) map[string]struct{} {
-	wctx, cancel := context.WithCancel(ctx)
-	defer cancel()
-
-	// remember which packs were found in the index
-	indexPackIDs := make(map[string]struct{})
-	for pb := range f.repo.Index().Each(wctx) {
-		idStr := pb.PackID.String()
-		// keep entry in packIDs as Each() returns individual index entries
-		matchingID := false
-		if _, ok := packIDs[idStr]; ok {
-			matchingID = true
-		} else {
-			if _, ok := packIDs[pb.PackID.Str()]; ok {
-				// expand id
-				delete(packIDs, pb.PackID.Str())
-				packIDs[idStr] = struct{}{}
-				matchingID = true
-			}
-		}
-		if matchingID {
-			f.blobIDs[pb.ID.String()] = struct{}{}
-			indexPackIDs[idStr] = struct{}{}
-		}
-	}
-
-	for id := range indexPackIDs {
-		delete(packIDs, id)
-	}
-
-	if len(indexPackIDs) > 0 {
-		list := make([]string, 0, len(indexPackIDs))
-		for h := range indexPackIDs {
-			list = append(list, h)
-		}
-		Warnf("some pack files are missing from the repository, getting their blobs from the repository index: %v\n\n", list)
-	}
-	return packIDs
-}
-
 func (f *Finder) findObjectPack(ctx context.Context, id string, t restic.BlobType) {
 	idx := f.repo.Index()
 
@@ -612,7 +563,7 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	if opts.PackID {
-		err := f.packsToBlobs(ctx, f.pat.pattern)
+		err := f.packsToBlobs(ctx, []string{f.pat.pattern[0]}) // TODO: support multiple packs
 		if err != nil {
 			return err
 		}

cmd/restic/cmd_forget.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"io"
 
-	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
 	"github.com/spf13/cobra"
 )
@@ -16,9 +15,8 @@ var cmdForget = &cobra.Command{
 	Long: `
 The "forget" command removes snapshots according to a policy. Please note that
 this command really only deletes the snapshot object in the repository, which
-is a reference to data stored there. In order to remove the unreferenced data
-after "forget" was run successfully, see the "prune" command. Please also read
-the documentation for "forget" to learn about important security considerations.
+is a reference to data stored there. In order to remove this (now unreferenced)
+data after 'forget' was run successfully, see the 'prune' command.
 
 EXIT STATUS
 ===========
@@ -33,19 +31,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 
 // ForgetOptions collects all options for the forget command.
 type ForgetOptions struct {
-	Last          int
-	Hourly        int
-	Daily         int
-	Weekly        int
-	Monthly       int
-	Yearly        int
-	Within        restic.Duration
-	WithinHourly  restic.Duration
-	WithinDaily   restic.Duration
-	WithinWeekly  restic.Duration
-	WithinMonthly restic.Duration
-	WithinYearly  restic.Duration
-	KeepTags      restic.TagLists
+	Last     int
+	Hourly   int
+	Daily    int
+	Weekly   int
+	Monthly  int
+	Yearly   int
+	Within   restic.Duration
+	KeepTags restic.TagLists
 
 	Hosts   []string
 	Tags    restic.TagLists
@@ -71,11 +64,6 @@ func init() {
 	f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots")
 	f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots")
 	f.VarP(&forgetOptions.Within, "keep-within", "", "keep snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
-	f.VarP(&forgetOptions.WithinHourly, "keep-within-hourly", "", "keep hourly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
-	f.VarP(&forgetOptions.WithinDaily, "keep-within-daily", "", "keep daily snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
-	f.VarP(&forgetOptions.WithinWeekly, "keep-within-weekly", "", "keep weekly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
-	f.VarP(&forgetOptions.WithinMonthly, "keep-within-monthly", "", "keep monthly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
-	f.VarP(&forgetOptions.WithinYearly, "keep-within-yearly", "", "keep yearly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
 
 	f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)")
 	f.StringArrayVar(&forgetOptions.Hosts, "host", nil, "only consider snapshots with the given `host` (can be specified multiple times)")
@@ -110,16 +98,10 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	if gopts.NoLock && !opts.DryRun {
-		return errors.Fatal("--no-lock is only applicable in combination with --dry-run for forget command")
-	}
-
-	if !opts.DryRun || !gopts.NoLock {
-		lock, err := lockRepoExclusive(gopts.ctx, repo)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
+	lock, err := lockRepoExclusive(gopts.ctx, repo)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
 	}
 
 	ctx, cancel := context.WithCancel(gopts.ctx)
@@ -146,19 +128,14 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		}
 
 		policy := restic.ExpirePolicy{
-			Last:          opts.Last,
-			Hourly:        opts.Hourly,
-			Daily:         opts.Daily,
-			Weekly:        opts.Weekly,
-			Monthly:       opts.Monthly,
-			Yearly:        opts.Yearly,
-			Within:        opts.Within,
-			WithinHourly:  opts.WithinHourly,
-			WithinDaily:   opts.WithinDaily,
-			WithinWeekly:  opts.WithinWeekly,
-			WithinMonthly: opts.WithinMonthly,
-			WithinYearly:  opts.WithinYearly,
-			Tags:          opts.KeepTags,
+			Last:    opts.Last,
+			Hourly:  opts.Hourly,
+			Daily:   opts.Daily,
+			Weekly:  opts.Weekly,
+			Monthly: opts.Monthly,
+			Yearly:  opts.Yearly,
+			Within:  opts.Within,
+			Tags:    opts.KeepTags,
 		}
 
 		if policy.Empty() && len(args) == 0 {

cmd/restic/cmd_generate.go

@@ -10,10 +10,10 @@ import (
 
 var cmdGenerate = &cobra.Command{
 	Use:   "generate [flags]",
-	Short: "Generate manual pages and auto-completion files (bash, fish, zsh)",
+	Short: "Generate manual pages and auto-completion files (bash, zsh)",
 	Long: `
 The "generate" command writes automatically generated files (like the man pages
-and the auto-completion files for bash, fish and zsh).
+and the auto-completion files for bash and zsh).
 
 EXIT STATUS
 ===========
@@ -27,7 +27,6 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 type generateOptions struct {
 	ManDir             string
 	BashCompletionFile string
-	FishCompletionFile string
 	ZSHCompletionFile  string
 }
 
@@ -38,7 +37,6 @@ func init() {
 	fs := cmdGenerate.Flags()
 	fs.StringVar(&genOpts.ManDir, "man", "", "write man pages to `directory`")
 	fs.StringVar(&genOpts.BashCompletionFile, "bash-completion", "", "write bash completion `file`")
-	fs.StringVar(&genOpts.FishCompletionFile, "fish-completion", "", "write fish completion `file`")
 	fs.StringVar(&genOpts.ZSHCompletionFile, "zsh-completion", "", "write zsh completion `file`")
 }
 
@@ -65,11 +63,6 @@ func writeBashCompletion(file string) error {
 	return cmdRoot.GenBashCompletionFile(file)
 }
 
-func writeFishCompletion(file string) error {
-	Verbosef("writing fish completion file to %v\n", file)
-	return cmdRoot.GenFishCompletionFile(file, true)
-}
-
 func writeZSHCompletion(file string) error {
 	Verbosef("writing zsh completion file to %v\n", file)
 	return cmdRoot.GenZshCompletionFile(file)
@@ -90,13 +83,6 @@ func runGenerate(cmd *cobra.Command, args []string) error {
 		}
 	}
 
-	if genOpts.FishCompletionFile != "" {
-		err := writeFishCompletion(genOpts.FishCompletionFile)
-		if err != nil {
-			return err
-		}
-	}
-
 	if genOpts.ZSHCompletionFile != "" {
 		err := writeZSHCompletion(genOpts.ZSHCompletionFile)
 		if err != nil {

cmd/restic/cmd_init.go

@@ -53,6 +53,11 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
+	be, err := create(repo, gopts.extended)
+	if err != nil {
+		return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
+	}
+
 	gopts.password, err = ReadPasswordTwice(gopts,
 		"enter password for new repository: ",
 		"enter password again: ")
@@ -60,11 +65,6 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	be, err := create(repo, gopts.extended)
-	if err != nil {
-		return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
-	}
-
 	s := repository.New(be)
 
 	err = s.Init(gopts.ctx, gopts.password, chunkerPolynomial)

cmd/restic/cmd_key.go

@@ -131,11 +131,6 @@ func addKey(gopts GlobalOptions, repo *repository.Repository) error {
 		return errors.Fatalf("creating new key failed: %v\n", err)
 	}
 
-	err = switchToNewKeyAndRemoveIfBroken(gopts.ctx, repo, id, pw)
-	if err != nil {
-		return err
-	}
-
 	Verbosef("saved new key as %s\n", id)
 
 	return nil
@@ -166,14 +161,8 @@ func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
 	if err != nil {
 		return errors.Fatalf("creating new key failed: %v\n", err)
 	}
-	oldID := repo.KeyName()
 
-	err = switchToNewKeyAndRemoveIfBroken(gopts.ctx, repo, id, pw)
-	if err != nil {
-		return err
-	}
-
-	h := restic.Handle{Type: restic.KeyFile, Name: oldID}
+	h := restic.Handle{Type: restic.KeyFile, Name: repo.KeyName()}
 	err = repo.Backend().Remove(gopts.ctx, h)
 	if err != nil {
 		return err
@@ -184,19 +173,6 @@ func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
 	return nil
 }
 
-func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repository, key *repository.Key, pw string) error {
-	// Verify new key to make sure it really works. A broken key can render the
-	// whole repository inaccessible
-	err := repo.SearchKey(ctx, pw, 0, key.Name())
-	if err != nil {
-		// the key is invalid, try to remove it
-		h := restic.Handle{Type: restic.KeyFile, Name: key.Name()}
-		_ = repo.Backend().Remove(ctx, h)
-		return errors.Fatalf("failed to access repository with new key: %v", err)
-	}
-	return nil
-}
-
 func runKey(gopts GlobalOptions, args []string) error {
 	if len(args) < 1 || (args[0] == "remove" && len(args) != 2) || (args[0] != "remove" && len(args) != 1) {
 		return errors.Fatal("wrong number of arguments")

cmd/restic/cmd_list.go

@@ -39,7 +39,7 @@ func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error {
 		return err
 	}
 
-	if !opts.NoLock && args[0] != "locks" {
+	if !opts.NoLock {
 		lock, err := lockRepo(opts.ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {

cmd/restic/cmd_ls.go

@@ -61,9 +61,9 @@ func init() {
 
 	flags := cmdLs.Flags()
 	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
-	flags.StringArrayVarP(&lsOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when snapshot ID \"latest\" is given (can be specified multiple times)")
-	flags.Var(&lsOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when snapshot ID \"latest\" is given (can be specified multiple times)")
-	flags.StringArrayVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when snapshot ID \"latest\" is given (can be specified multiple times)")
+	flags.StringArrayVarP(&lsOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)")
+	flags.Var(&lsOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot ID is given")
+	flags.StringArrayVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
 	flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories")
 }
 
@@ -74,49 +74,23 @@ type lsSnapshot struct {
 	StructType string     `json:"struct_type"` // "snapshot"
 }
 
-// Print node in our custom JSON format, followed by a newline.
-func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
-	n := &struct {
-		Name        string      `json:"name"`
-		Type        string      `json:"type"`
-		Path        string      `json:"path"`
-		UID         uint32      `json:"uid"`
-		GID         uint32      `json:"gid"`
-		Size        *uint64     `json:"size,omitempty"`
-		Mode        os.FileMode `json:"mode,omitempty"`
-		Permissions string      `json:"permissions,omitempty"`
-		ModTime     time.Time   `json:"mtime,omitempty"`
-		AccessTime  time.Time   `json:"atime,omitempty"`
-		ChangeTime  time.Time   `json:"ctime,omitempty"`
-		StructType  string      `json:"struct_type"` // "node"
-
-		size uint64 // Target for Size pointer.
-	}{
-		Name:        node.Name,
-		Type:        node.Type,
-		Path:        path,
-		UID:         node.UID,
-		GID:         node.GID,
-		size:        node.Size,
-		Mode:        node.Mode,
-		Permissions: node.Mode.String(),
-		ModTime:     node.ModTime,
-		AccessTime:  node.AccessTime,
-		ChangeTime:  node.ChangeTime,
-		StructType:  "node",
-	}
-	// Always print size for regular files, even when empty,
-	// but never for other types.
-	if node.Type == "file" {
-		n.Size = &n.size
-	}
-
-	return enc.Encode(n)
+type lsNode struct {
+	Name       string      `json:"name"`
+	Type       string      `json:"type"`
+	Path       string      `json:"path"`
+	UID        uint32      `json:"uid"`
+	GID        uint32      `json:"gid"`
+	Size       uint64      `json:"size,omitempty"`
+	Mode       os.FileMode `json:"mode,omitempty"`
+	ModTime    time.Time   `json:"mtime,omitempty"`
+	AccessTime time.Time   `json:"atime,omitempty"`
+	ChangeTime time.Time   `json:"ctime,omitempty"`
+	StructType string      `json:"struct_type"` // "node"
 }
 
 func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 	if len(args) == 0 {
-		return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'")
+		return errors.Fatal("no snapshot ID specified")
 	}
 
 	// extract any specific directories to walk
@@ -185,7 +159,7 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 		enc := json.NewEncoder(gopts.stdout)
 
 		printSnapshot = func(sn *restic.Snapshot) {
-			err := enc.Encode(lsSnapshot{
+			err = enc.Encode(lsSnapshot{
 				Snapshot:   sn,
 				ID:         sn.ID(),
 				ShortID:    sn.ID().Str(),
@@ -197,7 +171,19 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 		}
 
 		printNode = func(path string, node *restic.Node) {
-			err := lsNodeJSON(enc, path, node)
+			err = enc.Encode(lsNode{
+				Name:       node.Name,
+				Type:       node.Type,
+				Path:       path,
+				UID:        node.UID,
+				GID:        node.GID,
+				Size:       node.Size,
+				Mode:       node.Mode,
+				ModTime:    node.ModTime,
+				AccessTime: node.AccessTime,
+				ChangeTime: node.ChangeTime,
+				StructType: "node",
+			})
 			if err != nil {
 				Warnf("JSON encode failed: %v\n", err)
 			}

cmd/restic/cmd_ls_test.go

@@ -1,92 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"encoding/json"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/restic/restic/internal/restic"
-	rtest "github.com/restic/restic/internal/test"
-)
-
-func TestLsNodeJSON(t *testing.T) {
-	for _, c := range []struct {
-		path string
-		restic.Node
-		expect string
-	}{
-		// Mode is omitted when zero.
-		// Permissions, by convention is "-" per mode bit
-		{
-			path: "/bar/baz",
-			Node: restic.Node{
-				Name: "baz",
-				Type: "file",
-				Size: 12345,
-				UID:  10000000,
-				GID:  20000000,
-
-				User:  "nobody",
-				Group: "nobodies",
-				Links: 1,
-			},
-			expect: `{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
-		},
-
-		// Even empty files get an explicit size.
-		{
-			path: "/foo/empty",
-			Node: restic.Node{
-				Name: "empty",
-				Type: "file",
-				Size: 0,
-				UID:  1001,
-				GID:  1001,
-
-				User:  "not printed",
-				Group: "not printed",
-				Links: 0xF00,
-			},
-			expect: `{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
-		},
-
-		// Non-regular files do not get a size.
-		// Mode is printed in decimal, including the type bits.
-		{
-			path: "/foo/link",
-			Node: restic.Node{
-				Name:       "link",
-				Type:       "symlink",
-				Mode:       os.ModeSymlink | 0777,
-				LinkTarget: "not printed",
-			},
-			expect: `{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
-		},
-
-		{
-			path: "/some/directory",
-			Node: restic.Node{
-				Name:       "directory",
-				Type:       "dir",
-				Mode:       os.ModeDir | 0755,
-				ModTime:    time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC),
-				AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC),
-				ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC),
-			},
-			expect: `{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`,
-		},
-	} {
-		buf := new(bytes.Buffer)
-		enc := json.NewEncoder(buf)
-		err := lsNodeJSON(enc, c.path, &c.Node)
-		rtest.OK(t, err)
-		rtest.Equals(t, c.expect+"\n", buf.String())
-
-		// Sanity check: output must be valid JSON.
-		var v interface{}
-		err = json.NewDecoder(buf).Decode(&v)
-		rtest.OK(t, err)
-	}
-}

cmd/restic/cmd_mount.go

@@ -122,7 +122,6 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
 	mountOptions := []systemFuse.MountOption{
 		systemFuse.ReadOnly(),
 		systemFuse.FSName("restic"),
-		systemFuse.MaxReadahead(128 * 1024),
 	}
 
 	if opts.AllowOther {
@@ -162,8 +161,7 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
 	root := fuse.NewRoot(repo, cfg)
 
 	Printf("Now serving the repository at %s\n", mountpoint)
-	Printf("Use another terminal or tool to browse the contents of this folder.\n")
-	Printf("When finished, quit with Ctrl-c here or umount the mountpoint.\n")
+	Printf("When finished, quit with Ctrl-c or umount the mountpoint.\n")
 
 	debug.Log("serving mount at %v", mountpoint)
 	err = fs.Serve(c, root)

cmd/restic/cmd_prune.go

@@ -66,7 +66,6 @@ func addPruneOptions(c *cobra.Command) {
 }
 
 func verifyPruneOptions(opts *PruneOptions) error {
-	opts.MaxRepackBytes = math.MaxUint64
 	if len(opts.MaxRepackSize) > 0 {
 		size, err := parseSizeStr(opts.MaxRepackSize)
 		if err != nil {
@@ -120,6 +119,18 @@ func verifyPruneOptions(opts *PruneOptions) error {
 	return nil
 }
 
+func shortenStatus(maxLength int, s string) string {
+	if len(s) <= maxLength {
+		return s
+	}
+
+	if maxLength < 3 {
+		return s[:maxLength]
+	}
+
+	return s[:maxLength-3] + "..."
+}
+
 func runPrune(opts PruneOptions, gopts GlobalOptions) error {
 	err := verifyPruneOptions(&opts)
 	if err != nil {
@@ -238,7 +249,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 			"Integrity check failed: Data seems to be missing.\n"+
 			"Will not start prune to prevent (additional) data loss!\n"+
 			"Please report this error (along with the output of the 'prune' run) at\n"+
-			"https://github.com/restic/restic/issues/new/choose\n", usedBlobs)
+			"https://github.com/restic/restic/issues/new/choose", usedBlobs)
 		return errorIndexIncomplete
 	}
 
@@ -313,7 +324,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 			// Pack size does not fit and pack is needed => error
 			// If the pack is not needed, this is no error, the pack can
 			// and will be simply removed, see below.
-			Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic rebuild-index'.\n",
+			Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic rebuild-index'.",
 				id.Str(), p.unusedSize+p.usedSize, packSize)
 			return errorSizeNotMatching
 		}
@@ -419,7 +430,11 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 
 	for _, p := range repackCandidates {
 		reachedUnusedSizeAfter := (stats.size.unused-stats.size.remove-stats.size.repackrm < maxUnusedSizeAfter)
-		reachedRepackSize := stats.size.repack+p.unusedSize+p.usedSize >= opts.MaxRepackBytes
+
+		reachedRepackSize := false
+		if opts.MaxRepackBytes > 0 {
+			reachedRepackSize = stats.size.repack+p.unusedSize+p.usedSize > opts.MaxRepackBytes
+		}
 
 		switch {
 		case reachedRepackSize:
@@ -444,26 +459,26 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 		stats.size.repackrm += stats.size.duplicate
 	}
 
-	Verboseff("\nused:         %10d blobs / %s\n", stats.blobs.used, formatBytes(stats.size.used))
+	Verboseff("\nused:        %10d blobs / %s\n", stats.blobs.used, formatBytes(stats.size.used))
 	if stats.blobs.duplicate > 0 {
-		Verboseff("duplicates:   %10d blobs / %s\n", stats.blobs.duplicate, formatBytes(stats.size.duplicate))
+		Verboseff("duplicates:  %10d blobs / %s\n", stats.blobs.duplicate, formatBytes(stats.size.duplicate))
 	}
-	Verboseff("unused:       %10d blobs / %s\n", stats.blobs.unused, formatBytes(stats.size.unused))
+	Verboseff("unused:      %10d blobs / %s\n", stats.blobs.unused, formatBytes(stats.size.unused))
 	if stats.size.unref > 0 {
-		Verboseff("unreferenced:                    %s\n", formatBytes(stats.size.unref))
+		Verboseff("unreferenced:                   %s\n", formatBytes(stats.size.unref))
 	}
 	totalBlobs := stats.blobs.used + stats.blobs.unused + stats.blobs.duplicate
 	totalSize := stats.size.used + stats.size.duplicate + stats.size.unused + stats.size.unref
 	unusedSize := stats.size.duplicate + stats.size.unused
-	Verboseff("total:        %10d blobs / %s\n", totalBlobs, formatBytes(totalSize))
+	Verboseff("total:       %10d blobs / %s\n", totalBlobs, formatBytes(totalSize))
 	Verboseff("unused size: %s of total size\n", formatPercent(unusedSize, totalSize))
 
-	Verbosef("\nto repack:    %10d blobs / %s\n", stats.blobs.repack, formatBytes(stats.size.repack))
-	Verbosef("this removes: %10d blobs / %s\n", stats.blobs.repackrm, formatBytes(stats.size.repackrm))
-	Verbosef("to delete:    %10d blobs / %s\n", stats.blobs.remove, formatBytes(stats.size.remove+stats.size.unref))
+	Verbosef("\nto repack:   %10d blobs / %s\n", stats.blobs.repack, formatBytes(stats.size.repack))
+	Verbosef("this removes %10d blobs / %s\n", stats.blobs.repackrm, formatBytes(stats.size.repackrm))
+	Verbosef("to delete:   %10d blobs / %s\n", stats.blobs.remove, formatBytes(stats.size.remove+stats.size.unref))
 	totalPruneSize := stats.size.remove + stats.size.repackrm + stats.size.unref
-	Verbosef("total prune:  %10d blobs / %s\n", stats.blobs.remove+stats.blobs.repackrm, formatBytes(totalPruneSize))
-	Verbosef("remaining:    %10d blobs / %s\n", totalBlobs-(stats.blobs.remove+stats.blobs.repackrm), formatBytes(totalSize-totalPruneSize))
+	Verbosef("total prune: %10d blobs / %s\n", stats.blobs.remove+stats.blobs.repackrm, formatBytes(totalPruneSize))
+	Verbosef("remaining:   %10d blobs / %s\n", totalBlobs-(stats.blobs.remove+stats.blobs.repackrm), formatBytes(totalSize-totalPruneSize))
 	unusedAfter := unusedSize - stats.size.remove - stats.size.repackrm
 	Verbosef("unused size after prune: %s (%s of remaining size)\n",
 		formatBytes(unusedAfter), formatPercent(unusedAfter, totalSize-totalPruneSize))
@@ -472,11 +487,11 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 	Verboseff("partly used packs:  %10d\n", stats.packs.partlyUsed)
 	Verboseff("unused packs:       %10d\n\n", stats.packs.unused)
 
-	Verboseff("to keep:      %10d packs\n", stats.packs.keep)
-	Verboseff("to repack:    %10d packs\n", len(repackPacks))
-	Verboseff("to delete:    %10d packs\n", len(removePacks))
+	Verboseff("to keep:   %10d packs\n", stats.packs.keep)
+	Verboseff("to repack: %10d packs\n", len(repackPacks))
+	Verboseff("to delete: %10d packs\n", len(removePacks))
 	if len(removePacksFirst) > 0 {
-		Verboseff("to delete:    %10d unreferenced packs\n\n", len(removePacksFirst))
+		Verboseff("to delete: %10d unreferenced packs\n\n", len(removePacksFirst))
 	}
 
 	if opts.DryRun {

cmd/restic/cmd_rebuild_index.go

@@ -73,27 +73,7 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor
 		}
 	} else {
 		Verbosef("loading indexes...\n")
-		mi := repository.NewMasterIndex()
-		err := repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
-			if err != nil {
-				Warnf("removing invalid index %v: %v\n", id, err)
-				obsoleteIndexes = append(obsoleteIndexes, id)
-				return nil
-			}
-
-			mi.Insert(idx)
-			return nil
-		})
-		if err != nil {
-			return err
-		}
-
-		err = mi.MergeFinalIndexes()
-		if err != nil {
-			return err
-		}
-
-		err = repo.SetIndex(mi)
+		err := repo.LoadIndex(gopts.ctx)
 		if err != nil {
 			return err
 		}

cmd/restic/cmd_recover.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"context"
 	"os"
 	"time"
 
@@ -12,11 +11,11 @@ import (
 
 var cmdRecover = &cobra.Command{
 	Use:   "recover [flags]",
-	Short: "Recover data from the repository not referenced by snapshots",
+	Short: "Recover data from the repository",
 	Long: `
 The "recover" command builds a new snapshot from all directories it can find in
-the raw data of the repository which are not referenced in an existing snapshot.
-It can be used if, for example, a snapshot has been removed by accident with "forget".
+the raw data of the repository. It can be used if, for example, a snapshot has
+been removed by accident with "forget".
 
 EXIT STATUS
 ===========
@@ -60,14 +59,24 @@ func runRecover(gopts GlobalOptions) error {
 	trees := make(map[restic.ID]bool)
 
 	for blob := range repo.Index().Each(gopts.ctx) {
-		if blob.Type == restic.TreeBlob {
-			trees[blob.Blob.ID] = false
+		if blob.Blob.Type != restic.TreeBlob {
+			continue
 		}
+		trees[blob.Blob.ID] = false
 	}
 
-	Verbosef("load %d trees\n", len(trees))
-	bar := newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
+	cur := 0
+	max := len(trees)
+	Verbosef("load %d trees\n\n", len(trees))
+
 	for id := range trees {
+		cur++
+		Verbosef("\rtree (%v/%v)", cur, max)
+
+		if !trees[id] {
+			trees[id] = false
+		}
+
 		tree, err := repo.LoadTree(gopts.ctx, id)
 		if err != nil {
 			Warnf("unable to load tree %v: %v\n", id.Str(), err)
@@ -75,39 +84,28 @@ func runRecover(gopts GlobalOptions) error {
 		}
 
 		for _, node := range tree.Nodes {
-			if node.Type == "dir" && node.Subtree != nil {
-				trees[*node.Subtree] = true
+			if node.Type != "dir" || node.Subtree == nil {
+				continue
 			}
-		}
-		bar.Add(1)
-	}
-	bar.Done()
 
-	Verbosef("load snapshots\n")
-	err = restic.ForAllSnapshots(gopts.ctx, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error {
-		trees[*sn.Tree] = true
-		return nil
-	})
-	if err != nil {
-		return err
+			subtree := *node.Subtree
+			trees[subtree] = true
+		}
 	}
-	Verbosef("done\n")
+	Verbosef("\ndone\n")
 
 	roots := restic.NewIDSet()
 	for id, seen := range trees {
-		if !seen {
-			Verboseff("found root tree %v\n", id.Str())
-			roots.Insert(id)
+		if seen {
+			continue
 		}
-	}
-	Printf("\nfound %d unreferenced roots\n", len(roots))
 
-	if len(roots) == 0 {
-		Verbosef("no snapshot to write.\n")
-		return nil
+		roots.Insert(id)
 	}
 
-	tree := restic.NewTree(len(roots))
+	Verbosef("found %d roots\n", len(roots))
+
+	tree := restic.NewTree()
 	for id := range roots {
 		var subtreeID = id
 		node := restic.Node{
@@ -119,7 +117,7 @@ func runRecover(gopts GlobalOptions) error {
 			ModTime:    time.Now(),
 			ChangeTime: time.Now(),
 		}
-		err := tree.Insert(&node)
+		err = tree.Insert(&node)
 		if err != nil {
 			return err
 		}
@@ -135,23 +133,19 @@ func runRecover(gopts GlobalOptions) error {
 		return errors.Fatalf("unable to save blobs to the repo: %v", err)
 	}
 
-	return createSnapshot(gopts.ctx, "/recover", hostname, []string{"recovered"}, repo, &treeID)
-
-}
-
-func createSnapshot(ctx context.Context, name, hostname string, tags []string, repo restic.Repository, tree *restic.ID) error {
-	sn, err := restic.NewSnapshot([]string{name}, tags, hostname, time.Now())
+	sn, err := restic.NewSnapshot([]string{"/recover"}, []string{}, hostname, time.Now())
 	if err != nil {
 		return errors.Fatalf("unable to save snapshot: %v", err)
 	}
 
-	sn.Tree = tree
+	sn.Tree = &treeID
 
-	id, err := repo.SaveJSONUnpacked(ctx, restic.SnapshotFile, sn)
+	id, err := repo.SaveJSONUnpacked(gopts.ctx, restic.SnapshotFile, sn)
 	if err != nil {
 		return errors.Fatalf("unable to save snapshot: %v", err)
 	}
 
 	Printf("saved new snapshot %v\n", id.Str())
+
 	return nil
 }

cmd/restic/cmd_restore.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"strings"
-	"time"
 
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
@@ -118,7 +117,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 	var id restic.ID
 
 	if snapshotIDString == "latest" {
-		id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Hosts, nil)
+		id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Hosts)
 		if err != nil {
 			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Hosts:%v", err, opts.Paths, opts.Hosts)
 		}
@@ -203,7 +202,6 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 	if opts.Verify {
 		Verbosef("verifying files in %s\n", opts.Target)
 		var count int
-		t0 := time.Now()
 		count, err = res.VerifyFiles(ctx, opts.Target)
 		if err != nil {
 			return err
@@ -211,8 +209,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 		if totalErrors > 0 {
 			return errors.Fatalf("There were %d errors\n", totalErrors)
 		}
-		Verbosef("finished verifying %d files in %s (took %s)\n", count, opts.Target,
-			time.Since(t0).Round(time.Millisecond))
+		Verbosef("finished verifying %d files in %s\n", count, opts.Target)
 	}
 
 	return nil

cmd/restic/cmd_self_update.go

@@ -71,7 +71,7 @@ func runSelfUpdate(opts SelfUpdateOptions, gopts GlobalOptions, args []string) e
 		}
 	}
 
-	Verbosef("writing restic to %v\n", opts.Output)
+	Printf("writing restic to %v\n", opts.Output)
 
 	v, err := selfupdate.DownloadLatestStableRelease(gopts.ctx, opts.Output, version, Verbosef)
 	if err != nil {

cmd/restic/cmd_snapshots.go

@@ -36,8 +36,7 @@ type SnapshotOptions struct {
 	Tags    restic.TagLists
 	Paths   []string
 	Compact bool
-	Last    bool // This option should be removed in favour of Latest.
-	Latest  int
+	Last    bool
 	GroupBy string
 }
 
@@ -52,12 +51,6 @@ func init() {
 	f.StringArrayVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)")
 	f.BoolVarP(&snapshotOptions.Compact, "compact", "c", false, "use compact output format")
 	f.BoolVar(&snapshotOptions.Last, "last", false, "only show the last snapshot for each host and path")
-	err := f.MarkDeprecated("last", "use --latest 1")
-	if err != nil {
-		// MarkDeprecated only returns an error when the flag is not found
-		panic(err)
-	}
-	f.IntVar(&snapshotOptions.Latest, "latest", 0, "only show the last `n` snapshots for each host and path")
 	f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "string for grouping snapshots by host,paths,tags")
 }
 
@@ -89,11 +82,7 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 
 	for k, list := range snapshotGroups {
 		if opts.Last {
-			// This branch should be removed in the same time
-			// that --last.
-			list = FilterLastestSnapshots(list, 1)
-		} else if opts.Latest > 0 {
-			list = FilterLastestSnapshots(list, opts.Latest)
+			list = FilterLastSnapshots(list)
 		}
 		sort.Sort(sort.Reverse(list))
 		snapshotGroups[k] = list
@@ -136,22 +125,21 @@ func newFilterLastSnapshotsKey(sn *restic.Snapshot) filterLastSnapshotsKey {
 	return filterLastSnapshotsKey{sn.Hostname, strings.Join(paths, "|")}
 }
 
-// FilterLastestSnapshots filters a list of snapshots to only return
-// the limit last entries for each hostname and path. If the snapshot
-// contains multiple paths, they will be joined and treated as one
-// item.
-func FilterLastestSnapshots(list restic.Snapshots, limit int) restic.Snapshots {
+// FilterLastSnapshots filters a list of snapshots to only return the last
+// entry for each hostname and path. If the snapshot contains multiple paths,
+// they will be joined and treated as one item.
+func FilterLastSnapshots(list restic.Snapshots) restic.Snapshots {
 	// Sort the snapshots so that the newer ones are listed first
 	sort.SliceStable(list, func(i, j int) bool {
 		return list[i].Time.After(list[j].Time)
 	})
 
 	var results restic.Snapshots
-	seen := make(map[filterLastSnapshotsKey]int)
+	seen := make(map[filterLastSnapshotsKey]bool)
 	for _, sn := range list {
 		key := newFilterLastSnapshotsKey(sn)
-		if seen[key] < limit {
-			seen[key]++
+		if !seen[key] {
+			seen[key] = true
 			results = append(results, sn)
 		}
 	}

cmd/restic/delete.go

@@ -25,21 +25,16 @@ const numDeleteWorkers = 8
 func deleteFiles(gopts GlobalOptions, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
 	totalCount := len(fileList)
 	fileChan := make(chan restic.ID)
-	wg, ctx := errgroup.WithContext(gopts.ctx)
-	wg.Go(func() error {
-		defer close(fileChan)
+	go func() {
 		for id := range fileList {
-			select {
-			case fileChan <- id:
-			case <-ctx.Done():
-				return ctx.Err()
-			}
+			fileChan <- id
 		}
-		return nil
-	})
+		close(fileChan)
+	}()
 
 	bar := newProgressMax(!gopts.JSON && !gopts.Quiet, uint64(totalCount), "files deleted")
 	defer bar.Done()
+	wg, ctx := errgroup.WithContext(gopts.ctx)
 	for i := 0; i < numDeleteWorkers; i++ {
 		wg.Go(func() error {
 			for id := range fileChan {

cmd/restic/find.go

@@ -23,7 +23,7 @@ func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, hos
 			for _, s := range snapshotIDs {
 				if s == "latest" {
 					usedFilter = true
-					id, err = restic.FindLatestSnapshot(ctx, repo, paths, tags, hosts, nil)
+					id, err = restic.FindLatestSnapshot(ctx, repo, paths, tags, hosts)
 					if err != nil {
 						Warnf("Ignoring %q, no snapshot matched given filter (Paths:%v Tags:%v Hosts:%v)\n", s, paths, tags, hosts)
 						continue
@@ -31,7 +31,7 @@ func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, hos
 				} else {
 					id, err = restic.FindSnapshot(ctx, repo, s)
 					if err != nil {
-						Warnf("Ignoring %q: %v\n", s, err)
+						Warnf("Ignoring %q, it is not a snapshot id\n", s)
 						continue
 					}
 				}

cmd/restic/global.go

@@ -31,7 +31,6 @@ import (
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/textfile"
-	"github.com/restic/restic/internal/ui/termstatus"
 
 	"github.com/restic/restic/internal/errors"
 
@@ -40,7 +39,7 @@ import (
 	"golang.org/x/crypto/ssh/terminal"
 )
 
-var version = "0.13.0"
+var version = "0.12.0"
 
 // TimeFormat is the format used for all timestamps printed by restic.
 const TimeFormat = "2006-01-02 15:04:05"
@@ -61,7 +60,6 @@ type GlobalOptions struct {
 	CacheDir        string
 	NoCache         bool
 	CACerts         []string
-	InsecureTLS     bool
 	TLSClientCert   string
 	CleanupCache    bool
 
@@ -98,8 +96,6 @@ func init() {
 	var cancel context.CancelFunc
 	globalOptions.ctx, cancel = context.WithCancel(context.Background())
 	AddCleanupHandler(func() error {
-		// Must be called before the unlock cleanup handler to ensure that the latter is
-		// not blocked due to limited number of backend connections, see #1434
 		cancel()
 		return nil
 	})
@@ -118,13 +114,10 @@ func init() {
 	f.BoolVar(&globalOptions.NoCache, "no-cache", false, "do not use a local cache")
 	f.StringSliceVar(&globalOptions.CACerts, "cacert", nil, "`file` to load root certificates from (default: use system certificates)")
 	f.StringVar(&globalOptions.TLSClientCert, "tls-client-cert", "", "path to a `file` containing PEM encoded TLS client certificate and private key")
-	f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repo (insecure)")
 	f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories")
 	f.IntVar(&globalOptions.LimitUploadKb, "limit-upload", 0, "limits uploads to a maximum rate in KiB/s. (default: unlimited)")
 	f.IntVar(&globalOptions.LimitDownloadKb, "limit-download", 0, "limits downloads to a maximum rate in KiB/s. (default: unlimited)")
 	f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)")
-	// Use our "generate" command instead of the cobra provided "completion" command
-	cmdRoot.CompletionOptions.DisableDefaultCmd = true
 
 	restoreTerminal()
 }
@@ -149,13 +142,7 @@ func stdinIsTerminal() bool {
 }
 
 func stdoutIsTerminal() bool {
-	// mintty on windows can use pipes which behave like a posix terminal,
-	// but which are not a terminal handle
-	return terminal.IsTerminal(int(os.Stdout.Fd())) || stdoutCanUpdateStatus()
-}
-
-func stdoutCanUpdateStatus() bool {
-	return termstatus.CanUpdateStatus(os.Stdout.Fd())
+	return terminal.IsTerminal(int(os.Stdout.Fd()))
 }
 
 func stdoutTerminalWidth() int {
@@ -172,7 +159,7 @@ func stdoutTerminalWidth() int {
 // program execution must revert changes to the terminal configuration itself.
 // The terminal configuration is only restored while reading a password.
 func restoreTerminal() {
-	if !terminal.IsTerminal(int(os.Stdout.Fd())) {
+	if !stdoutIsTerminal() {
 		return
 	}
 
@@ -201,21 +188,16 @@ func restoreTerminal() {
 }
 
 // ClearLine creates a platform dependent string to clear the current
-// line, so it can be overwritten.
-//
-// w should be the terminal width, or 0 to let clearLine figure it out.
-func clearLine(w int) string {
-	if runtime.GOOS != "windows" {
-		return "\x1b[2K"
-	}
-
-	// ANSI sequences are not supported on Windows cmd shell.
-	if w <= 0 {
-		if w = stdoutTerminalWidth(); w <= 0 {
-			return ""
+// line, so it can be overwritten. ANSI sequences are not supported on
+// current windows cmd shell.
+func ClearLine() string {
+	if runtime.GOOS == "windows" {
+		if w := stdoutTerminalWidth(); w > 0 {
+			return strings.Repeat(" ", w-1) + "\r"
 		}
+		return ""
 	}
-	return strings.Repeat(" ", w-1) + "\r"
+	return "\x1b[2K"
 }
 
 // Printf writes the message to the configured stdout stream.
@@ -256,6 +238,31 @@ func Verboseff(format string, args ...interface{}) {
 	}
 }
 
+// PrintProgress wraps fmt.Printf to handle the difference in writing progress
+// information to terminals and non-terminal stdout
+func PrintProgress(format string, args ...interface{}) {
+	var (
+		message         string
+		carriageControl string
+	)
+	message = fmt.Sprintf(format, args...)
+
+	if !(strings.HasSuffix(message, "\r") || strings.HasSuffix(message, "\n")) {
+		if stdoutIsTerminal() {
+			carriageControl = "\r"
+		} else {
+			carriageControl = "\n"
+		}
+		message = fmt.Sprintf("%s%s", message, carriageControl)
+	}
+
+	if stdoutIsTerminal() {
+		message = fmt.Sprintf("%s%s", ClearLine(), message)
+	}
+
+	fmt.Print(message)
+}
+
 // Warnf writes the message to the configured stderr stream.
 func Warnf(format string, args ...interface{}) {
 	_, err := fmt.Fprintf(globalOptions.stderr, format, args...)
@@ -357,7 +364,7 @@ func ReadPassword(opts GlobalOptions, prompt string) (string, error) {
 	}
 
 	if len(password) == 0 {
-		return "", errors.Fatal("an empty password is not a password")
+		return "", errors.New("an empty password is not a password")
 	}
 
 	return password, nil
@@ -485,7 +492,7 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
 		return s, nil
 	}
 
-	if c.Created && !opts.JSON && stdoutIsTerminal() {
+	if c.Created && !opts.JSON {
 		Verbosef("created new cache in %v\n", c.Base)
 	}
 
@@ -504,9 +511,8 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
 
 	// cleanup old cache dirs if instructed to do so
 	if opts.CleanupCache {
-		if stdoutIsTerminal() && !opts.JSON {
-			Verbosef("removing %d old cache dirs from %v\n", len(oldCacheDirs), c.Base)
-		}
+		Printf("removing %d old cache dirs from %v\n", len(oldCacheDirs), c.Base)
+
 		for _, item := range oldCacheDirs {
 			dir := filepath.Join(c.Base, item.Name())
 			err = fs.RemoveAll(dir)
@@ -557,12 +563,6 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
 			cfg.Secret = os.Getenv("AWS_SECRET_ACCESS_KEY")
 		}
 
-		if cfg.KeyID == "" && cfg.Secret != "" {
-			return nil, errors.Fatalf("unable to open S3 backend: Key ID ($AWS_ACCESS_KEY_ID) is empty")
-		} else if cfg.KeyID != "" && cfg.Secret == "" {
-			return nil, errors.Fatalf("unable to open S3 backend: Secret ($AWS_SECRET_ACCESS_KEY) is empty")
-		}
-
 		if cfg.Region == "" {
 			cfg.Region = os.Getenv("AWS_DEFAULT_REGION")
 		}
@@ -682,7 +682,6 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
 	tropts := backend.TransportOptions{
 		RootCertFilenames:        globalOptions.CACerts,
 		TLSClientCertKeyFilename: globalOptions.TLSClientCert,
-		InsecureTLS:              globalOptions.InsecureTLS,
 	}
 	rt, err := backend.Transport(tropts)
 	if err != nil {
@@ -705,7 +704,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
 	case "azure":
 		be, err = azure.Open(cfg.(azure.Config), rt)
 	case "swift":
-		be, err = swift.Open(globalOptions.ctx, cfg.(swift.Config), rt)
+		be, err = swift.Open(cfg.(swift.Config), rt)
 	case "b2":
 		be, err = b2.Open(globalOptions.ctx, cfg.(b2.Config), rt)
 	case "rest":
@@ -763,7 +762,6 @@ func create(s string, opts options.Options) (restic.Backend, error) {
 	tropts := backend.TransportOptions{
 		RootCertFilenames:        globalOptions.CACerts,
 		TLSClientCertKeyFilename: globalOptions.TLSClientCert,
-		InsecureTLS:              globalOptions.InsecureTLS,
 	}
 	rt, err := backend.Transport(tropts)
 	if err != nil {
@@ -782,7 +780,7 @@ func create(s string, opts options.Options) (restic.Backend, error) {
 	case "azure":
 		return azure.Create(cfg.(azure.Config), rt)
 	case "swift":
-		return swift.Open(globalOptions.ctx, cfg.(swift.Config), rt)
+		return swift.Open(cfg.(swift.Config), rt)
 	case "b2":
 		return b2.Create(globalOptions.ctx, cfg.(b2.Config), rt)
 	case "rest":

cmd/restic/integration_helpers_test.go

@@ -67,7 +67,7 @@ func isSymlink(fi os.FileInfo) bool {
 
 func sameModTime(fi1, fi2 os.FileInfo) bool {
 	switch runtime.GOOS {
-	case "darwin", "freebsd", "openbsd", "netbsd", "solaris":
+	case "darwin", "freebsd", "openbsd", "netbsd":
 		if isSymlink(fi1) && isSymlink(fi2) {
 			return true
 		}

cmd/restic/integration_test.go

@@ -15,7 +15,6 @@ import (
 	"regexp"
 	"runtime"
 	"strings"
-	"sync"
 	"syscall"
 	"testing"
 	"time"
@@ -159,11 +158,8 @@ func testRunDiffOutput(gopts GlobalOptions, firstSnapshotID string, secondSnapsh
 	buf := bytes.NewBuffer(nil)
 
 	globalOptions.stdout = buf
-	oldStdout := gopts.stdout
-	gopts.stdout = buf
 	defer func() {
 		globalOptions.stdout = os.Stdout
-		gopts.stdout = oldStdout
 	}()
 
 	opts := DiffOptions{
@@ -349,57 +345,6 @@ func testBackup(t *testing.T, useFsSnapshot bool) {
 	testRunCheck(t, env.gopts)
 }
 
-func TestDryRunBackup(t *testing.T) {
-	env, cleanup := withTestEnvironment(t)
-	defer cleanup()
-
-	testSetupBackupData(t, env)
-	opts := BackupOptions{}
-	dryOpts := BackupOptions{DryRun: true}
-
-	// dry run before first backup
-	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
-	snapshotIDs := testRunList(t, "snapshots", env.gopts)
-	rtest.Assert(t, len(snapshotIDs) == 0,
-		"expected no snapshot, got %v", snapshotIDs)
-	packIDs := testRunList(t, "packs", env.gopts)
-	rtest.Assert(t, len(packIDs) == 0,
-		"expected no data, got %v", snapshotIDs)
-	indexIDs := testRunList(t, "index", env.gopts)
-	rtest.Assert(t, len(indexIDs) == 0,
-		"expected no index, got %v", snapshotIDs)
-
-	// first backup
-	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
-	snapshotIDs = testRunList(t, "snapshots", env.gopts)
-	packIDs = testRunList(t, "packs", env.gopts)
-	indexIDs = testRunList(t, "index", env.gopts)
-
-	// dry run between backups
-	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
-	snapshotIDsAfter := testRunList(t, "snapshots", env.gopts)
-	rtest.Equals(t, snapshotIDs, snapshotIDsAfter)
-	dataIDsAfter := testRunList(t, "packs", env.gopts)
-	rtest.Equals(t, packIDs, dataIDsAfter)
-	indexIDsAfter := testRunList(t, "index", env.gopts)
-	rtest.Equals(t, indexIDs, indexIDsAfter)
-
-	// second backup, implicit incremental
-	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
-	snapshotIDs = testRunList(t, "snapshots", env.gopts)
-	packIDs = testRunList(t, "packs", env.gopts)
-	indexIDs = testRunList(t, "index", env.gopts)
-
-	// another dry run
-	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
-	snapshotIDsAfter = testRunList(t, "snapshots", env.gopts)
-	rtest.Equals(t, snapshotIDs, snapshotIDsAfter)
-	dataIDsAfter = testRunList(t, "packs", env.gopts)
-	rtest.Equals(t, packIDs, dataIDsAfter)
-	indexIDsAfter = testRunList(t, "index", env.gopts)
-	rtest.Equals(t, indexIDs, indexIDsAfter)
-}
-
 func TestBackupNonExistingFile(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
@@ -429,11 +374,10 @@ func removePacksExcept(gopts GlobalOptions, t *testing.T, keep restic.IDSet, rem
 
 	// Get all tree packs
 	rtest.OK(t, r.LoadIndex(gopts.ctx))
-
 	treePacks := restic.NewIDSet()
-	for pb := range r.Index().Each(context.TODO()) {
-		if pb.Type == restic.TreeBlob {
-			treePacks.Insert(pb.PackID)
+	for _, idx := range r.Index().(*repository.MasterIndex).All() {
+		for _, id := range idx.TreePacks() {
+			treePacks.Insert(id)
 		}
 	}
 
@@ -492,10 +436,11 @@ func TestBackupTreeLoadError(t *testing.T) {
 	r, err := OpenRepository(env.gopts)
 	rtest.OK(t, err)
 	rtest.OK(t, r.LoadIndex(env.gopts.ctx))
-	treePacks := restic.NewIDSet()
-	for pb := range r.Index().Each(context.TODO()) {
-		if pb.Type == restic.TreeBlob {
-			treePacks.Insert(pb.PackID)
+	// collect tree packs of subdirectory
+	subTreePacks := restic.NewIDSet()
+	for _, idx := range r.Index().(*repository.MasterIndex).All() {
+		for _, id := range idx.TreePacks() {
+			subTreePacks.Insert(id)
 		}
 	}
 
@@ -503,7 +448,7 @@ func TestBackupTreeLoadError(t *testing.T) {
 	testRunCheck(t, env.gopts)
 
 	// delete the subdirectory pack first
-	for id := range treePacks {
+	for id := range subTreePacks {
 		rtest.OK(t, r.Backend().Remove(env.gopts.ctx, restic.Handle{Type: restic.PackFile, Name: id.String()}))
 	}
 	testRunRebuildIndex(t, env.gopts)
@@ -854,25 +799,6 @@ func TestCopyIncremental(t *testing.T) {
 		len(copiedSnapshotIDs), len(snapshotIDs))
 }
 
-func TestCopyUnstableJSON(t *testing.T) {
-	env, cleanup := withTestEnvironment(t)
-	defer cleanup()
-	env2, cleanup2 := withTestEnvironment(t)
-	defer cleanup2()
-
-	// contains a symlink created using `ln -s '../i/'$'\355\246\361''d/samba' broken-symlink`
-	datafile := filepath.Join("testdata", "copy-unstable-json.tar.gz")
-	rtest.SetupTarTestFixture(t, env.base, datafile)
-
-	testRunInit(t, env2.gopts)
-	testRunCopy(t, env.gopts, env2.gopts)
-	testRunCheck(t, env2.gopts)
-
-	copiedSnapshotIDs := testRunList(t, "snapshots", env2.gopts)
-	rtest.Assert(t, 1 == len(copiedSnapshotIDs), "still expected %v snapshot, found %v",
-		1, len(copiedSnapshotIDs))
-}
-
 func TestInitCopyChunkerParams(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
@@ -1088,41 +1014,6 @@ func TestKeyAddRemove(t *testing.T) {
 	testRunKeyAddNewKeyUserHost(t, env.gopts)
 }
 
-type emptySaveBackend struct {
-	restic.Backend
-}
-
-func (b *emptySaveBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
-	return b.Backend.Save(ctx, h, restic.NewByteReader([]byte{}, nil))
-}
-
-func TestKeyProblems(t *testing.T) {
-	env, cleanup := withTestEnvironment(t)
-	defer cleanup()
-
-	testRunInit(t, env.gopts)
-	env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
-		return &emptySaveBackend{r}, nil
-	}
-
-	testKeyNewPassword = "geheim2"
-	defer func() {
-		testKeyNewPassword = ""
-	}()
-
-	err := runKey(env.gopts, []string{"passwd"})
-	t.Log(err)
-	rtest.Assert(t, err != nil, "expected passwd change to fail")
-
-	err = runKey(env.gopts, []string{"add"})
-	t.Log(err)
-	rtest.Assert(t, err != nil, "expected key adding to fail")
-
-	t.Logf("testing access with initial password %q\n", env.gopts.password)
-	rtest.OK(t, runKey(env.gopts, []string{"list"}))
-	testRunCheck(t, env.gopts)
-}
-
 func testFileSize(filename string, size int64) error {
 	fi, err := os.Stat(filename)
 	if err != nil {
@@ -1420,7 +1311,7 @@ func TestFindJSON(t *testing.T) {
 	rtest.Assert(t, matches[0].Hits == 3, "expected hits to show 3 matches (%v)", datafile)
 }
 
-func testRebuildIndex(t *testing.T, backendTestHook backendWrapper) {
+func TestRebuildIndex(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
 
@@ -1440,10 +1331,8 @@ func testRebuildIndex(t *testing.T, backendTestHook backendWrapper) {
 		t.Fatalf("did not find hint for rebuild-index command")
 	}
 
-	env.gopts.backendTestHook = backendTestHook
 	testRunRebuildIndex(t, env.gopts)
 
-	env.gopts.backendTestHook = nil
 	out, err = testRunCheckOutput(env.gopts)
 	if len(out) != 0 {
 		t.Fatalf("expected no output from the checker, got: %v", out)
@@ -1454,57 +1343,9 @@ func testRebuildIndex(t *testing.T, backendTestHook backendWrapper) {
 	}
 }
 
-func TestRebuildIndex(t *testing.T) {
-	testRebuildIndex(t, nil)
-}
-
 func TestRebuildIndexAlwaysFull(t *testing.T) {
-	indexFull := repository.IndexFull
-	defer func() {
-		repository.IndexFull = indexFull
-	}()
 	repository.IndexFull = func(*repository.Index) bool { return true }
-	testRebuildIndex(t, nil)
-}
-
-// indexErrorBackend modifies the first index after reading.
-type indexErrorBackend struct {
-	restic.Backend
-	lock     sync.Mutex
-	hasErred bool
-}
-
-func (b *indexErrorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
-	return b.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error {
-		// protect hasErred
-		b.lock.Lock()
-		defer b.lock.Unlock()
-		if !b.hasErred && h.Type == restic.IndexFile {
-			b.hasErred = true
-			return consumer(errorReadCloser{rd})
-		}
-		return consumer(rd)
-	})
-}
-
-type errorReadCloser struct {
-	io.Reader
-}
-
-func (erd errorReadCloser) Read(p []byte) (int, error) {
-	n, err := erd.Reader.Read(p)
-	if n > 0 {
-		p[0] ^= 1
-	}
-	return n, err
-}
-
-func TestRebuildIndexDamage(t *testing.T) {
-	testRebuildIndex(t, func(r restic.Backend) (restic.Backend, error) {
-		return &indexErrorBackend{
-			Backend: r,
-		}, nil
-	})
+	TestRebuildIndex(t)
 }
 
 type appendOnlyBackend struct {
@@ -1747,7 +1588,7 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o
 
 // a listOnceBackend only allows listing once per filetype
 // listing filetypes more than once may cause problems with eventually consistent
-// backends (like e.g. Amazon S3) as the second listing may be inconsistent to what
+// backends (like e.g. AWS S3) as the second listing may be inconsistent to what
 // is expected by the first listing + some operations.
 type listOnceBackend struct {
 	restic.Backend
@@ -1975,8 +1816,10 @@ var diffOutputRegexPatterns = []string{
 	"Removed: +2[0-9]{2}\\.[0-9]{3} KiB",
 }
 
-func setupDiffRepo(t *testing.T) (*testEnvironment, func(), string, string) {
+func TestDiff(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
 	testRunInit(t, env.gopts)
 
 	datadir := filepath.Join(env.base, "testdata")
@@ -2012,82 +1855,19 @@ func setupDiffRepo(t *testing.T) (*testEnvironment, func(), string, string) {
 	testRunBackup(t, "", []string{datadir}, opts, env.gopts)
 	_, secondSnapshotID := lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts))
 
-	return env, cleanup, firstSnapshotID, secondSnapshotID
-}
-
-func TestDiff(t *testing.T) {
-	env, cleanup, firstSnapshotID, secondSnapshotID := setupDiffRepo(t)
-	defer cleanup()
-
-	// quiet suppresses the diff output except for the summary
-	env.gopts.Quiet = false
 	_, err := testRunDiffOutput(env.gopts, "", secondSnapshotID)
 	rtest.Assert(t, err != nil, "expected error on invalid snapshot id")
 
 	out, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
-	rtest.OK(t, err)
+	if err != nil {
+		t.Fatalf("expected no error from diff for test repository, got %v", err)
+	}
 
 	for _, pattern := range diffOutputRegexPatterns {
 		r, err := regexp.Compile(pattern)
 		rtest.Assert(t, err == nil, "failed to compile regexp %v", pattern)
 		rtest.Assert(t, r.MatchString(out), "expected pattern %v in output, got\n%v", pattern, out)
 	}
-
-	// check quiet output
-	env.gopts.Quiet = true
-	outQuiet, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
-	rtest.OK(t, err)
-
-	rtest.Assert(t, len(outQuiet) < len(out), "expected shorter output on quiet mode %v vs. %v", len(outQuiet), len(out))
-}
-
-type typeSniffer struct {
-	MessageType string `json:"message_type"`
-}
-
-func TestDiffJSON(t *testing.T) {
-	env, cleanup, firstSnapshotID, secondSnapshotID := setupDiffRepo(t)
-	defer cleanup()
-
-	// quiet suppresses the diff output except for the summary
-	env.gopts.Quiet = false
-	env.gopts.JSON = true
-	out, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
-	rtest.OK(t, err)
-
-	var stat DiffStatsContainer
-	var changes int
-
-	scanner := bufio.NewScanner(strings.NewReader(out))
-	for scanner.Scan() {
-		line := scanner.Text()
-		var sniffer typeSniffer
-		rtest.OK(t, json.Unmarshal([]byte(line), &sniffer))
-		switch sniffer.MessageType {
-		case "change":
-			changes++
-		case "statistics":
-			rtest.OK(t, json.Unmarshal([]byte(line), &stat))
-		default:
-			t.Fatalf("unexpected message type %v", sniffer.MessageType)
-		}
-	}
-	rtest.Equals(t, 9, changes)
-	rtest.Assert(t, stat.Added.Files == 2 && stat.Added.Dirs == 3 && stat.Added.DataBlobs == 2 &&
-		stat.Removed.Files == 1 && stat.Removed.Dirs == 2 && stat.Removed.DataBlobs == 1 &&
-		stat.ChangedFiles == 1, "unexpected statistics")
-
-	// check quiet output
-	env.gopts.Quiet = true
-	outQuiet, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
-	rtest.OK(t, err)
-
-	stat = DiffStatsContainer{}
-	rtest.OK(t, json.Unmarshal([]byte(outQuiet), &stat))
-	rtest.Assert(t, stat.Added.Files == 2 && stat.Added.Dirs == 3 && stat.Added.DataBlobs == 2 &&
-		stat.Removed.Files == 1 && stat.Removed.Dirs == 2 && stat.Removed.DataBlobs == 1 &&
-		stat.ChangedFiles == 1, "unexpected statistics")
-	rtest.Assert(t, stat.SourceSnapshot == firstSnapshotID && stat.TargetSnapshot == secondSnapshotID, "unexpected snapshot ids")
 }
 
 type writeToOnly struct {

cmd/restic/lock.go

@@ -16,7 +16,6 @@ var globalLocks struct {
 	cancelRefresh chan struct{}
 	refreshWG     sync.WaitGroup
 	sync.Mutex
-	sync.Once
 }
 
 func lockRepo(ctx context.Context, repo *repository.Repository) (*restic.Lock, error) {
@@ -28,12 +27,6 @@ func lockRepoExclusive(ctx context.Context, repo *repository.Repository) (*resti
 }
 
 func lockRepository(ctx context.Context, repo *repository.Repository, exclusive bool) (*restic.Lock, error) {
-	// make sure that a repository is unlocked properly and after cancel() was
-	// called by the cleanup handler in global.go
-	globalLocks.Do(func() {
-		AddCleanupHandler(unlockAll)
-	})
-
 	lockFn := restic.NewLock
 	if exclusive {
 		lockFn = restic.NewExclusiveLock
@@ -135,3 +128,7 @@ func unlockAll() error {
 
 	return nil
 }
+
+func init() {
+	AddCleanupHandler(unlockAll)
+}

cmd/restic/progress.go

@@ -4,17 +4,15 @@ import (
 	"fmt"
 	"os"
 	"strconv"
-	"strings"
 	"time"
 
 	"github.com/restic/restic/internal/ui/progress"
-	"github.com/restic/restic/internal/ui/termstatus"
 )
 
 // calculateProgressInterval returns the interval configured via RESTIC_PROGRESS_FPS
 // or if unset returns an interval for 60fps on interactive terminals and 0 (=disabled)
-// for non-interactive terminals or when run using the --quiet flag
-func calculateProgressInterval(show bool, json bool) time.Duration {
+// for non-interactive terminals
+func calculateProgressInterval() time.Duration {
 	interval := time.Second / 60
 	fps, err := strconv.ParseFloat(os.Getenv("RESTIC_PROGRESS_FPS"), 64)
 	if err == nil && fps > 0 {
@@ -22,7 +20,7 @@ func calculateProgressInterval(show bool, json bool) time.Duration {
 			fps = 60
 		}
 		interval = time.Duration(float64(time.Second) / fps)
-	} else if !json && !stdoutCanUpdateStatus() || !show {
+	} else if !stdoutIsTerminal() {
 		interval = 0
 	}
 	return interval
@@ -33,8 +31,7 @@ func newProgressMax(show bool, max uint64, description string) *progress.Counter
 	if !show {
 		return nil
 	}
-	interval := calculateProgressInterval(show, false)
-	canUpdateStatus := stdoutCanUpdateStatus()
+	interval := calculateProgressInterval()
 
 	return progress.New(interval, max, func(v uint64, max uint64, d time.Duration, final bool) {
 		var status string
@@ -45,36 +42,13 @@ func newProgressMax(show bool, max uint64, description string) *progress.Counter
 				formatDuration(d), formatPercent(v, max), v, max, description)
 		}
 
-		printProgress(status, canUpdateStatus)
+		if w := stdoutTerminalWidth(); w > 0 {
+			status = shortenStatus(w, status)
+		}
+
+		PrintProgress("%s", status)
 		if final {
 			fmt.Print("\n")
 		}
 	})
 }
-
-func printProgress(status string, canUpdateStatus bool) {
-	w := stdoutTerminalWidth()
-	if w > 0 {
-		if w < 3 {
-			status = termstatus.Truncate(status, w)
-		} else {
-			status = termstatus.Truncate(status, w-3) + "..."
-		}
-	}
-
-	var carriageControl, clear string
-
-	if canUpdateStatus {
-		clear = clearLine(w)
-	}
-
-	if !(strings.HasSuffix(status, "\r") || strings.HasSuffix(status, "\n")) {
-		if canUpdateStatus {
-			carriageControl = "\r"
-		} else {
-			carriageControl = "\n"
-		}
-	}
-
-	_, _ = os.Stdout.Write([]byte(clear + status + carriageControl))
-}

cmd/restic/secondary_repo.go

@@ -9,7 +9,6 @@ import (
 
 type secondaryRepoOptions struct {
 	Repo            string
-	RepositoryFile  string
 	password        string
 	PasswordFile    string
 	PasswordCommand string
@@ -18,25 +17,18 @@ type secondaryRepoOptions struct {
 
 func initSecondaryRepoOptions(f *pflag.FlagSet, opts *secondaryRepoOptions, repoPrefix string, repoUsage string) {
 	f.StringVarP(&opts.Repo, "repo2", "", os.Getenv("RESTIC_REPOSITORY2"), repoPrefix+" `repository` "+repoUsage+" (default: $RESTIC_REPOSITORY2)")
-	f.StringVarP(&opts.RepositoryFile, "repository-file2", "", os.Getenv("RESTIC_REPOSITORY_FILE2"), "`file` from which to read the "+repoPrefix+" repository location "+repoUsage+" (default: $RESTIC_REPOSITORY_FILE2)")
 	f.StringVarP(&opts.PasswordFile, "password-file2", "", os.Getenv("RESTIC_PASSWORD_FILE2"), "`file` to read the "+repoPrefix+" repository password from (default: $RESTIC_PASSWORD_FILE2)")
 	f.StringVarP(&opts.KeyHint, "key-hint2", "", os.Getenv("RESTIC_KEY_HINT2"), "key ID of key to try decrypting the "+repoPrefix+" repository first (default: $RESTIC_KEY_HINT2)")
 	f.StringVarP(&opts.PasswordCommand, "password-command2", "", os.Getenv("RESTIC_PASSWORD_COMMAND2"), "shell `command` to obtain the "+repoPrefix+" repository password from (default: $RESTIC_PASSWORD_COMMAND2)")
 }
 
 func fillSecondaryGlobalOpts(opts secondaryRepoOptions, gopts GlobalOptions, repoPrefix string) (GlobalOptions, error) {
-	if opts.Repo == "" && opts.RepositoryFile == "" {
-		return GlobalOptions{}, errors.Fatal("Please specify a " + repoPrefix + " repository location (--repo2 or --repository-file2)")
+	if opts.Repo == "" {
+		return GlobalOptions{}, errors.Fatal("Please specify a " + repoPrefix + " repository location (--repo2)")
 	}
-
-	if opts.Repo != "" && opts.RepositoryFile != "" {
-		return GlobalOptions{}, errors.Fatal("Options --repo2 and --repository-file2 are mutually exclusive, please specify only one")
-	}
-
 	var err error
 	dstGopts := gopts
 	dstGopts.Repo = opts.Repo
-	dstGopts.RepositoryFile = opts.RepositoryFile
 	dstGopts.PasswordFile = opts.PasswordFile
 	dstGopts.PasswordCommand = opts.PasswordCommand
 	dstGopts.KeyHint = opts.KeyHint

cmd/restic/secondary_repo_test.go

@@ -1,132 +0,0 @@
-package main
-
-import (
-	"io/ioutil"
-	"path/filepath"
-	"testing"
-
-	rtest "github.com/restic/restic/internal/test"
-)
-
-//TestFillSecondaryGlobalOpts tests valid and invalid data on fillSecondaryGlobalOpts-function
-func TestFillSecondaryGlobalOpts(t *testing.T) {
-	//secondaryRepoTestCase defines a struct for test cases
-	type secondaryRepoTestCase struct {
-		Opts     secondaryRepoOptions
-		DstGOpts GlobalOptions
-	}
-
-	//validSecondaryRepoTestCases is a list with test cases that must pass
-	var validSecondaryRepoTestCases = []secondaryRepoTestCase{
-		{
-			// Test if Repo and Password are parsed correctly.
-			Opts: secondaryRepoOptions{
-				Repo:     "backupDst",
-				password: "secretDst",
-			},
-			DstGOpts: GlobalOptions{
-				Repo:     "backupDst",
-				password: "secretDst",
-			},
-		},
-		{
-			// Test if RepositoryFile and PasswordFile are parsed correctly.
-			Opts: secondaryRepoOptions{
-				RepositoryFile: "backupDst",
-				PasswordFile:   "passwordFileDst",
-			},
-			DstGOpts: GlobalOptions{
-				RepositoryFile: "backupDst",
-				password:       "secretDst",
-				PasswordFile:   "passwordFileDst",
-			},
-		},
-		{
-			// Test if RepositoryFile and PasswordCommand are parsed correctly.
-			Opts: secondaryRepoOptions{
-				RepositoryFile:  "backupDst",
-				PasswordCommand: "echo secretDst",
-			},
-			DstGOpts: GlobalOptions{
-				RepositoryFile:  "backupDst",
-				password:        "secretDst",
-				PasswordCommand: "echo secretDst",
-			},
-		},
-	}
-
-	//invalidSecondaryRepoTestCases is a list with test cases that must fail
-	var invalidSecondaryRepoTestCases = []secondaryRepoTestCase{
-		{
-			// Test must fail on no repo given.
-			Opts: secondaryRepoOptions{},
-		},
-		{
-			// Test must fail as Repo and RepositoryFile are both given
-			Opts: secondaryRepoOptions{
-				Repo:           "backupDst",
-				RepositoryFile: "backupDst",
-			},
-		},
-		{
-			// Test must fail as PasswordFile and PasswordCommand are both given
-			Opts: secondaryRepoOptions{
-				Repo:            "backupDst",
-				PasswordFile:    "passwordFileDst",
-				PasswordCommand: "notEmpty",
-			},
-		},
-		{
-			// Test must fail as PasswordFile does not exist
-			Opts: secondaryRepoOptions{
-				Repo:         "backupDst",
-				PasswordFile: "NonExistingFile",
-			},
-		},
-		{
-			// Test must fail as PasswordCommand does not exist
-			Opts: secondaryRepoOptions{
-				Repo:            "backupDst",
-				PasswordCommand: "notEmpty",
-			},
-		},
-		{
-			// Test must fail as no password is given.
-			Opts: secondaryRepoOptions{
-				Repo: "backupDst",
-			},
-		},
-	}
-
-	//gOpts defines the Global options used in the secondary repository tests
-	var gOpts = GlobalOptions{
-		Repo:           "backupSrc",
-		RepositoryFile: "backupSrc",
-		password:       "secretSrc",
-		PasswordFile:   "passwordFileSrc",
-	}
-
-	//Create temp dir to create password file.
-	dir, cleanup := rtest.TempDir(t)
-	defer cleanup()
-
-	cleanup = rtest.Chdir(t, dir)
-	defer cleanup()
-
-	//Create temporary password file
-	err := ioutil.WriteFile(filepath.Join(dir, "passwordFileDst"), []byte("secretDst"), 0666)
-	rtest.OK(t, err)
-
-	// Test all valid cases
-	for _, testCase := range validSecondaryRepoTestCases {
-		DstGOpts, err := fillSecondaryGlobalOpts(testCase.Opts, gOpts, "destination")
-		rtest.OK(t, err)
-		rtest.Equals(t, DstGOpts, testCase.DstGOpts)
-	}
-
-	// Test all invalid cases
-	for _, testCase := range invalidSecondaryRepoTestCases {
-		_, err := fillSecondaryGlobalOpts(testCase.Opts, gOpts, "destination")
-		rtest.Assert(t, err != nil, "Expected error, but function did not return an error")
-	}
-}