Add version for 0.12.1

Reword changelog entries

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,13 +1,7 @@
-
-
 <!--
 Thank you very much for contributing code or documentation to restic! Please
 fill out the following questions to make it easier for us to review your
 changes.
-
-You do not need to check all the boxes below all at once, feel free to take
-your time and add more commits. If you're done and ready for review, please
-check the last box.
 -->
 
 What does this PR change? What problem does it solve?
@@ -17,8 +11,8 @@ What does this PR change? What problem does it solve?
 Describe the changes and their purpose here, as detailed as needed.
 -->
 
-Was the change discussed in an issue or in the forum before?
-------------------------------------------------------------
+Was the change previously discussed in an issue or on the forum?
+----------------------------------------------------------------
 
 <!--
 Link issues and relevant forum posts here.
@@ -30,11 +24,17 @@ is closed automatically when this PR is merged.
 Checklist
 ---------
 
-- [ ] I have read the [Contribution Guidelines](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#providing-patches)
-- [ ] I have enabled [maintainer edits for this PR](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/allowing-changes-to-a-pull-request-branch-created-from-a-fork)
-- [ ] I have added tests for all changes in this PR
-- [ ] I have added documentation for the changes (in the manual)
-- [ ] There's a new file in `changelog/unreleased/` that describes the changes for our users (template [here](https://github.com/restic/restic/blob/master/changelog/TEMPLATE))
-- [ ] I have run `gofmt` on the code in all commits
-- [ ] All commit messages are formatted in the same style as [the other commits in the repo](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#git-commits)
-- [ ] I'm done, this Pull Request is ready for review
+<!--
+You do not need to check all the boxes below all at once. Feel free to take
+your time and add more commits. If you're done and ready for review, please
+check the last box. Enable a checkbox by replacing [ ] with [x].
+-->
+
+- [ ] I have read the [contribution guidelines](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#providing-patches).
+- [ ] I have [enabled maintainer edits](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/allowing-changes-to-a-pull-request-branch-created-from-a-fork).
+- [ ] I have added tests for all code changes.
+- [ ] I have added documentation for relevant changes (in the manual).
+- [ ] There's a new file in `changelog/unreleased/` that describes the changes for our users (see [template](https://github.com/restic/restic/blob/master/changelog/TEMPLATE)).
+- [ ] I have run `gofmt` on the code in all commits.
+- [ ] All commit messages are formatted in the same style as [the other commits in the repo](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#git-commits).
+- [ ] I'm done! This pull request is ready for review.

.github/workflows/tests.yml

@@ -0,0 +1,271 @@
+name: test
+on:
+  # run tests on push to master, but not when other branches are pushed to
+  push:
+    branches:
+      - master
+
+  # run tests for all pull requests
+  pull_request:
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        # list of jobs to run:
+        include:
+          - job_name: Windows
+            go: 1.16.x
+            os: windows-latest
+
+          - job_name: macOS
+            go: 1.16.x
+            os: macOS-latest
+            test_fuse: false
+
+          - job_name: Linux
+            go: 1.16.x
+            os: ubuntu-latest
+            test_cloud_backends: true
+            test_fuse: true
+            check_changelog: true
+
+          - job_name: Linux
+            go: 1.15.x
+            os: ubuntu-latest
+            test_fuse: true
+
+          - job_name: Linux
+            go: 1.14.x
+            os: ubuntu-latest
+            test_fuse: true
+
+          - job_name: Linux
+            go: 1.13.x
+            os: ubuntu-latest
+            test_fuse: true
+
+    name: ${{ matrix.job_name }} Go ${{ matrix.go }}
+    runs-on: ${{ matrix.os }}
+
+    env:
+      GOPROXY: https://proxy.golang.org
+
+    steps:
+      - name: Set up Go ${{ matrix.go }}
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go }}
+
+      - name: Get programs (Linux/macOS)
+        run: |
+          echo "build Go tools"
+          go get github.com/restic/rest-server/...
+
+          echo "install minio server"
+          mkdir $HOME/bin
+          if [ "$RUNNER_OS" == "macOS" ]; then
+            wget --no-verbose -O $HOME/bin/minio https://dl.minio.io/server/minio/release/darwin-amd64/minio
+          else
+            wget --no-verbose -O $HOME/bin/minio https://dl.minio.io/server/minio/release/linux-amd64/minio
+          fi
+          chmod 755 $HOME/bin/minio
+
+          echo "install rclone"
+          if [ "$RUNNER_OS" == "macOS" ]; then
+            wget --no-verbose -O rclone.zip https://downloads.rclone.org/rclone-current-osx-amd64.zip
+          else
+            wget --no-verbose -O rclone.zip https://downloads.rclone.org/rclone-current-linux-amd64.zip
+          fi
+          unzip rclone.zip
+          cp rclone*/rclone $HOME/bin
+          chmod 755 $HOME/bin/rclone
+          rm -rf rclone*
+
+          # add $HOME/bin to path ($GOBIN was already added to the path by setup-go@v2)
+          echo $HOME/bin >> $GITHUB_PATH
+        if: matrix.os == 'ubuntu-latest' || matrix.os == 'macOS-latest'
+
+      - name: Get programs (Windows)
+        shell: powershell
+        run: |
+          $ProgressPreference = 'SilentlyContinue'
+
+          echo "build Go tools"
+          go get github.com/restic/rest-server/...
+
+          echo "install minio server"
+          mkdir $Env:USERPROFILE/bin
+          Invoke-WebRequest https://dl.minio.io/server/minio/release/windows-amd64/minio.exe -OutFile $Env:USERPROFILE/bin/minio.exe
+
+          echo "install rclone"
+          Invoke-WebRequest https://downloads.rclone.org/rclone-current-windows-amd64.zip -OutFile rclone.zip
+
+          unzip rclone.zip
+          copy rclone*/rclone.exe $Env:USERPROFILE/bin
+
+          # add $USERPROFILE/bin to path ($GOBIN was already added to the path by setup-go@v2)
+          echo $Env:USERPROFILE\bin >> $Env:GITHUB_PATH
+
+          echo "install tar"
+          cd $env:USERPROFILE
+          mkdir tar
+          cd tar
+
+          # install exactly these versions of tar and the libraries, other combinations might not work!
+
+          Invoke-WebRequest https://github.com/restic/test-assets/raw/master/tar-1.13-1-bin.zip -OutFile tar.zip
+          unzip tar.zip
+          Invoke-WebRequest https://github.com/restic/test-assets/raw/master/libintl-0.11.5-2-bin.zip -OutFile libintl.zip
+          unzip libintl.zip
+          Invoke-WebRequest https://github.com/restic/test-assets/raw/master/libiconv-1.8-1-bin.zip -OutFile libiconv.zip
+          unzip libiconv.zip
+
+          # add $USERPROFILE/tar/bin to path
+          echo $Env:USERPROFILE\tar\bin >> $Env:GITHUB_PATH
+        if: matrix.os == 'windows-latest'
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Build with build.go
+        run: |
+          go run build.go
+
+      - name: Run local Tests
+        env:
+          RESTIC_TEST_FUSE: ${{ matrix.test_fuse }}
+        run: |
+          go test -cover ./...
+
+      - name: Test cloud backends
+        env:
+          RESTIC_TEST_S3_KEY: ${{ secrets.RESTIC_TEST_S3_KEY }}
+          RESTIC_TEST_S3_SECRET: ${{ secrets.RESTIC_TEST_S3_SECRET }}
+          RESTIC_TEST_S3_REPOSITORY: ${{ secrets.RESTIC_TEST_S3_REPOSITORY }}
+          RESTIC_TEST_AZURE_ACCOUNT_NAME: ${{ secrets.RESTIC_TEST_AZURE_ACCOUNT_NAME }}
+          RESTIC_TEST_AZURE_ACCOUNT_KEY: ${{ secrets.RESTIC_TEST_AZURE_ACCOUNT_KEY }}
+          RESTIC_TEST_AZURE_REPOSITORY: ${{ secrets.RESTIC_TEST_AZURE_REPOSITORY }}
+          RESTIC_TEST_B2_ACCOUNT_ID: ${{ secrets.RESTIC_TEST_B2_ACCOUNT_ID }}
+          RESTIC_TEST_B2_ACCOUNT_KEY: ${{ secrets.RESTIC_TEST_B2_ACCOUNT_KEY }}
+          RESTIC_TEST_B2_REPOSITORY: ${{ secrets.RESTIC_TEST_B2_REPOSITORY }}
+          RESTIC_TEST_GS_REPOSITORY: ${{ secrets.RESTIC_TEST_GS_REPOSITORY }}
+          RESTIC_TEST_GS_PROJECT_ID: ${{ secrets.RESTIC_TEST_GS_PROJECT_ID }}
+          GOOGLE_PROJECT_ID: ${{ secrets.RESTIC_TEST_GS_PROJECT_ID }}
+          RESTIC_TEST_GS_APPLICATION_CREDENTIALS_B64: ${{ secrets.RESTIC_TEST_GS_APPLICATION_CREDENTIALS_B64 }}
+          RESTIC_TEST_OS_AUTH_URL: ${{ secrets.RESTIC_TEST_OS_AUTH_URL }}
+          RESTIC_TEST_OS_TENANT_NAME: ${{ secrets.RESTIC_TEST_OS_TENANT_NAME }}
+          RESTIC_TEST_OS_USERNAME: ${{ secrets.RESTIC_TEST_OS_USERNAME }}
+          RESTIC_TEST_OS_PASSWORD: ${{ secrets.RESTIC_TEST_OS_PASSWORD }}
+          RESTIC_TEST_OS_REGION_NAME: ${{ secrets.RESTIC_TEST_OS_REGION_NAME }}
+          RESTIC_TEST_SWIFT: ${{ secrets.RESTIC_TEST_SWIFT }}
+          # fail if any of the following tests cannot be run
+          RESTIC_TEST_DISALLOW_SKIP: "restic/backend/rest.TestBackendREST,\
+            restic/backend/sftp.TestBackendSFTP,\
+            restic/backend/s3.TestBackendMinio,\
+            restic/backend/rclone.TestBackendRclone,\
+            restic/backend/s3.TestBackendS3,\
+            restic/backend/swift.TestBackendSwift,\
+            restic/backend/b2.TestBackendB2,\
+            restic/backend/gs.TestBackendGS,\
+            restic/backend/azure.TestBackendAzure"
+        run: |
+          # prepare credentials for Google Cloud Storage tests in a temp file
+          export GOOGLE_APPLICATION_CREDENTIALS=$(mktemp --tmpdir restic-gcs-auth-XXXXXXX)
+          echo $RESTIC_TEST_GS_APPLICATION_CREDENTIALS_B64 | base64 -d > $GOOGLE_APPLICATION_CREDENTIALS
+          go test -cover -parallel 4 ./internal/backend/...
+
+        # only run cloud backend tests for pull requests from and pushes to our
+        # own repo, otherwise the secrets are not available
+        if: (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && matrix.test_cloud_backends
+
+      - name: Check changelog files with calens
+        run: |
+          echo "install calens"
+          go get github.com/restic/calens
+
+          echo "check changelog files"
+          calens
+        if: matrix.check_changelog
+
+  cross_compile:
+    strategy:
+
+      # ATTENTION: the list of architectures must be in sync with helpers/build-release-binaries/main.go!
+      matrix:
+        # run cross-compile in two batches parallel so the overall tests run faster
+        targets:
+          - "linux/386 linux/amd64 linux/arm linux/arm64 linux/ppc64le linux/mips linux/mipsle linux/mips64 linux/mips64le linux/s390x \
+            openbsd/386 openbsd/amd64"
+
+          - "freebsd/386 freebsd/amd64 freebsd/arm \
+            aix/ppc64 \
+            darwin/amd64 darwin/arm64 \
+            netbsd/386 netbsd/amd64 \
+            windows/386 windows/amd64 \
+            solaris/amd64"
+
+    env:
+      go: 1.16.x
+      GOPROXY: https://proxy.golang.org
+
+    runs-on: ubuntu-latest
+
+    name: Cross Compile for ${{ matrix.targets }}
+
+    steps:
+      - name: Set up Go ${{ env.go }}
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.go }}
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Install gox
+        run: |
+          go get github.com/mitchellh/gox
+
+      - name: Cross-compile with gox for ${{ matrix.targets }}
+        env:
+          GOFLAGS: "-trimpath"
+          GOX_ARCHS: "${{ matrix.targets }}"
+        run: |
+          mkdir build-output
+          gox -parallel 2 -verbose -osarch "$GOX_ARCHS" -output "build-output/{{.Dir}}_{{.OS}}_{{.Arch}}" ./cmd/restic
+          gox -parallel 2 -verbose -osarch "$GOX_ARCHS" -tags debug -output "build-output/{{.Dir}}_{{.OS}}_{{.Arch}}_debug" ./cmd/restic
+
+  lint:
+    name: lint
+    runs-on: ubuntu-latest
+    env:
+      go: 1.16.x
+    steps:
+      - name: Set up Go ${{ env.go }}
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.go }}
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v2
+        with:
+          # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
+          version: v1.36
+          # Optional: show only new issues if it's a pull request. The default value is `false`.
+          only-new-issues: true
+          args: --verbose --timeout 5m
+          skip-go-installation: true
+
+        # only run golangci-lint for pull requests, otherwise ALL hints get
+        # reported. We need to slowly address all issues until we can enable
+        # linting the master branch :)
+        if: github.event_name == 'pull_request'
+
+      - name: Check go.mod/go.sum
+        run: |
+          echo "check if go.mod and go.sum are up to date"
+          go mod tidy
+          git diff --exit-code go.mod go.sum

.gitignore

@@ -1,2 +1,3 @@
 /restic
 /.vagrant
+/.vscode

.golangci.yml

@@ -0,0 +1,57 @@
+# This is the configuration for golangci-lint for the restic project.
+#
+# A sample config with all settings is here:
+# https://github.com/golangci/golangci-lint/blob/master/.golangci.example.yml
+
+linters:
+  # only enable the linters listed below
+  disable-all: true
+  enable:
+    # make sure all errors returned by functions are handled
+    - errcheck
+
+    # find unused code
+    - deadcode
+
+    # show how code can be simplified
+    - gosimple
+
+    # # make sure code is formatted
+    - gofmt
+
+    # examine code and report suspicious constructs, such as Printf calls whose
+    # arguments do not align with the format string
+    - govet
+
+    # make sure names and comments are used according to the conventions
+    - golint
+
+    # detect when assignments to existing variables are not used
+    - ineffassign
+
+    # run static analysis and find errors
+    - staticcheck
+
+    # find unused variables, functions, structs, types, etc.
+    - unused
+
+    # find unused struct fields
+    - structcheck
+
+    # find unused global variables
+    - varcheck
+
+    # parse and typecheck code
+    - typecheck
+
+issues:
+  # don't use the default exclude rules, this hides (among others) ignored
+  # errors from Close() calls
+  exclude-use-default: false
+
+  # list of things to not warn about
+  exclude:
+    # golint: do not warn about missing comments for exported stuff
+    - exported (function|method|var|type|const) `.*` should have comment or be unexported
+    # golint: ignore constants in all caps
+    - don't use ALL_CAPS in Go names; use CamelCase

.hound.yml

@@ -1,2 +0,0 @@
-go:
-  enabled: true

.travis.yml

@@ -1,58 +0,0 @@
-language: go
-sudo: false
-
-matrix:
-  include:
-    - os: linux
-      go: "1.13.x"
-      env: RESTIC_TEST_FUSE=0 RESTIC_TEST_CLOUD_BACKENDS=0
-      cache:
-        directories:
-          - $HOME/.cache/go-build
-          - $HOME/gopath/pkg/mod
-
-    - os: linux
-      go: "1.14.x"
-      env: RESTIC_TEST_FUSE=0 RESTIC_TEST_CLOUD_BACKENDS=0
-      cache:
-        directories:
-          - $HOME/.cache/go-build
-          - $HOME/gopath/pkg/mod
-
-    # only run fuse and cloud backends tests on Travis for the latest Go on Linux
-    - os: linux
-      go: "1.15.x"
-      sudo: true
-      cache:
-        directories:
-          - $HOME/.cache/go-build
-          - $HOME/gopath/pkg/mod
-
-    - os: osx
-      go: "1.15.x"
-      env: RESTIC_TEST_FUSE=0 RESTIC_TEST_CLOUD_BACKENDS=0
-      cache:
-        directories:
-          - $HOME/Library/Caches/go-build
-          - $HOME/gopath/pkg/mod
-
-branches:
-  only:
-    - master
-
-notifications:
-  irc:
-    channels:
-      - "chat.freenode.net#restic"
-    on_success: change
-    on_failure: change
-    skip_join: true
-
-install:
-  - go version
-  - export GOBIN="$GOPATH/bin"
-  - export PATH="$PATH:$GOBIN"
-  - go env
-
-script:
-  - go run run_integration_tests.go

CHANGELOG.md

@@ -1,3 +1,921 @@
+Changelog for restic 0.12.1 (2021-08-03)
+=======================================
+
+The following sections list the changes in restic 0.12.1 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #2742: Improve error handling for rclone and REST backend over HTTP2
+ * Fix #3111: Fix terminal output redirection for PowerShell
+ * Fix #3214: Treat an empty password as a fatal error for repository init
+ * Fix #3267: `copy` failed to copy snapshots in rare cases
+ * Fix #3184: `backup --quiet` no longer prints status information
+ * Fix #3296: Fix crash of `check --read-data-subset=x%` run for an empty repository
+ * Fix #3302: Fix `fdopendir: not a directory` error for local backend
+ * Fix #3334: Print `created new cache` message only on a terminal
+ * Fix #3380: Fix crash of `backup --exclude='**'`
+ * Fix #3305: Fix possibly missing backup summary of JSON output in case of error
+ * Fix #3439: Correctly handle download errors during `restore`
+ * Chg #3247: Empty files now have size of 0 in `ls --json` output
+ * Enh #2780: Add release binaries for s390x architecture on Linux
+ * Enh #3293: Add `--repository-file2` option to `init` and `copy` command
+ * Enh #3312: Add auto-completion support for fish
+ * Enh #3336: SFTP backend now checks for disk space
+ * Enh #3377: Add release binaries for Apple Silicon
+ * Enh #3414: Add `--keep-within-hourly` option to restic forget
+ * Enh #3456: Support filtering and specifying untagged snapshots
+ * Enh #3167: Allow specifying limit of `snapshots` list
+ * Enh #3426: Optimize read performance of mount command
+ * Enh #3427: `find --pack` fallback to index if data file is missing
+
+Details
+-------
+
+ * Bugfix #2742: Improve error handling for rclone and REST backend over HTTP2
+
+   When retrieving data from the rclone / REST backend while also using HTTP2 restic did not detect
+   when no data was returned at all. This could cause for example the `check` command to report the
+   following error:
+
+   Pack ID does not match, want [...], got e3b0c442
+
+   This has been fixed by correctly detecting and retrying the incomplete download.
+
+   https://github.com/restic/restic/issues/2742
+   https://github.com/restic/restic/pull/3453
+   https://forum.restic.net/t/http2-stream-closed-connection-reset-context-canceled/3743/10
+
+ * Bugfix #3111: Fix terminal output redirection for PowerShell
+
+   When redirecting the output of restic using PowerShell on Windows, the output contained
+   terminal escape characters. This has been fixed by properly detecting the terminal type.
+
+   In addition, the mintty terminal now shows progress output for the backup command.
+
+   https://github.com/restic/restic/issues/3111
+   https://github.com/restic/restic/pull/3325
+
+ * Bugfix #3214: Treat an empty password as a fatal error for repository init
+
+   When attempting to initialize a new repository, if an empty password was supplied, the
+   repository would be created but the init command would return an error with a stack trace. Now,
+   if an empty password is provided, it is treated as a fatal error, and no repository is created.
+
+   https://github.com/restic/restic/issues/3214
+   https://github.com/restic/restic/pull/3283
+
+ * Bugfix #3267: `copy` failed to copy snapshots in rare cases
+
+   The `copy` command could in rare cases fail with the error message `SaveTree(...) returned
+   unexpected id ...`. This has been fixed.
+
+   On Linux/BSDs, the error could be caused by backing up symlinks with non-UTF-8 target paths.
+   Note that, due to limitations in the repository format, these are not stored properly and
+   should be avoided if possible.
+
+   https://github.com/restic/restic/issues/3267
+   https://github.com/restic/restic/pull/3310
+
+ * Bugfix #3184: `backup --quiet` no longer prints status information
+
+   A regression in the latest restic version caused the output of `backup --quiet` to contain
+   large amounts of backup progress information when run using an interactive terminal. This is
+   fixed now.
+
+   A workaround for this bug is to run restic as follows: `restic backup --quiet [..] | cat -`.
+
+   https://github.com/restic/restic/issues/3184
+   https://github.com/restic/restic/pull/3186
+
+ * Bugfix #3296: Fix crash of `check --read-data-subset=x%` run for an empty repository
+
+   The command `restic check --read-data-subset=x%` crashed when run for an empty repository.
+   This has been fixed.
+
+   https://github.com/restic/restic/issues/3296
+   https://github.com/restic/restic/pull/3309
+
+ * Bugfix #3302: Fix `fdopendir: not a directory` error for local backend
+
+   The `check`, `list packs`, `prune` and `rebuild-index` commands failed for the local backend
+   when the `data` folder in the repository contained files. This has been fixed.
+
+   https://github.com/restic/restic/issues/3302
+   https://github.com/restic/restic/pull/3308
+
+ * Bugfix #3334: Print `created new cache` message only on a terminal
+
+   The message `created new cache` was printed even when the output wasn't a terminal. That broke
+   piping `restic dump` output to tar or zip if cache directory didn't exist. The message is now
+   only printed on a terminal.
+
+   https://github.com/restic/restic/issues/3334
+   https://github.com/restic/restic/pull/3343
+
+ * Bugfix #3380: Fix crash of `backup --exclude='**'`
+
+   The exclude filter `**`, which excludes all files, caused restic to crash. This has been
+   corrected.
+
+   https://github.com/restic/restic/issues/3380
+   https://github.com/restic/restic/pull/3393
+
+ * Bugfix #3305: Fix possibly missing backup summary of JSON output in case of error
+
+   When using `--json` output it happened from time to time that the summary output was missing in
+   case an error occurred. This has been fixed.
+
+   https://github.com/restic/restic/pull/3305
+
+ * Bugfix #3439: Correctly handle download errors during `restore`
+
+   Due to a regression in restic 0.12.0, the `restore` command in some cases did not retry download
+   errors and only printed a warning. This has been fixed by retrying incomplete data downloads.
+
+   https://github.com/restic/restic/issues/3439
+   https://github.com/restic/restic/pull/3449
+
+ * Change #3247: Empty files now have size of 0 in `ls --json` output
+
+   The `ls --json` command used to omit the sizes of empty files in its output. It now reports a size
+   of zero explicitly for regular files, while omitting the size field for all other types.
+
+   https://github.com/restic/restic/issues/3247
+   https://github.com/restic/restic/pull/3257
+
+ * Enhancement #2780: Add release binaries for s390x architecture on Linux
+
+   We've added release binaries for Linux using the s390x architecture.
+
+   https://github.com/restic/restic/issues/2780
+   https://github.com/restic/restic/pull/3452
+
+ * Enhancement #3293: Add `--repository-file2` option to `init` and `copy` command
+
+   The `init` and `copy` command can now be used with the `--repository-file2` option or the
+   `$RESTIC_REPOSITORY_FILE2` environment variable. These to options are in addition to the
+   `--repo2` flag and allow you to read the destination repository from a file.
+
+   Using both `--repository-file` and `--repo2` options resulted in an error for the `copy` or
+   `init` command. The handling of this combination of options has been fixed. A workaround for
+   this issue is to only use `--repo` or `-r` and `--repo2` for `init` or `copy`.
+
+   https://github.com/restic/restic/issues/3293
+   https://github.com/restic/restic/pull/3294
+
+ * Enhancement #3312: Add auto-completion support for fish
+
+   The `generate` command now supports fish auto completion.
+
+   https://github.com/restic/restic/pull/3312
+
+ * Enhancement #3336: SFTP backend now checks for disk space
+
+   Backing up over SFTP previously spewed multiple generic "failure" messages when the remote
+   disk was full. It now checks for disk space before writing a file and fails immediately with a "no
+   space left on device" message.
+
+   https://github.com/restic/restic/issues/3336
+   https://github.com/restic/restic/pull/3345
+
+ * Enhancement #3377: Add release binaries for Apple Silicon
+
+   We've added release binaries for macOS on Apple Silicon (M1).
+
+   https://github.com/restic/restic/issues/3377
+   https://github.com/restic/restic/pull/3394
+
+ * Enhancement #3414: Add `--keep-within-hourly` option to restic forget
+
+   The `forget` command allowed keeping a given number of hourly backups or to keep all backups
+   within a given interval, but it was not possible to specify keeping hourly backups within a
+   given interval.
+
+   The new `--keep-within-hourly` option now offers this functionality. Similar options for
+   daily/weekly/monthly/yearly are also implemented, the new options are:
+
+   --keep-within-hourly <1y2m3d4h> --keep-within-daily <1y2m3d4h> --keep-within-weekly
+   <1y2m3d4h> --keep-within-monthly <1y2m3d4h> --keep-within-yearly <1y2m3d4h>
+
+   https://github.com/restic/restic/issues/3414
+   https://github.com/restic/restic/pull/3416
+   https://forum.restic.net/t/forget-policy/4014/11
+
+ * Enhancement #3456: Support filtering and specifying untagged snapshots
+
+   It was previously not possible to specify an empty tag with the `--tag` and `--keep-tag`
+   options. This has now been fixed, such that `--tag ''` and `--keep-tag ''` now matches
+   snapshots without tags. This allows e.g. the `snapshots` and `forget` commands to only
+   operate on untagged snapshots.
+
+   https://github.com/restic/restic/issues/3456
+   https://github.com/restic/restic/pull/3457
+
+ * Enhancement #3167: Allow specifying limit of `snapshots` list
+
+   The `--last` option allowed limiting the output of the `snapshots` command to the latest
+   snapshot for each host. The new `--latest n` option allows limiting the output to the latest `n`
+   snapshots.
+
+   This change deprecates the option `--last` in favour of `--latest 1`.
+
+   https://github.com/restic/restic/pull/3167
+
+ * Enhancement #3426: Optimize read performance of mount command
+
+   Reading large files in a mounted repository may be up to five times faster. This improvement
+   primarily applies to repositories stored at a backend that can be accessed with low latency,
+   like e.g. the local backend.
+
+   https://github.com/restic/restic/pull/3426
+
+ * Enhancement #3427: `find --pack` fallback to index if data file is missing
+
+   When investigating a repository with missing data files, it might be useful to determine
+   affected snapshots before running `rebuild-index`. Previously, `find --pack pack-id`
+   returned no data as it required accessing the data file. Now, if the necessary data is still
+   available in the repository index, it gets retrieved from there.
+
+   The command now also supports looking up multiple pack files in a single `find` run.
+
+   https://github.com/restic/restic/pull/3427
+   https://forum.restic.net/t/missing-packs-not-found/2600
+
+
+Changelog for restic 0.12.0 (2021-02-14)
+=======================================
+
+The following sections list the changes in restic 0.12.0 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #1681: Make `mount` not create missing mount point directory
+ * Fix #1800: Ignore `no data available` filesystem error during backup
+ * Fix #2563: Report the correct owner of directories in FUSE mounts
+ * Fix #2688: Make `backup` and `tag` commands separate tags by comma
+ * Fix #2739: Make the `cat` command respect the `--no-lock` option
+ * Fix #3087: The `--use-fs-snapshot` option now works on windows/386
+ * Fix #3100: Do not require gs bucket permissions when running `init`
+ * Fix #3111: Correctly detect output redirection for `backup` command on Windows
+ * Fix #3151: Don't create invalid snapshots when `backup` is interrupted
+ * Fix #3166: Improve error handling in the `restore` command
+ * Fix #3232: Correct statistics for overlapping targets
+ * Fix #3014: Fix sporadic stream reset between rclone and restic
+ * Fix #3152: Do not hang until foregrounded when completed in background
+ * Fix #3249: Improve error handling in `gs` backend
+ * Chg #3095: Deleting files on Google Drive now moves them to the trash
+ * Enh #2186: Allow specifying percentage in `check --read-data-subset`
+ * Enh #2453: Report permanent/fatal backend errors earlier
+ * Enh #2528: Add Alibaba/Aliyun OSS support in the `s3` backend
+ * Enh #2706: Configurable progress reports for non-interactive terminals
+ * Enh #2944: Add `backup` options `--files-from-{verbatim,raw}`
+ * Enh #3083: Allow usage of deprecated S3 `ListObjects` API
+ * Enh #3147: Support additional environment variables for Swift authentication
+ * Enh #3191: Add release binaries for MIPS architectures
+ * Enh #909: Back up mountpoints as empty directories
+ * Enh #3250: Add several more error checks
+ * Enh #2718: Improve `prune` performance and make it more customizable
+ * Enh #2495: Add option to let `backup` trust mtime without checking ctime
+ * Enh #2941: Speed up the repacking step of the `prune` command
+ * Enh #3006: Speed up the `rebuild-index` command
+ * Enh #3048: Add more checks for index and pack files in the `check` command
+ * Enh #2433: Make the `dump` command support `zip` format
+ * Enh #3099: Reduce memory usage of `check` command
+ * Enh #3106: Parallelize scan of snapshot content in `copy` and `prune`
+ * Enh #3130: Parallelize reading of locks and snapshots
+ * Enh #3254: Enable HTTP/2 for backend connections
+
+Details
+-------
+
+ * Bugfix #1681: Make `mount` not create missing mount point directory
+
+   When specifying a non-existent directory as mount point for the `mount` command, restic used
+   to create the specified directory automatically.
+
+   This has now changed such that restic instead gives an error when the specified directory for
+   the mount point does not exist.
+
+   https://github.com/restic/restic/issues/1681
+   https://github.com/restic/restic/pull/3008
+
+ * Bugfix #1800: Ignore `no data available` filesystem error during backup
+
+   Restic was unable to backup files on some filesystems, for example certain configurations of
+   CIFS on Linux which return a `no data available` error when reading extended attributes. These
+   errors are now ignored.
+
+   https://github.com/restic/restic/issues/1800
+   https://github.com/restic/restic/pull/3034
+
+ * Bugfix #2563: Report the correct owner of directories in FUSE mounts
+
+   Restic 0.10.0 changed the FUSE mount to always report the current user as the owner of
+   directories within the FUSE mount, which is incorrect.
+
+   This is now changed back to reporting the correct owner of a directory.
+
+   https://github.com/restic/restic/issues/2563
+   https://github.com/restic/restic/pull/3141
+
+ * Bugfix #2688: Make `backup` and `tag` commands separate tags by comma
+
+   Running `restic backup --tag foo,bar` previously created snapshots with one single tag
+   containing a comma (`foo,bar`) instead of two tags (`foo`, `bar`).
+
+   Similarly, the `tag` command's `--set`, `--add` and `--remove` options would treat
+   `foo,bar` as one tag instead of two tags. This was inconsistent with other commands and often
+   unexpected when one intended `foo,bar` to mean two tags.
+
+   To be consistent in all commands, restic now interprets `foo,bar` to mean two separate tags
+   (`foo` and `bar`) instead of one tag (`foo,bar`) everywhere, including in the `backup` and
+   `tag` commands.
+
+   NOTE: This change might result in unexpected behavior in cases where you use the `forget`
+   command and filter on tags like `foo,bar`. Snapshots previously backed up with `--tag
+   foo,bar` will still not match that filter, but snapshots saved from now on will match that
+   filter.
+
+   To replace `foo,bar` tags with `foo` and `bar` tags in old snapshots, you can first generate a
+   list of the relevant snapshots using a command like:
+
+   Restic snapshots --json --quiet | jq '.[] | select(contains({tags: ["foo,bar"]})) | .id'
+
+   And then use `restic tag --set foo --set bar snapshotID [...]` to set the new tags. Please adjust
+   the commands to include real tag names and any additional tags, as well as the list of snapshots
+   to process.
+
+   https://github.com/restic/restic/issues/2688
+   https://github.com/restic/restic/pull/2690
+   https://github.com/restic/restic/pull/3197
+
+ * Bugfix #2739: Make the `cat` command respect the `--no-lock` option
+
+   The `cat` command would not respect the `--no-lock` flag. This is now fixed.
+
+   https://github.com/restic/restic/issues/2739
+
+ * Bugfix #3087: The `--use-fs-snapshot` option now works on windows/386
+
+   Restic failed to create VSS snapshots on windows/386 with the following error:
+
+   GetSnapshotProperties() failed: E_INVALIDARG (0x80070057)
+
+   This is now fixed.
+
+   https://github.com/restic/restic/issues/3087
+   https://github.com/restic/restic/pull/3090
+
+ * Bugfix #3100: Do not require gs bucket permissions when running `init`
+
+   Restic used to require bucket level permissions for the `gs` backend in order to initialize a
+   restic repository.
+
+   It now allows a `gs` service account to initialize a repository if the bucket does exist and the
+   service account has permissions to write/read to that bucket.
+
+   https://github.com/restic/restic/issues/3100
+
+ * Bugfix #3111: Correctly detect output redirection for `backup` command on Windows
+
+   On Windows, since restic 0.10.0 the `backup` command did not properly detect when the output
+   was redirected to a file. This caused restic to output terminal control characters. This has
+   been fixed by correcting the terminal detection.
+
+   https://github.com/restic/restic/issues/3111
+   https://github.com/restic/restic/pull/3150
+
+ * Bugfix #3151: Don't create invalid snapshots when `backup` is interrupted
+
+   When canceling a backup run at a certain moment it was possible that restic created a snapshot
+   with an invalid "null" tree. This caused `check` and other operations to fail. The `backup`
+   command now properly handles interruptions and never saves a snapshot when interrupted.
+
+   https://github.com/restic/restic/issues/3151
+   https://github.com/restic/restic/pull/3164
+
+ * Bugfix #3166: Improve error handling in the `restore` command
+
+   The `restore` command used to not print errors while downloading file contents from the
+   repository. It also incorrectly exited with a zero error code even when there were errors
+   during the restore process. This has all been fixed and `restore` now returns with a non-zero
+   exit code when there's an error.
+
+   https://github.com/restic/restic/issues/3166
+   https://github.com/restic/restic/pull/3207
+
+ * Bugfix #3232: Correct statistics for overlapping targets
+
+   A user reported that restic's statistics and progress information during backup was not
+   correctly calculated when the backup targets (files/dirs to save) overlap. For example,
+   consider a directory `foo` which contains (among others) a file `foo/bar`. When `restic
+   backup foo foo/bar` was run, restic counted the size of the file `foo/bar` twice, so the
+   completeness percentage as well as the number of files was wrong. This is now corrected.
+
+   https://github.com/restic/restic/issues/3232
+   https://github.com/restic/restic/pull/3243
+
+ * Bugfix #3014: Fix sporadic stream reset between rclone and restic
+
+   Sometimes when using restic with the `rclone` backend, an error message similar to the
+   following would be printed:
+
+   Didn't finish writing GET request (wrote 0/xxx): http2: stream closed
+
+   It was found that this was caused by restic closing the connection to rclone to soon when
+   downloading data. A workaround has been added which waits for the end of the download before
+   closing the connection.
+
+   https://github.com/rclone/rclone/issues/2598
+   https://github.com/restic/restic/pull/3014
+
+ * Bugfix #3152: Do not hang until foregrounded when completed in background
+
+   On Linux, when running in the background restic failed to stop the terminal output of the
+   `backup` command after it had completed. This caused restic to hang until moved to the
+   foreground. This has now been fixed.
+
+   https://github.com/restic/restic/pull/3152
+   https://forum.restic.net/t/restic-alpine-container-cron-hangs-epoll-pwait/3334
+
+ * Bugfix #3249: Improve error handling in `gs` backend
+
+   The `gs` backend did not notice when the last step of completing a file upload failed. Under rare
+   circumstances, this could cause missing files in the backup repository. This has now been
+   fixed.
+
+   https://github.com/restic/restic/pull/3249
+
+ * Change #3095: Deleting files on Google Drive now moves them to the trash
+
+   When deleting files on Google Drive via the `rclone` backend, restic used to bypass the trash
+   folder required that one used the `-o rclone.args` option to enable usage of the trash folder.
+   This ensured that deleted files in Google Drive were not kept indefinitely in the trash folder.
+   However, since Google Drive's trash retention policy changed to deleting trashed files after
+   30 days, this is no longer needed.
+
+   Restic now leaves it up to rclone and its configuration to use or not use the trash folder when
+   deleting files. The default is to use the trash folder, as of rclone 1.53.2. To re-enable the
+   restic 0.11 behavior, set the `RCLONE_DRIVE_USE_TRASH` environment variable or change the
+   rclone configuration. See the rclone documentation for more details.
+
+   https://github.com/restic/restic/issues/3095
+   https://github.com/restic/restic/pull/3102
+
+ * Enhancement #2186: Allow specifying percentage in `check --read-data-subset`
+
+   We've enhanced the `check` command's `--read-data-subset` option to also accept a
+   percentage (e.g. `2.5%` or `10%`). This will check the given percentage of pack files (which
+   are randomly selected on each run).
+
+   https://github.com/restic/restic/issues/2186
+   https://github.com/restic/restic/pull/3038
+
+ * Enhancement #2453: Report permanent/fatal backend errors earlier
+
+   When encountering errors in reading from or writing to storage backends, restic retries the
+   failing operation up to nine times (for a total of ten attempts). It used to retry all backend
+   operations, but now detects some permanent error conditions so that it can report fatal errors
+   earlier.
+
+   Permanent failures include local disks being full, SSH connections dropping and permission
+   errors.
+
+   https://github.com/restic/restic/issues/2453
+   https://github.com/restic/restic/issues/3180
+   https://github.com/restic/restic/pull/3170
+   https://github.com/restic/restic/pull/3181
+
+ * Enhancement #2528: Add Alibaba/Aliyun OSS support in the `s3` backend
+
+   A new extended option `s3.bucket-lookup` has been added to support Alibaba/Aliyun OSS in the
+   `s3` backend. The option can be set to one of the following values:
+
+   - `auto` - Existing behaviour - `dns` - Use DNS style bucket access - `path` - Use path style
+   bucket access
+
+   To make the `s3` backend work with Alibaba/Aliyun OSS you must set `s3.bucket-lookup` to `dns`
+   and set the `s3.region` parameter. For example:
+
+   Restic -o s3.bucket-lookup=dns -o s3.region=oss-eu-west-1 -r
+   s3:https://oss-eu-west-1.aliyuncs.com/bucketname init
+
+   Note that `s3.region` must be set, otherwise the MinIO SDK tries to look it up and it seems that
+   Alibaba doesn't support that properly.
+
+   https://github.com/restic/restic/issues/2528
+   https://github.com/restic/restic/pull/2535
+
+ * Enhancement #2706: Configurable progress reports for non-interactive terminals
+
+   The `backup`, `check` and `prune` commands never printed any progress reports on
+   non-interactive terminals. This behavior is now configurable using the
+   `RESTIC_PROGRESS_FPS` environment variable. Use for example a value of `1` for an update
+   every second, or `0.01666` for an update every minute.
+
+   The `backup` command now also prints the current progress when restic receives a `SIGUSR1`
+   signal.
+
+   Setting the `RESTIC_PROGRESS_FPS` environment variable or sending a `SIGUSR1` signal
+   prints a status report even when `--quiet` was specified.
+
+   https://github.com/restic/restic/issues/2706
+   https://github.com/restic/restic/issues/3194
+   https://github.com/restic/restic/pull/3199
+
+ * Enhancement #2944: Add `backup` options `--files-from-{verbatim,raw}`
+
+   The new `backup` options `--files-from-verbatim` and `--files-from-raw` read a list of
+   files to back up from a file. Unlike the existing `--files-from` option, these options do not
+   interpret the listed filenames as glob patterns; instead, whitespace in filenames is
+   preserved as-is and no pattern expansion is done. Please see the documentation for specifics.
+
+   These new options are highly recommended over `--files-from`, when using a script to generate
+   the list of files to back up.
+
+   https://github.com/restic/restic/issues/2944
+   https://github.com/restic/restic/issues/3013
+
+ * Enhancement #3083: Allow usage of deprecated S3 `ListObjects` API
+
+   Some S3 API implementations, e.g. Ceph before version 14.2.5, have a broken `ListObjectsV2`
+   implementation which causes problems for restic when using their API endpoints. When a broken
+   server implementation is used, restic prints errors similar to the following:
+
+   List() returned error: Truncated response should have continuation token set
+
+   As a temporary workaround, restic now allows using the older `ListObjects` endpoint by
+   setting the `s3.list-objects-v1` extended option, for instance:
+
+   Restic -o s3.list-objects-v1=true snapshots
+
+   Please note that this option may be removed in future versions of restic.
+
+   https://github.com/restic/restic/issues/3083
+   https://github.com/restic/restic/pull/3085
+
+ * Enhancement #3147: Support additional environment variables for Swift authentication
+
+   The `swift` backend now supports the following additional environment variables for passing
+   authentication details to restic: `OS_USER_ID`, `OS_USER_DOMAIN_ID`,
+   `OS_PROJECT_DOMAIN_ID` and `OS_TRUST_ID`
+
+   Depending on the `openrc` configuration file these might be required when the user and project
+   domains differ from one another.
+
+   https://github.com/restic/restic/issues/3147
+   https://github.com/restic/restic/pull/3158
+
+ * Enhancement #3191: Add release binaries for MIPS architectures
+
+   We've added a few new architectures for Linux to the release binaries: `mips`, `mipsle`,
+   `mips64`, and `mip64le`. MIPS is mostly used for low-end embedded systems.
+
+   https://github.com/restic/restic/issues/3191
+   https://github.com/restic/restic/pull/3208
+
+ * Enhancement #909: Back up mountpoints as empty directories
+
+   When the `--one-file-system` option is specified to `restic backup`, it ignores all file
+   systems mounted below one of the target directories. This means that when a snapshot is
+   restored, users needed to manually recreate the mountpoint directories.
+
+   Restic now backs up mountpoints as empty directories and therefore implements the same
+   approach as `tar`.
+
+   https://github.com/restic/restic/issues/909
+   https://github.com/restic/restic/pull/3119
+
+ * Enhancement #3250: Add several more error checks
+
+   We've added a lot more error checks in places where errors were previously ignored (as hinted by
+   the static analysis program `errcheck` via `golangci-lint`).
+
+   https://github.com/restic/restic/pull/3250
+
+ * Enhancement #2718: Improve `prune` performance and make it more customizable
+
+   The `prune` command is now much faster. This is especially the case for remote repositories or
+   repositories with not much data to remove. Also the memory usage of the `prune` command is now
+   reduced.
+
+   Restic used to rebuild the index from scratch after pruning. This could lead to missing packs in
+   the index in some cases for eventually consistent backends such as e.g. AWS S3. This behavior is
+   now changed and the index rebuilding uses the information already known by `prune`.
+
+   By default, the `prune` command no longer removes all unused data. This behavior can be
+   fine-tuned by new options, like the acceptable amount of unused space or the maximum size of
+   data to reorganize. For more details, please see
+   https://restic.readthedocs.io/en/stable/060_forget.html .
+
+   Moreover, `prune` now accepts the `--dry-run` option and also running `forget --dry-run
+   --prune` will show what `prune` would do.
+
+   This enhancement also fixes several open issues, e.g.: -
+   https://github.com/restic/restic/issues/1140 -
+   https://github.com/restic/restic/issues/1599 -
+   https://github.com/restic/restic/issues/1985 -
+   https://github.com/restic/restic/issues/2112 -
+   https://github.com/restic/restic/issues/2227 -
+   https://github.com/restic/restic/issues/2305
+
+   https://github.com/restic/restic/pull/2718
+   https://github.com/restic/restic/pull/2842
+
+ * Enhancement #2495: Add option to let `backup` trust mtime without checking ctime
+
+   The `backup` command used to require that both `ctime` and `mtime` of a file matched with a
+   previously backed up version to determine that the file was unchanged. In other words, if
+   either `ctime` or `mtime` of the file had changed, it would be considered changed and restic
+   would read the file's content again to back up the relevant (changed) parts of it.
+
+   The new option `--ignore-ctime` makes restic look at `mtime` only, such that `ctime` changes
+   for a file does not cause restic to read the file's contents again.
+
+   The check for both `ctime` and `mtime` was introduced in restic 0.9.6 to make backups more
+   reliable in the face of programs that reset `mtime` (some Unix archivers do that), but it turned
+   out to often be expensive because it made restic read file contents even if only the metadata
+   (owner, permissions) of a file had changed. The new `--ignore-ctime` option lets the user
+   restore the 0.9.5 behavior when needed. The existing `--ignore-inode` option already turned
+   off this behavior, but also removed a different check.
+
+   Please note that changes in files' metadata are still recorded, regardless of the command line
+   options provided to the backup command.
+
+   https://github.com/restic/restic/issues/2495
+   https://github.com/restic/restic/issues/2558
+   https://github.com/restic/restic/issues/2819
+   https://github.com/restic/restic/pull/2823
+
+ * Enhancement #2941: Speed up the repacking step of the `prune` command
+
+   The repack step of the `prune` command, which moves still used file parts into new pack files
+   such that the old ones can be garbage collected later on, now processes multiple pack files in
+   parallel. This is especially beneficial for high latency backends or when using a fast network
+   connection.
+
+   https://github.com/restic/restic/pull/2941
+
+ * Enhancement #3006: Speed up the `rebuild-index` command
+
+   We've optimized the `rebuild-index` command. Now, existing index entries are used to
+   minimize the number of pack files that must be read. This speeds up the index rebuild a lot.
+
+   Additionally, the option `--read-all-packs` has been added, implementing the previous
+   behavior.
+
+   https://github.com/restic/restic/pull/3006
+   https://github.com/restic/restic/issue/2547
+
+ * Enhancement #3048: Add more checks for index and pack files in the `check` command
+
+   The `check` command run with the `--read-data` or `--read-data-subset` options used to only
+   verify only the pack file content - it did not check if the blobs within the pack are correctly
+   contained in the index.
+
+   A check for the latter is now in place, which can print the following error:
+
+   Blob ID is not contained in index or position is incorrect
+
+   Another test is also added, which compares pack file sizes computed from the index and the pack
+   header with the actual file size. This test is able to detect truncated pack files.
+
+   If the index is not correct, it can be rebuilt by using the `rebuild-index` command.
+
+   Having added these tests, `restic check` is now able to detect non-existing blobs which are
+   wrongly referenced in the index. This situation could have lead to missing data.
+
+   https://github.com/restic/restic/pull/3048
+   https://github.com/restic/restic/pull/3082
+
+ * Enhancement #2433: Make the `dump` command support `zip` format
+
+   Previously, restic could dump the contents of a whole folder structure only in the `tar`
+   format. The `dump` command now has a new flag to change output format to `zip`. Just pass
+   `--archive zip` as an option to `restic dump`.
+
+   https://github.com/restic/restic/pull/2433
+   https://github.com/restic/restic/pull/3081
+
+ * Enhancement #3099: Reduce memory usage of `check` command
+
+   The `check` command now requires less memory if it is run without the `--check-unused` option.
+
+   https://github.com/restic/restic/pull/3099
+
+ * Enhancement #3106: Parallelize scan of snapshot content in `copy` and `prune`
+
+   The `copy` and `prune` commands used to traverse the directories of snapshots one by one to find
+   used data. This snapshot traversal is now parallized which can speed up this step several
+   times.
+
+   In addition the `check` command now reports how many snapshots have already been processed.
+
+   https://github.com/restic/restic/pull/3106
+
+ * Enhancement #3130: Parallelize reading of locks and snapshots
+
+   Restic used to read snapshots sequentially. For repositories containing many snapshots this
+   slowed down commands which have to read all snapshots.
+
+   Now the reading of snapshots is parallelized. This speeds up for example `prune`, `backup` and
+   other commands that search for snapshots with certain properties or which have to find the
+   `latest` snapshot.
+
+   The speed up also applies to locks stored in the backup repository.
+
+   https://github.com/restic/restic/pull/3130
+   https://github.com/restic/restic/pull/3174
+
+ * Enhancement #3254: Enable HTTP/2 for backend connections
+
+   Go's HTTP library usually automatically chooses between HTTP/1.x and HTTP/2 depending on
+   what the server supports. But for compatibility this mechanism is disabled if DialContext is
+   used (which is the case for restic). This change allows restic's HTTP client to negotiate
+   HTTP/2 if supported by the server.
+
+   https://github.com/restic/restic/pull/3254
+
+
+Changelog for restic 0.11.0 (2020-11-05)
+=======================================
+
+The following sections list the changes in restic 0.11.0 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #1212: Restore timestamps and permissions on intermediate directories
+ * Fix #1756: Mark repository files as read-only when using the local backend
+ * Fix #2241: Hide password in REST backend repository URLs
+ * Fix #2319: Correctly dump directories into tar files
+ * Fix #2491: Don't require `self-update --output` placeholder file
+ * Fix #2834: Fix rare cases of backup command hanging forever
+ * Fix #2938: Fix manpage formatting
+ * Fix #2942: Make --exclude-larger-than handle disappearing files
+ * Fix #2951: Restic generate, help and self-update no longer check passwords
+ * Fix #2979: Make snapshots --json output [] instead of null when no snapshots
+ * Enh #2969: Optimize check for unchanged files during backup
+ * Enh #340: Add support for Volume Shadow Copy Service (VSS) on Windows
+ * Enh #2849: Authenticate to Google Cloud Storage with access token
+ * Enh #1458: New option --repository-file
+ * Enh #2978: Warn if parent snapshot cannot be loaded during backup
+
+Details
+-------
+
+ * Bugfix #1212: Restore timestamps and permissions on intermediate directories
+
+   When using the `--include` option of the restore command, restic restored timestamps and
+   permissions only on directories selected by the include pattern. Intermediate directories,
+   which are necessary to restore files located in sub- directories, were created with default
+   permissions. We've fixed the restore command to restore timestamps and permissions for these
+   directories as well.
+
+   https://github.com/restic/restic/issues/1212
+   https://github.com/restic/restic/issues/1402
+   https://github.com/restic/restic/pull/2906
+
+ * Bugfix #1756: Mark repository files as read-only when using the local backend
+
+   Files stored in a local repository were marked as writeable on the filesystem for non-Windows
+   systems, which did not prevent accidental file modifications outside of restic. In addition,
+   the local backend did not work with certain filesystems and network mounts which do not permit
+   modifications of file permissions.
+
+   Restic now marks files stored in a local repository as read-only on the filesystem on
+   non-Windows systems. The error handling is improved to support more filesystems.
+
+   https://github.com/restic/restic/issues/1756
+   https://github.com/restic/restic/issues/2157
+   https://github.com/restic/restic/pull/2989
+
+ * Bugfix #2241: Hide password in REST backend repository URLs
+
+   When using a password in the REST backend repository URL, the password could in some cases be
+   included in the output from restic, e.g. when initializing a repo or during an error.
+
+   The password is now replaced with "***" where applicable.
+
+   https://github.com/restic/restic/issues/2241
+   https://github.com/restic/restic/pull/2658
+
+ * Bugfix #2319: Correctly dump directories into tar files
+
+   The dump command previously wrote directories in a tar file in a way which can cause
+   compatibility problems. This caused, for example, 7zip on Windows to not open tar files
+   containing directories. In addition it was not possible to dump directories with extended
+   attributes. These compatibility problems are now corrected.
+
+   In addition, a tar file now includes the name of the owner and group of a file.
+
+   https://github.com/restic/restic/issues/2319
+   https://github.com/restic/restic/pull/3039
+
+ * Bugfix #2491: Don't require `self-update --output` placeholder file
+
+   `restic self-update --output /path/to/new-restic` used to require that new-restic was an
+   existing file, to be overwritten. Now it's possible to download an updated restic binary to a
+   new path, without first having to create a placeholder file.
+
+   https://github.com/restic/restic/issues/2491
+   https://github.com/restic/restic/pull/2937
+
+ * Bugfix #2834: Fix rare cases of backup command hanging forever
+
+   We've fixed an issue with the backup progress reporting which could cause restic to hang
+   forever right before finishing a backup.
+
+   https://github.com/restic/restic/issues/2834
+   https://github.com/restic/restic/pull/2963
+
+ * Bugfix #2938: Fix manpage formatting
+
+   The manpage formatting in restic v0.10.0 was garbled, which is fixed now.
+
+   https://github.com/restic/restic/issues/2938
+   https://github.com/restic/restic/pull/2977
+
+ * Bugfix #2942: Make --exclude-larger-than handle disappearing files
+
+   There was a small bug in the backup command's --exclude-larger-than option where files that
+   disappeared between scanning and actually backing them up to the repository caused a panic.
+   This is now fixed.
+
+   https://github.com/restic/restic/issues/2942
+
+ * Bugfix #2951: Restic generate, help and self-update no longer check passwords
+
+   The commands `restic cache`, `generate`, `help` and `self-update` don't need passwords, but
+   they previously did run the RESTIC_PASSWORD_COMMAND (if set in the environment), prompting
+   users to authenticate for no reason. They now skip running the password command.
+
+   https://github.com/restic/restic/issues/2951
+   https://github.com/restic/restic/pull/2987
+
+ * Bugfix #2979: Make snapshots --json output [] instead of null when no snapshots
+
+   Restic previously output `null` instead of `[]` for the `--json snapshots` command, when
+   there were no snapshots in the repository. This caused some minor problems when parsing the
+   output, but is now fixed such that `[]` is output when the list of snapshots is empty.
+
+   https://github.com/restic/restic/issues/2979
+   https://github.com/restic/restic/pull/2984
+
+ * Enhancement #2969: Optimize check for unchanged files during backup
+
+   During a backup restic skips processing files which have not changed since the last backup run.
+   Previously this required opening each file once which can be slow on network filesystems. The
+   backup command now checks for file changes before opening a file. This considerably reduces
+   the time to create a backup on network filesystems.
+
+   https://github.com/restic/restic/issues/2969
+   https://github.com/restic/restic/pull/2970
+
+ * Enhancement #340: Add support for Volume Shadow Copy Service (VSS) on Windows
+
+   Volume Shadow Copy Service allows read access to files that are locked by another process using
+   an exclusive lock through a filesystem snapshot. Restic was unable to backup those files
+   before. This update enables backing up these files.
+
+   This needs to be enabled explicitely using the --use-fs-snapshot option of the backup
+   command.
+
+   https://github.com/restic/restic/issues/340
+   https://github.com/restic/restic/pull/2274
+
+ * Enhancement #2849: Authenticate to Google Cloud Storage with access token
+
+   When using the GCS backend, it is now possible to authenticate with OAuth2 access tokens
+   instead of a credentials file by setting the GOOGLE_ACCESS_TOKEN environment variable.
+
+   https://github.com/restic/restic/pull/2849
+
+ * Enhancement #1458: New option --repository-file
+
+   We've added a new command-line option --repository-file as an alternative to -r. This allows
+   to read the repository URL from a file in order to prevent certain types of information leaks,
+   especially for URLs containing credentials.
+
+   https://github.com/restic/restic/issues/1458
+   https://github.com/restic/restic/issues/2900
+   https://github.com/restic/restic/pull/2910
+
+ * Enhancement #2978: Warn if parent snapshot cannot be loaded during backup
+
+   During a backup restic uses the parent snapshot to check whether a file was changed and has to be
+   backed up again. For this check the backup has to read the directories contained in the old
+   snapshot. If a tree blob cannot be loaded, restic now warns about this problem with the backup
+   repository.
+
+   https://github.com/restic/restic/pull/2978
+
+
 Changelog for restic 0.10.0 (2020-09-19)
 =======================================
 

CONTRIBUTING.md

@@ -13,10 +13,10 @@ bug fixes are most welcome. However even "minor" details as fixing spelling
 errors, improving documentation or pointing out usability issues are a great
 help also.
 
-
 The restic project uses the GitHub infrastructure (see the
 [project page](https://github.com/restic/restic)) for all related discussions
-as well as the `#restic` channel on `irc.freenode.net`.
+as well as the [forum](https://forum.restic.net/) and the `#restic` channel
+on [irc.libera.chat](https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:6697/#restic).
 
 If you want to find an area that currently needs improving have a look at the
 open issues listed at the
@@ -25,7 +25,10 @@ for discussing enhancement to the restic tools.
 
 If you are unsure what to do, please have a look at the issues, especially
 those tagged
-[minor complexity](https://github.com/restic/restic/labels/minor%20complexity).
+[minor complexity](https://github.com/restic/restic/labels/help%3A%20minor%20complexity)
+or [good first issue](https://github.com/restic/restic/labels/help%3A%20good%20first%20issue).
+If you are already a bit experienced with the restic internals, take a look
+at the issues tagged as [help wanted](https://github.com/restic/restic/labels/help%3A%20wanted).
 
 
 Reporting Bugs
@@ -63,7 +66,7 @@ Development Environment
 The repository contains the code written for restic in the directories
 `cmd/` and `internal/`.
 
-Restic requires Go version 1.12 or later for compiling. Clone the repo (without
+Restic requires Go version 1.13 or later for compiling. Clone the repo (without
 having `$GOPATH` set) and `cd` into the directory:
 
     $ unset GOPATH
@@ -74,7 +77,7 @@ Then use the `go` tool to build restic:
 
     $ go build ./cmd/restic
     $ ./restic version
-    restic 0.9.6-dev (compiled manually) compiled with go1.14 on linux/amd64
+    restic 0.10.0-dev (compiled manually) compiled with go1.15.2 on linux/amd64
 
 You can run all tests with the following command:
 
@@ -137,6 +140,14 @@ Installing the script `fmt-check` from https://github.com/edsrzf/gofmt-git-hook
 locally as a pre-commit hook checks formatting before committing automatically,
 just copy this script to `.git/hooks/pre-commit`.
 
+The project is using the program
+[`golangci-lint`](https://github.com/golangci/golangci-lint) to run a list of
+linters and checkers. It will be run on the code when you submit a PR. In order
+to check your code beforehand, you can run `golangci-lint run` manually.
+Eventually, we will enable `golangci-lint` for the whole code base. For now,
+you can ignore warnings printed for lines you did not modify, those will be
+ignored by the CI.
+
 For each pull request, several different systems run the integration tests on
 Linux, macOS and Windows. We won't merge any code that does not pass all tests
 for all systems, so when a tests fails, try to find out what's wrong and fix

README.md

@@ -0,0 +1,113 @@
+[![Documentation](https://readthedocs.org/projects/restic/badge/?version=latest)](https://restic.readthedocs.io/en/latest/?badge=latest)
+[![Build Status](https://github.com/restic/restic/workflows/test/badge.svg)](https://github.com/restic/restic/actions?query=workflow%3Atest)
+[![Go Report Card](https://goreportcard.com/badge/github.com/restic/restic)](https://goreportcard.com/report/github.com/restic/restic)
+
+# Introduction
+
+restic is a backup program that is fast, efficient and secure. It supports the three major operating systems (Linux, macOS, Windows) and a few smaller ones (FreeBSD, OpenBSD).
+
+For detailed usage and installation instructions check out the [documentation](https://restic.readthedocs.io/en/latest).
+
+You can ask questions in our [Discourse forum](https://forum.restic.net).
+
+Quick start
+-----------
+
+Once you've [installed](https://restic.readthedocs.io/en/latest/020_installation.html) restic, start
+off with creating a repository for your backups:
+
+    $ restic init --repo /tmp/backup
+    enter password for new backend:
+    enter password again:
+    created restic backend 085b3c76b9 at /tmp/backup
+    Please note that knowledge of your password is required to access the repository.
+    Losing your password means that your data is irrecoverably lost.
+
+and add some data:
+
+    $ restic --repo /tmp/backup backup ~/work
+    enter password for repository:
+    scan [/home/user/work]
+    scanned 764 directories, 1816 files in 0:00
+    [0:29] 100.00%  54.732 MiB/s  1.582 GiB / 1.582 GiB  2580 / 2580 items  0 errors  ETA 0:00
+    duration: 0:29, 54.47MiB/s
+    snapshot 40dc1520 saved
+
+Next you can either use `restic restore` to restore files or use `restic
+mount` to mount the repository via fuse and browse the files from previous
+snapshots.
+
+For more options check out the [online documentation](https://restic.readthedocs.io/en/latest/).
+
+# Backends
+
+Saving a backup on the same machine is nice but not a real backup strategy.
+Therefore, restic supports the following backends for storing backups natively:
+
+- [Local directory](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#local)
+- [sftp server (via SSH)](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#sftp)
+- [HTTP REST server](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server) ([protocol](doc/100_references.rst#rest-backend), [rest-server](https://github.com/restic/rest-server))
+- [AWS S3](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) (either from Amazon or using the [Minio](https://minio.io) server)
+- [OpenStack Swift](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#openstack-swift)
+- [BackBlaze B2](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#backblaze-b2)
+- [Microsoft Azure Blob Storage](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#microsoft-azure-blob-storage)
+- [Google Cloud Storage](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#google-cloud-storage)
+- And many other services via the [rclone](https://rclone.org) [Backend](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#other-services-via-rclone)
+
+# Design Principles
+
+Restic is a program that does backups right and was designed with the
+following principles in mind:
+
+-  **Easy:** Doing backups should be a frictionless process, otherwise
+   you might be tempted to skip it. Restic should be easy to configure
+   and use, so that, in the event of a data loss, you can just restore
+   it. Likewise, restoring data should not be complicated.
+
+-  **Fast**: Backing up your data with restic should only be limited by
+   your network or hard disk bandwidth so that you can backup your files
+   every day. Nobody does backups if it takes too much time. Restoring
+   backups should only transfer data that is needed for the files that
+   are to be restored, so that this process is also fast.
+
+-  **Verifiable**: Much more important than backup is restore, so restic
+   enables you to easily verify that all data can be restored.
+
+-  **Secure**: Restic uses cryptography to guarantee confidentiality and
+   integrity of your data. The location the backup data is stored is
+   assumed not to be a trusted environment (e.g. a shared space where
+   others like system administrators are able to access your backups).
+   Restic is built to secure your data against such attackers.
+
+-  **Efficient**: With the growth of data, additional snapshots should
+   only take the storage of the actual increment. Even more, duplicate
+   data should be de-duplicated before it is actually written to the
+   storage back end to save precious backup space.
+
+# Reproducible Builds
+
+The binaries released with each restic version starting at 0.6.1 are
+[reproducible](https://reproducible-builds.org/), which means that you can
+reproduce a byte identical version from the source code for that
+release. Instructions on how to do that are contained in the
+[builder repository](https://github.com/restic/builder).
+
+News
+----
+
+You can follow the restic project on Twitter [@resticbackup](https://twitter.com/resticbackup) or by subscribing to
+the [project blog](https://restic.net/blog/).
+
+License
+-------
+
+Restic is licensed under [BSD 2-Clause License](https://opensource.org/licenses/BSD-2-Clause). You can find the
+complete text in [``LICENSE``](LICENSE).
+
+Sponsorship
+-----------
+
+Backend integration tests for Google Cloud Storage and Microsoft Azure Blob
+Storage are sponsored by [AppsCode](https://appscode.com)!
+
+[![Sponsored by AppsCode](https://cdn.appscode.com/images/logo/appscode/ac-logo-color.png)](https://appscode.com)

README.rst

@@ -1,135 +0,0 @@
-|Documentation| |Build Status| |Build status| |Report Card| |Say Thanks| |Reviewed by Hound|
-
-Introduction
-------------
-
-restic is a backup program that is fast, efficient and secure. It supports the three major operating systems (Linux, macOS, Windows) and a few smaller ones (FreeBSD, OpenBSD).
-
-For detailed usage and installation instructions check out the `documentation <https://restic.readthedocs.io/en/latest>`__.
-
-You can ask questions in our `Discourse forum <https://forum.restic.net>`__.
-
-Quick start
------------
-
-Once you've `installed
-<https://restic.readthedocs.io/en/latest/020_installation.html>`__ restic, start
-off with creating a repository for your backups:
-
-.. code-block:: console
-
-    $ restic init --repo /tmp/backup
-    enter password for new backend:
-    enter password again:
-    created restic backend 085b3c76b9 at /tmp/backup
-    Please note that knowledge of your password is required to access the repository.
-    Losing your password means that your data is irrecoverably lost.
-
-and add some data:
-
-.. code-block:: console
-
-    $ restic --repo /tmp/backup backup ~/work
-    enter password for repository:
-    scan [/home/user/work]
-    scanned 764 directories, 1816 files in 0:00
-    [0:29] 100.00%  54.732 MiB/s  1.582 GiB / 1.582 GiB  2580 / 2580 items  0 errors  ETA 0:00
-    duration: 0:29, 54.47MiB/s
-    snapshot 40dc1520 saved
-
-Next you can either use ``restic restore`` to restore files or use ``restic
-mount`` to mount the repository via fuse and browse the files from previous
-snapshots.
-
-For more options check out the `online documentation <https://restic.readthedocs.io/en/latest/>`__.
-
-Backends
---------
-
-Saving a backup on the same machine is nice but not a real backup strategy.
-Therefore, restic supports the following backends for storing backups natively:
-
-- `Local directory <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#local>`__
-- `sftp server (via SSH) <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#sftp>`__
-- `HTTP REST server <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server>`__ (`protocol <doc/100_references.rst#rest-backend>`__ `rest-server <https://github.com/restic/rest-server>`__)
-- `AWS S3 <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3>`__ (either from Amazon or using the `Minio <https://minio.io>`__ server)
-- `OpenStack Swift <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#openstack-swift>`__
-- `BackBlaze B2 <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#backblaze-b2>`__
-- `Microsoft Azure Blob Storage <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#microsoft-azure-blob-storage>`__
-- `Google Cloud Storage <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#google-cloud-storage>`__
-- And many other services via the `rclone <https://rclone.org>`__ `Backend <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#other-services-via-rclone>`__
-
-Design Principles
------------------
-
-Restic is a program that does backups right and was designed with the
-following principles in mind:
-
--  **Easy:** Doing backups should be a frictionless process, otherwise
-   you might be tempted to skip it. Restic should be easy to configure
-   and use, so that, in the event of a data loss, you can just restore
-   it. Likewise, restoring data should not be complicated.
-
--  **Fast**: Backing up your data with restic should only be limited by
-   your network or hard disk bandwidth so that you can backup your files
-   every day. Nobody does backups if it takes too much time. Restoring
-   backups should only transfer data that is needed for the files that
-   are to be restored, so that this process is also fast.
-
--  **Verifiable**: Much more important than backup is restore, so restic
-   enables you to easily verify that all data can be restored.
-
--  **Secure**: Restic uses cryptography to guarantee confidentiality and
-   integrity of your data. The location the backup data is stored is
-   assumed not to be a trusted environment (e.g. a shared space where
-   others like system administrators are able to access your backups).
-   Restic is built to secure your data against such attackers.
-
--  **Efficient**: With the growth of data, additional snapshots should
-   only take the storage of the actual increment. Even more, duplicate
-   data should be de-duplicated before it is actually written to the
-   storage back end to save precious backup space.
-
-Reproducible Builds
--------------------
-
-The binaries released with each restic version starting at 0.6.1 are
-`reproducible <https://reproducible-builds.org/>`__, which means that you can
-easily reproduce a byte identical version from the source code for that
-release. Instructions on how to do that are contained in the
-`builder repository <https://github.com/restic/builder>`__.
-
-News
-----
-
-You can follow the restic project on Twitter `@resticbackup <https://twitter.com/resticbackup>`__ or by subscribing to
-the `development blog <https://restic.net/blog/>`__.
-
-License
--------
-
-Restic is licensed under `BSD 2-Clause License <https://opensource.org/licenses/BSD-2-Clause>`__. You can find the
-complete text in ``LICENSE``.
-
-Sponsorship
------------
-
-Backend integration tests for Google Cloud Storage and Microsoft Azure Blob
-Storage are sponsored by `AppsCode <https://appscode.com>`__!
-
-|AppsCode|
-
-.. |Documentation| image:: https://readthedocs.org/projects/restic/badge/?version=latest
-   :target: https://restic.readthedocs.io/en/latest/?badge=latest
-.. |Build Status| image:: https://travis-ci.com/restic/restic.svg?branch=master
-   :target: https://travis-ci.com/restic/restic
-.. |Build status| image:: https://ci.appveyor.com/api/projects/status/nuy4lfbgfbytw92q/branch/master?svg=true
-   :target: https://ci.appveyor.com/project/fd0/restic/branch/master
-.. |Report Card| image:: https://goreportcard.com/badge/github.com/restic/restic
-   :target: https://goreportcard.com/report/github.com/restic/restic
-.. |Say Thanks| image:: https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg
-   :target: https://saythanks.io/to/restic
-.. |AppsCode| image:: https://cdn.appscode.com/images/logo/appscode/ac-logo-color.png
-   :target: https://appscode.com
-.. |Reviewed by Hound| image:: https://img.shields.io/badge/Reviewed_by-Hound-8E64B0.svg
-   :target: https://houndci.com

VERSION

@@ -1 +1 @@
-0.10.0
+0.12.1

appveyor.yml

@@ -1,32 +0,0 @@
-clone_folder: c:\restic
-
-environment:
-  GOPATH: c:\gopath
-
-branches:
-  only:
-    - master
-
-cache:
-  - '%LocalAppData%\go-build'
-
-init:
-  - ps: >-
-      $app = Get-WmiObject -Class Win32_Product -Filter "Vendor = 'http://golang.org'"
-
-      if ($app) {
-        $app.Uninstall()
-      }
-
-install:
-  - rmdir c:\go /s /q
-  - appveyor DownloadFile https://dl.google.com/go/go1.15.2.windows-amd64.msi
-  - msiexec /i go1.15.2.windows-amd64.msi /q
-  - go version
-  - go env
-  - appveyor DownloadFile http://sourceforge.netcologne.de/project/gnuwin32/tar/1.13-1/tar-1.13-1-bin.zip -FileName tar.zip
-  - 7z x tar.zip bin/tar.exe
-  - set PATH=bin/;%PATH%
-
-build_script:
-  - go run run_integration_tests.go

changelog/0.11.0_2020-11-05/issue-1212

@@ -0,0 +1,11 @@
+Bugfix: Restore timestamps and permissions on intermediate directories
+
+When using the `--include` option of the restore command, restic restored
+timestamps and permissions only on directories selected by the include pattern.
+Intermediate directories, which are necessary to restore files located in sub-
+directories, were created with default permissions. We've fixed the restore
+command to restore timestamps and permissions for these directories as well.
+
+https://github.com/restic/restic/issues/1212
+https://github.com/restic/restic/issues/1402
+https://github.com/restic/restic/pull/2906

changelog/0.11.0_2020-11-05/issue-1756

@@ -0,0 +1,15 @@
+Bugfix: Mark repository files as read-only when using the local backend
+
+Files stored in a local repository were marked as writeable on the
+filesystem for non-Windows systems, which did not prevent accidental file
+modifications outside of restic. In addition, the local backend did not work
+with certain filesystems and network mounts which do not permit modifications
+of file permissions.
+
+restic now marks files stored in a local repository as read-only on the
+filesystem on non-Windows systems. The error handling is improved to support
+more filesystems.
+
+https://github.com/restic/restic/issues/1756
+https://github.com/restic/restic/issues/2157
+https://github.com/restic/restic/pull/2989

changelog/0.11.0_2020-11-05/issue-2241

@@ -0,0 +1,10 @@
+Bugfix: Hide password in REST backend repository URLs
+
+When using a password in the REST backend repository URL,
+the password could in some cases be included in the output
+from restic, e.g. when initializing a repo or during an error.
+
+The password is now replaced with "***" where applicable.
+
+https://github.com/restic/restic/issues/2241
+https://github.com/restic/restic/pull/2658

changelog/0.11.0_2020-11-05/issue-2319

@@ -0,0 +1,12 @@
+Bugfix: Correctly dump directories into tar files
+
+The dump command previously wrote directories in a tar file in a way which
+can cause compatibility problems. This caused, for example, 7zip on Windows
+to not open tar files containing directories. In addition it was not possible
+to dump directories with extended attributes. These compatibility problems
+are now corrected.
+
+In addition, a tar file now includes the name of the owner and group of a file.
+
+https://github.com/restic/restic/issues/2319
+https://github.com/restic/restic/pull/3039

changelog/0.11.0_2020-11-05/issue-2491

@@ -0,0 +1,9 @@
+Bugfix: Don't require `self-update --output` placeholder file
+
+`restic self-update --output /path/to/new-restic` used to require that
+new-restic was an existing file, to be overwritten. Now it's possible
+to download an updated restic binary to a new path, without first
+having to create a placeholder file.
+
+https://github.com/restic/restic/issues/2491
+https://github.com/restic/restic/pull/2937

changelog/0.11.0_2020-11-05/issue-2834

@@ -0,0 +1,7 @@
+Bugfix: Fix rare cases of backup command hanging forever
+
+We've fixed an issue with the backup progress reporting which could cause
+restic to hang forever right before finishing a backup.
+
+https://github.com/restic/restic/issues/2834
+https://github.com/restic/restic/pull/2963

changelog/0.11.0_2020-11-05/issue-2938

@@ -0,0 +1,6 @@
+Bugfix: Fix manpage formatting
+
+The manpage formatting in restic v0.10.0 was garbled, which is fixed now.
+
+https://github.com/restic/restic/issues/2938
+https://github.com/restic/restic/pull/2977

changelog/0.11.0_2020-11-05/issue-2942

@@ -0,0 +1,7 @@
+Bugfix: Make --exclude-larger-than handle disappearing files
+
+There was a small bug in the backup command's --exclude-larger-than
+option where files that disappeared between scanning and actually
+backing them up to the repository caused a panic. This is now fixed.
+
+https://github.com/restic/restic/issues/2942

changelog/0.11.0_2020-11-05/issue-2951

@@ -0,0 +1,9 @@
+Bugfix: restic generate, help and self-update no longer check passwords
+
+The commands `restic cache`, `generate`, `help` and `self-update` don't need
+passwords, but they previously did run the RESTIC_PASSWORD_COMMAND (if set in
+the environment), prompting users to authenticate for no reason. They now skip
+running the password command.
+
+https://github.com/restic/restic/issues/2951
+https://github.com/restic/restic/pull/2987

changelog/0.11.0_2020-11-05/issue-2969

@@ -0,0 +1,9 @@
+Enhancement: Optimize check for unchanged files during backup
+
+During a backup restic skips processing files which have not changed since the last backup run.
+Previously this required opening each file once which can be slow on network filesystems. The
+backup command now checks for file changes before opening a file. This considerably reduces
+the time to create a backup on network filesystems.
+
+https://github.com/restic/restic/issues/2969
+https://github.com/restic/restic/pull/2970

changelog/0.11.0_2020-11-05/issue-2979

@@ -0,0 +1,9 @@
+Bugfix: Make snapshots --json output [] instead of null when no snapshots
+
+Restic previously output `null` instead of `[]` for the `--json snapshots`
+command, when there were no snapshots in the repository. This caused some
+minor problems when parsing the output, but is now fixed such that `[]` is
+output when the list of snapshots is empty.
+
+https://github.com/restic/restic/issues/2979
+https://github.com/restic/restic/pull/2984

changelog/0.11.0_2020-11-05/issue-340

@@ -0,0 +1,12 @@
+Enhancement: Add support for Volume Shadow Copy Service (VSS) on Windows
+
+Volume Shadow Copy Service allows read access to files that are locked by
+another process using an exclusive lock through a filesystem snapshot. Restic
+was unable to backup those files before. This update enables backing up these
+files.
+
+This needs to be enabled explicitely using the --use-fs-snapshot option of the
+backup command.
+
+https://github.com/restic/restic/issues/340
+https://github.com/restic/restic/pull/2274

changelog/0.11.0_2020-11-05/pull-2849

@@ -0,0 +1,7 @@
+Enhancement: Authenticate to Google Cloud Storage with access token
+
+When using the GCS backend, it is now possible to authenticate with OAuth2
+access tokens instead of a credentials file by setting the GOOGLE_ACCESS_TOKEN
+environment variable.
+
+https://github.com/restic/restic/pull/2849

changelog/0.11.0_2020-11-05/pull-2910

@@ -0,0 +1,10 @@
+Enhancement: New option --repository-file
+
+We've added a new command-line option --repository-file as an alternative
+to -r. This allows to read the repository URL from a file in order to
+prevent certain types of information leaks, especially for URLs containing
+credentials.
+
+https://github.com/restic/restic/issues/1458
+https://github.com/restic/restic/issues/2900
+https://github.com/restic/restic/pull/2910

changelog/0.11.0_2020-11-05/pull-2978

@@ -0,0 +1,8 @@
+Enhancement: Warn if parent snapshot cannot be loaded during backup
+
+During a backup restic uses the parent snapshot to check whether a file was
+changed and has to be backed up again. For this check the backup has to read
+the directories contained in the old snapshot. If a tree blob cannot be
+loaded, restic now warns about this problem with the backup repository.
+
+https://github.com/restic/restic/pull/2978

changelog/0.12.0_2021-02-14/issue-1681

@@ -0,0 +1,10 @@
+Bugfix: Make `mount` not create missing mount point directory
+
+When specifying a non-existent directory as mount point for the `mount`
+command, restic used to create the specified directory automatically.
+
+This has now changed such that restic instead gives an error when the
+specified directory for the mount point does not exist.
+
+https://github.com/restic/restic/issues/1681
+https://github.com/restic/restic/pull/3008

changelog/0.12.0_2021-02-14/issue-1800

@@ -0,0 +1,8 @@
+Bugfix: Ignore `no data available` filesystem error during backup
+
+Restic was unable to backup files on some filesystems, for example certain
+configurations of CIFS on Linux which return a `no data available` error
+when reading extended attributes. These errors are now ignored.
+
+https://github.com/restic/restic/issues/1800
+https://github.com/restic/restic/pull/3034

changelog/0.12.0_2021-02-14/issue-2186

@@ -0,0 +1,8 @@
+Enhancement: Allow specifying percentage in `check --read-data-subset`
+
+We've enhanced the `check` command's `--read-data-subset` option to also accept
+a percentage (e.g. `2.5%` or `10%`). This will check the given percentage of
+pack files (which are randomly selected on each run).
+
+https://github.com/restic/restic/issues/2186
+https://github.com/restic/restic/pull/3038

changelog/0.12.0_2021-02-14/issue-2453

@@ -0,0 +1,14 @@
+Enhancement: Report permanent/fatal backend errors earlier
+
+When encountering errors in reading from or writing to storage backends,
+restic retries the failing operation up to nine times (for a total of ten
+attempts). It used to retry all backend operations, but now detects some
+permanent error conditions so that it can report fatal errors earlier.
+
+Permanent failures include local disks being full, SSH connections
+dropping and permission errors.
+
+https://github.com/restic/restic/issues/2453
+https://github.com/restic/restic/pull/3170
+https://github.com/restic/restic/issues/3180
+https://github.com/restic/restic/pull/3181

changelog/0.12.0_2021-02-14/issue-2528

@@ -0,0 +1,21 @@
+Enhancement: Add Alibaba/Aliyun OSS support in the `s3` backend
+
+A new extended option `s3.bucket-lookup` has been added to support
+Alibaba/Aliyun OSS in the `s3` backend. The option can be set to one
+of the following values:
+
+- `auto` - Existing behaviour
+- `dns` - Use DNS style bucket access
+- `path` - Use path style bucket access
+
+To make the `s3` backend work with Alibaba/Aliyun OSS you must set
+`s3.bucket-lookup` to `dns` and set the `s3.region` parameter. For
+example:
+
+    restic -o s3.bucket-lookup=dns -o s3.region=oss-eu-west-1 -r s3:https://oss-eu-west-1.aliyuncs.com/bucketname init
+
+Note that `s3.region` must be set, otherwise the MinIO SDK tries to
+look it up and it seems that Alibaba doesn't support that properly.
+
+https://github.com/restic/restic/issues/2528
+https://github.com/restic/restic/pull/2535

changelog/0.12.0_2021-02-14/issue-2563

@@ -0,0 +1,9 @@
+Bugfix: Report the correct owner of directories in FUSE mounts
+
+Restic 0.10.0 changed the FUSE mount to always report the current user
+as the owner of directories within the FUSE mount, which is incorrect.
+
+This is now changed back to reporting the correct owner of a directory.
+
+https://github.com/restic/restic/issues/2563
+https://github.com/restic/restic/pull/3141

changelog/0.12.0_2021-02-14/issue-2688

@@ -0,0 +1,31 @@
+Bugfix: Make `backup` and `tag` commands separate tags by comma
+
+Running `restic backup --tag foo,bar` previously created snapshots with one
+single tag containing a comma (`foo,bar`) instead of two tags (`foo`, `bar`).
+
+Similarly, the `tag` command's `--set`, `--add` and `--remove` options would
+treat `foo,bar` as one tag instead of two tags. This was inconsistent with
+other commands and often unexpected when one intended `foo,bar` to mean two
+tags.
+
+To be consistent in all commands, restic now interprets `foo,bar` to mean two
+separate tags (`foo` and `bar`) instead of one tag (`foo,bar`) everywhere,
+including in the `backup` and `tag` commands.
+
+NOTE: This change might result in unexpected behavior in cases where you use
+the `forget` command and filter on tags like `foo,bar`. Snapshots previously
+backed up with `--tag foo,bar` will still not match that filter, but snapshots
+saved from now on will match that filter.
+
+To replace `foo,bar` tags with `foo` and `bar` tags in old snapshots, you can
+first generate a list of the relevant snapshots using a command like:
+
+    restic snapshots --json --quiet | jq '.[] | select(contains({tags: ["foo,bar"]})) | .id'
+
+and then use `restic tag --set foo --set bar snapshotID [...]` to set the new
+tags. Please adjust the commands to include real tag names and any additional
+tags, as well as the list of snapshots to process.
+
+https://github.com/restic/restic/issues/2688
+https://github.com/restic/restic/pull/2690
+https://github.com/restic/restic/pull/3197

changelog/0.12.0_2021-02-14/issue-2706

@@ -0,0 +1,17 @@
+Enhancement: Configurable progress reports for non-interactive terminals
+
+The `backup`, `check` and `prune` commands never printed any progress
+reports on non-interactive terminals. This behavior is now configurable
+using the `RESTIC_PROGRESS_FPS` environment variable. Use for example a
+value of `1` for an update every second, or `0.01666` for an update every
+minute.
+
+The `backup` command now also prints the current progress when restic
+receives a `SIGUSR1` signal.
+
+Setting the `RESTIC_PROGRESS_FPS` environment variable or sending a `SIGUSR1`
+signal prints a status report even when `--quiet` was specified.
+
+https://github.com/restic/restic/issues/2706
+https://github.com/restic/restic/issues/3194
+https://github.com/restic/restic/pull/3199

changelog/0.12.0_2021-02-14/issue-2739

@@ -0,0 +1,5 @@
+Bugfix: Make the `cat` command respect the `--no-lock` option
+
+The `cat` command would not respect the `--no-lock` flag. This is now fixed.
+
+https://github.com/restic/restic/issues/2739

changelog/0.12.0_2021-02-14/issue-2944

@@ -0,0 +1,13 @@
+Enhancement: Add `backup` options `--files-from-{verbatim,raw}`
+
+The new `backup` options `--files-from-verbatim` and `--files-from-raw` read a
+list of files to back up from a file. Unlike the existing `--files-from`
+option, these options do not interpret the listed filenames as glob patterns;
+instead, whitespace in filenames is preserved as-is and no pattern expansion is
+done. Please see the documentation for specifics.
+
+These new options are highly recommended over `--files-from`, when using a
+script to generate the list of files to back up.
+
+https://github.com/restic/restic/issues/2944
+https://github.com/restic/restic/issues/3013

changelog/0.12.0_2021-02-14/issue-3083

@@ -0,0 +1,18 @@
+Enhancement: Allow usage of deprecated S3 `ListObjects` API
+
+Some S3 API implementations, e.g. Ceph before version 14.2.5, have a broken
+`ListObjectsV2` implementation which causes problems for restic when using
+their API endpoints. When a broken server implementation is used, restic prints
+errors similar to the following:
+
+    List() returned error: Truncated response should have continuation token set
+
+As a temporary workaround, restic now allows using the older `ListObjects`
+endpoint by setting the `s3.list-objects-v1` extended option, for instance:
+
+    restic -o s3.list-objects-v1=true snapshots
+
+Please note that this option may be removed in future versions of restic.
+
+https://github.com/restic/restic/issues/3083
+https://github.com/restic/restic/pull/3085

changelog/0.12.0_2021-02-14/issue-3090

@@ -0,0 +1,10 @@
+Bugfix: The `--use-fs-snapshot` option now works on windows/386
+
+Restic failed to create VSS snapshots on windows/386 with the following error:
+
+    GetSnapshotProperties() failed: E_INVALIDARG (0x80070057)
+
+This is now fixed.
+
+https://github.com/restic/restic/issues/3087
+https://github.com/restic/restic/pull/3090

changelog/0.12.0_2021-02-14/issue-3095

@@ -0,0 +1,17 @@
+Change: Deleting files on Google Drive now moves them to the trash
+
+When deleting files on Google Drive via the `rclone` backend, restic used to
+bypass the trash folder required that one used the `-o rclone.args` option to
+enable usage of the trash folder. This ensured that deleted files in Google
+Drive were not kept indefinitely in the trash folder. However, since Google
+Drive's trash retention policy changed to deleting trashed files after 30 days,
+this is no longer needed.
+
+Restic now leaves it up to rclone and its configuration to use or not use the
+trash folder when deleting files. The default is to use the trash folder, as
+of rclone 1.53.2. To re-enable the restic 0.11 behavior, set the
+`RCLONE_DRIVE_USE_TRASH` environment variable or change the rclone
+configuration. See the rclone documentation for more details.
+
+https://github.com/restic/restic/issues/3095
+https://github.com/restic/restic/pull/3102

changelog/0.12.0_2021-02-14/issue-3100

@@ -0,0 +1,10 @@
+Bugfix: Do not require gs bucket permissions when running `init`
+
+Restic used to require bucket level permissions for the `gs` backend 
+in order to initialize a restic repository.
+
+It now allows a `gs` service account to initialize a repository if the
+bucket does exist and the service account has permissions to write/read
+to that bucket.
+
+https://github.com/restic/restic/issues/3100

changelog/0.12.0_2021-02-14/issue-3111

@@ -0,0 +1,9 @@
+Bugfix: Correctly detect output redirection for `backup` command on Windows
+
+On Windows, since restic 0.10.0 the `backup` command did not properly detect
+when the output was redirected to a file. This caused restic to output
+terminal control characters. This has been fixed by correcting the terminal
+detection.
+
+https://github.com/restic/restic/issues/3111
+https://github.com/restic/restic/pull/3150

changelog/0.12.0_2021-02-14/issue-3147

@@ -0,0 +1,11 @@
+Enhancement: Support additional environment variables for Swift authentication
+
+The `swift` backend now supports the following additional environment variables
+for passing authentication details to restic:
+`OS_USER_ID`, `OS_USER_DOMAIN_ID`, `OS_PROJECT_DOMAIN_ID` and `OS_TRUST_ID`
+
+Depending on the `openrc` configuration file these might be required when the
+user and project domains differ from one another.
+
+https://github.com/restic/restic/issues/3147
+https://github.com/restic/restic/pull/3158

changelog/0.12.0_2021-02-14/issue-3151

@@ -0,0 +1,9 @@
+Bugfix: Don't create invalid snapshots when `backup` is interrupted
+
+When canceling a backup run at a certain moment it was possible that
+restic created a snapshot with an invalid "null" tree. This caused
+`check` and other operations to fail. The `backup` command now properly
+handles interruptions and never saves a snapshot when interrupted.
+
+https://github.com/restic/restic/issues/3151
+https://github.com/restic/restic/pull/3164

changelog/0.12.0_2021-02-14/issue-3166

@@ -0,0 +1,9 @@
+Bugfix: Improve error handling in the `restore` command
+
+The `restore` command used to not print errors while downloading file contents
+from the repository. It also incorrectly exited with a zero error code even
+when there were errors during the restore process. This has all been fixed and
+`restore` now returns with a non-zero exit code when there's an error.
+
+https://github.com/restic/restic/issues/3166
+https://github.com/restic/restic/pull/3207

changelog/0.12.0_2021-02-14/issue-3191

@@ -0,0 +1,8 @@
+Enhancement: Add release binaries for MIPS architectures
+
+We've added a few new architectures for Linux to the release binaries: `mips`,
+`mipsle`, `mips64`, and `mip64le`. MIPS is mostly used for low-end embedded
+systems.
+
+https://github.com/restic/restic/issues/3191
+https://github.com/restic/restic/pull/3208

changelog/0.12.0_2021-02-14/issue-3232

@@ -0,0 +1,11 @@
+Bugfix: Correct statistics for overlapping targets
+
+A user reported that restic's statistics and progress information during backup
+was not correctly calculated when the backup targets (files/dirs to save)
+overlap. For example, consider a directory `foo` which contains (among others)
+a file `foo/bar`. When `restic backup foo foo/bar` was run, restic counted the
+size of the file `foo/bar` twice, so the completeness percentage as well as the
+number of files was wrong. This is now corrected.
+
+https://github.com/restic/restic/issues/3232
+https://github.com/restic/restic/pull/3243

changelog/0.12.0_2021-02-14/issue-909

@@ -0,0 +1,12 @@
+Enhancement: Back up mountpoints as empty directories
+
+When the `--one-file-system` option is specified to `restic backup`, it
+ignores all file systems mounted below one of the target directories. This
+means that when a snapshot is restored, users needed to manually recreate
+the mountpoint directories.
+
+Restic now backs up mountpoints as empty directories and therefore implements
+the same approach as `tar`.
+
+https://github.com/restic/restic/issues/909
+https://github.com/restic/restic/pull/3119

changelog/0.12.0_2021-02-14/pr-3250

@@ -0,0 +1,6 @@
+Enhancement: Add several more error checks
+
+We've added a lot more error checks in places where errors were previously
+ignored (as hinted by the static analysis program `errcheck` via `golangci-lint`).
+
+https://github.com/restic/restic/pull/3250

changelog/0.12.0_2021-02-14/pull-2718

@@ -0,0 +1,29 @@
+Enhancement: Improve `prune` performance and make it more customizable
+
+The `prune` command is now much faster. This is especially the case for remote
+repositories or repositories with not much data to remove. Also the memory
+usage of the `prune` command is now reduced.
+
+Restic used to rebuild the index from scratch after pruning. This could lead
+to missing packs in the index in some cases for eventually consistent backends
+such as e.g. AWS S3. This behavior is now changed and the index rebuilding
+uses the information already known by `prune`.
+
+By default, the `prune` command no longer removes all unused data. This
+behavior can be fine-tuned by new options, like the acceptable amount of
+unused space or the maximum size of data to reorganize. For more details,
+please see https://restic.readthedocs.io/en/stable/060_forget.html .
+
+Moreover, `prune` now accepts the `--dry-run` option and also running
+`forget --dry-run --prune` will show what `prune` would do.
+
+This enhancement also fixes several open issues, e.g.:
+- https://github.com/restic/restic/issues/1140
+- https://github.com/restic/restic/issues/1599
+- https://github.com/restic/restic/issues/1985
+- https://github.com/restic/restic/issues/2112
+- https://github.com/restic/restic/issues/2227
+- https://github.com/restic/restic/issues/2305
+
+https://github.com/restic/restic/pull/2718
+https://github.com/restic/restic/pull/2842

changelog/0.12.0_2021-02-14/pull-2823

@@ -0,0 +1,27 @@
+Enhancement: Add option to let `backup` trust mtime without checking ctime
+
+The `backup` command used to require that both `ctime` and `mtime` of a file
+matched with a previously backed up version to determine that the file was
+unchanged. In other words, if either `ctime` or `mtime` of the file had
+changed, it would be considered changed and restic would read the file's
+content again to back up the relevant (changed) parts of it.
+
+The new option `--ignore-ctime` makes restic look at `mtime` only, such that
+`ctime` changes for a file does not cause restic to read the file's contents
+again.
+
+The check for both `ctime` and `mtime` was introduced in restic 0.9.6 to make
+backups more reliable in the face of programs that reset `mtime` (some Unix
+archivers do that), but it turned out to often be expensive because it made
+restic read file contents even if only the metadata (owner, permissions) of
+a file had changed. The new `--ignore-ctime` option lets the user restore the
+0.9.5 behavior when needed. The existing `--ignore-inode` option already
+turned off this behavior, but also removed a different check.
+
+Please note that changes in files' metadata are still recorded, regardless of
+the command line options provided to the backup command.
+
+https://github.com/restic/restic/issues/2495
+https://github.com/restic/restic/issues/2558
+https://github.com/restic/restic/issues/2819
+https://github.com/restic/restic/pull/2823

changelog/0.12.0_2021-02-14/pull-2941

@@ -0,0 +1,8 @@
+Enhancement: Speed up the repacking step of the `prune` command
+
+The repack step of the `prune` command, which moves still used file parts into
+new pack files such that the old ones can be garbage collected later on, now
+processes multiple pack files in parallel. This is especially beneficial for
+high latency backends or when using a fast network connection.
+
+https://github.com/restic/restic/pull/2941

changelog/0.12.0_2021-02-14/pull-3006

@@ -0,0 +1,11 @@
+Enhancement: Speed up the `rebuild-index` command
+
+We've optimized the `rebuild-index` command. Now, existing index entries are used
+to minimize the number of pack files that must be read. This speeds up the index
+rebuild a lot.
+
+Additionally, the option `--read-all-packs` has been added, implementing the
+previous behavior.
+
+https://github.com/restic/restic/issue/2547
+https://github.com/restic/restic/pull/3006

changelog/0.12.0_2021-02-14/pull-3014

@@ -0,0 +1,13 @@
+Bugfix: Fix sporadic stream reset between rclone and restic
+
+Sometimes when using restic with the `rclone` backend, an error message
+similar to the following would be printed:
+
+    Didn't finish writing GET request (wrote 0/xxx): http2: stream closed
+
+It was found that this was caused by restic closing the connection to rclone
+to soon when downloading data. A workaround has been added which waits for
+the end of the download before closing the connection.
+
+https://github.com/restic/restic/pull/3014
+https://github.com/rclone/rclone/issues/2598

changelog/0.12.0_2021-02-14/pull-3048

@@ -0,0 +1,23 @@
+Enhancement: Add more checks for index and pack files in the `check` command
+
+The `check` command run with the `--read-data` or `--read-data-subset` options
+used to only verify only the pack file content - it did not check if the blobs
+within the pack are correctly contained in the index.
+
+A check for the latter is now in place, which can print the following error:
+
+    Blob ID is not contained in index or position is incorrect
+
+Another test is also added, which compares pack file sizes computed from the
+index and the pack header with the actual file size. This test is able to
+detect truncated pack files.
+
+If the index is not correct, it can be rebuilt by using the `rebuild-index`
+command.
+
+Having added these tests, `restic check` is now able to detect non-existing
+blobs which are wrongly referenced in the index. This situation could have
+lead to missing data.
+
+https://github.com/restic/restic/pull/3048
+https://github.com/restic/restic/pull/3082

changelog/0.12.0_2021-02-14/pull-3081

@@ -0,0 +1,8 @@
+Enhancement: Make the `dump` command support `zip` format
+
+Previously, restic could dump the contents of a whole folder structure only
+in the `tar` format. The `dump` command now has a new flag to change output
+format to `zip`. Just pass `--archive zip` as an option to `restic dump`.
+
+https://github.com/restic/restic/pull/2433
+https://github.com/restic/restic/pull/3081

changelog/0.12.0_2021-02-14/pull-3099

@@ -0,0 +1,6 @@
+Enhancement: Reduce memory usage of `check` command
+
+The `check` command now requires less memory if it is run without the
+`--check-unused` option.
+
+https://github.com/restic/restic/pull/3099

changelog/0.12.0_2021-02-14/pull-3106

@@ -0,0 +1,10 @@
+Enhancement: Parallelize scan of snapshot content in `copy` and `prune`
+
+The `copy` and `prune` commands used to traverse the directories of
+snapshots one by one to find used data. This snapshot traversal is
+now parallized which can speed up this step several times.
+
+In addition the `check` command now reports how many snapshots have
+already been processed.
+
+https://github.com/restic/restic/pull/3106

changelog/0.12.0_2021-02-14/pull-3130

@@ -0,0 +1,13 @@
+Enhancement: Parallelize reading of locks and snapshots
+
+Restic used to read snapshots sequentially. For repositories containing
+many snapshots this slowed down commands which have to read all snapshots.
+
+Now the reading of snapshots is parallelized. This speeds up for example
+`prune`, `backup` and other commands that search for snapshots with certain
+properties or which have to find the `latest` snapshot.
+
+The speed up also applies to locks stored in the backup repository.
+
+https://github.com/restic/restic/pull/3130
+https://github.com/restic/restic/pull/3174

changelog/0.12.0_2021-02-14/pull-3152

@@ -0,0 +1,8 @@
+Bugfix: Do not hang until foregrounded when completed in background
+
+On Linux, when running in the background restic failed to stop the terminal
+output of the `backup` command after it had completed. This caused restic to
+hang until moved to the foreground. This has now been fixed.
+
+https://github.com/restic/restic/pull/3152
+https://forum.restic.net/t/restic-alpine-container-cron-hangs-epoll-pwait/3334

changelog/0.12.0_2021-02-14/pull-3249

@@ -0,0 +1,7 @@
+Bugfix: Improve error handling in `gs` backend
+
+The `gs` backend did not notice when the last step of completing a
+file upload failed. Under rare circumstances, this could cause
+missing files in the backup repository. This has now been fixed.
+
+https://github.com/restic/restic/pull/3249

changelog/0.12.0_2021-02-14/pull-3254

@@ -0,0 +1,8 @@
+Enhancement: Enable HTTP/2 for backend connections
+
+Go's HTTP library usually automatically chooses between HTTP/1.x and HTTP/2
+depending on what the server supports. But for compatibility this mechanism
+is disabled if DialContext is used (which is the case for restic). This change
+allows restic's HTTP client to negotiate HTTP/2 if supported by the server.
+
+https://github.com/restic/restic/pull/3254

changelog/0.12.1_2021-08-03/issue-2742

@@ -0,0 +1,13 @@
+Bugfix: Improve error handling for rclone and REST backend over HTTP2
+
+When retrieving data from the rclone / REST backend while also using HTTP2
+restic did not detect when no data was returned at all. This could cause
+for example the `check` command to report the following error:
+
+    Pack ID does not match, want [...], got e3b0c442
+
+This has been fixed by correctly detecting and retrying the incomplete download.
+
+https://github.com/restic/restic/issues/2742
+https://github.com/restic/restic/pull/3453
+https://forum.restic.net/t/http2-stream-closed-connection-reset-context-canceled/3743/10

changelog/0.12.1_2021-08-03/issue-2780

@@ -0,0 +1,6 @@
+Enhancement: Add release binaries for s390x architecture on Linux
+
+We've added release binaries for Linux using the s390x architecture.
+
+https://github.com/restic/restic/issues/2780
+https://github.com/restic/restic/pull/3452

changelog/0.12.1_2021-08-03/issue-3111

@@ -0,0 +1,11 @@
+Bugfix: Fix terminal output redirection for PowerShell
+
+When redirecting the output of restic using PowerShell on Windows, the
+output contained terminal escape characters. This has been fixed by
+properly detecting the terminal type.
+
+In addition, the mintty terminal now shows progress output for the backup
+command.
+
+https://github.com/restic/restic/issues/3111
+https://github.com/restic/restic/pull/3325

changelog/0.12.1_2021-08-03/issue-3214

@@ -0,0 +1,9 @@
+Bugfix: Treat an empty password as a fatal error for repository init
+
+When attempting to initialize a new repository, if an empty password was
+supplied, the repository would be created but the init command would return
+an error with a stack trace. Now, if an empty password is provided, it is
+treated as a fatal error, and no repository is created.
+
+https://github.com/restic/restic/issues/3214
+https://github.com/restic/restic/pull/3283

changelog/0.12.1_2021-08-03/issue-3247

@@ -0,0 +1,8 @@
+Change: Empty files now have size of 0 in `ls --json` output
+
+The `ls --json` command used to omit the sizes of empty files in its
+output. It now reports a size of zero explicitly for regular files,
+while omitting the size field for all other types.
+
+https://github.com/restic/restic/issues/3247
+https://github.com/restic/restic/pull/3257

changelog/0.12.1_2021-08-03/issue-3267

@@ -0,0 +1,11 @@
+Bugfix: `copy` failed to copy snapshots in rare cases
+
+The `copy` command could in rare cases fail with the error message `SaveTree(...)
+returned unexpected id ...`.  This has been fixed.
+
+On Linux/BSDs, the error could be caused by backing up symlinks with non-UTF-8
+target paths. Note that, due to limitations in the repository format, these are
+not stored properly and should be avoided if possible.
+
+https://github.com/restic/restic/issues/3267
+https://github.com/restic/restic/pull/3310

changelog/0.12.1_2021-08-03/issue-3284

@@ -0,0 +1,11 @@
+Bugfix: `backup --quiet` no longer prints status information
+
+A regression in the latest restic version caused the output of `backup --quiet`
+to contain large amounts of backup progress information when run using an
+interactive terminal. This is fixed now.
+
+A workaround for this bug is to run restic as follows:
+`restic backup --quiet [..] | cat -`.
+
+https://github.com/restic/restic/issues/3184
+https://github.com/restic/restic/pull/3186

changelog/0.12.1_2021-08-03/issue-3293

@@ -0,0 +1,14 @@
+Enhancement: Add `--repository-file2` option to `init` and `copy` command
+
+The `init` and `copy` command can now be used with the `--repository-file2`
+option or the `$RESTIC_REPOSITORY_FILE2` environment variable.
+These to options are in addition to the `--repo2` flag and allow you to read
+the destination repository from a file.
+
+Using both `--repository-file` and `--repo2` options resulted in an error for
+the `copy` or `init` command. The handling of this combination of options has
+been fixed. A workaround for this issue is to only use `--repo` or `-r` and
+`--repo2` for `init` or `copy`.
+
+https://github.com/restic/restic/issues/3293
+https://github.com/restic/restic/pull/3294

changelog/0.12.1_2021-08-03/issue-3296

@@ -0,0 +1,7 @@
+Bugfix: Fix crash of `check --read-data-subset=x%` run for an empty repository
+
+The command `restic check --read-data-subset=x%` crashed when run for an empty
+repository. This has been fixed.
+
+https://github.com/restic/restic/issues/3296
+https://github.com/restic/restic/pull/3309

changelog/0.12.1_2021-08-03/issue-3302

@@ -0,0 +1,8 @@
+Bugfix: Fix `fdopendir: not a directory` error for local backend
+
+The `check`, `list packs`, `prune` and `rebuild-index` commands failed
+for the local backend when the `data` folder in the repository contained
+files. This has been fixed.
+
+https://github.com/restic/restic/issues/3302
+https://github.com/restic/restic/pull/3308

changelog/0.12.1_2021-08-03/issue-3312

@@ -0,0 +1,5 @@
+Enhancement: Add auto-completion support for fish
+
+The `generate` command now supports fish auto completion.
+
+https://github.com/restic/restic/pull/3312

changelog/0.12.1_2021-08-03/issue-3334

@@ -0,0 +1,8 @@
+Bugfix: Print `created new cache` message only on a terminal
+
+The message `created new cache` was printed even when the output wasn't a
+terminal. That broke piping `restic dump` output to tar or zip if cache
+directory didn't exist. The message is now only printed on a terminal.
+
+https://github.com/restic/restic/issues/3334
+https://github.com/restic/restic/pull/3343

changelog/0.12.1_2021-08-03/issue-3336

@@ -0,0 +1,8 @@
+Enhancement: SFTP backend now checks for disk space
+
+Backing up over SFTP previously spewed multiple generic "failure" messages
+when the remote disk was full. It now checks for disk space before writing
+a file and fails immediately with a "no space left on device" message.
+
+https://github.com/restic/restic/issues/3336
+https://github.com/restic/restic/pull/3345

changelog/0.12.1_2021-08-03/issue-3377

@@ -0,0 +1,6 @@
+Enhancement: Add release binaries for Apple Silicon
+
+We've added release binaries for macOS on Apple Silicon (M1).
+
+https://github.com/restic/restic/issues/3377
+https://github.com/restic/restic/pull/3394

changelog/0.12.1_2021-08-03/issue-3380

@@ -0,0 +1,7 @@
+Bugfix: Fix crash of `backup --exclude='**'`
+
+The exclude filter `**`, which excludes all files, caused restic to crash. This
+has been corrected.
+
+https://github.com/restic/restic/issues/3380
+https://github.com/restic/restic/pull/3393

changelog/0.12.1_2021-08-03/issue-3414

@@ -0,0 +1,20 @@
+Enhancement: Add `--keep-within-hourly` option to restic forget
+
+The `forget` command allowed keeping a given number of hourly
+backups or to keep all backups within a given interval, but it
+was not possible to specify keeping hourly backups within a given
+interval.
+
+The new `--keep-within-hourly` option now offers this functionality.
+Similar options for daily/weekly/monthly/yearly are also implemented,
+the new options are:
+
+    --keep-within-hourly <1y2m3d4h>
+    --keep-within-daily <1y2m3d4h>
+    --keep-within-weekly <1y2m3d4h>
+    --keep-within-monthly <1y2m3d4h>
+    --keep-within-yearly <1y2m3d4h>
+
+https://github.com/restic/restic/issues/3414
+https://github.com/restic/restic/pull/3416
+https://forum.restic.net/t/forget-policy/4014/11

changelog/0.12.1_2021-08-03/issue-3456

@@ -0,0 +1,9 @@
+Enhancement: Support filtering and specifying untagged snapshots
+
+It was previously not possible to specify an empty tag with the `--tag` and
+`--keep-tag` options. This has now been fixed, such that `--tag ''` and
+`--keep-tag ''` now matches snapshots without tags. This allows e.g. the
+`snapshots` and `forget` commands to only operate on untagged snapshots.
+
+https://github.com/restic/restic/issues/3456
+https://github.com/restic/restic/pull/3457

changelog/0.12.1_2021-08-03/pull-3167

@@ -0,0 +1,9 @@
+Enhancement: Allow specifying limit of `snapshots` list
+
+The `--last` option allowed limiting the output of the `snapshots`
+command to the latest snapshot for each host. The new `--latest n`
+option allows limiting the output to the latest `n` snapshots.
+
+This change deprecates the option `--last` in favour of `--latest 1`.
+
+https://github.com/restic/restic/pull/3167

changelog/0.12.1_2021-08-03/pull-3305

@@ -0,0 +1,6 @@
+Bugfix: Fix possibly missing backup summary of JSON output in case of error
+
+When using `--json` output it happened from time to time that the summary
+output was missing in case an error occurred. This has been fixed.
+
+https://github.com/restic/restic/pull/3305

changelog/0.12.1_2021-08-03/pull-3426

@@ -0,0 +1,7 @@
+Enhancement: Optimize read performance of mount command
+
+Reading large files in a mounted repository may be up to five times faster.
+This improvement primarily applies to repositories stored at a backend that can
+be accessed with low latency, like e.g. the local backend.
+
+https://github.com/restic/restic/pull/3426

changelog/0.12.1_2021-08-03/pull-3427

@@ -0,0 +1,13 @@
+Enhancement: `find --pack` fallback to index if data file is missing
+
+When investigating a repository with missing data files, it might be useful to
+determine affected snapshots before running `rebuild-index`. Previously,
+`find --pack pack-id` returned no data as it required accessing the data file.
+Now, if the necessary data is still available in the repository index, it gets
+retrieved from there.
+
+The command now also supports looking up multiple pack files in a single `find`
+run.
+
+https://github.com/restic/restic/pull/3427
+https://forum.restic.net/t/missing-packs-not-found/2600

changelog/0.12.1_2021-08-03/pull-3449

@@ -0,0 +1,8 @@
+Bugfix: Correctly handle download errors during `restore`
+
+Due to a regression in restic 0.12.0, the `restore` command in some cases did
+not retry download errors and only printed a warning. This has been fixed by
+retrying incomplete data downloads.
+
+https://github.com/restic/restic/issues/3439
+https://github.com/restic/restic/pull/3449

changelog/TEMPLATE

@@ -1,11 +1,21 @@
-Bugfix: Fix behavior for foobar (in present tense)
+# The first line must start with Bugfix:, Enhancement: or Change:,
+# including the colon. Use present use. Remove lines starting with '#'
+# from this template.
+Enhancement: Allow custom bar in the foo command
 
-We've fixed the behavior for foobar, a long-standing annoyance for restic
-users.
+# Describe the problem in the past tense, the new behavior in the present
+# tense. Mention the affected commands, backends, operating systems, etc.
+# Focus on user-facing behavior, not the implementation.
 
-The text in the paragraphs is written in past tense. The last section is a list
-of issue URLs, PR URLs and other URLs. The first issue ID (or the first PR ID,
-in case there aren't any issue links) is used as the primary ID.
+Restic foo always used the system-wide bar when deciding how to frob an
+item in the baz backend. It now permits selecting the bar with --bar or
+the environment variable RESTIC_BAR. The system-wide bar is still the
+default.
+
+# The last section is a list of issue, PR and forum URLs.
+# The first issue ID determines the filename for the changelog entry:
+# changelog/unreleased/issue-1234. If there are no relevant issue links,
+# use the PR ID and call the file pull-55555.
 
 https://github.com/restic/restic/issues/1234
 https://github.com/restic/restic/pull/55555

changelog/unreleased/.gitignore

@@ -0,0 +1,2 @@
+# this file is here so the unreleased/ directory is tracked within git, even if
+# no other changelog files are present.

cmd/restic/cmd_backup.go

@@ -10,7 +10,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"strconv"
+	"runtime"
 	"strings"
 	"time"
 
@@ -55,24 +55,16 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
 	},
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		if backupOptions.Stdin {
-			for _, filename := range backupOptions.FilesFrom {
-				if filename == "-" {
-					return errors.Fatal("cannot use both `--stdin` and `--files-from -`")
-				}
-			}
-		}
-
 		var t tomb.Tomb
 		term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet)
 		t.Go(func() error { term.Run(t.Context(globalOptions.ctx)); return nil })
 
 		err := runBackup(backupOptions, globalOptions, term, args)
-		if err != nil {
-			return err
-		}
 		t.Kill(nil)
-		return t.Wait()
+		if werr := t.Wait(); werr != nil {
+			panic(fmt.Sprintf("term.Run() returned err: %v", err))
+		}
+		return err
 	},
 }
 
@@ -90,18 +82,22 @@ type BackupOptions struct {
 	ExcludeLargerThan       string
 	Stdin                   bool
 	StdinFilename           string
-	Tags                    []string
+	Tags                    restic.TagLists
 	Host                    string
 	FilesFrom               []string
+	FilesFromVerbatim       []string
+	FilesFromRaw            []string
 	TimeStamp               string
 	WithAtime               bool
 	IgnoreInode             bool
+	IgnoreCtime             bool
+	UseFsSnapshot           bool
 }
 
 var backupOptions BackupOptions
 
 // ErrInvalidSourceData is used to report an incomplete backup
-var ErrInvalidSourceData = errors.New("failed to read all source data during backup")
+var ErrInvalidSourceData = errors.New("at least one source file could not be read")
 
 func init() {
 	cmdRoot.AddCommand(cmdBackup)
@@ -113,22 +109,32 @@ func init() {
 	f.StringArrayVar(&backupOptions.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
 	f.StringArrayVar(&backupOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
 	f.StringArrayVar(&backupOptions.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns")
-	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems")
+	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems, don't cross filesystem boundaries and subvolumes")
 	f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes `filename[:header]`, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)")
 	f.BoolVar(&backupOptions.ExcludeCaches, "exclude-caches", false, `excludes cache directories that are marked with a CACHEDIR.TAG file. See https://bford.info/cachedir/ for the Cache Directory Tagging Standard`)
 	f.StringVar(&backupOptions.ExcludeLargerThan, "exclude-larger-than", "", "max `size` of the files to be backed up (allowed suffixes: k/K, m/M, g/G, t/T)")
 	f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
 	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "`filename` to use when reading from stdin")
-	f.StringArrayVar(&backupOptions.Tags, "tag", nil, "add a `tag` for the new snapshot (can be specified multiple times)")
+	f.Var(&backupOptions.Tags, "tag", "add `tags` for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times)")
 
 	f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag")
 	f.StringVar(&backupOptions.Host, "hostname", "", "set the `hostname` for the snapshot manually")
-	f.MarkDeprecated("hostname", "use --host")
+	err := f.MarkDeprecated("hostname", "use --host")
+	if err != nil {
+		// MarkDeprecated only returns an error when the flag could not be found
+		panic(err)
+	}
 
-	f.StringArrayVar(&backupOptions.FilesFrom, "files-from", nil, "read the files to backup from `file` (can be combined with file args/can be specified multiple times)")
+	f.StringArrayVar(&backupOptions.FilesFrom, "files-from", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
+	f.StringArrayVar(&backupOptions.FilesFromVerbatim, "files-from-verbatim", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
+	f.StringArrayVar(&backupOptions.FilesFromRaw, "files-from-raw", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
 	f.StringVar(&backupOptions.TimeStamp, "time", "", "`time` of the backup (ex. '2012-11-01 22:08:41') (default: now)")
 	f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories")
 	f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number changes when checking for modified files")
+	f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files")
+	if runtime.GOOS == "windows" {
+		f.BoolVar(&backupOptions.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)")
+	}
 }
 
 // filterExisting returns a slice of all existing items, or an error if no
@@ -151,11 +157,13 @@ func filterExisting(items []string) (result []string, err error) {
 	return
 }
 
-// readFromFile will read all lines from the given filename and return them as
-// a string array, if filename is empty readFromFile returns and empty string
-// array. If filename is a dash (-), readFromFile will read the lines from the
+// readLines reads all lines from the named file and returns them as a
+// string slice.
+//
+// If filename is empty, readPatternsFromFile returns an empty slice.
+// If filename is a dash (-), readPatternsFromFile will read the lines from the
 // standard input.
-func readLinesFromFile(filename string) ([]string, error) {
+func readLines(filename string) ([]string, error) {
 	if filename == "" {
 		return nil, nil
 	}
@@ -179,29 +187,72 @@ func readLinesFromFile(filename string) ([]string, error) {
 
 	scanner := bufio.NewScanner(bytes.NewReader(data))
 	for scanner.Scan() {
-		line := strings.TrimSpace(scanner.Text())
-		// ignore empty lines
-		if line == "" {
-			continue
-		}
-		// strip comments
-		if strings.HasPrefix(line, "#") {
-			continue
-		}
-		lines = append(lines, line)
+		lines = append(lines, scanner.Text())
 	}
 
 	if err := scanner.Err(); err != nil {
 		return nil, err
 	}
-
 	return lines, nil
 }
 
+// readFilenamesFromFileRaw reads a list of filenames from the given file,
+// or stdin if filename is "-". Each filename is terminated by a zero byte,
+// which is stripped off.
+func readFilenamesFromFileRaw(filename string) (names []string, err error) {
+	f := os.Stdin
+	if filename != "-" {
+		if f, err = os.Open(filename); err != nil {
+			return nil, err
+		}
+	}
+
+	names, err = readFilenamesRaw(f)
+	if err != nil {
+		// ignore subsequent errors
+		_ = f.Close()
+		return nil, err
+	}
+
+	err = f.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	return names, nil
+}
+
+func readFilenamesRaw(r io.Reader) (names []string, err error) {
+	br := bufio.NewReader(r)
+	for {
+		name, err := br.ReadString(0)
+		switch err {
+		case nil:
+		case io.EOF:
+			if name == "" {
+				return names, nil
+			}
+			return nil, errors.Fatal("--files-from-raw: trailing zero byte missing")
+		default:
+			return nil, err
+		}
+
+		name = name[:len(name)-1]
+		if name == "" {
+			// The empty filename is never valid. Handle this now to
+			// prevent downstream code from erroneously backing up
+			// filepath.Clean("") == ".".
+			return nil, errors.Fatal("--files-from-raw: empty filename in listing")
+		}
+		names = append(names, name)
+	}
+}
+
 // Check returns an error when an invalid combination of options was set.
 func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
 	if gopts.password == "" {
-		for _, filename := range opts.FilesFrom {
+		filesFrom := append(append(opts.FilesFrom, opts.FilesFromVerbatim...), opts.FilesFromRaw...)
+		for _, filename := range filesFrom {
 			if filename == "-" {
 				return errors.Fatal("unable to read password from stdin when data is to be read from stdin, use --password-file or $RESTIC_PASSWORD")
 			}
@@ -212,6 +263,12 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
 		if len(opts.FilesFrom) > 0 {
 			return errors.Fatal("--stdin and --files-from cannot be used together")
 		}
+		if len(opts.FilesFromVerbatim) > 0 {
+			return errors.Fatal("--stdin and --files-from-verbatim cannot be used together")
+		}
+		if len(opts.FilesFromRaw) > 0 {
+			return errors.Fatal("--stdin and --files-from-raw cannot be used together")
+		}
 
 		if len(args) > 0 {
 			return errors.Fatal("--stdin was specified and files/dirs were listed as arguments")
@@ -351,15 +408,19 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 		return nil, nil
 	}
 
-	var lines []string
 	for _, file := range opts.FilesFrom {
-		fromfile, err := readLinesFromFile(file)
+		fromfile, err := readLines(file)
 		if err != nil {
 			return nil, err
 		}
 
 		// expand wildcards
 		for _, line := range fromfile {
+			line = strings.TrimSpace(line)
+			if line == "" || line[0] == '#' { // '#' marks a comment.
+				continue
+			}
+
 			var expanded []string
 			expanded, err := filepath.Glob(line)
 			if err != nil {
@@ -368,19 +429,38 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 			if len(expanded) == 0 {
 				Warnf("pattern %q does not match any files, skipping\n", line)
 			}
-			lines = append(lines, expanded...)
+			targets = append(targets, expanded...)
 		}
 	}
 
-	// merge files from files-from into normal args so we can reuse the normal
-	// args checks and have the ability to use both files-from and args at the
-	// same time
-	args = append(args, lines...)
-	if len(args) == 0 && !opts.Stdin {
+	for _, file := range opts.FilesFromVerbatim {
+		fromfile, err := readLines(file)
+		if err != nil {
+			return nil, err
+		}
+		for _, line := range fromfile {
+			if line == "" {
+				continue
+			}
+			targets = append(targets, line)
+		}
+	}
+
+	for _, file := range opts.FilesFromRaw {
+		fromfile, err := readFilenamesFromFileRaw(file)
+		if err != nil {
+			return nil, err
+		}
+		targets = append(targets, fromfile...)
+	}
+
+	// Merge args into files-from so we can reuse the normal args checks
+	// and have the ability to use both files-from and args at the same time.
+	targets = append(targets, args...)
+	if len(targets) == 0 && !opts.Stdin {
 		return nil, errors.Fatal("nothing to backup, please specify target files/dirs")
 	}
 
-	targets = args
 	targets, err = filterExisting(targets)
 	if err != nil {
 		return nil, err
@@ -394,7 +474,7 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string) (parentID *restic.ID, err error) {
 	// Force using a parent
 	if !opts.Force && opts.Parent != "" {
-		id, err := restic.FindSnapshot(repo, opts.Parent)
+		id, err := restic.FindSnapshot(ctx, repo, opts.Parent)
 		if err != nil {
 			return nil, errors.Fatalf("invalid id %q: %v", opts.Parent, err)
 		}
@@ -481,22 +561,14 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}()
 	gopts.stdout, gopts.stderr = p.Stdout(), p.Stderr()
 
-	if s, ok := os.LookupEnv("RESTIC_PROGRESS_FPS"); ok {
-		fps, err := strconv.Atoi(s)
-		if err == nil && fps >= 1 {
-			if fps > 60 {
-				fps = 60
-			}
-			p.SetMinUpdatePause(time.Second / time.Duration(fps))
-		}
-	}
+	p.SetMinUpdatePause(calculateProgressInterval(!gopts.Quiet))
 
 	t.Go(func() error { return p.Run(t.Context(gopts.ctx)) })
 
 	if !gopts.JSON {
 		p.V("lock repository")
 	}
-	lock, err := lockRepo(repo)
+	lock, err := lockRepo(gopts.ctx, repo)
 	defer unlockRepo(lock)
 	if err != nil {
 		return err
@@ -527,8 +599,12 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		return err
 	}
 
-	if !gopts.JSON && parentSnapshotID != nil {
-		p.V("using parent snapshot %v\n", parentSnapshotID.Str())
+	if !gopts.JSON {
+		if parentSnapshotID != nil {
+			p.P("using parent snapshot %v\n", parentSnapshotID.Str())
+		} else {
+			p.P("no parent snapshot found, will read all files\n")
+		}
 	}
 
 	selectByNameFilter := func(item string) bool {
@@ -550,6 +626,25 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}
 
 	var targetFS fs.FS = fs.Local{}
+	if runtime.GOOS == "windows" && opts.UseFsSnapshot {
+		if err = fs.HasSufficientPrivilegesForVSS(); err != nil {
+			return err
+		}
+
+		errorHandler := func(item string, err error) error {
+			return p.Error(item, nil, err)
+		}
+
+		messageHandler := func(msg string, args ...interface{}) {
+			if !gopts.JSON {
+				p.P(msg, args...)
+			}
+		}
+
+		localVss := fs.NewLocalVss(errorHandler, messageHandler)
+		defer localVss.DeleteSnapshots()
+		targetFS = localVss
+	}
 	if opts.Stdin {
 		if !gopts.JSON {
 			p.V("read data from stdin")
@@ -587,7 +682,15 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	arch.CompleteItem = p.CompleteItem
 	arch.StartFile = p.StartFile
 	arch.CompleteBlob = p.CompleteBlob
-	arch.IgnoreInode = opts.IgnoreInode
+
+	if opts.IgnoreInode {
+		// --ignore-inode implies --ignore-ctime: on FUSE, the ctime is not
+		// reliable either.
+		arch.ChangeIgnoreFlags |= archiver.ChangeIgnoreCtime | archiver.ChangeIgnoreInode
+	}
+	if opts.IgnoreCtime {
+		arch.ChangeIgnoreFlags |= archiver.ChangeIgnoreCtime
+	}
 
 	if parentSnapshotID == nil {
 		parentSnapshotID = &restic.ID{}
@@ -595,7 +698,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 
 	snapshotOpts := archiver.SnapshotOptions{
 		Excludes:       opts.Excludes,
-		Tags:           opts.Tags,
+		Tags:           opts.Tags.Flatten(),
 		Time:           timeStamp,
 		Hostname:       opts.Host,
 		ParentSnapshot: *parentSnapshotID,
@@ -605,15 +708,17 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		p.V("start backup on %v", targets)
 	}
 	_, id, err := arch.Snapshot(gopts.ctx, targets, snapshotOpts)
-	if err != nil {
-		return errors.Fatalf("unable to save snapshot: %v", err)
-	}
 
 	// cleanly shutdown all running goroutines
 	t.Kill(nil)
 
 	// let's see if one returned an error
-	err = t.Wait()
+	werr := t.Wait()
+
+	// return original error
+	if err != nil {
+		return errors.Fatalf("unable to save snapshot: %v", err)
+	}
 
 	// Report finished execution
 	p.Finish(id)
@@ -625,5 +730,5 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}
 
 	// Return error if any
-	return err
+	return werr
 }

cmd/restic/cmd_backup_test.go

@@ -0,0 +1,113 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sort"
+	"strings"
+	"testing"
+
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func TestCollectTargets(t *testing.T) {
+	dir, cleanup := rtest.TempDir(t)
+	defer cleanup()
+
+	fooSpace := "foo "
+	barStar := "bar*"              // Must sort before the others, below.
+	if runtime.GOOS == "windows" { // Doesn't allow "*" or trailing space.
+		fooSpace = "foo"
+		barStar = "bar"
+	}
+
+	var expect []string
+	for _, filename := range []string{
+		barStar, "baz", "cmdline arg", fooSpace,
+		"fromfile", "fromfile-raw", "fromfile-verbatim", "quux",
+	} {
+		// All mentioned files must exist for collectTargets.
+		f, err := os.Create(filepath.Join(dir, filename))
+		rtest.OK(t, err)
+		rtest.OK(t, f.Close())
+
+		expect = append(expect, f.Name())
+	}
+
+	f1, err := os.Create(filepath.Join(dir, "fromfile"))
+	rtest.OK(t, err)
+	// Empty lines should be ignored. A line starting with '#' is a comment.
+	fmt.Fprintf(f1, "\n%s*\n # here's a comment\n", f1.Name())
+	rtest.OK(t, f1.Close())
+
+	f2, err := os.Create(filepath.Join(dir, "fromfile-verbatim"))
+	rtest.OK(t, err)
+	for _, filename := range []string{fooSpace, barStar} {
+		// Empty lines should be ignored. CR+LF is allowed.
+		fmt.Fprintf(f2, "%s\r\n\n", filepath.Join(dir, filename))
+	}
+	rtest.OK(t, f2.Close())
+
+	f3, err := os.Create(filepath.Join(dir, "fromfile-raw"))
+	rtest.OK(t, err)
+	for _, filename := range []string{"baz", "quux"} {
+		fmt.Fprintf(f3, "%s\x00", filepath.Join(dir, filename))
+	}
+	rtest.OK(t, err)
+	rtest.OK(t, f3.Close())
+
+	opts := BackupOptions{
+		FilesFrom:         []string{f1.Name()},
+		FilesFromVerbatim: []string{f2.Name()},
+		FilesFromRaw:      []string{f3.Name()},
+	}
+
+	targets, err := collectTargets(opts, []string{filepath.Join(dir, "cmdline arg")})
+	rtest.OK(t, err)
+	sort.Strings(targets)
+	rtest.Equals(t, expect, targets)
+}
+
+func TestReadFilenamesRaw(t *testing.T) {
+	// These should all be returned exactly as-is.
+	expected := []string{
+		"\xef\xbb\xbf/utf-8-bom",
+		"/absolute",
+		"../.././relative",
+		"\t\t leading and trailing space   \t\t",
+		"newline\nin filename",
+		"not UTF-8: \x80\xff/simple",
+		` / *[]* \ `,
+	}
+
+	var buf bytes.Buffer
+	for _, name := range expected {
+		buf.WriteString(name)
+		buf.WriteByte(0)
+	}
+
+	got, err := readFilenamesRaw(&buf)
+	rtest.OK(t, err)
+	rtest.Equals(t, expected, got)
+
+	// Empty input is ok.
+	got, err = readFilenamesRaw(strings.NewReader(""))
+	rtest.OK(t, err)
+	rtest.Equals(t, 0, len(got))
+
+	// An empty filename is an error.
+	_, err = readFilenamesRaw(strings.NewReader("foo\x00\x00"))
+	rtest.Assert(t, err != nil, "no error for zero byte")
+	rtest.Assert(t, strings.Contains(err.Error(), "empty filename"),
+		"wrong error message: %v", err.Error())
+
+	// No trailing NUL byte is an error, because it likely means we're
+	// reading a line-oriented text file (someone forgot -print0).
+	_, err = readFilenamesRaw(strings.NewReader("simple.txt"))
+	rtest.Assert(t, err != nil, "no error for zero byte")
+	rtest.Assert(t, strings.Contains(err.Error(), "zero byte"),
+		"wrong error message: %v", err.Error())
+}

cmd/restic/cmd_cache.go

@@ -51,7 +51,7 @@ func init() {
 
 func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
 	if len(args) > 0 {
-		return errors.Fatal("the cache command has no arguments")
+		return errors.Fatal("the cache command expects no arguments, only options - please see `restic help cache` for usage and flags")
 	}
 
 	if gopts.NoCache {
@@ -148,7 +148,7 @@ func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
 		})
 	}
 
-	tab.Write(gopts.stdout)
+	_ = tab.Write(gopts.stdout)
 	Printf("%d cache dirs in %s\n", len(dirs), cachedir)
 
 	return nil

cmd/restic/cmd_cat.go

@@ -42,10 +42,13 @@ func runCat(gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	lock, err := lockRepo(repo)
-	defer unlockRepo(lock)
-	if err != nil {
-		return err
+	if !gopts.NoLock {
+		lock, err := lockRepo(gopts.ctx, repo)
+		if err != nil {
+			return err
+		}
+
+		defer unlockRepo(lock)
 	}
 
 	tpe := args[0]
@@ -59,14 +62,13 @@ func runCat(gopts GlobalOptions, args []string) error {
 			}
 
 			// find snapshot id with prefix
-			id, err = restic.FindSnapshot(repo, args[1])
+			id, err = restic.FindSnapshot(gopts.ctx, repo, args[1])
 			if err != nil {
 				return errors.Fatalf("could not find snapshot: %v\n", err)
 			}
 		}
 	}
 
-	// handle all types that don't need an index
 	switch tpe {
 	case "config":
 		buf, err := json.MarshalIndent(repo.Config(), "", "  ")
@@ -139,15 +141,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 
 		Println(string(buf))
 		return nil
-	}
 
-	// load index, handle all the other types
-	err = repo.LoadIndex(gopts.ctx)
-	if err != nil {
-		return err
-	}
-
-	switch tpe {
 	case "pack":
 		h := restic.Handle{Type: restic.PackFile, Name: id.String()}
 		buf, err := backend.LoadAll(gopts.ctx, nil, repo.Backend(), h)
@@ -164,8 +158,14 @@ func runCat(gopts GlobalOptions, args []string) error {
 		return err
 
 	case "blob":
+		err = repo.LoadIndex(gopts.ctx)
+		if err != nil {
+			return err
+		}
+
 		for _, t := range []restic.BlobType{restic.DataBlob, restic.TreeBlob} {
-			if !repo.Index().Has(id, t) {
+			bh := restic.BlobHandle{ID: id, Type: t}
+			if !repo.Index().Has(bh) {
 				continue
 			}
 

cmd/restic/cmd_check.go

@@ -1,10 +1,11 @@
 package main
 
 import (
-	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/spf13/cobra"
 
@@ -53,25 +54,38 @@ func init() {
 
 	f := cmdCheck.Flags()
 	f.BoolVar(&checkOptions.ReadData, "read-data", false, "read all data blobs")
-	f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read subset n of m data packs (format: `n/m`)")
+	f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read a `subset` of data packs, specified as 'n/t' for specific subset or either 'x%' or 'x.y%' for random subset")
 	f.BoolVar(&checkOptions.CheckUnused, "check-unused", false, "find unused blobs")
 	f.BoolVar(&checkOptions.WithCache, "with-cache", false, "use the cache")
 }
 
 func checkFlags(opts CheckOptions) error {
 	if opts.ReadData && opts.ReadDataSubset != "" {
-		return errors.Fatalf("check flags --read-data and --read-data-subset cannot be used together")
+		return errors.Fatal("check flags --read-data and --read-data-subset cannot be used together")
 	}
 	if opts.ReadDataSubset != "" {
 		dataSubset, err := stringToIntSlice(opts.ReadDataSubset)
-		if err != nil || len(dataSubset) != 2 {
-			return errors.Fatalf("check flag --read-data-subset must have two positive integer values, e.g. --read-data-subset=1/2")
-		}
-		if dataSubset[0] == 0 || dataSubset[1] == 0 || dataSubset[0] > dataSubset[1] {
-			return errors.Fatalf("check flag --read-data-subset=n/t values must be positive integers, and n <= t, e.g. --read-data-subset=1/2")
-		}
-		if dataSubset[1] > totalBucketsMax {
-			return errors.Fatalf("check flag --read-data-subset=n/t t must be at most %d", totalBucketsMax)
+		argumentError := errors.Fatal("check flag --read-data-subset must have two positive integer values or a percentage, e.g. --read-data-subset=1/2 or --read-data-subset=2.5%%")
+		if err == nil {
+			if len(dataSubset) != 2 {
+				return argumentError
+			}
+			if dataSubset[0] == 0 || dataSubset[1] == 0 || dataSubset[0] > dataSubset[1] {
+				return errors.Fatal("check flag --read-data-subset=n/t values must be positive integers, and n <= t, e.g. --read-data-subset=1/2")
+			}
+			if dataSubset[1] > totalBucketsMax {
+				return errors.Fatalf("check flag --read-data-subset=n/t t must be at most %d", totalBucketsMax)
+			}
+		} else {
+			percentage, err := parsePercentage(opts.ReadDataSubset)
+			if err != nil {
+				return argumentError
+			}
+
+			if percentage <= 0.0 || percentage > 100.0 {
+				return errors.Fatal(
+					"check flag --read-data-subset=n% n must be above 0.0% and at most 100.0%")
+			}
 		}
 	}
 
@@ -98,6 +112,21 @@ func stringToIntSlice(param string) (split []uint, err error) {
 	return result, nil
 }
 
+// ParsePercentage parses a percentage string of the form "X%" where X is a float constant,
+// and returns the value of that constant. It does not check the range of the value.
+func parsePercentage(s string) (float64, error) {
+	if !strings.HasSuffix(s, "%") {
+		return 0, errors.Errorf(`parsePercentage: %q does not end in "%%"`, s)
+	}
+	s = s[:len(s)-1]
+
+	p, err := strconv.ParseFloat(s, 64)
+	if err != nil {
+		return 0, errors.Errorf("parsePercentage: %v", err)
+	}
+	return p, nil
+}
+
 // prepareCheckCache configures a special cache directory for check.
 //
 //  * if --with-cache is specified, the default cache is used
@@ -142,7 +171,7 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func())
 
 func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 	if len(args) != 0 {
-		return errors.Fatal("check has no arguments")
+		return errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags")
 	}
 
 	cleanup := prepareCheckCache(opts, &gopts)
@@ -158,14 +187,14 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 
 	if !gopts.NoLock {
 		Verbosef("create exclusive lock for repository\n")
-		lock, err := lockRepoExclusive(repo)
+		lock, err := lockRepoExclusive(gopts.ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
 		}
 	}
 
-	chkr := checker.New(repo)
+	chkr := checker.New(repo, opts.CheckUnused)
 
 	Verbosef("load indexes\n")
 	hints, errs := chkr.LoadIndex(gopts.ctx)
@@ -212,7 +241,11 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 
 	Verbosef("check snapshots, trees and blobs\n")
 	errChan = make(chan error)
-	go chkr.Structure(gopts.ctx, errChan)
+	go func() {
+		bar := newProgressMax(!gopts.Quiet, 0, "snapshots")
+		defer bar.Done()
+		chkr.Structure(gopts.ctx, bar, errChan)
+	}()
 
 	for err := range errChan {
 		errorsFound = true
@@ -227,29 +260,15 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	if opts.CheckUnused {
-		for _, id := range chkr.UnusedBlobs() {
+		for _, id := range chkr.UnusedBlobs(gopts.ctx) {
 			Verbosef("unused blob %v\n", id)
 			errorsFound = true
 		}
 	}
 
-	doReadData := func(bucket, totalBuckets uint) {
-		packs := restic.IDSet{}
-		for pack := range chkr.GetPacks() {
-			// If we ever check more than the first byte
-			// of pack, update totalBucketsMax.
-			if (uint(pack[0]) % totalBuckets) == (bucket - 1) {
-				packs.Insert(pack)
-			}
-		}
+	doReadData := func(packs map[restic.ID]int64) {
 		packCount := uint64(len(packs))
 
-		if packCount < chkr.CountPacks() {
-			Verbosef(fmt.Sprintf("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, packCount, chkr.CountPacks(), totalBuckets))
-		} else {
-			Verbosef("read all data\n")
-		}
-
 		p := newProgressMax(!gopts.Quiet, packCount, "packs")
 		errChan := make(chan error)
 
@@ -259,14 +278,31 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 			errorsFound = true
 			Warnf("%v\n", err)
 		}
+		p.Done()
 	}
 
 	switch {
 	case opts.ReadData:
-		doReadData(1, 1)
+		Verbosef("read all data\n")
+		doReadData(selectPacksByBucket(chkr.GetPacks(), 1, 1))
 	case opts.ReadDataSubset != "":
-		dataSubset, _ := stringToIntSlice(opts.ReadDataSubset)
-		doReadData(dataSubset[0], dataSubset[1])
+		var packs map[restic.ID]int64
+		dataSubset, err := stringToIntSlice(opts.ReadDataSubset)
+		if err == nil {
+			bucket := dataSubset[0]
+			totalBuckets := dataSubset[1]
+			packs = selectPacksByBucket(chkr.GetPacks(), bucket, totalBuckets)
+			packCount := uint64(len(packs))
+			Verbosef("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, packCount, chkr.CountPacks(), totalBuckets)
+		} else {
+			percentage, _ := parsePercentage(opts.ReadDataSubset)
+			packs = selectRandomPacksByPercentage(chkr.GetPacks(), percentage)
+			Verbosef("read %.1f%% of data packs\n", percentage)
+		}
+		if packs == nil {
+			return errors.Fatal("internal error: failed to select packs to check")
+		}
+		doReadData(packs)
 	}
 
 	if errorsFound {
@@ -277,3 +313,42 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 
 	return nil
 }
+
+// selectPacksByBucket selects subsets of packs by ranges of buckets.
+func selectPacksByBucket(allPacks map[restic.ID]int64, bucket, totalBuckets uint) map[restic.ID]int64 {
+	packs := make(map[restic.ID]int64)
+	for pack, size := range allPacks {
+		// If we ever check more than the first byte
+		// of pack, update totalBucketsMax.
+		if (uint(pack[0]) % totalBuckets) == (bucket - 1) {
+			packs[pack] = size
+		}
+	}
+	return packs
+}
+
+// selectRandomPacksByPercentage selects the given percentage of packs which are randomly choosen.
+func selectRandomPacksByPercentage(allPacks map[restic.ID]int64, percentage float64) map[restic.ID]int64 {
+	packCount := len(allPacks)
+	packsToCheck := int(float64(packCount) * (percentage / 100.0))
+	if packCount > 0 && packsToCheck < 1 {
+		packsToCheck = 1
+	}
+	timeNs := time.Now().UnixNano()
+	r := rand.New(rand.NewSource(timeNs))
+	idx := r.Perm(packCount)
+
+	var keys []restic.ID
+	for k := range allPacks {
+		keys = append(keys, k)
+	}
+
+	packs := make(map[restic.ID]int64)
+
+	for i := 0; i < packsToCheck; i++ {
+		id := keys[idx[i]]
+		packs[id] = allPacks[id]
+	}
+
+	return packs
+}

cmd/restic/cmd_check_test.go

@@ -0,0 +1,131 @@
+package main
+
+import (
+	"math"
+	"reflect"
+	"testing"
+
+	"github.com/restic/restic/internal/restic"
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func TestParsePercentage(t *testing.T) {
+	testCases := []struct {
+		input       string
+		output      float64
+		expectError bool
+	}{
+		{"0%", 0.0, false},
+		{"1%", 1.0, false},
+		{"100%", 100.0, false},
+		{"123%", 123.0, false},
+		{"123.456%", 123.456, false},
+		{"0.742%", 0.742, false},
+		{"-100%", -100.0, false},
+		{" 1%", 0.0, true},
+		{"1 %", 0.0, true},
+		{"1% ", 0.0, true},
+	}
+	for _, testCase := range testCases {
+		output, err := parsePercentage(testCase.input)
+
+		if testCase.expectError {
+			rtest.Assert(t, err != nil, "Expected error for case %s", testCase.input)
+			rtest.Assert(t, output == 0.0, "Expected output to be 0.0, got %s", output)
+		} else {
+			rtest.Assert(t, err == nil, "Expected no error for case %s", testCase.input)
+			rtest.Assert(t, math.Abs(testCase.output-output) < 0.00001, "Expected %f, got %f",
+				testCase.output, output)
+		}
+	}
+}
+
+func TestStringToIntSlice(t *testing.T) {
+	testCases := []struct {
+		input       string
+		output      []uint
+		expectError bool
+	}{
+		{"3/5", []uint{3, 5}, false},
+		{"1/100", []uint{1, 100}, false},
+		{"abc", nil, true},
+		{"1/a", nil, true},
+		{"/", nil, true},
+	}
+	for _, testCase := range testCases {
+		output, err := stringToIntSlice(testCase.input)
+
+		if testCase.expectError {
+			rtest.Assert(t, err != nil, "Expected error for case %s", testCase.input)
+			rtest.Assert(t, output == nil, "Expected output to be nil, got %s", output)
+		} else {
+			rtest.Assert(t, err == nil, "Expected no error for case %s", testCase.input)
+			rtest.Assert(t, len(output) == 2, "Invalid output length for case %s", testCase.input)
+			rtest.Assert(t, reflect.DeepEqual(output, testCase.output), "Expected %f, got %f",
+				testCase.output, output)
+		}
+	}
+}
+
+func TestSelectPacksByBucket(t *testing.T) {
+	var testPacks = make(map[restic.ID]int64)
+	for i := 1; i <= 10; i++ {
+		id := restic.NewRandomID()
+		// ensure relevant part of generated id is reproducable
+		id[0] = byte(i)
+		testPacks[id] = 0
+	}
+
+	selectedPacks := selectPacksByBucket(testPacks, 0, 10)
+	rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs")
+
+	for i := uint(1); i <= 5; i++ {
+		selectedPacks = selectPacksByBucket(testPacks, i, 5)
+		rtest.Assert(t, len(selectedPacks) == 2, "Expected 2 selected packs")
+	}
+
+	selectedPacks = selectPacksByBucket(testPacks, 1, 1)
+	rtest.Assert(t, len(selectedPacks) == 10, "Expected 10 selected packs")
+	for testPack := range testPacks {
+		_, ok := selectedPacks[testPack]
+		rtest.Assert(t, ok, "Expected input and output to be equal")
+	}
+}
+
+func TestSelectRandomPacksByPercentage(t *testing.T) {
+	var testPacks = make(map[restic.ID]int64)
+	for i := 1; i <= 10; i++ {
+		testPacks[restic.NewRandomID()] = 0
+	}
+
+	selectedPacks := selectRandomPacksByPercentage(testPacks, 0.0)
+	rtest.Assert(t, len(selectedPacks) == 1, "Expected 1 selected packs")
+
+	selectedPacks = selectRandomPacksByPercentage(testPacks, 10.0)
+	rtest.Assert(t, len(selectedPacks) == 1, "Expected 1 selected pack")
+	for pack := range selectedPacks {
+		_, ok := testPacks[pack]
+		rtest.Assert(t, ok, "Unexpected selection")
+	}
+
+	selectedPacks = selectRandomPacksByPercentage(testPacks, 50.0)
+	rtest.Assert(t, len(selectedPacks) == 5, "Expected 5 selected packs")
+	for pack := range selectedPacks {
+		_, ok := testPacks[pack]
+		rtest.Assert(t, ok, "Unexpected item in selection")
+	}
+
+	selectedPacks = selectRandomPacksByPercentage(testPacks, 100.0)
+	rtest.Assert(t, len(selectedPacks) == 10, "Expected 10 selected packs")
+	for testPack := range testPacks {
+		_, ok := selectedPacks[testPack]
+		rtest.Assert(t, ok, "Expected input and output to be equal")
+	}
+}
+
+func TestSelectNoRandomPacksByPercentage(t *testing.T) {
+	// that the a repository without pack files works
+	var testPacks = make(map[restic.ID]int64)
+	selectedPacks := selectRandomPacksByPercentage(testPacks, 10.0)
+	rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs")
+}

cmd/restic/cmd_copy.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/restic"
+	"golang.org/x/sync/errgroup"
 
 	"github.com/spf13/cobra"
 )
@@ -14,12 +15,19 @@ var cmdCopy = &cobra.Command{
 	Use:   "copy [flags] [snapshotID ...]",
 	Short: "Copy snapshots from one repository to another",
 	Long: `
-The "copy" command copies one or more snapshots from one repository to another
-repository. Note that this will have to read (download) and write (upload) the
-entire snapshot(s) due to the different encryption keys on the source and
-destination, and that transferred files are not re-chunked, which may break
-their deduplication. This can be mitigated by the "--copy-chunker-params"
-option when initializing a new destination repository using the "init" command.
+The "copy" command copies one or more snapshots from one repository to another.
+
+NOTE: This process will have to both download (read) and upload (write) the
+entire snapshot(s) due to the different encryption keys used in the source and
+destination repositories. This /may incur higher bandwidth usage and costs/ than
+expected during normal backup runs.
+
+NOTE: The copying process does not re-chunk files, which may break deduplication
+between the files copied and files already stored in the destination repository.
+This means that copied files, which existed in both the source and destination
+repository, /may occupy up to twice their space/ in the destination repository.
+This can be mitigated by the "--copy-chunker-params" option when initializing a
+new destination repository using the "init" command.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runCopy(copyOptions, globalOptions, args)
@@ -65,13 +73,13 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	srcLock, err := lockRepo(srcRepo)
+	srcLock, err := lockRepo(ctx, srcRepo)
 	defer unlockRepo(srcLock)
 	if err != nil {
 		return err
 	}
 
-	dstLock, err := lockRepo(dstRepo)
+	dstLock, err := lockRepo(ctx, dstRepo)
 	defer unlockRepo(dstLock)
 	if err != nil {
 		return err
@@ -96,12 +104,8 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 		dstSnapshotByOriginal[*sn.ID()] = append(dstSnapshotByOriginal[*sn.ID()], sn)
 	}
 
-	cloner := &treeCloner{
-		srcRepo:      srcRepo,
-		dstRepo:      dstRepo,
-		visitedTrees: restic.NewIDSet(),
-		buf:          nil,
-	}
+	// remember already processed trees across all snapshots
+	visitedTrees := restic.NewIDSet()
 
 	for sn := range FindFilteredSnapshots(ctx, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) {
 		Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
@@ -126,7 +130,7 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 		}
 		Verbosef("  copy started, this may take a while...\n")
 
-		if err := cloner.copyTree(ctx, *sn.Tree); err != nil {
+		if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree); err != nil {
 			return err
 		}
 		debug.Log("tree copied")
@@ -170,64 +174,67 @@ func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool {
 	return true
 }
 
-type treeCloner struct {
-	srcRepo      restic.Repository
-	dstRepo      restic.Repository
-	visitedTrees restic.IDSet
-	buf          []byte
-}
+func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository,
+	visitedTrees restic.IDSet, rootTreeID restic.ID) error {
 
-func (t *treeCloner) copyTree(ctx context.Context, treeID restic.ID) error {
-	// We have already processed this tree
-	if t.visitedTrees.Has(treeID) {
+	wg, ctx := errgroup.WithContext(ctx)
+
+	treeStream := restic.StreamTrees(ctx, wg, srcRepo, restic.IDs{rootTreeID}, func(treeID restic.ID) bool {
+		visited := visitedTrees.Has(treeID)
+		visitedTrees.Insert(treeID)
+		return visited
+	}, nil)
+
+	wg.Go(func() error {
+		// reused buffer
+		var buf []byte
+
+		for tree := range treeStream {
+			if tree.Error != nil {
+				return fmt.Errorf("LoadTree(%v) returned error %v", tree.ID.Str(), tree.Error)
+			}
+
+			// Do we already have this tree blob?
+			if !dstRepo.Index().Has(restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob}) {
+				// copy raw tree bytes to avoid problems if the serialization changes
+				var err error
+				buf, err = srcRepo.LoadBlob(ctx, restic.TreeBlob, tree.ID, buf)
+				if err != nil {
+					return fmt.Errorf("LoadBlob(%v) for tree returned error %v", tree.ID, err)
+				}
+
+				_, _, err = dstRepo.SaveBlob(ctx, restic.TreeBlob, buf, tree.ID, false)
+				if err != nil {
+					return fmt.Errorf("SaveBlob(%v) for tree returned error %v", tree.ID.Str(), err)
+				}
+			}
+
+			// TODO: parallelize blob down/upload
+
+			for _, entry := range tree.Nodes {
+				// Recursion into directories is handled by StreamTrees
+				// Copy the blobs for this file.
+				for _, blobID := range entry.Content {
+					// Do we already have this data blob?
+					if dstRepo.Index().Has(restic.BlobHandle{ID: blobID, Type: restic.DataBlob}) {
+						continue
+					}
+					debug.Log("Copying blob %s\n", blobID.Str())
+					var err error
+					buf, err = srcRepo.LoadBlob(ctx, restic.DataBlob, blobID, buf)
+					if err != nil {
+						return fmt.Errorf("LoadBlob(%v) returned error %v", blobID, err)
+					}
+
+					_, _, err = dstRepo.SaveBlob(ctx, restic.DataBlob, buf, blobID, false)
+					if err != nil {
+						return fmt.Errorf("SaveBlob(%v) returned error %v", blobID, err)
+					}
+				}
+			}
+
+		}
 		return nil
-	}
-
-	tree, err := t.srcRepo.LoadTree(ctx, treeID)
-	if err != nil {
-		return fmt.Errorf("LoadTree(%v) returned error %v", treeID.Str(), err)
-	}
-	t.visitedTrees.Insert(treeID)
-
-	// Do we already have this tree blob?
-	if !t.dstRepo.Index().Has(treeID, restic.TreeBlob) {
-		newTreeID, err := t.dstRepo.SaveTree(ctx, tree)
-		if err != nil {
-			return fmt.Errorf("SaveTree(%v) returned error %v", treeID.Str(), err)
-		}
-		// Assurance only.
-		if newTreeID != treeID {
-			return fmt.Errorf("SaveTree(%v) returned unexpected id %s", treeID.Str(), newTreeID.Str())
-		}
-	}
-
-	// TODO: parellize this stuff, likely only needed inside a tree.
-
-	for _, entry := range tree.Nodes {
-		// If it is a directory, recurse
-		if entry.Type == "dir" && entry.Subtree != nil {
-			if err := t.copyTree(ctx, *entry.Subtree); err != nil {
-				return err
-			}
-		}
-		// Copy the blobs for this file.
-		for _, blobID := range entry.Content {
-			// Do we already have this data blob?
-			if t.dstRepo.Index().Has(blobID, restic.DataBlob) {
-				continue
-			}
-			debug.Log("Copying blob %s\n", blobID.Str())
-			t.buf, err = t.srcRepo.LoadBlob(ctx, restic.DataBlob, blobID, t.buf)
-			if err != nil {
-				return fmt.Errorf("LoadBlob(%v) returned error %v", blobID, err)
-			}
-
-			_, _, err = t.dstRepo.SaveBlob(ctx, restic.DataBlob, t.buf, blobID, false)
-			if err != nil {
-				return fmt.Errorf("SaveBlob(%v) returned error %v", blobID, err)
-			}
-		}
-	}
-
-	return nil
+	})
+	return wg.Wait()
 }

cmd/restic/cmd_debug.go

@@ -4,12 +4,21 @@ package main
 
 import (
 	"context"
+	"crypto/aes"
+	"crypto/cipher"
 	"encoding/json"
 	"fmt"
 	"io"
+	"os"
+	"runtime"
+	"sort"
+	"time"
 
 	"github.com/spf13/cobra"
+	"golang.org/x/sync/errgroup"
 
+	"github.com/restic/restic/internal/backend"
+	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/pack"
 	"github.com/restic/restic/internal/repository"
@@ -39,9 +48,17 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 	},
 }
 
+var tryRepair bool
+var repairByte bool
+var extractPack bool
+
 func init() {
 	cmdRoot.AddCommand(cmdDebug)
 	cmdDebug.AddCommand(cmdDebugDump)
+	cmdDebug.AddCommand(cmdDebugExamine)
+	cmdDebugExamine.Flags().BoolVar(&extractPack, "extract-pack", false, "write blobs to the current directory")
+	cmdDebugExamine.Flags().BoolVar(&tryRepair, "try-repair", false, "try to repair broken blobs with single bit flips")
+	cmdDebugExamine.Flags().BoolVar(&repairByte, "repair-byte", false, "try to repair broken blobs by trying bytes")
 }
 
 func prettyPrintJSON(wr io.Writer, item interface{}) error {
@@ -54,9 +71,8 @@ func prettyPrintJSON(wr io.Writer, item interface{}) error {
 	return err
 }
 
-func debugPrintSnapshots(repo *repository.Repository, wr io.Writer) error {
-	return repo.List(context.TODO(), restic.SnapshotFile, func(id restic.ID, size int64) error {
-		snapshot, err := restic.LoadSnapshot(context.TODO(), repo, id)
+func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
+	return restic.ForAllSnapshots(ctx, repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error {
 		if err != nil {
 			return err
 		}
@@ -82,12 +98,12 @@ type Blob struct {
 	Offset uint            `json:"offset"`
 }
 
-func printPacks(repo *repository.Repository, wr io.Writer) error {
+func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
 
-	return repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
+	return repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
 		h := restic.Handle{Type: restic.PackFile, Name: id.String()}
 
-		blobs, err := pack.List(repo.Key(), restic.ReaderAt(repo.Backend(), h), size)
+		blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), size)
 		if err != nil {
 			Warnf("error for pack %v: %v\n", id.Str(), err)
 			return nil
@@ -110,11 +126,9 @@ func printPacks(repo *repository.Repository, wr io.Writer) error {
 	})
 }
 
-func dumpIndexes(repo restic.Repository, wr io.Writer) error {
-	return repo.List(context.TODO(), restic.IndexFile, func(id restic.ID, size int64) error {
+func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error {
+	return repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
 		Printf("index_id: %v\n", id)
-
-		idx, err := repository.LoadIndex(context.TODO(), repo, id)
 		if err != nil {
 			return err
 		}
@@ -134,7 +148,7 @@ func runDebugDump(gopts GlobalOptions, args []string) error {
 	}
 
 	if !gopts.NoLock {
-		lock, err := lockRepo(repo)
+		lock, err := lockRepo(gopts.ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -145,20 +159,20 @@ func runDebugDump(gopts GlobalOptions, args []string) error {
 
 	switch tpe {
 	case "indexes":
-		return dumpIndexes(repo, gopts.stdout)
+		return dumpIndexes(gopts.ctx, repo, gopts.stdout)
 	case "snapshots":
-		return debugPrintSnapshots(repo, gopts.stdout)
+		return debugPrintSnapshots(gopts.ctx, repo, gopts.stdout)
 	case "packs":
-		return printPacks(repo, gopts.stdout)
+		return printPacks(gopts.ctx, repo, gopts.stdout)
 	case "all":
 		Printf("snapshots:\n")
-		err := debugPrintSnapshots(repo, gopts.stdout)
+		err := debugPrintSnapshots(gopts.ctx, repo, gopts.stdout)
 		if err != nil {
 			return err
 		}
 
 		Printf("\nindexes:\n")
-		err = dumpIndexes(repo, gopts.stdout)
+		err = dumpIndexes(gopts.ctx, repo, gopts.stdout)
 		if err != nil {
 			return err
 		}
@@ -168,3 +182,367 @@ func runDebugDump(gopts GlobalOptions, args []string) error {
 		return errors.Fatalf("no such type %q", tpe)
 	}
 }
+
+var cmdDebugExamine = &cobra.Command{
+	Use:               "examine pack-ID...",
+	Short:             "Examine a pack file",
+	DisableAutoGenTag: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runDebugExamine(globalOptions, args)
+	},
+}
+
+func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, bytewise bool) []byte {
+	if bytewise {
+		Printf("        trying to repair blob by finding a broken byte\n")
+	} else {
+		Printf("        trying to repair blob with single bit flip\n")
+	}
+
+	ch := make(chan int)
+	var wg errgroup.Group
+	done := make(chan struct{})
+	var fixed []byte
+	var found bool
+
+	workers := runtime.GOMAXPROCS(0)
+	Printf("         spinning up %d worker functions\n", runtime.GOMAXPROCS(0))
+	for i := 0; i < workers; i++ {
+		wg.Go(func() error {
+			// make a local copy of the buffer
+			buf := make([]byte, len(input))
+			copy(buf, input)
+
+			testFlip := func(idx int, pattern byte) bool {
+				// flip bits
+				buf[idx] ^= pattern
+
+				nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
+				plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil)
+				if err == nil {
+					Printf("\n")
+					Printf("        blob could be repaired by XORing byte %v with 0x%02x\n", idx, pattern)
+					Printf("        hash is %v\n", restic.Hash(plaintext))
+					close(done)
+					found = true
+					fixed = plaintext
+					return true
+				}
+
+				// flip bits back
+				buf[idx] ^= pattern
+				return false
+			}
+
+			for i := range ch {
+				if bytewise {
+					for j := 0; j < 255; j++ {
+						if testFlip(i, byte(j)) {
+							return nil
+						}
+					}
+				} else {
+					for j := 0; j < 7; j++ {
+						// flip each bit once
+						if testFlip(i, (1 << uint(j))) {
+							return nil
+						}
+					}
+				}
+			}
+			return nil
+		})
+	}
+
+	wg.Go(func() error {
+		defer close(ch)
+
+		start := time.Now()
+		info := time.Now()
+		for i := range input {
+			select {
+			case ch <- i:
+			case <-done:
+				Printf("     done after %v\n", time.Since(start))
+				return nil
+			}
+
+			if time.Since(info) > time.Second {
+				secs := time.Since(start).Seconds()
+				gps := float64(i) / secs
+				remaining := len(input) - i
+				eta := time.Duration(float64(remaining)/gps) * time.Second
+
+				Printf("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v",
+					i, len(input), float32(i)/float32(len(input))*100, gps, eta)
+				info = time.Now()
+			}
+		}
+		return nil
+	})
+	err := wg.Wait()
+	if err != nil {
+		panic("all go rountines can only return nil")
+	}
+
+	if !found {
+		Printf("\n        blob could not be repaired\n")
+	}
+	return fixed
+}
+
+func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte {
+	// strip signature at the end
+	l := len(buf)
+	nonce, ct := buf[:16], buf[16:l-16]
+	out := make([]byte, len(ct))
+
+	c, err := aes.NewCipher(k.EncryptionKey[:])
+	if err != nil {
+		panic(fmt.Sprintf("unable to create cipher: %v", err))
+	}
+	e := cipher.NewCTR(c, nonce)
+	e.XORKeyStream(out, ct)
+
+	return out
+}
+
+func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list []restic.Blob) error {
+	be := repo.Backend()
+	h := restic.Handle{
+		Name: pack.String(),
+		Type: restic.PackFile,
+	}
+	for _, blob := range list {
+		Printf("      loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length)
+		buf := make([]byte, blob.Length)
+		err := be.Load(ctx, h, int(blob.Length), int64(blob.Offset), func(rd io.Reader) error {
+			n, err := io.ReadFull(rd, buf)
+			if err != nil {
+				return fmt.Errorf("read error after %d bytes: %v", n, err)
+			}
+			return nil
+		})
+		if err != nil {
+			Warnf("error read: %v\n", err)
+			continue
+		}
+
+		key := repo.Key()
+
+		nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
+		plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
+		if err != nil {
+			Warnf("error decrypting blob: %v\n", err)
+			var plain []byte
+			if tryRepair || repairByte {
+				plain = tryRepairWithBitflip(ctx, key, buf, repairByte)
+			}
+			var prefix string
+			if plain != nil {
+				id := restic.Hash(plain)
+				if !id.Equal(blob.ID) {
+					Printf("         repaired blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plain), id, blob.ID)
+					prefix = "repaired-wrong-hash-"
+				} else {
+					Printf("         successfully repaired blob (length %v), hash is %v, ID matches\n", len(plain), id)
+					prefix = "repaired-"
+				}
+			} else {
+				plain = decryptUnsigned(ctx, key, buf)
+				prefix = "damaged-"
+			}
+			err = storePlainBlob(blob.ID, prefix, plain)
+			if err != nil {
+				return err
+			}
+			continue
+		}
+
+		id := restic.Hash(plaintext)
+		var prefix string
+		if !id.Equal(blob.ID) {
+			Printf("         successfully decrypted blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plaintext), id, blob.ID)
+			prefix = "wrong-hash-"
+		} else {
+			Printf("         successfully decrypted blob (length %v), hash is %v, ID matches\n", len(plaintext), id)
+			prefix = "correct-"
+		}
+		if extractPack {
+			err = storePlainBlob(id, prefix, plaintext)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func storePlainBlob(id restic.ID, prefix string, plain []byte) error {
+	filename := fmt.Sprintf("%s%s.bin", prefix, id)
+	f, err := os.Create(filename)
+	if err != nil {
+		return err
+	}
+
+	_, err = f.Write(plain)
+	if err != nil {
+		_ = f.Close()
+		return err
+	}
+
+	err = f.Close()
+	if err != nil {
+		return err
+	}
+
+	Printf("decrypt of blob %v stored at %v\n", id, filename)
+	return nil
+}
+
+func runDebugExamine(gopts GlobalOptions, args []string) error {
+	ids := make([]restic.ID, 0)
+	for _, name := range args {
+		id, err := restic.ParseID(name)
+		if err != nil {
+			Warnf("error: %v\n", err)
+			continue
+		}
+		ids = append(ids, id)
+	}
+
+	if len(ids) == 0 {
+		return errors.Fatal("no pack files to examine")
+	}
+
+	repo, err := OpenRepository(gopts)
+	if err != nil {
+		return err
+	}
+
+	if !gopts.NoLock {
+		lock, err := lockRepo(gopts.ctx, repo)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	}
+
+	err = repo.LoadIndex(gopts.ctx)
+	if err != nil {
+		return err
+	}
+
+	for _, id := range ids {
+		err := examinePack(gopts.ctx, repo, id)
+		if err != nil {
+			Warnf("error: %v\n", err)
+		}
+		if err == context.Canceled {
+			break
+		}
+	}
+	return nil
+}
+
+func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) error {
+	Printf("examine %v\n", id)
+
+	h := restic.Handle{
+		Type: restic.PackFile,
+		Name: id.String(),
+	}
+	fi, err := repo.Backend().Stat(ctx, h)
+	if err != nil {
+		return err
+	}
+	Printf("  file size is %v\n", fi.Size)
+
+	buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h)
+	if err != nil {
+		return err
+	}
+	gotID := restic.Hash(buf)
+	if !id.Equal(gotID) {
+		Printf("  wanted hash %v, got %v\n", id, gotID)
+	} else {
+		Printf("  hash for file content matches\n")
+	}
+
+	Printf("  ========================================\n")
+	Printf("  looking for info in the indexes\n")
+
+	blobsLoaded := false
+	// examine all data the indexes have for the pack file
+	for _, idx := range repo.Index().(*repository.MasterIndex).All() {
+		idxIDs, err := idx.IDs()
+		if err != nil {
+			idxIDs = restic.IDs{}
+		}
+
+		blobs := idx.ListPack(id)
+		if len(blobs) == 0 {
+			continue
+		}
+
+		Printf("    index %v:\n", idxIDs)
+
+		// convert list of blobs to []restic.Blob
+		var list []restic.Blob
+		for _, b := range blobs {
+			list = append(list, b.Blob)
+		}
+		checkPackSize(list, fi.Size)
+
+		err = loadBlobs(ctx, repo, id, list)
+		if err != nil {
+			Warnf("error: %v\n", err)
+		} else {
+			blobsLoaded = true
+		}
+	}
+
+	Printf("  ========================================\n")
+	Printf("  inspect the pack itself\n")
+
+	blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), fi.Size)
+	if err != nil {
+		return fmt.Errorf("pack %v: %v", id.Str(), err)
+	}
+	checkPackSize(blobs, fi.Size)
+
+	if !blobsLoaded {
+		return loadBlobs(ctx, repo, id, blobs)
+	}
+	return nil
+}
+
+func checkPackSize(blobs []restic.Blob, fileSize int64) {
+	// track current size and offset
+	var size, offset uint64
+
+	sort.Slice(blobs, func(i, j int) bool {
+		return blobs[i].Offset < blobs[j].Offset
+	})
+
+	for _, pb := range blobs {
+		Printf("      %v blob %v, offset %-6d, raw length %-6d\n", pb.Type, pb.ID, pb.Offset, pb.Length)
+		if offset != uint64(pb.Offset) {
+			Printf("      hole in file, want offset %v, got %v\n", offset, pb.Offset)
+		}
+		offset += uint64(pb.Length)
+		size += uint64(pb.Length)
+	}
+
+	// compute header size, per blob: 1 byte type, 4 byte length, 32 byte id
+	size += uint64(restic.CiphertextLength(len(blobs) * (1 + 4 + 32)))
+	// length in uint32 little endian
+	size += 4
+
+	if uint64(fileSize) != size {
+		Printf("      file sizes do not match: computed %v from index, file size is %v\n", size, fileSize)
+	} else {
+		Printf("      file sizes match\n")
+	}
+}

cmd/restic/cmd_diff.go

@@ -53,11 +53,10 @@ func init() {
 }
 
 func loadSnapshot(ctx context.Context, repo *repository.Repository, desc string) (*restic.Snapshot, error) {
-	id, err := restic.FindSnapshot(repo, desc)
+	id, err := restic.FindSnapshot(ctx, repo, desc)
 	if err != nil {
-		return nil, err
+		return nil, errors.Fatal(err.Error())
 	}
-
 	return restic.LoadSnapshot(ctx, repo, id)
 }
 
@@ -332,7 +331,7 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	if !gopts.NoLock {
-		lock, err := lockRepo(repo)
+		lock, err := lockRepo(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -365,6 +364,8 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	stats := NewDiffStats()
+	stats.BlobsBefore.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn1.Tree})
+	stats.BlobsAfter.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn2.Tree})
 
 	err = c.diffTree(ctx, stats, "/", *sn1.Tree, *sn2.Tree)
 	if err != nil {

cmd/restic/cmd_dump.go

@@ -21,8 +21,8 @@ var cmdDump = &cobra.Command{
 	Long: `
 The "dump" command extracts files from a snapshot from the repository. If a
 single file is selected, it prints its contents to stdout. Folders are output
-as a tar file containing the contents of the specified folder.  Pass "/" as
-file name to dump the whole snapshot as a tar file.
+as a tar (default) or zip file containing the contents of the specified folder.
+Pass "/" as file name to dump the whole snapshot as an archive file.
 
 The special snapshot "latest" can be used to use the latest snapshot in the
 repository.
@@ -40,9 +40,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 
 // DumpOptions collects all options for the dump command.
 type DumpOptions struct {
-	Hosts []string
-	Paths []string
-	Tags  restic.TagLists
+	Hosts   []string
+	Paths   []string
+	Tags    restic.TagLists
+	Archive string
 }
 
 var dumpOptions DumpOptions
@@ -54,6 +55,7 @@ func init() {
 	flags.StringArrayVarP(&dumpOptions.Hosts, "host", "H", nil, `only consider snapshots for this host when the snapshot ID is "latest" (can be specified multiple times)`)
 	flags.Var(&dumpOptions.Tags, "tag", "only consider snapshots which include this `taglist` for snapshot ID \"latest\"")
 	flags.StringArrayVar(&dumpOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
+	flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"")
 }
 
 func splitPath(p string) []string {
@@ -65,8 +67,7 @@ func splitPath(p string) []string {
 	return append(s, f)
 }
 
-func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string) error {
-
+func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string, writeDump dump.WriteDump) error {
 	if tree == nil {
 		return fmt.Errorf("called with a nil tree")
 	}
@@ -81,10 +82,10 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
 	// If we print / we need to assume that there are multiple nodes at that
 	// level in the tree.
 	if pathComponents[0] == "" {
-		if err := checkStdoutTar(); err != nil {
+		if err := checkStdoutArchive(); err != nil {
 			return err
 		}
-		return dump.WriteTar(ctx, repo, tree, "/", os.Stdout)
+		return writeDump(ctx, repo, tree, "/", os.Stdout)
 	}
 
 	item := filepath.Join(prefix, pathComponents[0])
@@ -100,16 +101,16 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
 				if err != nil {
 					return errors.Wrapf(err, "cannot load subtree for %q", item)
 				}
-				return printFromTree(ctx, subtree, repo, item, pathComponents[1:])
+				return printFromTree(ctx, subtree, repo, item, pathComponents[1:], writeDump)
 			case dump.IsDir(node):
-				if err := checkStdoutTar(); err != nil {
+				if err := checkStdoutArchive(); err != nil {
 					return err
 				}
 				subtree, err := repo.LoadTree(ctx, *node.Subtree)
 				if err != nil {
 					return err
 				}
-				return dump.WriteTar(ctx, repo, subtree, item, os.Stdout)
+				return writeDump(ctx, repo, subtree, item, os.Stdout)
 			case l > 1:
 				return fmt.Errorf("%q should be a dir, but is a %q", item, node.Type)
 			case !dump.IsFile(node):
@@ -127,6 +128,16 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 		return errors.Fatal("no file and no snapshot ID specified")
 	}
 
+	var wd dump.WriteDump
+	switch opts.Archive {
+	case "tar":
+		wd = dump.WriteTar
+	case "zip":
+		wd = dump.WriteZip
+	default:
+		return fmt.Errorf("unknown archive format %q", opts.Archive)
+	}
+
 	snapshotIDString := args[0]
 	pathToPrint := args[1]
 
@@ -140,7 +151,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	if !gopts.NoLock {
-		lock, err := lockRepo(repo)
+		lock, err := lockRepo(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -160,7 +171,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Hosts:%v", err, opts.Paths, opts.Hosts)
 		}
 	} else {
-		id, err = restic.FindSnapshot(repo, snapshotIDString)
+		id, err = restic.FindSnapshot(ctx, repo, snapshotIDString)
 		if err != nil {
 			Exitf(1, "invalid id %q: %v", snapshotIDString, err)
 		}
@@ -176,7 +187,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 		Exitf(2, "loading tree for snapshot %q failed: %v", snapshotIDString, err)
 	}
 
-	err = printFromTree(ctx, tree, repo, "/", splittedPath)
+	err = printFromTree(ctx, tree, repo, "/", splittedPath, wd)
 	if err != nil {
 		Exitf(2, "cannot dump file: %v", err)
 	}
@@ -184,7 +195,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 	return nil
 }
 
-func checkStdoutTar() error {
+func checkStdoutArchive() error {
 	if stdoutIsTerminal() {
 		return fmt.Errorf("stdout is the terminal, please redirect output")
 	}

cmd/restic/cmd_find.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"sort"
 	"strings"
 	"time"
 
@@ -40,8 +41,6 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 	},
 }
 
-const shortStr = 8 // Length of short IDs: 4 bytes as hex strings
-
 // FindOptions bundles all options for the find command.
 type FindOptions struct {
 	Oldest             string
@@ -386,15 +385,14 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 				idStr := id.String()
 				if _, ok := f.blobIDs[idStr]; !ok {
 					// Look for short ID form
-					if _, ok := f.blobIDs[idStr[:shortStr]]; !ok {
+					if _, ok := f.blobIDs[id.Str()]; !ok {
 						continue
 					}
 					// Replace the short ID with the long one
 					f.blobIDs[idStr] = struct{}{}
-					delete(f.blobIDs, idStr[:shortStr])
+					delete(f.blobIDs, id.Str())
 				}
 				f.out.PrintObject("blob", idStr, nodepath, parentTreeID.String(), sn)
-				break
 			}
 		}
 
@@ -402,6 +400,8 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 	})
 }
 
+var errAllPacksFound = errors.New("all packs found")
+
 // packsToBlobs converts the list of pack IDs to a list of blob IDs that
 // belong to those packs.
 func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
@@ -413,20 +413,18 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
 		f.blobIDs = make(map[string]struct{})
 	}
 
-	allPacksFound := false
-	packsFound := 0
-
 	debug.Log("Looking for packs...")
 	err := f.repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
-		if allPacksFound {
-			return nil
-		}
 		idStr := id.String()
 		if _, ok := packIDs[idStr]; !ok {
 			// Look for short ID form
-			if _, ok := packIDs[idStr[:shortStr]]; !ok {
+			if _, ok := packIDs[id.Str()]; !ok {
 				return nil
 			}
+			delete(packIDs, id.Str())
+		} else {
+			// forget found id
+			delete(packIDs, idStr)
 		}
 		debug.Log("Found pack %s", idStr)
 		blobs, _, err := f.repo.ListPack(ctx, id, size)
@@ -437,25 +435,75 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
 			f.blobIDs[b.ID.String()] = struct{}{}
 		}
 		// Stop searching when all packs have been found
-		packsFound++
-		if packsFound >= len(packIDs) {
-			allPacksFound = true
+		if len(packIDs) == 0 {
+			return errAllPacksFound
 		}
 		return nil
 	})
 
-	if err != nil {
+	if err != nil && err != errAllPacksFound {
 		return err
 	}
 
-	if !allPacksFound {
-		return errors.Fatal("unable to find all specified pack(s)")
+	if err != errAllPacksFound {
+		// try to resolve unknown pack ids from the index
+		packIDs = f.indexPacksToBlobs(ctx, packIDs)
+	}
+
+	if len(packIDs) > 0 {
+		list := make([]string, 0, len(packIDs))
+		for h := range packIDs {
+			list = append(list, h)
+		}
+
+		sort.Strings(list)
+		return errors.Fatalf("unable to find pack(s): %v", list)
 	}
 
 	debug.Log("%d blobs found", len(f.blobIDs))
 	return nil
 }
 
+func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struct{}) map[string]struct{} {
+	wctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	// remember which packs were found in the index
+	indexPackIDs := make(map[string]struct{})
+	for pb := range f.repo.Index().Each(wctx) {
+		idStr := pb.PackID.String()
+		// keep entry in packIDs as Each() returns individual index entries
+		matchingID := false
+		if _, ok := packIDs[idStr]; ok {
+			matchingID = true
+		} else {
+			if _, ok := packIDs[pb.PackID.Str()]; ok {
+				// expand id
+				delete(packIDs, pb.PackID.Str())
+				packIDs[idStr] = struct{}{}
+				matchingID = true
+			}
+		}
+		if matchingID {
+			f.blobIDs[pb.ID.String()] = struct{}{}
+			indexPackIDs[idStr] = struct{}{}
+		}
+	}
+
+	for id := range indexPackIDs {
+		delete(packIDs, id)
+	}
+
+	if len(indexPackIDs) > 0 {
+		list := make([]string, 0, len(indexPackIDs))
+		for h := range indexPackIDs {
+			list = append(list, h)
+		}
+		Warnf("some pack files are missing from the repository, getting their blobs from the repository index: %v\n\n", list)
+	}
+	return packIDs
+}
+
 func (f *Finder) findObjectPack(ctx context.Context, id string, t restic.BlobType) {
 	idx := f.repo.Index()
 
@@ -465,7 +513,7 @@ func (f *Finder) findObjectPack(ctx context.Context, id string, t restic.BlobTyp
 		return
 	}
 
-	blobs := idx.Lookup(rid, t)
+	blobs := idx.Lookup(restic.BlobHandle{ID: rid, Type: t})
 	if len(blobs) == 0 {
 		Printf("Object %s not found in the index\n", rid.Str())
 		return
@@ -529,7 +577,7 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	if !gopts.NoLock {
-		lock, err := lockRepo(repo)
+		lock, err := lockRepo(gopts.ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -564,7 +612,10 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	if opts.PackID {
-		f.packsToBlobs(ctx, []string{f.pat.pattern[0]}) // TODO: support multiple packs
+		err := f.packsToBlobs(ctx, f.pat.pattern)
+		if err != nil {
+			return err
+		}
 	}
 
 	for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, opts.Snapshots) {

cmd/restic/cmd_forget.go

@@ -31,14 +31,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 
 // ForgetOptions collects all options for the forget command.
 type ForgetOptions struct {
-	Last     int
-	Hourly   int
-	Daily    int
-	Weekly   int
-	Monthly  int
-	Yearly   int
-	Within   restic.Duration
-	KeepTags restic.TagLists
+	Last          int
+	Hourly        int
+	Daily         int
+	Weekly        int
+	Monthly       int
+	Yearly        int
+	Within        restic.Duration
+	WithinHourly  restic.Duration
+	WithinDaily   restic.Duration
+	WithinWeekly  restic.Duration
+	WithinMonthly restic.Duration
+	WithinYearly  restic.Duration
+	KeepTags      restic.TagLists
 
 	Hosts   []string
 	Tags    restic.TagLists
@@ -64,31 +69,46 @@ func init() {
 	f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots")
 	f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots")
 	f.VarP(&forgetOptions.Within, "keep-within", "", "keep snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
+	f.VarP(&forgetOptions.WithinHourly, "keep-within-hourly", "", "keep hourly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
+	f.VarP(&forgetOptions.WithinDaily, "keep-within-daily", "", "keep daily snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
+	f.VarP(&forgetOptions.WithinWeekly, "keep-within-weekly", "", "keep weekly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
+	f.VarP(&forgetOptions.WithinMonthly, "keep-within-monthly", "", "keep monthly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
+	f.VarP(&forgetOptions.WithinYearly, "keep-within-yearly", "", "keep yearly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
 
 	f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)")
 	f.StringArrayVar(&forgetOptions.Hosts, "host", nil, "only consider snapshots with the given `host` (can be specified multiple times)")
 	f.StringArrayVar(&forgetOptions.Hosts, "hostname", nil, "only consider snapshots with the given `hostname` (can be specified multiple times)")
-	f.MarkDeprecated("hostname", "use --host")
+	err := f.MarkDeprecated("hostname", "use --host")
+	if err != nil {
+		// MarkDeprecated only returns an error when the flag is not found
+		panic(err)
+	}
 
 	f.Var(&forgetOptions.Tags, "tag", "only consider snapshots which include this `taglist` in the format `tag[,tag,...]` (can be specified multiple times)")
 
 	f.StringArrayVar(&forgetOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` (can be specified multiple times)")
-	f.BoolVarP(&forgetOptions.Compact, "compact", "c", false, "use compact format")
+	f.BoolVarP(&forgetOptions.Compact, "compact", "c", false, "use compact output format")
 
 	f.StringVarP(&forgetOptions.GroupBy, "group-by", "g", "host,paths", "string for grouping snapshots by host,paths,tags")
 	f.BoolVarP(&forgetOptions.DryRun, "dry-run", "n", false, "do not delete anything, just print what would be done")
 	f.BoolVar(&forgetOptions.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed")
 
 	f.SortFlags = false
+	addPruneOptions(cmdForget)
 }
 
 func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
+	err := verifyPruneOptions(&pruneOptions)
+	if err != nil {
+		return err
+	}
+
 	repo, err := OpenRepository(gopts)
 	if err != nil {
 		return err
 	}
 
-	lock, err := lockRepoExclusive(repo)
+	lock, err := lockRepoExclusive(gopts.ctx, repo)
 	defer unlockRepo(lock)
 	if err != nil {
 		return err
@@ -118,14 +138,19 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		}
 
 		policy := restic.ExpirePolicy{
-			Last:    opts.Last,
-			Hourly:  opts.Hourly,
-			Daily:   opts.Daily,
-			Weekly:  opts.Weekly,
-			Monthly: opts.Monthly,
-			Yearly:  opts.Yearly,
-			Within:  opts.Within,
-			Tags:    opts.KeepTags,
+			Last:          opts.Last,
+			Hourly:        opts.Hourly,
+			Daily:         opts.Daily,
+			Weekly:        opts.Weekly,
+			Monthly:       opts.Monthly,
+			Yearly:        opts.Yearly,
+			Within:        opts.Within,
+			WithinHourly:  opts.WithinHourly,
+			WithinDaily:   opts.WithinDaily,
+			WithinWeekly:  opts.WithinWeekly,
+			WithinMonthly: opts.WithinMonthly,
+			WithinYearly:  opts.WithinYearly,
+			Tags:          opts.KeepTags,
 		}
 
 		if policy.Empty() && len(args) == 0 {
@@ -204,8 +229,12 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	if len(removeSnIDs) > 0 && opts.Prune && !opts.DryRun {
-		return pruneRepository(gopts, repo)
+	if len(removeSnIDs) > 0 && opts.Prune {
+		if !gopts.JSON {
+			Verbosef("%d snapshots have been removed, running prune\n", len(removeSnIDs))
+		}
+		pruneOptions.DryRun = opts.DryRun
+		return runPruneWithRepo(pruneOptions, gopts, repo, removeSnIDs)
 	}
 
 	return nil

cmd/restic/cmd_generate.go

@@ -10,10 +10,10 @@ import (
 
 var cmdGenerate = &cobra.Command{
 	Use:   "generate [flags]",
-	Short: "Generate manual pages and auto-completion files (bash, zsh)",
+	Short: "Generate manual pages and auto-completion files (bash, fish, zsh)",
 	Long: `
 The "generate" command writes automatically generated files (like the man pages
-and the auto-completion files for bash and zsh).
+and the auto-completion files for bash, fish and zsh).
 
 EXIT STATUS
 ===========
@@ -27,6 +27,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 type generateOptions struct {
 	ManDir             string
 	BashCompletionFile string
+	FishCompletionFile string
 	ZSHCompletionFile  string
 }
 
@@ -37,6 +38,7 @@ func init() {
 	fs := cmdGenerate.Flags()
 	fs.StringVar(&genOpts.ManDir, "man", "", "write man pages to `directory`")
 	fs.StringVar(&genOpts.BashCompletionFile, "bash-completion", "", "write bash completion `file`")
+	fs.StringVar(&genOpts.FishCompletionFile, "fish-completion", "", "write fish completion `file`")
 	fs.StringVar(&genOpts.ZSHCompletionFile, "zsh-completion", "", "write zsh completion `file`")
 }
 
@@ -63,6 +65,11 @@ func writeBashCompletion(file string) error {
 	return cmdRoot.GenBashCompletionFile(file)
 }
 
+func writeFishCompletion(file string) error {
+	Verbosef("writing fish completion file to %v\n", file)
+	return cmdRoot.GenFishCompletionFile(file, true)
+}
+
 func writeZSHCompletion(file string) error {
 	Verbosef("writing zsh completion file to %v\n", file)
 	return cmdRoot.GenZshCompletionFile(file)
@@ -83,6 +90,13 @@ func runGenerate(cmd *cobra.Command, args []string) error {
 		}
 	}
 
+	if genOpts.FishCompletionFile != "" {
+		err := writeFishCompletion(genOpts.FishCompletionFile)
+		if err != nil {
+			return err
+		}
+	}
+
 	if genOpts.ZSHCompletionFile != "" {
 		err := writeZSHCompletion(genOpts.ZSHCompletionFile)
 		if err != nil {