Add version for 0.9.3

Output directory size in cache command

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,27 @@
+<!--
+
+Welcome! If you have a question or are unsure if you should open an issue,
+please use the forum instead!
+
+   https://forum.restic.net
+
+The forum is a better place for questions about restic or general suggestions
+and topics, e.g. usage or documentation questions! This issue tracker is mainly
+for tracking bugs and feature requests directly relating to the development of
+the software itself, rather than the project.
+
+Thanks for understanding, and for contributing to the project!
+-->
+
+
+Output of `restic version`
+--------------------------
+
+<!--
+Please add the version of restic you're currently using here, this helps us
+later to see what has changed in restic when we revisit this issue after some
+time.
+-->
+
+Describe the issue
+------------------

.gitignore

@@ -1,3 +1,2 @@
 /restic
 /.vagrant
-/doc/_build

.travis.yml

@@ -6,15 +6,35 @@ matrix:
     - os: linux
       go: "1.9.x"
       env: RESTIC_TEST_FUSE=0 RESTIC_TEST_CLOUD_BACKENDS=0 RESTIC_BUILD_SOLARIS=0
+      cache:
+        directories:
+          - $HOME/.cache/go-build
+          - $HOME/gopath/pkg/mod
+
+    - os: linux
+      go: "1.10.x"
+      env: RESTIC_TEST_FUSE=0 RESTIC_TEST_CLOUD_BACKENDS=0
+      cache:
+        directories:
+          - $HOME/.cache/go-build
+          - $HOME/gopath/pkg/mod
 
     # only run fuse and cloud backends tests on Travis for the latest Go on Linux
     - os: linux
-      go: "1.10.x"
+      go: "1.11.x"
       sudo: true
+      cache:
+        directories:
+          - $HOME/.cache/go-build
+          - $HOME/gopath/pkg/mod
 
     - os: osx
-      go: "1.10.x"
+      go: "1.11.x"
       env: RESTIC_TEST_FUSE=0 RESTIC_TEST_CLOUD_BACKENDS=0
+      cache:
+        directories:
+          - $HOME/Library/Caches/go-build
+          - $HOME/gopath/pkg/mod
 
 branches:
   only:
@@ -38,4 +58,4 @@ script:
   - go run run_integration_tests.go
 
 after_success:
-  - bash <(curl -s https://codecov.io/bash) -f all.cov
+  - test -r all.cov && bash <(curl -s https://codecov.io/bash) -f all.cov

CHANGELOG.md

@@ -1,3 +1,161 @@
+Changelog for restic 0.9.3 (2018-10-13)
+=======================================
+
+The following sections list the changes in restic 0.9.3 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #1935: Remove truncated files from cache
+ * Fix #1978: Do not return an error when the scanner is faster than backup
+ * Enh #1766: Restore: suppress lchown errors when not running as root
+ * Enh #1909: Reject files/dirs by name first
+ * Enh #1940: Add directory filter to ls command
+ * Enh #1967: Use `--host` everywhere
+ * Enh #2028: Display size of cache directories
+ * Enh #1777: Improve the `find` command
+ * Enh #1876: Display reason why forget keeps snapshots
+ * Enh #1891: Accept glob in paths loaded via --files-from
+ * Enh #1920: Vendor dependencies with Go 1.11 Modules
+ * Enh #1949: Add new command `self-update`
+ * Enh #1953: Ls: Add JSON output support for restic ls cmd
+
+Details
+-------
+
+ * Bugfix #1935: Remove truncated files from cache
+
+   When a file in the local cache is truncated, and restic tries to access data beyond the end of the
+   (cached) file, it used to return an error "EOF". This is now fixed, such truncated files are
+   removed and the data is fetched directly from the backend.
+
+   https://github.com/restic/restic/issues/1935
+
+ * Bugfix #1978: Do not return an error when the scanner is faster than backup
+
+   When restic makes a backup, there's a background task called "scanner" which collects
+   information on how many files and directories are to be saved, in order to display progress
+   information to the user. When the backup finishes faster than the scanner, it is aborted
+   because the result is not needed any more. This logic contained a bug, where quitting the
+   scanner process was treated as an error, and caused restic to print an unhelpful error message
+   ("context canceled").
+
+   https://github.com/restic/restic/issues/1978
+   https://github.com/restic/restic/pull/1991
+
+ * Enhancement #1766: Restore: suppress lchown errors when not running as root
+
+   Like "cp" and "rsync" do, restic now only reports errors for changing the ownership of files
+   during restore if it is run as root, on non-Windows operating systems. On Windows, the error
+   is reported as usual.
+
+   https://github.com/restic/restic/issues/1766
+
+ * Enhancement #1909: Reject files/dirs by name first
+
+   The current scanner/archiver code had an architectural limitation: it always ran the
+   `lstat()` system call on all files and directories before a decision to include/exclude the
+   file/dir was made. This lead to a lot of unnecessary system calls for items that could have been
+   rejected by their name or path only.
+
+   We've changed the archiver/scanner implementation so that it now first rejects by name/path,
+   and only runs the system call on the remaining items. This reduces the number of `lstat()`
+   system calls a lot (depending on the exclude settings).
+
+   https://github.com/restic/restic/issues/1909
+   https://github.com/restic/restic/pull/1912
+
+ * Enhancement #1940: Add directory filter to ls command
+
+   The ls command can now be filtered by directories, so that only files in the given directories
+   will be shown. If the --recursive flag is specified, then ls will traverse subfolders and list
+   their files as well.
+
+   It used to be possible to specify multiple snapshots, but that has been replaced by only one
+   snapshot and the possibility of specifying multiple directories.
+
+   Specifying directories constrains the walk, which can significantly speed up the listing.
+
+   https://github.com/restic/restic/issues/1940
+   https://github.com/restic/restic/pull/1941
+
+ * Enhancement #1967: Use `--host` everywhere
+
+   We now use the flag `--host` for all commands which need a host name, using `--hostname` (e.g.
+   for `restic backup`) still works, but will print a deprecation warning. Also, add the short
+   option `-H` where possible.
+
+   https://github.com/restic/restic/issues/1967
+
+ * Enhancement #2028: Display size of cache directories
+
+   The `cache` command now by default shows the size of the individual cache directories. It can be
+   disabled with `--no-size`.
+
+   https://github.com/restic/restic/issues/2028
+   https://github.com/restic/restic/pull/2033
+
+ * Enhancement #1777: Improve the `find` command
+
+   We've updated the `find` command to support multiple patterns.
+
+   `restic find` is now able to list the snapshots containing a specific tree or blob, or even the
+   snapshots that contain blobs belonging to a given pack. A list of IDs can be given, as long as they
+   all have the same type.
+
+   The command `find` can also display the pack IDs the blobs belong to, if the `--show-pack-id`
+   flag is provided.
+
+   https://github.com/restic/restic/issues/1777
+   https://github.com/restic/restic/pull/1780
+
+ * Enhancement #1876: Display reason why forget keeps snapshots
+
+   We've added a column to the list of snapshots `forget` keeps which details the reasons to keep a
+   particuliar snapshot. This makes debugging policies for forget much easier. Please remember
+   to always try things out with `--dry-run`!
+
+   https://github.com/restic/restic/pull/1876
+
+ * Enhancement #1891: Accept glob in paths loaded via --files-from
+
+   Before that, behaviour was different if paths were appended to command line or from a file,
+   because wild card characters were expanded by shell if appended to command line, but not
+   expanded if loaded from file.
+
+   https://github.com/restic/restic/issues/1891
+
+ * Enhancement #1920: Vendor dependencies with Go 1.11 Modules
+
+   Until now, we've used `dep` for managing dependencies, we've now switch to using Go modules.
+   For users this does not change much, only if you want to compile restic without downloading
+   anything with Go 1.11, then you need to run: `go build -mod=vendor build.go`
+
+   https://github.com/restic/restic/pull/1920
+
+ * Enhancement #1949: Add new command `self-update`
+
+   We have added a new command called `self-update` which downloads the latest released version
+   of restic from GitHub and replaces the current binary with it. It does not rely on any external
+   program (so it'll work everywhere), but still verifies the GPG signature using the embedded
+   GPG public key.
+
+   By default, the `self-update` command is hidden behind the `selfupdate` built tag, which is
+   only set when restic is built using `build.go` (including official releases). The reason for
+   this is that downstream distributions will then not include the command by default, so users
+   are encouraged to use the platform-specific distribution mechanism.
+
+   https://github.com/restic/restic/pull/1949
+
+ * Enhancement #1953: Ls: Add JSON output support for restic ls cmd
+
+   We've implemented listing files in the repository with JSON as output, just pass `--json` as an
+   option to `restic ls`. This makes the output of the command machine readable.
+
+   https://github.com/restic/restic/pull/1953
+
+
 Changelog for restic 0.9.2 (2018-08-06)
 =======================================
 

CONTRIBUTING.md

@@ -46,12 +46,15 @@ Remember, the easier it is for us to reproduce the bug, the earlier it will be
 corrected!
 
 In addition, you can compile restic with debug support by running
-`go run build.go -tags debug` and instructing it to create a debug log by
-setting the environment variable `DEBUG_LOG` to a file, e.g. like this:
+`go run -mod=vendor build.go -tags debug` and instructing it to create a debug
+log by setting the environment variable `DEBUG_LOG` to a file, e.g. like this:
 
     $ export DEBUG_LOG=/tmp/restic-debug.log
     $ restic backup ~/work
 
+For Go < 1.11, you need to remove the `-mod=vendor` option from the build
+command.
+
 Please be aware that the debug log file will contain potentially sensitive
 things like file and directory names, so please either redact it before
 uploading it somewhere or post only the parts that are really relevant.
@@ -60,9 +63,37 @@ uploading it somewhere or post only the parts that are really relevant.
 Development Environment
 =======================
 
-In order to compile restic with the `go` tool directly, it needs to be checked
-out at the right path within a `GOPATH`. The concept of a `GOPATH` is explained
-in ["How to write Go code"](https://golang.org/doc/code.html).
+The repository contains several sets of directories with code: `cmd/` and
+`internal/` contain the code written for restic, whereas `vendor/` contains
+copies of libraries restic depends on. The libraries are managed with the
+command `go mod vendor`.
+
+Go >= 1.11
+----------
+
+For Go version 1.11 or later, you should clone the repo (without having
+`$GOPATH` set) and `cd` into the directory:
+
+    $ unset GOPATH
+    $ git clone https://github.com/restic/restic
+    $ cd restic
+
+Then use the `go` tool to build restic:
+
+    $ go build ./cmd/restic
+    $ ./restic version
+    restic 0.9.2-dev (compiled manually) compiled with go1.11 on linux/amd64
+
+You can run all tests with the following command:
+
+    $ go test ./...
+
+Go < 1.11
+---------
+
+In order to compile restic with Go before 1.11, it needs to be checked out at
+the right path within a `GOPATH`. The concept of a `GOPATH` is explained in
+["How to write Go code"](https://golang.org/doc/code.html).
 
 If you do not have a directory with Go code yet, executing the following
 instructions in your shell will create one for you and check out the restic
@@ -83,12 +114,7 @@ You can then build restic as follows:
 
 The following commands can be used to run all the tests:
 
-    $ go test ./cmd/... ./internal/...
-
-The repository contains two sets of directories with code: `cmd/` and
-`internal/` contain the code written for restic, whereas `vendor/` contains
-copies of libraries restic depends on. The libraries are managed with the
-[`dep`](https://github.com/golang/dep) tool.
+    $ go test ./...
 
 Providing Patches
 =================
@@ -141,13 +167,14 @@ run
 
     gofmt -w **/*.go
 
-in the project root directory before committing. Installing the script
-`fmt-check` from https://github.com/edsrzf/gofmt-git-hook locally as a
-pre-commit hook checks formatting before committing automatically, just copy
-this script to `.git/hooks/pre-commit`.
+in the project root directory before committing. For each Pull Request, the
+formatting is tested with `gofmt` for the latest stable version of Go.
+Installing the script `fmt-check` from https://github.com/edsrzf/gofmt-git-hook
+locally as a pre-commit hook checks formatting before committing automatically,
+just copy this script to `.git/hooks/pre-commit`.
 
 For each pull request, several different systems run the integration tests on
-Linux, OS X and Windows. We won't merge any code that does not pass all tests
+Linux, macOS and Windows. We won't merge any code that does not pass all tests
 for all systems, so when a tests fails, try to find out what's wrong and fix
 it. If you need help on this, please leave a comment in the pull request, and
 we'll be glad to assist. Having a PR with failing integration tests is nothing

Gopkg.lock

@@ -1,474 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  branch = "master"
-  digest = "1:94e9caf404409a2990cfd22aca37d758494c098eff3e2c37fda1abed862e74dd"
-  name = "bazil.org/fuse"
-  packages = [
-    ".",
-    "fs",
-    "fuseutil",
-  ]
-  pruneopts = "UT"
-  revision = "65cc252bf6691cb3c7014bcb2c8dc29de91e3a7e"
-
-[[projects]]
-  digest = "1:5c3894b2aa4d6bead0ceeea6831b305d62879c871780e7b76296ded1b004bc57"
-  name = "cloud.google.com/go"
-  packages = ["compute/metadata"]
-  pruneopts = "UT"
-  revision = "aad3f485ee528456e0768f20397b4d9dd941e755"
-  version = "v0.25.0"
-
-[[projects]]
-  digest = "1:46ea9487304f4b3c787f54483ecb13a338d686dcd670db0ab1a112ed0ae2128e"
-  name = "github.com/Azure/azure-sdk-for-go"
-  packages = [
-    "storage",
-    "version",
-  ]
-  pruneopts = "UT"
-  revision = "4e8cbbfb1aeab140cd0fa97fd16b64ee18c3ca6a"
-  version = "v19.1.0"
-
-[[projects]]
-  digest = "1:27d0cd1a78fc836f7c0f07749d029a5f7895c84ad066187b08b70e9d1830098e"
-  name = "github.com/Azure/go-autorest"
-  packages = [
-    "autorest",
-    "autorest/adal",
-    "autorest/azure",
-    "autorest/date",
-    "logger",
-    "version",
-  ]
-  pruneopts = "UT"
-  revision = "dd94e014aaf16d1df746762e392aa201c1b4c461"
-  version = "v10.15.0"
-
-[[projects]]
-  digest = "1:2209584c0f7c9b68c23374e659357ab546e1b70eec2761f03280f69a8fd23d77"
-  name = "github.com/cenkalti/backoff"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "2ea60e5f094469f9e65adb9cd103795b73ae743e"
-  version = "v2.0.0"
-
-[[projects]]
-  digest = "1:7cb4fdca4c251b3ef8027c90ea35f70c7b661a593b9eeae34753c65499098bb1"
-  name = "github.com/cpuguy83/go-md2man"
-  packages = ["md2man"]
-  pruneopts = "UT"
-  revision = "20f5889cbdc3c73dbd2862796665e7c465ade7d1"
-  version = "v1.0.8"
-
-[[projects]]
-  digest = "1:76dc72490af7174349349838f2fe118996381b31ea83243812a97e5a0fd5ed55"
-  name = "github.com/dgrijalva/jwt-go"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e"
-  version = "v3.2.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:6f9339c912bbdda81302633ad7e99a28dfa5a639c864061f1929510a9a64aa74"
-  name = "github.com/dustin/go-humanize"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "9f541cc9db5d55bce703bd99987c9d5cb8eea45e"
-
-[[projects]]
-  digest = "1:c7edfbb6320d6a93240d663dc52bca92bed4c116abe54c35679eec4e7cc2bd77"
-  name = "github.com/elithrar/simple-scrypt"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "d150773194090feb6c897805a7bcea8d49544e2c"
-  version = "v1.3.0"
-
-[[projects]]
-  digest = "1:fe8a03a8222d5b913f256972933d26d24ad7c8286692a42943bc01633cc8fce3"
-  name = "github.com/go-ini/ini"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "358ee7663966325963d4e8b2e1fbd570c5195153"
-  version = "v1.38.1"
-
-[[projects]]
-  digest = "1:15042ad3498153684d09f393bbaec6b216c8eec6d61f63dff711de7d64ed8861"
-  name = "github.com/golang/protobuf"
-  packages = ["proto"]
-  pruneopts = "UT"
-  revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265"
-  version = "v1.1.0"
-
-[[projects]]
-  digest = "1:2e3c336fc7fde5c984d2841455a658a6d626450b1754a854b3b32e7a8f49a07a"
-  name = "github.com/google/go-cmp"
-  packages = [
-    "cmp",
-    "cmp/internal/diff",
-    "cmp/internal/function",
-    "cmp/internal/value",
-  ]
-  pruneopts = "UT"
-  revision = "3af367b6b30c263d47e8895973edcca9a49cf029"
-  version = "v0.2.0"
-
-[[projects]]
-  digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be"
-  name = "github.com/inconshreveable/mousetrap"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
-  version = "v1.0"
-
-[[projects]]
-  digest = "1:190ff84d9b2ed6589088f178cba8edb4b8ecb334df4572421fb016be1ac20463"
-  name = "github.com/juju/ratelimit"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "59fac5042749a5afb9af70e813da1dd5474f0167"
-  version = "1.0.1"
-
-[[projects]]
-  digest = "1:9cedee824c21326bd26950bd9e1ffe9dc4e7ca03dc8634d0e6f954ee6a383172"
-  name = "github.com/kr/fs"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "1455def202f6e05b95cc7bfc7e8ae67ae5141eba"
-  version = "v0.1.0"
-
-[[projects]]
-  digest = "1:1faa76bd9bffce9c25eaca0597afb67bd05a21ae57fe4378154ce8855ef163d1"
-  name = "github.com/kurin/blazer"
-  packages = [
-    "b2",
-    "base",
-    "internal/b2assets",
-    "internal/b2types",
-    "internal/blog",
-    "x/window",
-  ]
-  pruneopts = "UT"
-  revision = "caf65aa76491dc533bac68ad3243ce72fa4e0a0a"
-  version = "v0.5.1"
-
-[[projects]]
-  digest = "1:4e878df5f4e9fd625bf9c9aac77ef7cbfa4a74c01265505527c23470c0e40300"
-  name = "github.com/marstr/guid"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "8bd9a64bf37eb297b492a4101fb28e80ac0b290f"
-  version = "v1.1.0"
-
-[[projects]]
-  digest = "1:d4d17353dbd05cb52a2a52b7fe1771883b682806f68db442b436294926bbfafb"
-  name = "github.com/mattn/go-isatty"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39"
-  version = "v0.0.3"
-
-[[projects]]
-  digest = "1:95c73c666919be2843b955eafc83f58c136312b74f79c703152f4c4a95fd64dc"
-  name = "github.com/minio/minio-go"
-  packages = [
-    ".",
-    "pkg/credentials",
-    "pkg/encrypt",
-    "pkg/s3signer",
-    "pkg/s3utils",
-    "pkg/set",
-  ]
-  pruneopts = "UT"
-  revision = "70799fe8dae6ecfb6c7d7e9e048fce27f23a1992"
-  version = "v6.0.5"
-
-[[projects]]
-  branch = "master"
-  digest = "1:8eb17c2ec4df79193ae65b621cd1c0c4697db3bc317fe6afdc76d7f2746abd05"
-  name = "github.com/mitchellh/go-homedir"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "3864e76763d94a6df2f9960b16a20a33da9f9a66"
-
-[[projects]]
-  digest = "1:928de5172dd3563964d1b88a4ee3775cf72e16f1efabb482ab6d0e0bab86ee69"
-  name = "github.com/ncw/swift"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "b2a7479cf26fa841ff90dd932d0221cb5c50782d"
-  version = "v1.0.39"
-
-[[projects]]
-  digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747"
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
-
-[[projects]]
-  digest = "1:cfa0d7741863a0e1d30e0ccdd4b48a96a471cdb47892303de8b92c3713af3e77"
-  name = "github.com/pkg/profile"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "5b67d428864e92711fcbd2f8629456121a56d91f"
-  version = "v1.2.1"
-
-[[projects]]
-  digest = "1:23ed92ba5d90a2dfe817f3895027ccef796e79c30be5125d48e17afdcc395d73"
-  name = "github.com/pkg/sftp"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "57673e38ea946592a59c26592b7e6fbda646975b"
-  version = "v1.8.0"
-
-[[projects]]
-  digest = "1:0d67664e93e366f072ac9672feea29bfc63c9f90f005e9e8a0df1954153f5a14"
-  name = "github.com/pkg/xattr"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "ae385d07bb53f092fcc7daaf738d8513df084931"
-  version = "v0.3.1"
-
-[[projects]]
-  digest = "1:13ecc4000f49cf0aa3ee56fffcc93119c8edffacfff08674c80d2757d8c33a83"
-  name = "github.com/restic/chunker"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "db83917be3b88cc307464b7d8a221c173e34a0db"
-  version = "v0.2.0"
-
-[[projects]]
-  digest = "1:8bc629776d035c003c7814d4369521afe67fdb8efc4b5f66540d29343b98cf23"
-  name = "github.com/russross/blackfriday"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "55d61fa8aa702f59229e6cff85793c22e580eaf5"
-  version = "v1.5.1"
-
-[[projects]]
-  digest = "1:274f67cb6fed9588ea2521ecdac05a6d62a8c51c074c1fccc6a49a40ba80e925"
-  name = "github.com/satori/go.uuid"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "f58768cc1a7a7e77a3bd49e98cdd21419399b6a3"
-  version = "v1.2.0"
-
-[[projects]]
-  digest = "1:d867dfa6751c8d7a435821ad3b736310c2ed68945d05b50fb9d23aee0540c8cc"
-  name = "github.com/sirupsen/logrus"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "3e01752db0189b9157070a0e1668a620f9a85da2"
-  version = "v1.0.6"
-
-[[projects]]
-  digest = "1:e01b05ba901239c783dfe56450bcde607fc858908529868259c9a8765dc176d0"
-  name = "github.com/spf13/cobra"
-  packages = [
-    ".",
-    "doc",
-  ]
-  pruneopts = "UT"
-  revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385"
-  version = "v0.0.3"
-
-[[projects]]
-  digest = "1:9424f440bba8f7508b69414634aef3b2b3a877e522d8a4624692412805407bb7"
-  name = "github.com/spf13/pflag"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "583c0c0531f06d5278b7d917446061adc344b5cd"
-  version = "v1.0.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:eefb1f49ec07e71206d4c9ea1a3e634cad331c2180733e4121b8ae39e8e92ecb"
-  name = "golang.org/x/crypto"
-  packages = [
-    "argon2",
-    "blake2b",
-    "curve25519",
-    "ed25519",
-    "ed25519/internal/edwards25519",
-    "internal/chacha20",
-    "internal/subtle",
-    "pbkdf2",
-    "poly1305",
-    "scrypt",
-    "ssh",
-    "ssh/terminal",
-  ]
-  pruneopts = "UT"
-  revision = "c126467f60eb25f8f27e5a981f32a87e3965053f"
-
-[[projects]]
-  branch = "master"
-  digest = "1:8356aa7bdcb10a210b814b64ff76d61de7c36ac4cb6263de3af5e3e2e546956d"
-  name = "golang.org/x/net"
-  packages = [
-    "context",
-    "context/ctxhttp",
-    "http/httpguts",
-    "http2",
-    "http2/hpack",
-    "idna",
-  ]
-  pruneopts = "UT"
-  revision = "32f9bdbd7df18e8641d215e7ea68be88b971feb0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:bea0314c10bd362ab623af4880d853b5bad3b63d0ab9945c47e461b8d04203ed"
-  name = "golang.org/x/oauth2"
-  packages = [
-    ".",
-    "google",
-    "internal",
-    "jws",
-    "jwt",
-  ]
-  pruneopts = "UT"
-  revision = "3d292e4d0cdc3a0113e6d207bb137145ef1de42f"
-
-[[projects]]
-  branch = "master"
-  digest = "1:39ebcc2b11457b703ae9ee2e8cca0f68df21969c6102cb3b705f76cca0ea0239"
-  name = "golang.org/x/sync"
-  packages = ["errgroup"]
-  pruneopts = "UT"
-  revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca"
-
-[[projects]]
-  branch = "master"
-  digest = "1:a220a85c72a6cb7339c412cb2b117019a7fd94007cdfffb3b5b1d058227a2bf8"
-  name = "golang.org/x/sys"
-  packages = [
-    "cpu",
-    "unix",
-    "windows",
-  ]
-  pruneopts = "UT"
-  revision = "bd9dbc187b6e1dacfdd2722a87e83093c2d7bd6e"
-
-[[projects]]
-  digest = "1:e5a8511f063c38c51ab9ab80e718e9149f692652aeb4e393a8c020dd1bf38ca2"
-  name = "golang.org/x/text"
-  packages = [
-    "collate",
-    "collate/build",
-    "encoding",
-    "encoding/internal",
-    "encoding/internal/identifier",
-    "encoding/unicode",
-    "internal/colltab",
-    "internal/gen",
-    "internal/tag",
-    "internal/triegen",
-    "internal/ucd",
-    "internal/utf8internal",
-    "language",
-    "runes",
-    "secure/bidirule",
-    "transform",
-    "unicode/bidi",
-    "unicode/cldr",
-    "unicode/norm",
-    "unicode/rangetable",
-  ]
-  pruneopts = "UT"
-  revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
-  version = "v0.3.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:fb983acae7bd9c3ed9aadc1b1241d9e559ed21dbf84c17a0dda663ca169ccd69"
-  name = "google.golang.org/api"
-  packages = [
-    "gensupport",
-    "googleapi",
-    "googleapi/internal/uritemplates",
-    "storage/v1",
-  ]
-  pruneopts = "UT"
-  revision = "31ca0e01cd791f07750cb23fc99327721f753290"
-
-[[projects]]
-  digest = "1:c8907869850adaa8bd7631887948d0684f3787d0912f1c01ab72581a6c34432e"
-  name = "google.golang.org/appengine"
-  packages = [
-    ".",
-    "internal",
-    "internal/app_identity",
-    "internal/base",
-    "internal/datastore",
-    "internal/log",
-    "internal/modules",
-    "internal/remote_api",
-    "internal/urlfetch",
-    "urlfetch",
-  ]
-  pruneopts = "UT"
-  revision = "b1f26356af11148e710935ed1ac8a7f5702c7612"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "v2"
-  digest = "1:5bb148b78468350091db2ffbb2370f35cc6dcd74d9378a31b1c7b86ff7528f08"
-  name = "gopkg.in/tomb.v2"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "d5d1b5820637886def9eef33e03a27a9f166942c"
-
-[[projects]]
-  digest = "1:342378ac4dcb378a5448dd723f0784ae519383532f5e70ade24132c4c8693202"
-  name = "gopkg.in/yaml.v2"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
-  version = "v2.2.1"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  input-imports = [
-    "bazil.org/fuse",
-    "bazil.org/fuse/fs",
-    "github.com/Azure/azure-sdk-for-go/storage",
-    "github.com/cenkalti/backoff",
-    "github.com/elithrar/simple-scrypt",
-    "github.com/google/go-cmp/cmp",
-    "github.com/juju/ratelimit",
-    "github.com/kurin/blazer/b2",
-    "github.com/mattn/go-isatty",
-    "github.com/minio/minio-go",
-    "github.com/minio/minio-go/pkg/credentials",
-    "github.com/ncw/swift",
-    "github.com/pkg/errors",
-    "github.com/pkg/profile",
-    "github.com/pkg/sftp",
-    "github.com/pkg/xattr",
-    "github.com/restic/chunker",
-    "github.com/spf13/cobra",
-    "github.com/spf13/cobra/doc",
-    "github.com/spf13/pflag",
-    "golang.org/x/crypto/poly1305",
-    "golang.org/x/crypto/scrypt",
-    "golang.org/x/crypto/ssh/terminal",
-    "golang.org/x/net/context",
-    "golang.org/x/net/context/ctxhttp",
-    "golang.org/x/net/http2",
-    "golang.org/x/oauth2/google",
-    "golang.org/x/sync/errgroup",
-    "golang.org/x/sys/unix",
-    "golang.org/x/text/encoding/unicode",
-    "google.golang.org/api/googleapi",
-    "google.golang.org/api/storage/v1",
-    "gopkg.in/tomb.v2",
-  ]
-  solver-name = "gps-cdcl"
-  solver-version = 1

Gopkg.toml

@@ -1,25 +0,0 @@
-
-# Gopkg.toml example
-#
-# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
-# for detailed Gopkg.toml documentation.
-#
-# required = ["github.com/user/thing/cmd/thing"]
-# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
-#
-# [[constraint]]
-#   name = "github.com/user/project"
-#   version = "1.0.0"
-#
-# [[constraint]]
-#   name = "github.com/user/project2"
-#   branch = "dev"
-#   source = "github.com/myfork/project2"
-#
-# [[override]]
-#  name = "github.com/x/y"
-#  version = "2.4.0"
-
-[prune]
-  unused-packages = true
-  go-tests = true

Makefile

@@ -3,7 +3,7 @@
 all: restic
 
 restic:
-	go run build.go
+	go run -mod=vendor build.go || go run build.go
 
 clean:
 	rm -f restic

VERSION

@@ -1 +1 @@
-0.9.2
+0.9.3

appveyor.yml

@@ -7,6 +7,9 @@ branches:
   only:
     - master
 
+cache:
+  - '%LocalAppData%\go-build'
+
 init:
   - ps: >-
       $app = Get-WmiObject -Class Win32_Product -Filter "Vendor = 'http://golang.org'"
@@ -17,8 +20,8 @@ init:
 
 install:
   - rmdir c:\go /s /q
-  - appveyor DownloadFile https://dl.google.com/go/go1.10.windows-amd64.msi
-  - msiexec /i go1.10.windows-amd64.msi /q
+  - appveyor DownloadFile https://dl.google.com/go/go1.11.windows-amd64.msi
+  - msiexec /i go1.11.windows-amd64.msi /q
   - go version
   - go env
   - appveyor DownloadFile http://sourceforge.netcologne.de/project/gnuwin32/tar/1.13-1/tar-1.13-1-bin.zip -FileName tar.zip
@@ -26,4 +29,4 @@ install:
   - set PATH=bin/;%PATH%
 
 build_script:
-  - go run run_integration_tests.go
+  - go run -mod=vendor run_integration_tests.go

build.go

@@ -1,3 +1,18 @@
+// Description
+//
+// This program aims to make building Go programs for end users easier by just
+// calling it with `go run`, without having to setup a GOPATH.
+//
+// For Go < 1.11, it'll create a new GOPATH in a temporary directory, then run
+// `go build` on the package configured as Main in the Config struct.
+//
+// For Go >= 1.11 if the file go.mod is present, it'll use Go modules and not
+// setup a GOPATH. It builds the package configured as Main in the Config
+// struct with `go build -mod=vendor` to use the vendored dependencies.
+// The variable GOPROXY is set to `off` so that no network calls are made. All
+// files are copied to a temporary directory before `go build` is called within
+// that directory.
+
 // BSD 2-Clause License
 //
 // Copyright (c) 2016-2018, Alexander Neumann <alexander@bumpern.de>
@@ -37,7 +52,6 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
-	"path"
 	"path/filepath"
 	"runtime"
 	"strconv"
@@ -46,23 +60,22 @@ import (
 
 // config contains the configuration for the program to build.
 var config = Config{
-	Name:      "restic",                              // name of the program executable and directory
-	Namespace: "github.com/restic/restic",            // subdir of GOPATH, e.g. "github.com/foo/bar"
-	Main:      "github.com/restic/restic/cmd/restic", // package name for the main package
-	Tests: []string{ // tests to run
-		"github.com/restic/restic/internal/...",
-		"github.com/restic/restic/cmd/...",
-	},
-	MinVersion: GoVersion{Major: 1, Minor: 9, Patch: 0}, // minimum Go version supported
+	Name:             "restic",                                // name of the program executable and directory
+	Namespace:        "github.com/restic/restic",              // subdir of GOPATH, e.g. "github.com/foo/bar"
+	Main:             "./cmd/restic",                          // package name for the main package
+	DefaultBuildTags: []string{"selfupdate"},                  // specify build tags which are always used
+	Tests:            []string{"./..."},                       // tests to run
+	MinVersion:       GoVersion{Major: 1, Minor: 9, Patch: 0}, // minimum Go version supported
 }
 
 // Config configures the build.
 type Config struct {
-	Name       string
-	Namespace  string
-	Main       string
-	Tests      []string
-	MinVersion GoVersion
+	Name             string
+	Namespace        string
+	Main             string
+	DefaultBuildTags []string
+	Tests            []string
+	MinVersion       GoVersion
 }
 
 var (
@@ -71,41 +84,12 @@ var (
 	runTests   bool
 	enableCGO  bool
 	enablePIE  bool
+	goVersion  = ParseGoVersion(runtime.Version())
 )
 
-// specialDir returns true if the file begins with a special character ('.' or '_').
-func specialDir(name string) bool {
-	if name == "." {
-		return false
-	}
-
-	base := filepath.Base(name)
-	if base == "vendor" || base[0] == '_' || base[0] == '.' {
-		return true
-	}
-
-	return false
-}
-
-// excludePath returns true if the file should not be copied to the new GOPATH.
-func excludePath(name string) bool {
-	ext := path.Ext(name)
-	if ext == ".go" || ext == ".s" || ext == ".h" {
-		return false
-	}
-
-	parentDir := filepath.Base(filepath.Dir(name))
-	if parentDir == "testdata" {
-		return false
-	}
-
-	return true
-}
-
-// updateGopath builds a valid GOPATH at dst, with all Go files in src/ copied
-// to dst/prefix/, so calling
+// copy all Go files in src to dst, creating directories on the fly, so calling
 //
-//   updateGopath("/tmp/gopath", "/home/u/restic", "github.com/restic/restic")
+//   copy("/tmp/gopath/src/github.com/restic/restic", "/home/u/restic")
 //
 // with "/home/u/restic" containing the file "foo.go" yields the following tree
 // at "/tmp/gopath":
@@ -116,19 +100,15 @@ func excludePath(name string) bool {
 //           └── restic
 //               └── restic
 //                   └── foo.go
-func updateGopath(dst, src, prefix string) error {
-	verbosePrintf("copy contents of %v to %v\n", src, filepath.Join(dst, prefix))
+func copy(dst, src string) error {
+	verbosePrintf("copy contents of %v to %v\n", src, dst)
 	return filepath.Walk(src, func(name string, fi os.FileInfo, err error) error {
 		if name == src {
 			return err
 		}
 
-		if specialDir(name) {
-			if fi.IsDir() {
-				return filepath.SkipDir
-			}
-
-			return nil
+		if name == ".git" {
+			return filepath.SkipDir
 		}
 
 		if err != nil {
@@ -139,17 +119,13 @@ func updateGopath(dst, src, prefix string) error {
 			return nil
 		}
 
-		if excludePath(name) {
-			return nil
-		}
-
 		intermediatePath, err := filepath.Rel(src, name)
 		if err != nil {
 			return err
 		}
 
 		fileSrc := filepath.Join(src, intermediatePath)
-		fileDst := filepath.Join(dst, "src", prefix, intermediatePath)
+		fileDst := filepath.Join(dst, intermediatePath)
 
 		return copyFile(fileDst, fileSrc)
 	})
@@ -164,6 +140,15 @@ func directoryExists(dirname string) bool {
 	return stat.IsDir()
 }
 
+func fileExists(filename string) bool {
+	stat, err := os.Stat(filename)
+	if err != nil && os.IsNotExist(err) {
+		return false
+	}
+
+	return stat.Mode().IsRegular()
+}
+
 // copyFile creates dst from src, preserving file attributes and timestamps.
 func copyFile(dst, src string) error {
 	fi, err := os.Stat(src)
@@ -183,30 +168,34 @@ func copyFile(dst, src string) error {
 
 	fdst, err := os.Create(dst)
 	if err != nil {
+		_ = fsrc.Close()
 		return err
 	}
 
-	if _, err = io.Copy(fdst, fsrc); err != nil {
+	_, err = io.Copy(fdst, fsrc)
+	if err != nil {
+		_ = fsrc.Close()
+		_ = fdst.Close()
 		return err
 	}
 
-	if err == nil {
-		err = fsrc.Close()
+	err = fdst.Close()
+	if err != nil {
+		_ = fsrc.Close()
+		return err
 	}
 
-	if err == nil {
-		err = fdst.Close()
+	err = fsrc.Close()
+	if err != nil {
+		return err
 	}
 
-	if err == nil {
-		err = os.Chmod(dst, fi.Mode())
+	err = os.Chmod(dst, fi.Mode())
+	if err != nil {
+		return err
 	}
 
-	if err == nil {
-		err = os.Chtimes(dst, fi.ModTime(), fi.ModTime())
-	}
-
-	return nil
+	return os.Chtimes(dst, fi.ModTime(), fi.ModTime())
 }
 
 // die prints the message with fmt.Fprintf() to stderr and exits with an error
@@ -222,7 +211,7 @@ func showUsage(output io.Writer) {
 	fmt.Fprintf(output, "OPTIONS:\n")
 	fmt.Fprintf(output, "  -v     --verbose       output more messages\n")
 	fmt.Fprintf(output, "  -t     --tags          specify additional build tags\n")
-	fmt.Fprintf(output, "  -k     --keep-gopath   do not remove the GOPATH after build\n")
+	fmt.Fprintf(output, "  -k     --keep-tempdir  do not remove the temporary directory after build\n")
 	fmt.Fprintf(output, "  -T     --test          run tests\n")
 	fmt.Fprintf(output, "  -o     --output        set output file name\n")
 	fmt.Fprintf(output, "         --enable-cgo    use CGO to link against libc\n")
@@ -241,11 +230,20 @@ func verbosePrintf(message string, args ...interface{}) {
 	fmt.Printf("build: "+message, args...)
 }
 
-// cleanEnv returns a clean environment with GOPATH and GOBIN removed (if
-// present).
+// cleanEnv returns a clean environment with GOPATH, GOBIN and GO111MODULE
+// removed (if present).
 func cleanEnv() (env []string) {
+	removeKeys := map[string]struct{}{
+		"GOPATH":      struct{}{},
+		"GOBIN":       struct{}{},
+		"GO111MODULE": struct{}{},
+	}
+
 	for _, v := range os.Environ() {
-		if strings.HasPrefix(v, "GOPATH=") || strings.HasPrefix(v, "GOBIN=") {
+		data := strings.SplitN(v, "=", 2)
+		name := data[0]
+
+		if _, ok := removeKeys[name]; ok {
 			continue
 		}
 
@@ -256,17 +254,17 @@ func cleanEnv() (env []string) {
 }
 
 // build runs "go build args..." with GOPATH set to gopath.
-func build(cwd string, ver GoVersion, goos, goarch, goarm, gopath string, args ...string) error {
+func build(cwd string, env map[string]string, args ...string) error {
 	a := []string{"build"}
 
-	if ver.AtLeast(GoVersion{1, 10, 0}) {
+	if goVersion.AtLeast(GoVersion{1, 10, 0}) {
 		verbosePrintf("Go version is at least 1.10, using new syntax for -gcflags\n")
 		// use new prefix
-		a = append(a, "-asmflags", fmt.Sprintf("all=-trimpath=%s", gopath))
-		a = append(a, "-gcflags", fmt.Sprintf("all=-trimpath=%s", gopath))
+		a = append(a, "-asmflags", fmt.Sprintf("all=-trimpath=%s", cwd))
+		a = append(a, "-gcflags", fmt.Sprintf("all=-trimpath=%s", cwd))
 	} else {
-		a = append(a, "-asmflags", fmt.Sprintf("-trimpath=%s", gopath))
-		a = append(a, "-gcflags", fmt.Sprintf("-trimpath=%s", gopath))
+		a = append(a, "-asmflags", fmt.Sprintf("-trimpath=%s", cwd))
+		a = append(a, "-gcflags", fmt.Sprintf("-trimpath=%s", cwd))
 	}
 	if enablePIE {
 		a = append(a, "-buildmode=pie")
@@ -274,9 +272,9 @@ func build(cwd string, ver GoVersion, goos, goarch, goarm, gopath string, args .
 
 	a = append(a, args...)
 	cmd := exec.Command("go", a...)
-	cmd.Env = append(cleanEnv(), "GOPATH="+gopath, "GOARCH="+goarch, "GOOS="+goos)
-	if goarm != "" {
-		cmd.Env = append(cmd.Env, "GOARM="+goarm)
+	cmd.Env = append(cleanEnv(), "GOPROXY=off")
+	for k, v := range env {
+		cmd.Env = append(cmd.Env, k+"="+v)
 	}
 	if !enableCGO {
 		cmd.Env = append(cmd.Env, "CGO_ENABLED=0")
@@ -285,20 +283,30 @@ func build(cwd string, ver GoVersion, goos, goarch, goarm, gopath string, args .
 	cmd.Dir = cwd
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
-	verbosePrintf("go %s\n", a)
+
+	verbosePrintf("chdir %q\n", cwd)
+	verbosePrintf("go %q\n", a)
 
 	return cmd.Run()
 }
 
 // test runs "go test args..." with GOPATH set to gopath.
-func test(cwd, gopath string, args ...string) error {
-	args = append([]string{"test"}, args...)
+func test(cwd string, env map[string]string, args ...string) error {
+	args = append([]string{"test", "-count", "1"}, args...)
 	cmd := exec.Command("go", args...)
-	cmd.Env = append(cleanEnv(), "GOPATH="+gopath)
+	cmd.Env = append(cleanEnv(), "GOPROXY=off")
+	for k, v := range env {
+		cmd.Env = append(cmd.Env, k+"="+v)
+	}
+	if !enableCGO {
+		cmd.Env = append(cmd.Env, "CGO_ENABLED=0")
+	}
 	cmd.Dir = cwd
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
-	verbosePrintf("go %s\n", args)
+
+	verbosePrintf("chdir %q\n", cwd)
+	verbosePrintf("go %q\n", args)
 
 	return cmd.Run()
 }
@@ -446,22 +454,24 @@ func (v GoVersion) String() string {
 }
 
 func main() {
-	ver := ParseGoVersion(runtime.Version())
-	if !ver.AtLeast(config.MinVersion) {
-		fmt.Fprintf(os.Stderr, "%s detected, this program requires at least %s\n", ver, config.MinVersion)
+	if !goVersion.AtLeast(config.MinVersion) {
+		fmt.Fprintf(os.Stderr, "%s detected, this program requires at least %s\n", goVersion, config.MinVersion)
 		os.Exit(1)
 	}
 
-	buildTags := []string{}
+	buildTags := config.DefaultBuildTags
 
 	skipNext := false
 	params := os.Args[1:]
 
-	targetGOOS := runtime.GOOS
-	targetGOARCH := runtime.GOARCH
-	targetGOARM := ""
+	goEnv := map[string]string{}
+	buildEnv := map[string]string{
+		"GOOS":   runtime.GOOS,
+		"GOARCH": runtime.GOARCH,
+		"GOARM":  "",
+	}
 
-	gopath := ""
+	tempdir := ""
 
 	var outputFilename string
 
@@ -481,13 +491,13 @@ func main() {
 				die("-t given but no tag specified")
 			}
 			skipNext = true
-			buildTags = strings.Split(params[i+1], " ")
+			buildTags = append(buildTags, strings.Split(params[i+1], " ")...)
 		case "-o", "--output":
 			skipNext = true
 			outputFilename = params[i+1]
 		case "--tempdir":
 			skipNext = true
-			gopath = params[i+1]
+			tempdir = params[i+1]
 		case "-T", "--test":
 			runTests = true
 		case "--enable-cgo":
@@ -496,13 +506,13 @@ func main() {
 			enablePIE = true
 		case "--goos":
 			skipNext = true
-			targetGOOS = params[i+1]
+			buildEnv["GOOS"] = params[i+1]
 		case "--goarch":
 			skipNext = true
-			targetGOARCH = params[i+1]
+			buildEnv["GOARCH"] = params[i+1]
 		case "--goarm":
 			skipNext = true
-			targetGOARM = params[i+1]
+			buildEnv["GOARM"] = params[i+1]
 		case "-h":
 			showUsage(os.Stdout)
 			return
@@ -513,12 +523,7 @@ func main() {
 		}
 	}
 
-	verbosePrintf("detected Go version %v\n", ver)
-
-	if len(buildTags) == 0 {
-		verbosePrintf("adding build-tag release\n")
-		buildTags = []string{"release"}
-	}
+	verbosePrintf("detected Go version %v\n", goVersion)
 
 	for i := range buildTags {
 		buildTags[i] = strings.TrimSpace(buildTags[i])
@@ -531,50 +536,16 @@ func main() {
 		die("Getwd(): %v\n", err)
 	}
 
-	if gopath == "" {
-		gopath, err = ioutil.TempDir("", fmt.Sprintf("%v-build-", config.Name))
-		if err != nil {
-			die("TempDir(): %v\n", err)
-		}
-	}
-
-	verbosePrintf("create GOPATH at %v\n", gopath)
-	if err = updateGopath(gopath, root, config.Namespace); err != nil {
-		die("copying files from %v/src to %v/src failed: %v\n", root, gopath, err)
-	}
-
-	vendor := filepath.Join(root, "vendor")
-	if directoryExists(vendor) {
-		if err = updateGopath(gopath, vendor, filepath.Join(config.Namespace, "vendor")); err != nil {
-			die("copying files from %v to %v failed: %v\n", root, gopath, err)
-		}
-	}
-
-	defer func() {
-		if !keepGopath {
-			verbosePrintf("remove %v\n", gopath)
-			if err = os.RemoveAll(gopath); err != nil {
-				die("remove GOPATH at %s failed: %v\n", err)
-			}
-		} else {
-			verbosePrintf("leaving temporary GOPATH at %v\n", gopath)
-		}
-	}()
-
 	if outputFilename == "" {
 		outputFilename = config.Name
-		if targetGOOS == "windows" {
+		if buildEnv["GOOS"] == "windows" {
 			outputFilename += ".exe"
 		}
 	}
 
-	cwd, err := os.Getwd()
-	if err != nil {
-		die("Getwd() returned %v\n", err)
-	}
 	output := outputFilename
 	if !filepath.IsAbs(output) {
-		output = filepath.Join(cwd, output)
+		output = filepath.Join(root, output)
 	}
 
 	version := getVersion()
@@ -585,13 +556,65 @@ func main() {
 	ldflags := "-s -w " + constants.LDFlags()
 	verbosePrintf("ldflags: %s\n", ldflags)
 
-	args := []string{
-		"-tags", strings.Join(buildTags, " "),
-		"-ldflags", ldflags,
-		"-o", output, config.Main,
+	var (
+		buildArgs []string
+		testArgs  []string
+	)
+
+	mainPackage := config.Main
+	if strings.HasPrefix(mainPackage, config.Namespace) {
+		mainPackage = strings.Replace(mainPackage, config.Namespace, "./", 1)
 	}
 
-	err = build(filepath.Join(gopath, "src"), ver, targetGOOS, targetGOARCH, targetGOARM, gopath, args...)
+	buildTarget := filepath.FromSlash(mainPackage)
+	buildCWD := ""
+
+	if goVersion.AtLeast(GoVersion{1, 11, 0}) && fileExists("go.mod") {
+		verbosePrintf("Go >= 1.11 and 'go.mod' found, building with modules\n")
+		buildCWD = root
+
+		buildArgs = append(buildArgs, "-mod=vendor")
+		testArgs = append(testArgs, "-mod=vendor")
+	} else {
+		if tempdir == "" {
+			tempdir, err = ioutil.TempDir("", fmt.Sprintf("%v-build-", config.Name))
+			if err != nil {
+				die("TempDir(): %v\n", err)
+			}
+		}
+
+		verbosePrintf("Go < 1.11 or 'go.mod' not found, create GOPATH at %v\n", tempdir)
+		targetdir := filepath.Join(tempdir, "src", filepath.FromSlash(config.Namespace))
+		if err = copy(targetdir, root); err != nil {
+			die("copying files from %v to %v/src failed: %v\n", root, tempdir, err)
+		}
+
+		defer func() {
+			if !keepGopath {
+				verbosePrintf("remove %v\n", tempdir)
+				if err = os.RemoveAll(tempdir); err != nil {
+					die("remove GOPATH at %s failed: %v\n", tempdir, err)
+				}
+			} else {
+				verbosePrintf("leaving temporary GOPATH at %v\n", tempdir)
+			}
+		}()
+
+		buildCWD = targetdir
+
+		goEnv["GOPATH"] = tempdir
+		buildEnv["GOPATH"] = tempdir
+	}
+
+	verbosePrintf("environment:\n  go: %v\n  build: %v\n", goEnv, buildEnv)
+
+	buildArgs = append(buildArgs,
+		"-tags", strings.Join(buildTags, " "),
+		"-ldflags", ldflags,
+		"-o", output, buildTarget,
+	)
+
+	err = build(buildCWD, buildEnv, buildArgs...)
 	if err != nil {
 		die("build failed: %v\n", err)
 	}
@@ -599,7 +622,9 @@ func main() {
 	if runTests {
 		verbosePrintf("running tests\n")
 
-		err = test(cwd, gopath, config.Tests...)
+		testArgs = append(testArgs, config.Tests...)
+
+		err = test(buildCWD, goEnv, testArgs...)
 		if err != nil {
 			die("running tests failed: %v\n", err)
 		}

changelog/0.9.3_2018-10-13/issue-1766

@@ -0,0 +1,7 @@
+Enhancement: restore: suppress lchown errors when not running as root
+
+Like "cp" and "rsync" do, restic now only reports errors for changing
+the ownership of files during restore if it is run as root, on non-Windows
+operating systems. On Windows, the error is reported as usual.
+
+https://github.com/restic/restic/issues/1766

changelog/0.9.3_2018-10-13/issue-1909

@@ -0,0 +1,14 @@
+Enhancement: Reject files/dirs by name first
+
+The current scanner/archiver code had an architectural limitation: it always
+ran the `lstat()` system call on all files and directories before a decision to
+include/exclude the file/dir was made. This lead to a lot of unnecessary system
+calls for items that could have been rejected by their name or path only.
+
+We've changed the archiver/scanner implementation so that it now first rejects
+by name/path, and only runs the system call on the remaining items. This
+reduces the number of `lstat()` system calls a lot (depending on the exclude
+settings).
+
+https://github.com/restic/restic/issues/1909
+https://github.com/restic/restic/pull/1912

changelog/0.9.3_2018-10-13/issue-1935

@@ -0,0 +1,8 @@
+Bugfix: Remove truncated files from cache
+
+When a file in the local cache is truncated, and restic tries to access data
+beyond the end of the (cached) file, it used to return an error "EOF". This is
+now fixed, such truncated files are removed and the data is fetched directly
+from the backend.
+
+https://github.com/restic/restic/issues/1935

changelog/0.9.3_2018-10-13/issue-1941

@@ -0,0 +1,15 @@
+Enhancement: Add directory filter to ls command
+
+The ls command can now be filtered by directories, so that only files in the
+given directories will be shown. If the --recursive flag is specified, then
+ls will traverse subfolders and list their files as well.
+
+It used to be possible to specify multiple snapshots, but that has been
+replaced by only one snapshot and the possibility of specifying multiple
+directories.
+
+Specifying directories constrains the walk, which can significantly speed up
+the listing.
+
+https://github.com/restic/restic/issues/1940
+https://github.com/restic/restic/pull/1941

changelog/0.9.3_2018-10-13/issue-1967

@@ -0,0 +1,7 @@
+Enhancement: Use `--host` everywhere
+
+We now use the flag `--host` for all commands which need a host name, using
+`--hostname` (e.g. for `restic backup`) still works, but will print a
+deprecation warning. Also, add the short option `-H` where possible.
+
+https://github.com/restic/restic/issues/1967

changelog/0.9.3_2018-10-13/issue-1978

@@ -0,0 +1,12 @@
+Bugfix: Do not return an error when the scanner is faster than backup
+
+When restic makes a backup, there's a background task called "scanner" which
+collects information on how many files and directories are to be saved, in
+order to display progress information to the user. When the backup finishes
+faster than the scanner, it is aborted because the result is not needed any
+more. This logic contained a bug, where quitting the scanner process was
+treated as an error, and caused restic to print an unhelpful error message
+("context canceled").
+
+https://github.com/restic/restic/issues/1978
+https://github.com/restic/restic/pull/1991

changelog/0.9.3_2018-10-13/issue-2028

@@ -0,0 +1,7 @@
+Enhancement: Display size of cache directories
+
+The `cache` command now by default shows the size of the individual cache
+directories. It can be disabled with `--no-size`.
+
+https://github.com/restic/restic/issues/2028
+https://github.com/restic/restic/pull/2033

changelog/0.9.3_2018-10-13/pull-1780

@@ -0,0 +1,13 @@
+Enhancement: Improve the `find` command
+
+We've updated the `find` command to support multiple patterns.
+
+`restic find` is now able to list the snapshots containing a specific tree
+or blob, or even the snapshots that contain blobs belonging to a given pack.
+A list of IDs can be given, as long as they all have the same type.
+
+The command `find` can also display the pack IDs the blobs belong to, if
+the `--show-pack-id` flag is provided.
+
+https://github.com/restic/restic/issues/1777
+https://github.com/restic/restic/pull/1780

changelog/0.9.3_2018-10-13/pull-1876

@@ -0,0 +1,7 @@
+Enhancement: Display reason why forget keeps snapshots
+
+We've added a column to the list of snapshots `forget` keeps which details the
+reasons to keep a particuliar snapshot. This makes debugging policies for
+forget much easier. Please remember to always try things out with `--dry-run`!
+
+https://github.com/restic/restic/pull/1876

changelog/0.9.3_2018-10-13/pull-1891

@@ -0,0 +1,7 @@
+Enhancement: Accept glob in paths loaded via --files-from
+
+Before that, behaviour was different if paths were appended to command line or
+from a file, because wild card characters were expanded by shell if appended to
+command line, but not expanded if loaded from file.
+
+https://github.com/restic/restic/issues/1891

changelog/0.9.3_2018-10-13/pull-1920

@@ -0,0 +1,8 @@
+Enhancement: Vendor dependencies with Go 1.11 Modules
+
+Until now, we've used `dep` for managing dependencies, we've now switch to
+using Go modules. For users this does not change much, only if you want to
+compile restic without downloading anything with Go 1.11, then you need to run:
+`go build -mod=vendor build.go`
+
+https://github.com/restic/restic/pull/1920

changelog/0.9.3_2018-10-13/pull-1949

@@ -0,0 +1,15 @@
+Enhancement: Add new command `self-update`
+
+We have added a new command called `self-update` which downloads the
+latest released version of restic from GitHub and replaces the current
+binary with it. It does not rely on any external program (so it'll work
+everywhere), but still verifies the GPG signature using the embedded GPG
+public key.
+
+By default, the `self-update` command is hidden behind the `selfupdate`
+built tag, which is only set when restic is built using `build.go` (including
+official releases). The reason for this is that downstream distributions will
+then not include the command by default, so users are encouraged to use the
+platform-specific distribution mechanism.
+
+https://github.com/restic/restic/pull/1949

changelog/0.9.3_2018-10-13/pull-1953

@@ -0,0 +1,7 @@
+Enhancement: ls: Add JSON output support for restic ls cmd
+
+We've implemented listing files in the repository with JSON as output, just
+pass `--json` as an option to `restic ls`. This makes the output of the command
+machine readable.
+
+https://github.com/restic/restic/pull/1953

cmd/restic/cmd_backup.go

@@ -4,8 +4,10 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -32,13 +34,13 @@ The "backup" command creates a new snapshot and saves the files and directories
 given as the arguments.
 `,
 	PreRun: func(cmd *cobra.Command, args []string) {
-		if backupOptions.Hostname == "" {
+		if backupOptions.Host == "" {
 			hostname, err := os.Hostname()
 			if err != nil {
 				debug.Log("os.Hostname() returned err: %v", err)
 				return
 			}
-			backupOptions.Hostname = hostname
+			backupOptions.Host = hostname
 		}
 	},
 	DisableAutoGenTag: true,
@@ -72,7 +74,7 @@ type BackupOptions struct {
 	Stdin            bool
 	StdinFilename    string
 	Tags             []string
-	Hostname         string
+	Host             string
 	FilesFrom        string
 	TimeStamp        string
 	WithAtime        bool
@@ -94,7 +96,11 @@ func init() {
 	f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
 	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "file name to use when reading from stdin")
 	f.StringArrayVar(&backupOptions.Tags, "tag", nil, "add a `tag` for the new snapshot (can be specified multiple times)")
-	f.StringVar(&backupOptions.Hostname, "hostname", "", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag")
+
+	f.StringVar(&backupOptions.Host, "host", "H", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag")
+	f.StringVar(&backupOptions.Host, "hostname", "", "set the `hostname` for the snapshot manually")
+	f.MarkDeprecated("hostname", "use --host")
+
 	f.StringVar(&backupOptions.FilesFrom, "files-from", "", "read the files to backup from file (can be combined with file args)")
 	f.StringVar(&backupOptions.TimeStamp, "time", "", "time of the backup (ex. '2012-11-01 22:08:41') (default: now)")
 	f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories")
@@ -186,18 +192,9 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
 	return nil
 }
 
-// collectRejectFuncs returns a list of all functions which may reject data
-// from being saved in a snapshot
-func collectRejectFuncs(opts BackupOptions, repo *repository.Repository, targets []string) (fs []RejectFunc, err error) {
-	// allowed devices
-	if opts.ExcludeOtherFS && !opts.Stdin {
-		f, err := rejectByDevice(targets)
-		if err != nil {
-			return nil, err
-		}
-		fs = append(fs, f)
-	}
-
+// collectRejectByNameFuncs returns a list of all functions which may reject data
+// from being saved in a snapshot based on path only
+func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, targets []string) (fs []RejectByNameFunc, err error) {
 	// exclude restic cache
 	if repo.Cache != nil {
 		f, err := rejectResticCache(repo)
@@ -237,6 +234,21 @@ func collectRejectFuncs(opts BackupOptions, repo *repository.Repository, targets
 	return fs, nil
 }
 
+// collectRejectFuncs returns a list of all functions which may reject data
+// from being saved in a snapshot based on path and file info
+func collectRejectFuncs(opts BackupOptions, repo *repository.Repository, targets []string) (fs []RejectFunc, err error) {
+	// allowed devices
+	if opts.ExcludeOtherFS && !opts.Stdin {
+		f, err := rejectByDevice(targets)
+		if err != nil {
+			return nil, err
+		}
+		fs = append(fs, f)
+	}
+
+	return fs, nil
+}
+
 // readExcludePatternsFromFiles reads all exclude files and returns the list of
 // exclude patterns. For each line, leading and trailing white space is removed
 // and comment lines are ignored. For each remaining pattern, environment
@@ -295,10 +307,21 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 		return nil, err
 	}
 
+	// expand wildcards
+	var lines []string
+	for _, line := range fromfile {
+		var expanded []string
+		expanded, err := filepath.Glob(line)
+		if err != nil {
+			return nil, errors.WithMessage(err, fmt.Sprintf("pattern: %s", line))
+		}
+		lines = append(lines, expanded...)
+	}
+
 	// merge files from files-from into normal args so we can reuse the normal
 	// args checks and have the ability to use both files-from and args at the
 	// same time
-	args = append(args, fromfile...)
+	args = append(args, lines...)
 	if len(args) == 0 && !opts.Stdin {
 		return nil, errors.Fatal("nothing to backup, please specify target files/dirs")
 	}
@@ -327,7 +350,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
 
 	// Find last snapshot to set it as parent, if not already set
 	if !opts.Force && parentID == nil {
-		id, err := restic.FindLatestSnapshot(ctx, repo, targets, []restic.TagList{}, opts.Hostname)
+		id, err := restic.FindLatestSnapshot(ctx, repo, targets, []restic.TagList{}, opts.Host)
 		if err == nil {
 			parentID = &id
 		} else if err != restic.ErrNoSnapshotFound {
@@ -393,7 +416,13 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		return err
 	}
 
-	// rejectFuncs collect functions that can reject items from the backup
+	// rejectByNameFuncs collect functions that can reject items from the backup based on path only
+	rejectByNameFuncs, err := collectRejectByNameFuncs(opts, repo, targets)
+	if err != nil {
+		return err
+	}
+
+	// rejectFuncs collect functions that can reject items from the backup based on path and file info
 	rejectFuncs, err := collectRejectFuncs(opts, repo, targets)
 	if err != nil {
 		return err
@@ -414,6 +443,15 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		p.V("using parent snapshot %v\n", parentSnapshotID.Str())
 	}
 
+	selectByNameFilter := func(item string) bool {
+		for _, reject := range rejectByNameFuncs {
+			if reject(item) {
+				return false
+			}
+		}
+		return true
+	}
+
 	selectFilter := func(item string, fi os.FileInfo) bool {
 		for _, reject := range rejectFuncs {
 			if reject(item, fi) {
@@ -436,6 +474,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}
 
 	sc := archiver.NewScanner(targetFS)
+	sc.SelectByName = selectByNameFilter
 	sc.Select = selectFilter
 	sc.Error = p.ScannerError
 	sc.Result = p.ReportTotal
@@ -444,6 +483,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	t.Go(func() error { return sc.Scan(t.Context(gopts.ctx), targets) })
 
 	arch := archiver.New(repo, targetFS, archiver.Options{})
+	arch.SelectByName = selectByNameFilter
 	arch.Select = selectFilter
 	arch.WithAtime = opts.WithAtime
 	arch.Error = p.Error
@@ -459,7 +499,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		Excludes:       opts.Excludes,
 		Tags:           opts.Tags,
 		Time:           timeStamp,
-		Hostname:       opts.Hostname,
+		Hostname:       opts.Host,
 		ParentSnapshot: *parentSnapshotID,
 	}
 

cmd/restic/cmd_cache.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 	"sort"
 	"time"
@@ -9,6 +10,7 @@ import (
 	"github.com/restic/restic/internal/cache"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/fs"
+	"github.com/restic/restic/internal/ui/table"
 	"github.com/spf13/cobra"
 )
 
@@ -28,6 +30,7 @@ The "cache" command allows listing and cleaning local cache directories.
 type CacheOptions struct {
 	Cleanup bool
 	MaxAge  uint
+	NoSize  bool
 }
 
 var cacheOptions CacheOptions
@@ -38,6 +41,7 @@ func init() {
 	f := cmdCache.Flags()
 	f.BoolVar(&cacheOptions.Cleanup, "cleanup", false, "remove old cache directories")
 	f.UintVar(&cacheOptions.MaxAge, "max-age", 30, "max age in `days` for cache directories to be considered old")
+	f.BoolVar(&cacheOptions.NoSize, "no-size", false, "do not output the size of the cache directories")
 }
 
 func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
@@ -85,9 +89,22 @@ func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
 		return nil
 	}
 
-	tab := NewTable()
-	tab.Header = fmt.Sprintf("%-14s   %-16s %s", "Repository ID", "Last Used", "Old")
-	tab.RowFormat = "%-14s   %-16s %s"
+	tab := table.New()
+
+	type data struct {
+		ID   string
+		Last string
+		Old  string
+		Size string
+	}
+
+	tab.AddColumn("Repo ID", "{{ .ID }}")
+	tab.AddColumn("Last Used", "{{ .Last }}")
+	tab.AddColumn("Old", "{{ .Old }}")
+
+	if !opts.NoSize {
+		tab.AddColumn("Size", "{{ .Size }}")
+	}
 
 	dirs, err := cache.All(cachedir)
 	if err != nil {
@@ -109,14 +126,41 @@ func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
 			old = "yes"
 		}
 
-		tab.Rows = append(tab.Rows, []interface{}{
+		var size string
+		if !opts.NoSize {
+			bytes, err := dirSize(filepath.Join(cachedir, entry.Name()))
+			if err != nil {
+				return err
+			}
+			size = fmt.Sprintf("%11s", formatBytes(uint64(bytes)))
+		}
+
+		tab.AddRow(data{
 			entry.Name()[:10],
 			fmt.Sprintf("%d days ago", uint(time.Since(entry.ModTime()).Hours()/24)),
 			old,
+			size,
 		})
 	}
 
 	tab.Write(gopts.stdout)
+	Printf("%d cache dirs in %s\n", len(dirs), cachedir)
 
 	return nil
 }
+
+func dirSize(path string) (int64, error) {
+	var size int64
+	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
+		if err != nil || info == nil {
+			return err
+		}
+
+		if !info.IsDir() {
+			size += info.Size()
+		}
+
+		return nil
+	})
+	return size, err
+}

cmd/restic/cmd_diff.go

@@ -21,11 +21,11 @@ The "diff" command shows differences from the first to the second snapshot. The
 first characters in each line display what has happened to a particular file or
 directory:
 
- +  The item was added
- -  The item was removed
- U  The metadata (access mode, timestamps, ...) for the item was updated
- M  The file's content was modified
- T  The type was changed, e.g. a file was made a symlink
+* +  The item was added
+* -  The item was removed
+* U  The metadata (access mode, timestamps, ...) for the item was updated
+* M  The file's content was modified
+* T  The type was changed, e.g. a file was made a symlink
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {

cmd/restic/cmd_find.go

@@ -16,27 +16,38 @@ import (
 )
 
 var cmdFind = &cobra.Command{
-	Use:   "find [flags] PATTERN",
-	Short: "Find a file or directory",
+	Use:   "find [flags] PATTERN...",
+	Short: "Find a file, a directory or restic IDs",
 	Long: `
 The "find" command searches for files or directories in snapshots stored in the
-repo. `,
+repo.
+It can also be used to search for restic blobs or trees for troubleshooting.`,
+	Example: `restic find config.json
+restic find --json "*.yml" "*.json"
+restic find --json --blob 420f620f b46ebe8a ddd38656
+restic find --show-pack-id --blob 420f620f
+restic find --tree 577c2bc9 f81f2e22 a62827a9
+restic find --pack 025c1d06`,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runFind(findOptions, globalOptions, args)
 	},
 }
 
+const shortStr = 8 // Length of short IDs: 4 bytes as hex strings
+
 // FindOptions bundles all options for the find command.
 type FindOptions struct {
-	Oldest          string
-	Newest          string
-	Snapshots       []string
-	CaseInsensitive bool
-	ListLong        bool
-	Host            string
-	Paths           []string
-	Tags            restic.TagLists
+	Oldest             string
+	Newest             string
+	Snapshots          []string
+	BlobID, TreeID     bool
+	PackID, ShowPackID bool
+	CaseInsensitive    bool
+	ListLong           bool
+	Host               string
+	Paths              []string
+	Tags               restic.TagLists
 }
 
 var findOptions FindOptions
@@ -48,6 +59,10 @@ func init() {
 	f.StringVarP(&findOptions.Oldest, "oldest", "O", "", "oldest modification date/time")
 	f.StringVarP(&findOptions.Newest, "newest", "N", "", "newest modification date/time")
 	f.StringArrayVarP(&findOptions.Snapshots, "snapshot", "s", nil, "snapshot `id` to search in (can be given multiple times)")
+	f.BoolVar(&findOptions.BlobID, "blob", false, "pattern is a blob-ID")
+	f.BoolVar(&findOptions.TreeID, "tree", false, "pattern is a tree-ID")
+	f.BoolVar(&findOptions.PackID, "pack", false, "pattern is a pack-ID")
+	f.BoolVar(&findOptions.ShowPackID, "show-pack-id", false, "display the pack-ID the blobs belong to (with --blob)")
 	f.BoolVarP(&findOptions.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern")
 	f.BoolVarP(&findOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
 
@@ -58,7 +73,7 @@ func init() {
 
 type findPattern struct {
 	oldest, newest time.Time
-	pattern        string
+	pattern        []string
 	ignoreCase     bool
 }
 
@@ -95,7 +110,7 @@ type statefulOutput struct {
 	hits     int
 }
 
-func (s *statefulOutput) PrintJSON(path string, node *restic.Node) {
+func (s *statefulOutput) PrintPatternJSON(path string, node *restic.Node) {
 	type findNode restic.Node
 	b, err := json.Marshal(struct {
 		// Add these attributes
@@ -139,7 +154,7 @@ func (s *statefulOutput) PrintJSON(path string, node *restic.Node) {
 	s.hits++
 }
 
-func (s *statefulOutput) PrintNormal(path string, node *restic.Node) {
+func (s *statefulOutput) PrintPatternNormal(path string, node *restic.Node) {
 	if s.newsn != s.oldsn {
 		if s.oldsn != nil {
 			Verbosef("\n")
@@ -150,11 +165,62 @@ func (s *statefulOutput) PrintNormal(path string, node *restic.Node) {
 	Printf(formatNode(path, node, s.ListLong) + "\n")
 }
 
-func (s *statefulOutput) Print(path string, node *restic.Node) {
+func (s *statefulOutput) PrintPattern(path string, node *restic.Node) {
 	if s.JSON {
-		s.PrintJSON(path, node)
+		s.PrintPatternJSON(path, node)
 	} else {
-		s.PrintNormal(path, node)
+		s.PrintPatternNormal(path, node)
+	}
+}
+
+func (s *statefulOutput) PrintObjectJSON(kind, id, nodepath, treeID string, sn *restic.Snapshot) {
+	b, err := json.Marshal(struct {
+		// Add these attributes
+		ObjectType string    `json:"object_type"`
+		ID         string    `json:"id"`
+		Path       string    `json:"path"`
+		ParentTree string    `json:"parent_tree,omitempty"`
+		SnapshotID string    `json:"snapshot"`
+		Time       time.Time `json:"time,omitempty"`
+	}{
+		ObjectType: kind,
+		ID:         id,
+		Path:       nodepath,
+		SnapshotID: sn.ID().String(),
+		ParentTree: treeID,
+		Time:       sn.Time,
+	})
+	if err != nil {
+		Warnf("Marshall failed: %v\n", err)
+		return
+	}
+	if !s.inuse {
+		Printf("[")
+		s.inuse = true
+	}
+	if s.hits > 0 {
+		Printf(",")
+	}
+	Printf(string(b))
+	s.hits++
+}
+
+func (s *statefulOutput) PrintObjectNormal(kind, id, nodepath, treeID string, sn *restic.Snapshot) {
+	Printf("Found %s %s\n", kind, id)
+	if kind == "blob" {
+		Printf(" ... in file %s\n", nodepath)
+		Printf("     (tree %s)\n", treeID)
+	} else {
+		Printf(" ... path %s\n", nodepath)
+	}
+	Printf(" ... in snapshot %s (%s)\n", sn.ID().Str(), sn.Time.Format(TimeFormat))
+}
+
+func (s *statefulOutput) PrintObject(kind, id, nodepath, treeID string, sn *restic.Snapshot) {
+	if s.JSON {
+		s.PrintObjectJSON(kind, id, nodepath, treeID, sn)
+	} else {
+		s.PrintObjectNormal(kind, id, nodepath, treeID, sn)
 	}
 }
 
@@ -179,6 +245,9 @@ type Finder struct {
 	pat         findPattern
 	out         statefulOutput
 	ignoreTrees restic.IDSet
+	blobIDs     map[string]struct{}
+	treeIDs     map[string]struct{}
+	itemsFound  int
 }
 
 func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error {
@@ -189,7 +258,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 	}
 
 	f.out.newsn = sn
-	return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(nodepath string, node *restic.Node, err error) (bool, error) {
+	return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
 		if err != nil {
 			return false, err
 		}
@@ -203,9 +272,17 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 			normalizedNodepath = strings.ToLower(nodepath)
 		}
 
-		foundMatch, err := filter.Match(f.pat.pattern, normalizedNodepath)
-		if err != nil {
-			return false, err
+		var foundMatch bool
+
+		for _, pat := range f.pat.pattern {
+			found, err := filter.Match(pat, normalizedNodepath)
+			if err != nil {
+				return false, err
+			}
+			if found {
+				foundMatch = true
+				break
+			}
 		}
 
 		var (
@@ -213,9 +290,16 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 			errIfNoMatch    error
 		)
 		if node.Type == "dir" {
-			childMayMatch, err := filter.ChildMatch(f.pat.pattern, normalizedNodepath)
-			if err != nil {
-				return false, err
+			var childMayMatch bool
+			for _, pat := range f.pat.pattern {
+				mayMatch, err := filter.ChildMatch(pat, normalizedNodepath)
+				if err != nil {
+					return false, err
+				}
+				if mayMatch {
+					childMayMatch = true
+					break
+				}
 			}
 
 			if !childMayMatch {
@@ -241,20 +325,158 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 		}
 
 		debug.Log("    found match\n")
-		f.out.Print(nodepath, node)
+		f.out.PrintPattern(nodepath, node)
 		return false, nil
 	})
 }
 
+func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
+	debug.Log("searching IDs in snapshot %s", sn.ID())
+
+	if sn.Tree == nil {
+		return errors.Errorf("snapshot %v has no tree", sn.ID().Str())
+	}
+
+	f.out.newsn = sn
+	return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
+		if err != nil {
+			return false, err
+		}
+
+		if node == nil {
+			return false, nil
+		}
+
+		if node.Type == "dir" && f.treeIDs != nil {
+			treeID := node.Subtree
+			found := false
+			if _, ok := f.treeIDs[treeID.Str()]; ok {
+				found = true
+			} else if _, ok := f.treeIDs[treeID.String()]; ok {
+				found = true
+			}
+			if found {
+				f.out.PrintObject("tree", treeID.String(), nodepath, "", sn)
+				f.itemsFound++
+				// Terminate if we have found all trees (and we are not
+				// looking for blobs)
+				if f.itemsFound >= len(f.treeIDs) && f.blobIDs == nil {
+					// Return an error to terminate the Walk
+					return true, errors.New("OK")
+				}
+			}
+		}
+
+		if node.Type == "file" && f.blobIDs != nil {
+			for _, id := range node.Content {
+				idStr := id.String()
+				if _, ok := f.blobIDs[idStr]; !ok {
+					// Look for short ID form
+					if _, ok := f.blobIDs[idStr[:shortStr]]; !ok {
+						continue
+					}
+					// Replace the short ID with the long one
+					f.blobIDs[idStr] = struct{}{}
+					delete(f.blobIDs, idStr[:shortStr])
+				}
+				f.out.PrintObject("blob", idStr, nodepath, parentTreeID.String(), sn)
+				break
+			}
+		}
+
+		return false, nil
+	})
+}
+
+// packsToBlobs converts the list of pack IDs to a list of blob IDs that
+// belong to those packs.
+func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
+	packIDs := make(map[string]struct{})
+	for _, p := range packs {
+		packIDs[p] = struct{}{}
+	}
+	if f.blobIDs == nil {
+		f.blobIDs = make(map[string]struct{})
+	}
+
+	allPacksFound := false
+	packsFound := 0
+
+	debug.Log("Looking for packs...")
+	err := f.repo.List(ctx, restic.DataFile, func(id restic.ID, size int64) error {
+		if allPacksFound {
+			return nil
+		}
+		idStr := id.String()
+		if _, ok := packIDs[idStr]; !ok {
+			// Look for short ID form
+			if _, ok := packIDs[idStr[:shortStr]]; !ok {
+				return nil
+			}
+		}
+		debug.Log("Found pack %s", idStr)
+		blobs, _, err := f.repo.ListPack(ctx, id, size)
+		if err != nil {
+			return err
+		}
+		for _, b := range blobs {
+			f.blobIDs[b.ID.String()] = struct{}{}
+		}
+		// Stop searching when all packs have been found
+		packsFound++
+		if packsFound >= len(packIDs) {
+			allPacksFound = true
+		}
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	if !allPacksFound {
+		return errors.Fatal("unable to find all specified pack(s)")
+	}
+
+	debug.Log("%d blobs found", len(f.blobIDs))
+	return nil
+}
+
+func (f *Finder) findBlobsPacks(ctx context.Context) {
+	idx := f.repo.Index()
+	for i := range f.blobIDs {
+		rid, err := restic.ParseID(i)
+		if err != nil {
+			Printf("Note: cannot find pack for blob '%s', unable to parse ID: %v\n", i, err)
+			continue
+		}
+
+		blobs, found := idx.Lookup(rid, restic.DataBlob)
+		if !found {
+			Printf("Blob %s not found in the index\n", rid.Str())
+			continue
+		}
+
+		for _, b := range blobs {
+			if b.ID.Equal(rid) {
+				Printf("Blob belongs to pack %s\n ... Pack %s: %s\n", b.PackID, b.PackID.Str(), b.String())
+				break
+			}
+		}
+	}
+}
+
 func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
-	if len(args) != 1 {
+	if len(args) == 0 {
 		return errors.Fatal("wrong number of arguments")
 	}
 
 	var err error
-	pat := findPattern{pattern: args[0]}
+	pat := findPattern{pattern: args}
 	if opts.CaseInsensitive {
-		pat.pattern = strings.ToLower(pat.pattern)
+		for i := range pat.pattern {
+			pat.pattern[i] = strings.ToLower(pat.pattern[i])
+		}
 		pat.ignoreCase = true
 	}
 
@@ -270,6 +492,14 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
+	// Check at most only one kind of IDs is provided: currently we
+	// can't mix types
+	if (opts.BlobID && opts.TreeID) ||
+		(opts.BlobID && opts.PackID) ||
+		(opts.TreeID && opts.PackID) {
+		return errors.Fatal("cannot have several ID types")
+	}
+
 	repo, err := OpenRepository(gopts)
 	if err != nil {
 		return err
@@ -296,12 +526,40 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 		out:         statefulOutput{ListLong: opts.ListLong, JSON: globalOptions.JSON},
 		ignoreTrees: restic.NewIDSet(),
 	}
+
+	if opts.BlobID {
+		f.blobIDs = make(map[string]struct{})
+		for _, pat := range f.pat.pattern {
+			f.blobIDs[pat] = struct{}{}
+		}
+	}
+	if opts.TreeID {
+		f.treeIDs = make(map[string]struct{})
+		for _, pat := range f.pat.pattern {
+			f.treeIDs[pat] = struct{}{}
+		}
+	}
+
+	if opts.PackID {
+		f.packsToBlobs(ctx, []string{f.pat.pattern[0]}) // TODO: support multiple packs
+	}
+
 	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, opts.Snapshots) {
+		if f.blobIDs != nil || f.treeIDs != nil {
+			if err = f.findIDs(ctx, sn); err != nil && err.Error() != "OK" {
+				return err
+			}
+			continue
+		}
 		if err = f.findInSnapshot(ctx, sn); err != nil {
 			return err
 		}
 	}
 	f.out.Finish()
 
+	if opts.ShowPackID && f.blobIDs != nil {
+		f.findBlobsPacks(ctx)
+	}
+
 	return nil
 }

cmd/restic/cmd_forget.go

@@ -59,14 +59,15 @@ func init() {
 	f.IntVarP(&forgetOptions.Weekly, "keep-weekly", "w", 0, "keep the last `n` weekly snapshots")
 	f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots")
 	f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots")
-	f.VarP(&forgetOptions.Within, "keep-within", "", "keep snapshots that were created within `duration` before the newest (e.g. 1y5m7d)")
+	f.VarP(&forgetOptions.Within, "keep-within", "", "keep snapshots that are older than `duration` (eg. 1y5m7d) relative to the latest snapshot")
 
 	f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)")
-	// Sadly the commonly used shortcut `H` is already used.
 	f.StringVar(&forgetOptions.Host, "host", "", "only consider snapshots with the given `host`")
-	// Deprecated since 2017-03-07.
-	f.StringVar(&forgetOptions.Host, "hostname", "", "only consider snapshots with the given `hostname` (deprecated)")
+	f.StringVar(&forgetOptions.Host, "hostname", "", "only consider snapshots with the given `hostname`")
+	f.MarkDeprecated("hostname", "use --host")
+
 	f.Var(&forgetOptions.Tags, "tag", "only consider snapshots which include this `taglist` in the format `tag[,tag,...]` (can be specified multiple times)")
+
 	f.StringArrayVar(&forgetOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` (can be specified multiple times)")
 	f.BoolVarP(&forgetOptions.Compact, "compact", "c", false, "use compact format")
 
@@ -206,17 +207,17 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 			}
 			Verbosef(":\n\n")
 
-			keep, remove := restic.ApplyPolicy(snapshotGroup, policy)
+			keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy)
 
 			if len(keep) != 0 && !gopts.Quiet {
 				Printf("keep %d snapshots:\n", len(keep))
-				PrintSnapshots(globalOptions.stdout, keep, opts.Compact)
+				PrintSnapshots(globalOptions.stdout, keep, reasons, opts.Compact)
 				Printf("\n")
 			}
 
 			if len(remove) != 0 && !gopts.Quiet {
 				Printf("remove %d snapshots:\n", len(remove))
-				PrintSnapshots(globalOptions.stdout, remove, opts.Compact)
+				PrintSnapshots(globalOptions.stdout, remove, nil, opts.Compact)
 				Printf("\n")
 			}
 

cmd/restic/cmd_key.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"io/ioutil"
 	"os"
 	"strings"
@@ -11,6 +10,7 @@ import (
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui/table"
 
 	"github.com/spf13/cobra"
 )
@@ -36,53 +36,18 @@ func init() {
 	flags.StringVarP(&newPasswordFile, "new-password-file", "", "", "the file from which to load a new password")
 }
 
-type keyInfo struct {
-	Current  bool   `json:"current"`
-	ID       string `json:"id"`
-	UserName string `json:"userName"`
-	HostName string `json:"hostName"`
-	Created  string `json:"created"`
-}
-
-func (ki keyInfo) CurrentStr() string {
-	if ki.Current {
-		return "*"
-	}
-	return " "
-}
-
 func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions) error {
-	var (
-		appendKey func(keyInfo)
-		printKeys func() error
-	)
-
-	switch gopts.JSON {
-	case true:
-		var keys []keyInfo
-
-		appendKey = func(key keyInfo) {
-			keys = append(keys, key)
-		}
-
-		printKeys = func() error {
-			return json.NewEncoder(gopts.stdout).Encode(keys)
-		}
-	default:
-		tab := NewTable()
-		tab.Header = fmt.Sprintf(" %-10s  %-10s  %-10s  %s", "ID", "User", "Host", "Created")
-		tab.RowFormat = "%s%-10s  %-10s  %-10s  %s"
-
-		appendKey = func(key keyInfo) {
-			tab.Rows = append(tab.Rows, []interface{}{key.CurrentStr(), key.ID, key.UserName, key.HostName, key.Created})
-		}
-
-		printKeys = func() error {
-			return tab.Write(globalOptions.stdout)
-		}
+	type keyInfo struct {
+		Current  bool   `json:"current"`
+		ID       string `json:"id"`
+		UserName string `json:"userName"`
+		HostName string `json:"hostName"`
+		Created  string `json:"created"`
 	}
 
-	if err := s.List(ctx, restic.KeyFile, func(id restic.ID, size int64) error {
+	var keys []keyInfo
+
+	err := s.List(ctx, restic.KeyFile, func(id restic.ID, size int64) error {
 		k, err := repository.LoadKey(ctx, s, id.String())
 		if err != nil {
 			Warnf("LoadKey() failed: %v\n", err)
@@ -97,14 +62,29 @@ func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions
 			Created:  k.Created.Format(TimeFormat),
 		}
 
-		appendKey(key)
-
+		keys = append(keys, key)
 		return nil
-	}); err != nil {
+	})
+
+	if err != nil {
 		return err
 	}
 
-	return printKeys()
+	if gopts.JSON {
+		return json.NewEncoder(globalOptions.stdout).Encode(keys)
+	}
+
+	tab := table.New()
+	tab.AddColumn(" ID", "{{if .Current}}*{{else}} {{end}}{{ .ID }}")
+	tab.AddColumn("User", "{{ .UserName }}")
+	tab.AddColumn("Host", "{{ .HostName }}")
+	tab.AddColumn("Created", "{{ .Created }}")
+
+	for _, key := range keys {
+		tab.AddRow(key)
+	}
+
+	return tab.Write(globalOptions.stdout)
 }
 
 // testKeyNewPassword is used to set a new password during integration testing.

cmd/restic/cmd_ls.go

@@ -2,21 +2,37 @@ package main
 
 import (
 	"context"
+	"encoding/json"
+	"os"
+	"strings"
+	"time"
 
 	"github.com/spf13/cobra"
 
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/walker"
 )
 
 var cmdLs = &cobra.Command{
-	Use:   "ls [flags] [snapshot-ID ...]",
+	Use:   "ls [flags] [snapshotID] [dir...]",
 	Short: "List files in a snapshot",
 	Long: `
-The "ls" command allows listing files and directories in a snapshot.
+The "ls" command lists files and directories in a snapshot.
 
-The special snapshot-ID "latest" can be used to list files and directories of the latest snapshot in the repository.
+The special snapshot ID "latest" can be used to list files and
+directories of the latest snapshot in the repository. The
+--host flag can be used in conjunction to select the latest
+snapshot originating from a certain host only.
+
+File listings can optionally be filtered by directories. Any
+positional arguments after the snapshot ID are interpreted as
+absolute directory paths, and only files inside those directories
+will be listed. If the --recursive flag is used, then the filter
+will allow traversing into matching directories' subfolders.
+Any directory paths specified must be absolute (starting with
+a path separator); paths use the forward slash '/' as separator.
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -26,10 +42,11 @@ The special snapshot-ID "latest" can be used to list files and directories of th
 
 // LsOptions collects all options for the ls command.
 type LsOptions struct {
-	ListLong bool
-	Host     string
-	Tags     restic.TagLists
-	Paths    []string
+	ListLong  bool
+	Host      string
+	Tags      restic.TagLists
+	Paths     []string
+	Recursive bool
 }
 
 var lsOptions LsOptions
@@ -39,10 +56,33 @@ func init() {
 
 	flags := cmdLs.Flags()
 	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
-
 	flags.StringVarP(&lsOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
 	flags.Var(&lsOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot ID is given")
 	flags.StringArrayVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
+	flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories")
+}
+
+type lsSnapshot struct {
+	*restic.Snapshot
+
+	ID         *restic.ID `json:"id"`
+	ShortID    string     `json:"short_id"`
+	Nodes      []lsNode   `json:"nodes"`
+	StructType string     `json:"struct_type"` // "snapshot"
+}
+
+type lsNode struct {
+	Name       string      `json:"name"`
+	Type       string      `json:"type"`
+	Path       string      `json:"path"`
+	UID        uint32      `json:"uid"`
+	GID        uint32      `json:"gid"`
+	Size       uint64      `json:"size,omitempty"`
+	Mode       os.FileMode `json:"mode,omitempty"`
+	ModTime    time.Time   `json:"mtime,omitempty"`
+	AccessTime time.Time   `json:"atime,omitempty"`
+	ChangeTime time.Time   `json:"ctime,omitempty"`
+	StructType string      `json:"struct_type"` // "node"
 }
 
 func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
@@ -50,6 +90,51 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 		return errors.Fatal("Invalid arguments, either give one or more snapshot IDs or set filters.")
 	}
 
+	// extract any specific directories to walk
+	var dirs []string
+	if len(args) > 1 {
+		dirs = args[1:]
+		for _, dir := range dirs {
+			if !strings.HasPrefix(dir, "/") {
+				return errors.Fatal("All path filters must be absolute, starting with a forward slash '/'")
+			}
+		}
+	}
+
+	withinDir := func(nodepath string) bool {
+		if len(dirs) == 0 {
+			return true
+		}
+
+		for _, dir := range dirs {
+			// we're within one of the selected dirs, example:
+			//   nodepath: "/test/foo"
+			//   dir:      "/test"
+			if fs.HasPathPrefix(dir, nodepath) {
+				return true
+			}
+		}
+		return false
+	}
+
+	approachingMatchingTree := func(nodepath string) bool {
+		if len(dirs) == 0 {
+			return true
+		}
+
+		for _, dir := range dirs {
+			// the current node path is a prefix for one of the
+			// directories, so we're interested in something deeper in the
+			// tree. Example:
+			//   nodepath: "/test"
+			//   dir:      "/test/foo"
+			if fs.HasPathPrefix(nodepath, dir) {
+				return true
+			}
+		}
+		return false
+	}
+
 	repo, err := OpenRepository(gopts)
 	if err != nil {
 		return err
@@ -61,23 +146,99 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 
 	ctx, cancel := context.WithCancel(gopts.ctx)
 	defer cancel()
-	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
-		Verbosef("snapshot %s of %v at %s):\n", sn.ID().Str(), sn.Paths, sn.Time)
 
-		err := walker.Walk(ctx, repo, *sn.Tree, nil, func(nodepath string, node *restic.Node, err error) (bool, error) {
+	var (
+		printSnapshot func(sn *restic.Snapshot)
+		printNode     func(path string, node *restic.Node)
+		printFinish   func() error
+	)
+
+	if gopts.JSON {
+		var lssnapshots []lsSnapshot
+
+		printSnapshot = func(sn *restic.Snapshot) {
+			lss := lsSnapshot{
+				Snapshot:   sn,
+				ID:         sn.ID(),
+				ShortID:    sn.ID().Str(),
+				StructType: "snapshot",
+			}
+			lssnapshots = append(lssnapshots, lss)
+		}
+
+		printNode = func(path string, node *restic.Node) {
+			lsn := lsNode{
+				Name:       node.Name,
+				Type:       node.Type,
+				Path:       path,
+				UID:        node.UID,
+				GID:        node.GID,
+				Size:       node.Size,
+				Mode:       node.Mode,
+				ModTime:    node.ModTime,
+				AccessTime: node.AccessTime,
+				ChangeTime: node.ChangeTime,
+				StructType: "node",
+			}
+			s := &lssnapshots[len(lssnapshots)-1]
+			s.Nodes = append(s.Nodes, lsn)
+		}
+
+		printFinish = func() error {
+			return json.NewEncoder(gopts.stdout).Encode(lssnapshots)
+		}
+	} else {
+		// default output methods
+		printSnapshot = func(sn *restic.Snapshot) {
+			Verbosef("snapshot %s of %v filtered by %v at %s):\n", sn.ID().Str(), sn.Paths, dirs, sn.Time)
+		}
+		printNode = func(path string, node *restic.Node) {
+			Printf("%s\n", formatNode(path, node, lsOptions.ListLong))
+		}
+		printFinish = func() error {
+			return nil
+		}
+	}
+
+	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args[:1]) {
+		printSnapshot(sn)
+
+		err := walker.Walk(ctx, repo, *sn.Tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
 			if err != nil {
 				return false, err
 			}
-
 			if node == nil {
 				return false, nil
 			}
-			Printf("%s\n", formatNode(nodepath, node, lsOptions.ListLong))
+
+			if withinDir(nodepath) {
+				// if we're within a dir, print the node
+				printNode(nodepath, node)
+
+				// if recursive listing is requested, signal the walker that it
+				// should continue walking recursively
+				if opts.Recursive {
+					return false, nil
+				}
+			}
+
+			// if there's an upcoming match deeper in the tree (but we're not
+			// there yet), signal the walker to descend into any subdirs
+			if approachingMatchingTree(nodepath) {
+				return false, nil
+			}
+
+			// otherwise, signal the walker to not walk recursively into any
+			// subdirs
+			if node.Type == "dir" {
+				return false, walker.SkipNode
+			}
 			return false, nil
 		})
+
 		if err != nil {
 			return err
 		}
 	}
-	return nil
+	return printFinish()
 }

cmd/restic/cmd_prune.go

@@ -149,8 +149,8 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
 		len(idx.Packs), blobs, formatBytes(uint64(stats.bytes)))
 
 	blobCount := make(map[restic.BlobHandle]int)
-	duplicateBlobs := 0
-	duplicateBytes := 0
+	var duplicateBlobs uint64
+	var duplicateBytes uint64
 
 	// find duplicate blobs
 	for _, p := range idx.Packs {
@@ -161,7 +161,7 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
 
 			if blobCount[h] > 1 {
 				duplicateBlobs++
-				duplicateBytes += int(entry.Length)
+				duplicateBytes += uint64(entry.Length)
 			}
 		}
 	}
@@ -252,7 +252,7 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
 				continue
 			}
 
-			removeBytes += int(blob.Length)
+			removeBytes += uint64(blob.Length)
 		}
 
 		if hasActiveBlob {

cmd/restic/cmd_self_update.go

@@ -0,0 +1,51 @@
+// +build selfupdate
+
+package main
+
+import (
+	"os"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/selfupdate"
+	"github.com/spf13/cobra"
+)
+
+var cmdSelfUpdate = &cobra.Command{
+	Use:   "self-update [flags]",
+	Short: "Update the restic binary",
+	Long: `
+The command "update-restic" downloads the latest stable release of restic from
+GitHub and replaces the currently running binary. After download, the
+authenticity of the binary is verified using the GPG signature on the release
+files.
+`,
+	DisableAutoGenTag: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runSelfUpdate(selfUpdateOptions, globalOptions, args)
+	},
+}
+
+// SelfUpdateOptions collects all options for the update-restic command.
+type SelfUpdateOptions struct {
+	Output string
+}
+
+var selfUpdateOptions SelfUpdateOptions
+
+func init() {
+	cmdRoot.AddCommand(cmdSelfUpdate)
+
+	flags := cmdSelfUpdate.Flags()
+	flags.StringVar(&selfUpdateOptions.Output, "output", os.Args[0], "Save the downloaded file as `filename`")
+}
+
+func runSelfUpdate(opts SelfUpdateOptions, gopts GlobalOptions, args []string) error {
+	v, err := selfupdate.DownloadLatestStableRelease(gopts.ctx, opts.Output, Verbosef)
+	if err != nil {
+		return errors.Fatalf("unable to update restic: %v", err)
+	}
+
+	Printf("successfully updated restic to version %v\n", v)
+
+	return nil
+}

cmd/restic/cmd_snapshots.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui/table"
 	"github.com/spf13/cobra"
 )
 
@@ -81,7 +82,7 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 		}
 		return nil
 	}
-	PrintSnapshots(gopts.stdout, list, opts.Compact)
+	PrintSnapshots(gopts.stdout, list, nil, opts.Compact)
 
 	return nil
 }
@@ -123,7 +124,16 @@ func FilterLastSnapshots(list restic.Snapshots) restic.Snapshots {
 }
 
 // PrintSnapshots prints a text table of the snapshots in list to stdout.
-func PrintSnapshots(stdout io.Writer, list restic.Snapshots, compact bool) {
+func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.KeepReason, compact bool) {
+	// keep the reasons a snasphot is being kept in a map, so that it doesn't
+	// get lost when the list of snapshots is sorted
+	keepReasons := make(map[restic.ID]restic.KeepReason, len(reasons))
+	if len(reasons) > 0 {
+		for i, sn := range list {
+			id := sn.ID()
+			keepReasons[*id] = reasons[i]
+		}
+	}
 
 	// always sort the snapshots so that the newer ones are listed last
 	sort.SliceStable(list, func(i, j int) bool {
@@ -143,71 +153,72 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, compact bool) {
 		}
 	}
 
-	tab := NewTable()
-	if !compact {
-		tab.Header = fmt.Sprintf("%-8s  %-19s  %-*s  %-*s  %-3s %s", "ID", "Date", -maxHost, "Host", -maxTag, "Tags", "", "Directory")
-		tab.RowFormat = fmt.Sprintf("%%-8s  %%-19s  %%%ds  %%%ds  %%-3s %%s", -maxHost, -maxTag)
+	tab := table.New()
+
+	if compact {
+		tab.AddColumn("ID", "{{ .ID }}")
+		tab.AddColumn("Time", "{{ .Timestamp }}")
+		tab.AddColumn("Host", "{{ .Hostname }}")
+		tab.AddColumn("Tags  ", `{{ join .Tags "\n" }}`)
 	} else {
-		tab.Header = fmt.Sprintf("%-8s  %-19s  %-*s  %-*s", "ID", "Date", -maxHost, "Host", -maxTag, "Tags")
-		tab.RowFormat = fmt.Sprintf("%%-8s  %%-19s  %%%ds  %%s", -maxHost)
+		tab.AddColumn("ID", "{{ .ID }}")
+		tab.AddColumn("Time", "{{ .Timestamp }}")
+		tab.AddColumn("Host      ", "{{ .Hostname }}")
+		tab.AddColumn("Tags      ", `{{ join .Tags "," }}`)
+		if len(reasons) > 0 {
+			tab.AddColumn("Reasons", `{{ join .Reasons "\n" }}`)
+		}
+		tab.AddColumn("Paths", `{{ join .Paths "\n" }}`)
 	}
 
+	type snapshot struct {
+		ID        string
+		Timestamp string
+		Hostname  string
+		Tags      []string
+		Reasons   []string
+		Paths     []string
+	}
+
+	var multiline bool
 	for _, sn := range list {
-		if len(sn.Paths) == 0 {
-			continue
+		data := snapshot{
+			ID:        sn.ID().Str(),
+			Timestamp: sn.Time.Format(TimeFormat),
+			Hostname:  sn.Hostname,
+			Tags:      sn.Tags,
+			Paths:     sn.Paths,
 		}
 
-		firstTag := ""
-		if len(sn.Tags) > 0 {
-			firstTag = sn.Tags[0]
+		if len(reasons) > 0 {
+			id := sn.ID()
+			data.Reasons = keepReasons[*id].Matches
 		}
 
-		rows := len(sn.Paths)
-		if rows < len(sn.Tags) {
-			rows = len(sn.Tags)
+		if len(sn.Paths) > 1 {
+			multiline = true
 		}
 
-		treeElement := "   "
-		if rows != 1 {
-			treeElement = "┌──"
-		}
-
-		if !compact {
-			tab.Rows = append(tab.Rows, []interface{}{sn.ID().Str(), sn.Time.Format(TimeFormat), sn.Hostname, firstTag, treeElement, sn.Paths[0]})
-		} else {
-			allTags := ""
-			for _, tag := range sn.Tags {
-				allTags += tag + " "
-			}
-			tab.Rows = append(tab.Rows, []interface{}{sn.ID().Str(), sn.Time.Format(TimeFormat), sn.Hostname, allTags})
-			continue
-		}
-
-		if len(sn.Tags) > rows {
-			rows = len(sn.Tags)
-		}
-
-		for i := 1; i < rows; i++ {
-			path := ""
-			if len(sn.Paths) > i {
-				path = sn.Paths[i]
-			}
-
-			tag := ""
-			if len(sn.Tags) > i {
-				tag = sn.Tags[i]
-			}
-
-			treeElement := "│"
-			if i == (rows - 1) {
-				treeElement = "└──"
-			}
-
-			tab.Rows = append(tab.Rows, []interface{}{"", "", "", tag, treeElement, path})
-		}
+		tab.AddRow(data)
 	}
 
-	tab.Footer = fmt.Sprintf("%d snapshots", len(list))
+	tab.AddFooter(fmt.Sprintf("%d snapshots", len(list)))
+
+	if multiline {
+		// print an additional blank line between snapshots
+
+		var last int
+		tab.PrintData = func(w io.Writer, idx int, s string) error {
+			var err error
+			if idx == last {
+				_, err = fmt.Fprintf(w, "%s\n", s)
+			} else {
+				_, err = fmt.Fprintf(w, "\n%s\n", s)
+			}
+			last = idx
+			return err
+		}
+	}
 
 	tab.Write(stdout)
 }

cmd/restic/cmd_stats.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/walker"
 	"github.com/spf13/cobra"
@@ -29,17 +30,13 @@ to calculate.
 
 The modes are:
 
-  restore-size: (default) Counts the size of the restored files.
-
-  files-by-contents: Counts total size of files, where a file is
-                     considered unique if it has unique contents.
-
-  raw-data: Counts the size of blobs in the repository, regardless
-			of how many files reference them.
-
-  blobs-per-file: A combination of files-by-contents and raw-data.
-
-Refer to the online manual for more details about each mode.
+* restore-size: (default) Counts the size of the restored files.
+* files-by-contents: Counts total size of files, where a file is
+   considered unique if it has unique contents.
+* raw-data: Counts the size of blobs in the repository, regardless of
+  how many files reference them.
+* blobs-per-file: A combination of files-by-contents and raw-data.
+* Refer to the online manual for more details about each mode.
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -51,7 +48,7 @@ func init() {
 	cmdRoot.AddCommand(cmdStats)
 	f := cmdStats.Flags()
 	f.StringVar(&countMode, "mode", countModeRestoreSize, "counting mode: restore-size (default), files-by-contents, blobs-per-file, or raw-data")
-	f.StringVar(&snapshotByHost, "host", "", "filter latest snapshot by this hostname")
+	f.StringVar(&snapshotByHost, "host", "H", "filter latest snapshot by this hostname")
 }
 
 func runStats(gopts GlobalOptions, args []string) error {
@@ -80,6 +77,10 @@ func runStats(gopts GlobalOptions, args []string) error {
 		}
 	}
 
+	if !gopts.JSON {
+		Printf("scanning...\n")
+	}
+
 	// create a container for the stats (and other needed state)
 	stats := &statsContainer{
 		uniqueFiles: make(map[fileID]struct{}),
@@ -95,18 +96,18 @@ func runStats(gopts GlobalOptions, args []string) error {
 		if snapshotIDString == "latest" {
 			sID, err = restic.FindLatestSnapshot(ctx, repo, []string{}, []restic.TagList{}, snapshotByHost)
 			if err != nil {
-				Exitf(1, "latest snapshot for criteria not found: %v", err)
+				return errors.Fatalf("latest snapshot for criteria not found: %v", err)
 			}
 		} else {
 			sID, err = restic.FindSnapshot(repo, snapshotIDString)
 			if err != nil {
-				return err
+				return errors.Fatalf("error loading snapshot: %v", err)
 			}
 		}
 
 		snapshot, err := restic.LoadSnapshot(ctx, repo, sID)
 		if err != nil {
-			return err
+			return errors.Fatalf("error loading snapshot from repo: %v", err)
 		}
 
 		err = statsWalkSnapshot(ctx, snapshot, repo, stats)
@@ -144,6 +145,15 @@ func runStats(gopts GlobalOptions, args []string) error {
 		return nil
 	}
 
+	// inform the user what was scanned and how it was scanned
+	snapshotsScanned := snapshotIDString
+	if snapshotsScanned == "latest" {
+		snapshotsScanned = "the latest snapshot"
+	} else if snapshotsScanned == "" {
+		snapshotsScanned = "all snapshots"
+	}
+	Printf("Stats for %s in %s mode:\n", snapshotsScanned, countMode)
+
 	if stats.TotalBlobCount > 0 {
 		Printf("  Total Blob Count:   %d\n", stats.TotalBlobCount)
 	}
@@ -174,7 +184,7 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest
 }
 
 func statsWalkTree(repo restic.Repository, stats *statsContainer) walker.WalkFunc {
-	return func(npath string, node *restic.Node, nodeErr error) (bool, error) {
+	return func(_ restic.ID, npath string, node *restic.Node, nodeErr error) (bool, error) {
 		if nodeErr != nil {
 			return true, nodeErr
 		}

cmd/restic/exclude.go

@@ -60,15 +60,20 @@ func (rc *rejectionCache) Store(dir string, rejected bool) {
 	rc.m[dir] = rejected
 }
 
+// RejectByNameFunc is a function that takes a filename of a
+// file that would be included in the backup. The function returns true if it
+// should be excluded (rejected) from the backup.
+type RejectByNameFunc func(path string) bool
+
 // RejectFunc is a function that takes a filename and os.FileInfo of a
 // file that would be included in the backup. The function returns true if it
 // should be excluded (rejected) from the backup.
 type RejectFunc func(path string, fi os.FileInfo) bool
 
-// rejectByPattern returns a RejectFunc which rejects files that match
+// rejectByPattern returns a RejectByNameFunc which rejects files that match
 // one of the patterns.
-func rejectByPattern(patterns []string) RejectFunc {
-	return func(item string, fi os.FileInfo) bool {
+func rejectByPattern(patterns []string) RejectByNameFunc {
+	return func(item string) bool {
 		matched, _, err := filter.List(patterns, item)
 		if err != nil {
 			Warnf("error for exclude pattern: %v", err)
@@ -83,14 +88,14 @@ func rejectByPattern(patterns []string) RejectFunc {
 	}
 }
 
-// rejectIfPresent returns a RejectFunc which itself returns whether a path
-// should be excluded. The RejectFunc considers a file to be excluded when
+// rejectIfPresent returns a RejectByNameFunc which itself returns whether a path
+// should be excluded. The RejectByNameFunc considers a file to be excluded when
 // it resides in a directory with an exclusion file, that is specified by
 // excludeFileSpec in the form "filename[:content]". The returned error is
 // non-nil if the filename component of excludeFileSpec is empty. If rc is
-// non-nil, it is going to be used in the RejectFunc to expedite the evaluation
+// non-nil, it is going to be used in the RejectByNameFunc to expedite the evaluation
 // of a directory based on previous visits.
-func rejectIfPresent(excludeFileSpec string) (RejectFunc, error) {
+func rejectIfPresent(excludeFileSpec string) (RejectByNameFunc, error) {
 	if excludeFileSpec == "" {
 		return nil, errors.New("name for exclusion tagfile is empty")
 	}
@@ -107,7 +112,7 @@ func rejectIfPresent(excludeFileSpec string) (RejectFunc, error) {
 	}
 	debug.Log("using %q as exclusion tagfile", tf)
 	rc := &rejectionCache{}
-	fn := func(filename string, _ os.FileInfo) bool {
+	fn := func(filename string) bool {
 		return isExcludedByFile(filename, tf, tc, rc)
 	}
 	return fn, nil
@@ -252,11 +257,11 @@ func rejectByDevice(samples []string) (RejectFunc, error) {
 	}, nil
 }
 
-// rejectResticCache returns a RejectFunc that rejects the restic cache
+// rejectResticCache returns a RejectByNameFunc that rejects the restic cache
 // directory (if set).
-func rejectResticCache(repo *repository.Repository) (RejectFunc, error) {
+func rejectResticCache(repo *repository.Repository) (RejectByNameFunc, error) {
 	if repo.Cache == nil {
-		return func(string, os.FileInfo) bool {
+		return func(string) bool {
 			return false
 		}, nil
 	}
@@ -266,7 +271,7 @@ func rejectResticCache(repo *repository.Repository) (RejectFunc, error) {
 		return nil, errors.New("cacheBase is empty string")
 	}
 
-	return func(item string, _ os.FileInfo) bool {
+	return func(item string) bool {
 		if fs.HasPathPrefix(cacheBase, item) {
 			debug.Log("rejecting restic cache directory %v", item)
 			return true

cmd/restic/exclude_test.go

@@ -27,7 +27,7 @@ func TestRejectByPattern(t *testing.T) {
 	for _, tc := range tests {
 		t.Run("", func(t *testing.T) {
 			reject := rejectByPattern(patterns)
-			res := reject(tc.filename, nil)
+			res := reject(tc.filename)
 			if res != tc.reject {
 				t.Fatalf("wrong result for filename %v: want %v, got %v",
 					tc.filename, tc.reject, res)
@@ -140,8 +140,8 @@ func TestMultipleIsExcludedByFile(t *testing.T) {
 		if err != nil {
 			return err
 		}
-		excludedByFoo := fooExclude(p, fi)
-		excludedByBar := barExclude(p, fi)
+		excludedByFoo := fooExclude(p)
+		excludedByBar := barExclude(p)
 		excluded := excludedByFoo || excludedByBar
 		// the log message helps debugging in case the test fails
 		t.Logf("%q: %v || %v = %v", p, excludedByFoo, excludedByBar, excluded)

cmd/restic/global.go

@@ -36,7 +36,10 @@ import (
 	"golang.org/x/crypto/ssh/terminal"
 )
 
-var version = "compiled manually"
+var version = "0.9.3"
+
+// TimeFormat is the format used for all timestamps printed by restic.
+const TimeFormat = "2006-01-02 15:04:05"
 
 // GlobalOptions hold all global options for restic.
 type GlobalOptions struct {
@@ -64,7 +67,7 @@ type GlobalOptions struct {
 	//  0 means: don't print any messages except errors, this is used when --quiet is specified
 	//  1 is the default: print essential messages
 	//  2 means: print more messages, report minor things, this is used when --verbose is specified
-	//  3 means: print very detailed debug messages, this is used when --debug is specified
+	//  3 means: print very detailed debug messages, this is used when --verbose 2 is specified
 	verbosity uint
 
 	Options []string
@@ -92,7 +95,7 @@ func init() {
 	f.CountVarP(&globalOptions.Verbose, "verbose", "v", "be verbose (specify --verbose multiple times or level `n`)")
 	f.BoolVar(&globalOptions.NoLock, "no-lock", false, "do not lock the repo, this allows some operations on read-only repos")
 	f.BoolVarP(&globalOptions.JSON, "json", "", false, "set output mode to JSON for commands that support it")
-	f.StringVar(&globalOptions.CacheDir, "cache-dir", "", "set the cache directory")
+	f.StringVar(&globalOptions.CacheDir, "cache-dir", "", "set the cache directory. (default: use system default cache directory)")
 	f.BoolVar(&globalOptions.NoCache, "no-cache", false, "do not use a local cache")
 	f.StringSliceVar(&globalOptions.CACerts, "cacert", nil, "`file` to load root certificates from (default: use system certificates)")
 	f.StringVar(&globalOptions.TLSClientCert, "tls-client-cert", "", "path to a file containing PEM encoded TLS client certificate and private key")
@@ -373,6 +376,10 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
 		return s, nil
 	}
 
+	if c.Created && !opts.JSON {
+		Verbosef("created new cache in %v\n", c.Base)
+	}
+
 	// start using the cache
 	s.UseCache(c)
 

cmd/restic/table.go

@@ -1,63 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"io"
-	"strings"
-)
-
-// Table contains data for a table to be printed.
-type Table struct {
-	Header string
-	Rows   [][]interface{}
-	Footer string
-
-	RowFormat string
-}
-
-// NewTable initializes a new Table.
-func NewTable() Table {
-	return Table{
-		Rows: [][]interface{}{},
-	}
-}
-
-func (t Table) printSeparationLine(w io.Writer) error {
-	_, err := fmt.Fprintln(w, strings.Repeat("-", 70))
-	return err
-}
-
-// Write prints the table to w.
-func (t Table) Write(w io.Writer) error {
-	_, err := fmt.Fprintln(w, t.Header)
-	if err != nil {
-		return err
-	}
-
-	err = t.printSeparationLine(w)
-	if err != nil {
-		return err
-	}
-
-	for _, row := range t.Rows {
-		_, err = fmt.Fprintf(w, t.RowFormat+"\n", row...)
-		if err != nil {
-			return err
-		}
-	}
-
-	err = t.printSeparationLine(w)
-	if err != nil {
-		return err
-	}
-
-	_, err = fmt.Fprintln(w, t.Footer)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// TimeFormat is the format used for all timestamps printed by restic.
-const TimeFormat = "2006-01-02 15:04:05"

contrib/restic.spec

@@ -0,0 +1,72 @@
+Name: restic	
+Version: 0.9.2git.20180812	
+Release: 4%{?dist}
+Summary: restic is a backup program that is fast, efficient and secure.
+
+%global debug_package %{nil}
+
+Group: Applications/Archiving
+License: BSD 2-Clause
+URL: https://restic.net/	
+Source0: %{name}-%{version}.tar.gz
+
+BuildRequires:	golang 
+Requires: /bin/bash	
+
+%description
+restic is a program that does backups right. The design goals are:
+
+Easy: Doing backups should be a frictionless process, otherwise you are tempted to skip it. Restic should be easy to configure and use, so that in the unlikely event of a data loss you can just restore it. Likewise, restoring data should not be complicated.
+
+Fast: Backing up your data with restic should only be limited by your network or hard disk bandwidth so that you can backup your files every day. Nobody does backups if it takes too much time. Restoring backups should only transfer data that is needed for the files that are to be restored, so that this process is also fast.
+
+Verifiable: Much more important than backup is restore, so restic enables you to easily verify that all data can be restored.
+
+Secure: Restic uses cryptography to guarantee confidentiality and integrity of your data. The location where the backup data is stored is assumed to be an untrusted environment (e.g. a shared space where others like system administrators are able to access your backups). Restic is built to secure your data against such attackers, by encrypting it with AES-256 in counter mode and authenticating it using Poly1305-AES.
+
+Efficient: With the growth of data, additional snapshots should only take the storage of the actual increment. Even more, duplicate data should be de-duplicated before it is actually written to the storage backend to save precious backup space.
+
+Versatile storage: Users can provide many different places to store the backups. Local, SFTP, Restics REST-Server, Amazon S3, Minio, Openstack Swift, Backblaze B2, Microsoft Azure Blob Storage, Google Cloud Storage and more by the usage of rclone.
+
+Free: restic is free software and licensed under the BSD 2-Clause License and actively developed on GitHub.
+
+%prep
+%setup -q
+
+
+%build
+make %{?_smp_mflags}
+
+
+%install
+mkdir -p %{buildroot}%{_bindir}
+mkdir -p %{buildroot}%{_mandir}/man1
+mkdir -p %{buildroot}%{_datarootdir}/zsh/site-functions
+mkdir -p %{buildroot}%{_datarootdir}/bash-completion/completions
+install -p -m 644 doc/man/* %{buildroot}%{_mandir}/man1/
+install -p -m 644 doc/zsh-completion.zsh %{buildroot}%{_datarootdir}/zsh/site-functions/_restic
+install -p -m 644 doc/bash-completion.sh %{buildroot}%{_datarootdir}/bash-completion/completions/restic
+install -p -m 755 %{name} %{buildroot}%{_bindir}
+
+%files
+%doc LICENSE
+%doc README.rst
+%{_bindir}/%{name}
+%dir %{_datadir}/zsh/site-functions
+%{_datadir}/zsh/site-functions/_restic
+%dir %{_datadir}/bash-completion/
+%dir %{_datadir}/bash-completion/completions
+%{_datadir}/bash-completion/completions/restic
+%{_mandir}/man1/restic*.*
+
+
+%changelog
+* Sun Aug 12 2018 Luc De Louw <luc@delouw.ch> - 0.9.2git.20180812-4
+- %license does not work with RHEL6, using %doc instead
+* Sun Aug 12 2018 Luc De Louw <luc@delouw.ch> - 0.9.2git.20180812-3
+- Better description
+* Sun Aug 12 2018 Luc De Louw <luc@delouw.ch> - 0.9.2git.20180812-2
+- Initial RPM build
+* Sun Aug 12 2018 Luc De Louw <luc@delouw.ch> - 0.9.2git.20180812-1
+- Initial RPM build
+

doc.go

@@ -0,0 +1,11 @@
+// Package restic gives a (very brief) introduction to the structure of source code.
+//
+// Overview
+//
+// The packages are structured so that cmd/ contains the main package for the
+// restic binary, and internal/ contains almost all code in library form. We've
+// chosen to use the internal/ path so that the packages cannot be imported by
+// other programs. This was done on purpose, at the moment restic is a
+// command-line program and not a library. This may be revisited at a later
+// point in time.
+package restic

doc/.gitignore

@@ -0,0 +1,2 @@
+_build
+.doctrees

doc/020_installation.rst

@@ -24,16 +24,8 @@ These are up to date binaries, built in a reproducible and verifiable way, that
 you can download and run without having to do additional installation work.
 
 Please see the :ref:`official_binaries` section below for various downloads.
-
-Mac OS X
-========
-
-If you are using Mac OS X, you can install restic using the
-`homebrew <http://brew.sh/>`__ package manager:
-
-.. code-block:: console
-
-    $ brew install restic
+Official binaries can be updated in place by using the ``restic self-update``
+command.
 
 Arch Linux
 ==========
@@ -45,17 +37,6 @@ which can be installed from AUR, e.g. with ``pacaur``:
 
     $ pacaur -S restic-git
 
-Nix & NixOS
-===========
-
-If you are using `Nix <https://nixos.org/nix/>`__ or `NixOS <https://nixos.org/>`__
-there is a package available named ``restic``.
-It can be installed uisng ``nix-env``:
-
-.. code-block:: console
-
-    $ nix-env --install restic 
-
 Debian
 ======
 
@@ -70,7 +51,62 @@ installed from the official repos, e.g. with ``apt-get``:
 .. warning:: Please be aware that, at the time of writing, Debian *stable*
    has ``restic`` version 0.3.3 which is very old. The *testing* and *unstable*
    branches have recent versions of ``restic``.
-   
+
+Fedora
+======
+
+restic can be installed using ``dnf``:
+
+.. code-block:: console
+
+    $ dnf install restic
+
+If you used restic from copr previously, remove the copr repo as follows to
+avoid any conflicts:
+
+.. code-block:: console
+
+   $ dnf copr remove copart/restic
+
+macOS
+=====
+
+If you are using macOS, you can install restic using the
+`homebrew <http://brew.sh/>`__ package manager:
+
+.. code-block:: console
+
+    $ brew install restic
+
+Nix & NixOS
+===========
+
+If you are using `Nix <https://nixos.org/nix/>`__ or `NixOS <https://nixos.org/>`__
+there is a package available named ``restic``.
+It can be installed using ``nix-env``:
+
+.. code-block:: console
+
+    $ nix-env --install restic
+
+OpenBSD
+=======
+
+On OpenBSD 6.3 and greater, you can install restic using ``pkg_add``:
+
+.. code-block:: console
+
+    # pkg_add restic
+
+FreeBSD
+=======
+
+On FreeBSD (11 and probably later versions), you can install restic using ``pkg install``:
+
+.. code-block:: console
+
+    # pkg install restic
+
 RHEL & CentOS
 =============
 
@@ -94,17 +130,6 @@ For CentOS7 use:
 
     $ yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/copart/restic/repo/epel-7/copart-restic-epel-7.repo
 
-Fedora
-======
-
-restic can be installed via copr repository.
-
-.. code-block:: console
-
-    $ dnf install dnf-plugin-core
-    $ dnf copr enable copart/restic
-    $ dnf install restic
-
 Solus
 =====
 
@@ -113,15 +138,19 @@ restic can be installed from the official repo of Solus via the ``eopkg`` packag
 .. code-block:: console
 
     $ eopkg install restic
-    
-OpenBSD
+
+Windows
 =======
 
-On OpenBSD 6.3 and greater, you can install restic using ``pkg_add``:
+restic can be installed using `Scoop <https://scoop.sh/>`__:
 
 .. code-block:: console
 
-    # pkg_add restic
+    scoop install restic
+
+Using this installation method, ``restic.exe`` will automatically be available
+in the ``PATH``. It can be called from cmd.exe or PowerShell by typing ``restic``.
+
 
 .. _official_binaries:
 
@@ -138,6 +167,38 @@ are considered stable and releases are made regularly in a controlled manner.
 There's both pre-compiled binaries for different platforms as well as the source
 code available for download. Just download and run the one matching your system.
 
+The official binaries can be updated in place using the ``restic self-update``
+command:
+
+.. code-block:: console
+
+    $ restic version
+    restic 0.9.1 compiled with go1.10.3 on linux/amd64
+
+    $ restic self-update
+    find latest release of restic at GitHub
+    latest version is 0.9.2
+    download file SHA256SUMS
+    download SHA256SUMS
+    download file SHA256SUMS
+    download SHA256SUMS.asc
+    GPG signature verification succeeded
+    download restic_0.9.2_linux_amd64.bz2
+    downloaded restic_0.9.2_linux_amd64.bz2
+    saved 12115904 bytes in ./restic
+    successfully updated restic to version 0.9.2
+
+    $ restic version
+    restic 0.9.2 compiled with go1.10.3 on linux/amd64
+
+The ``self-update`` command uses the GPG signature on the files uploaded to
+GitHub to verify their authenticity. No external programs are necessary.
+
+.. note:: Please be aware that the user executing the ``restic self-update``
+   command must have the permission to replace the restic binary.
+   If you want to save the downloaded restic binary into a different file, pass
+   the file name via the option ``--output``.
+
 Unstable Builds
 ===============
 
@@ -187,6 +248,13 @@ In order to build restic from source, execute the following steps:
 
     $ cd restic
 
+    $ go run -mod=vendor build.go
+
+For Go versions < 1.11, the option ``-mod=vendor`` needs to be removed, like
+this:
+
+.. code-block:: console
+
     $ go run build.go
 
 You can easily cross-compile restic for all supported platforms, just
@@ -195,12 +263,14 @@ supply the target OS and platform via the command-line options like this
 
 .. code-block:: console
 
-    $ go run build.go --goos windows --goarch amd64
+    $ go run -mod=vendor build.go --goos windows --goarch amd64
 
-    $ go run build.go --goos freebsd --goarch 386
+    $ go run -mod=vendor build.go --goos freebsd --goarch 386
+
+    $ go run -mod=vendor build.go --goos linux --goarch arm --goarm 6
+
+Again, for Go < 1.11 ``-mod=vendor`` needs to be removed.
 
-    $ go run build.go --goos linux --goarch arm --goarm 6
-    
 The resulting binary is statically linked and does not require any
 libraries.
 

doc/040_backup.rst

@@ -33,18 +33,26 @@ again:
     processed 1.720 GiB in 0:12
     Files:        5307 new,     0 changed,     0 unmodified
     Dirs:         1867 new,     0 changed,     0 unmodified
-    Added:      1.700 GiB
+    Added:      1.200 GiB
     snapshot 40dc1520 saved
 
 As you can see, restic created a backup of the directory and was pretty
 fast! The specific snapshot just created is identified by a sequence of
 hexadecimal characters, ``40dc1520`` in this case.
 
-If you don't pass the ``--verbose`` option, restic will print less data. You'll still get a nice live status display. Be aware that the live status shows the processed files and not the transferred data. Transferred volume might be lower (due to deduplication) or higher.
+You can see that restic tells us it processed 1.720 GiB of data, this is the
+size of the files and directories in ``~/work`` on the local file system. It
+also tells us that only 1.200 GiB was added to the repository. This means that
+some of the data was duplicate and restic was able to efficiently reduce it.
+
+If you don't pass the ``--verbose`` option, restic will print less data. You'll
+still get a nice live status display. Be aware that the live status shows the
+processed files and not the transferred data. Transferred volume might be lower
+(due to de-duplication) or higher.
 
 If you run the command again, restic will create another snapshot of
-your data, but this time it's even faster. This is de-duplication at
-work!
+your data, but this time it's even faster and no new data was added to the
+repository (since all data is already there). This is de-duplication at work!
 
 .. code-block:: console
 
@@ -113,13 +121,18 @@ can compute which parts of the files need to be saved. When you backup
 the same directory again (maybe with new or changed files) restic will
 find the old snapshot in the repo and by default only reads those files
 that are new or have been modified since the last snapshot. This is
-decided based on the modify date of the file in the file system.
+decided based on the following attributes of the file in the file system:
+
+ * Type (file, symlink, or directory?)
+ * Modification time
+ * Size
+ * Inode number (internal number used to reference a file in a file system)
 
 Now is a good time to run ``restic check`` to verify that all data
 is properly stored in the repository. You should run this command regularly
 to make sure the internal structure of the repository is free of errors.
 
-Including and Excluding Files 
+Including and Excluding Files
 *****************************
 
 You can exclude folders and files by specifying exclude patterns, currently
@@ -179,7 +192,7 @@ For this, the special wildcard ``**`` can be used to match arbitrary
 sub-directories: The pattern ``foo/**/bar`` matches:
 
  * ``/dir1/foo/dir2/bar/file``
- * ``/foo/bar/file`` 
+ * ``/foo/bar/file``
  * ``/tmp/foo/bar``
 
 By specifying the option ``--one-file-system`` you can instruct restic
@@ -314,3 +327,51 @@ some additional data in the repository, but the snapshot will never be
 created as it would only be written at the very (successful) end of
 the backup operation.  Previous snapshots will still be there and will still
 work.
+
+
+Environment Variables
+*********************
+
+In addition to command-line options, restic supports passing various options in
+environment variables. The following list of environment variables:
+
+.. code-block:: console
+
+    RESTIC_REPOSITORY                   Location of repository (replaces -r)
+    RESTIC_PASSWORD_FILE                Location of password file (replaces --password-file)
+    RESTIC_PASSWORD                     The actual password for the repository
+
+    AWS_ACCESS_KEY_ID                   Amazon S3 access key ID
+    AWS_SECRET_ACCESS_KEY               Amazon S3 secret access key
+
+    ST_AUTH                             Auth URL for keystone v1 authentication
+    ST_USER                             Username for keystone v1 authentication
+    ST_KEY                              Password for keystone v1 authentication
+
+    OS_AUTH_URL                         Auth URL for keystone authentication
+    OS_REGION_NAME                      Region name for keystone authentication
+    OS_USERNAME                         Username for keystone authentication
+    OS_PASSWORD                         Password for keystone authentication
+    OS_TENANT_ID                        Tenant ID for keystone v2 authentication
+    OS_TENANT_NAME                      Tenant name for keystone v2 authentication
+
+    OS_USER_DOMAIN_NAME                 User domain name for keystone authentication
+    OS_PROJECT_NAME                     Project name for keystone authentication
+    OS_PROJECT_DOMAIN_NAME              PRoject domain name for keystone authentication
+
+    OS_STORAGE_URL                      Storage URL for token authentication
+    OS_AUTH_TOKEN                       Auth token for token authentication
+
+    B2_ACCOUNT_ID                       Account ID or applicationKeyId for Backblaze B2
+    B2_ACCOUNT_KEY                      Account Key or applicationKey for Backblaze B2
+
+    AZURE_ACCOUNT_NAME                  Account name for Azure
+    AZURE_ACCOUNT_KEY                   Account key for Azure
+
+    GOOGLE_PROJECT_ID                   Project ID for Google Cloud Storage
+    GOOGLE_APPLICATION_CREDENTIALS      Application Credentials for Google Cloud Storage (e.g. $HOME/.config/gs-secret-restic-key.json)
+
+    RCLONE_BWLIMIT                      rclone bandwidth limit
+
+
+

doc/050_restore.rst

@@ -68,7 +68,9 @@ command to serve the repository with FUSE:
     Don't forget to umount after quitting!
 
 Mounting repositories via FUSE is not possible on OpenBSD, Solaris/illumos
-and Windows.
+and Windows. For Linux, the ``fuse`` kernel module needs to be loaded. For
+FreeBSD, you may need to install FUSE and load the kernel module (``kldload
+fuse``).
 
 Restic supports storage and preservation of hard links. However, since
 hard links exist in the scope of a filesystem by definition, restoring

doc/090_participating.rst

@@ -22,7 +22,9 @@ The program can be built with debug support like this:
 
 .. code-block:: console
 
-    $ go run build.go -tags debug
+    $ go run build.go -mod=vendor -tags debug
+
+For Go < 1.11, the option ``-mod=vendor`` needs to be removed.
 
 Afterwards, extensive debug messages are written to the file in
 environment variable ``DEBUG_LOG``, e.g.:

doc/Makefile

@@ -17,4 +17,9 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: autobuild
+
+autobuild:
+	sphinx-autobuild -b html -i '.doctrees/*' . _build

doc/bash-completion.sh

@@ -274,8 +274,8 @@ _restic_backup()
     flags+=("--help")
     flags+=("-h")
     local_nonpersistent_flags+=("--help")
-    flags+=("--hostname=")
-    local_nonpersistent_flags+=("--hostname=")
+    flags+=("--host=")
+    local_nonpersistent_flags+=("--host=")
     flags+=("--one-file-system")
     flags+=("-x")
     local_nonpersistent_flags+=("--one-file-system")
@@ -337,6 +337,8 @@ _restic_cache()
     local_nonpersistent_flags+=("--help")
     flags+=("--max-age=")
     local_nonpersistent_flags+=("--max-age=")
+    flags+=("--no-size")
+    local_nonpersistent_flags+=("--no-size")
     flags+=("--cacert=")
     flags+=("--cache-dir=")
     flags+=("--cleanup-cache")
@@ -561,6 +563,8 @@ _restic_find()
     flags_with_completion=()
     flags_completion=()
 
+    flags+=("--blob")
+    local_nonpersistent_flags+=("--blob")
     flags+=("--help")
     flags+=("-h")
     local_nonpersistent_flags+=("--help")
@@ -579,13 +583,19 @@ _restic_find()
     flags+=("--oldest=")
     two_word_flags+=("-O")
     local_nonpersistent_flags+=("--oldest=")
+    flags+=("--pack")
+    local_nonpersistent_flags+=("--pack")
     flags+=("--path=")
     local_nonpersistent_flags+=("--path=")
+    flags+=("--show-pack-id")
+    local_nonpersistent_flags+=("--show-pack-id")
     flags+=("--snapshot=")
     two_word_flags+=("-s")
     local_nonpersistent_flags+=("--snapshot=")
     flags+=("--tag=")
     local_nonpersistent_flags+=("--tag=")
+    flags+=("--tree")
+    local_nonpersistent_flags+=("--tree")
     flags+=("--cacert=")
     flags+=("--cache-dir=")
     flags+=("--cleanup-cache")
@@ -649,8 +659,6 @@ _restic_forget()
     local_nonpersistent_flags+=("--keep-tag=")
     flags+=("--host=")
     local_nonpersistent_flags+=("--host=")
-    flags+=("--hostname=")
-    local_nonpersistent_flags+=("--hostname=")
     flags+=("--tag=")
     local_nonpersistent_flags+=("--tag=")
     flags+=("--path=")
@@ -895,6 +903,8 @@ _restic_ls()
     local_nonpersistent_flags+=("--long")
     flags+=("--path=")
     local_nonpersistent_flags+=("--path=")
+    flags+=("--recursive")
+    local_nonpersistent_flags+=("--recursive")
     flags+=("--tag=")
     local_nonpersistent_flags+=("--tag=")
     flags+=("--cacert=")
@@ -1168,6 +1178,50 @@ _restic_restore()
     noun_aliases=()
 }
 
+_restic_self-update()
+{
+    last_command="restic_self-update"
+
+    command_aliases=()
+
+    commands=()
+
+    flags=()
+    two_word_flags=()
+    local_nonpersistent_flags=()
+    flags_with_completion=()
+    flags_completion=()
+
+    flags+=("--help")
+    flags+=("-h")
+    local_nonpersistent_flags+=("--help")
+    flags+=("--output=")
+    local_nonpersistent_flags+=("--output=")
+    flags+=("--cacert=")
+    flags+=("--cache-dir=")
+    flags+=("--cleanup-cache")
+    flags+=("--json")
+    flags+=("--limit-download=")
+    flags+=("--limit-upload=")
+    flags+=("--no-cache")
+    flags+=("--no-lock")
+    flags+=("--option=")
+    two_word_flags+=("-o")
+    flags+=("--password-file=")
+    two_word_flags+=("-p")
+    flags+=("--quiet")
+    flags+=("-q")
+    flags+=("--repo=")
+    two_word_flags+=("-r")
+    flags+=("--tls-client-cert=")
+    flags+=("--verbose")
+    flags+=("-v")
+
+    must_have_one_flag=()
+    must_have_one_noun=()
+    noun_aliases=()
+}
+
 _restic_snapshots()
 {
     last_command="restic_snapshots"
@@ -1434,6 +1488,7 @@ _restic_root_command()
     commands+=("prune")
     commands+=("rebuild-index")
     commands+=("restore")
+    commands+=("self-update")
     commands+=("snapshots")
     commands+=("stats")
     commands+=("tag")

doc/cache.rst

@@ -10,7 +10,7 @@ Versions
 
 The cache directory is selected according to the `XDG base dir specification
 <http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html>`__.
-Each repository has its own cache sub-directory, consting of the repository ID
+Each repository has its own cache sub-directory, consisting of the repository ID
 which is chosen at ``init``. All cache directories for different repos are
 independent of each other.
 

doc/developer_information.rst

@@ -0,0 +1,123 @@
+Developer Information
+#####################
+
+Reproducible Builds
+*******************
+
+This section describes how to reproduce the official released binaries for
+restic for version 0.9.3 and later. The binary produced depends on the
+following things:
+
+ * The source code for the release
+ * The exact version of the official `Go compiler <https://golang.org>`__ used to produce the binaries (running ``restic version`` will print this)
+ * The architecture and operating system the Go compiler runs on (Linux, ``amd64``)
+ * The path where the source code is extracted to (``/restic``)
+ * The path to the Go compiler (``/usr/local/go``)
+ * The build tags (for official binaries, it's the tag ``selfupdate``)
+ * The environment variables (mostly ``$GOOS``, ``$GOARCH``, ``$CGO_ENABLED``)
+
+In addition, The compressed ZIP files for Windows depends on the modification
+timestamp of the binary contained in it. In order to reproduce the exact same
+ZIP file every time, we update the timestamp of the file ``VERSION`` in the
+source code archive and set the timezone to Europe/Berlin.
+
+In the following example, we'll use the file ``restic-0.9.3.tar.gz`` and Go
+1.11.1 to reproduce the released binaries.
+
+1. Download and extract the Go compiler into ``/usr/local/go``:
+
+.. code::
+
+    $ cd /usr/local
+    $ curl -L https://dl.google.com/go/go1.11.1.linux-amd64.tar.gz | tar xz
+
+2. Extract the restic source code into ``/restic``
+
+.. code::
+
+    $ mkdir /restic
+    $ cd /restic
+    $ TZ=Europe/Berlin curl -L https://github.com/restic/restic/releases/download/v0.9.3/restic-0.9.3.tar.gz | tar xz --strip-components=1
+
+3. Build the binaries for Windows and Linux:
+
+.. code::
+
+    $ export PATH=/usr/local/go/bin:$PATH
+    $ go version
+    go version go1.11.1 linux/amd64
+
+    $ GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -mod=vendor -ldflags "-s -w" -tags selfupdate -o restic_linux_amd64 ./cmd/restic
+    $ bzip2 restic_linux_amd64
+
+    $ GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -mod=vendor -ldflags "-s -w" -tags selfupdate -o restic_windows_amd64.exe ./cmd/restic
+    $ touch --reference VERSION restic_windows_amd64.exe
+    $ TZ=Europe/Berlin zip -q -X restic_windows_amd64.zip restic_windows_amd64.exe
+
+Building the Official Binaries
+******************************
+
+The released binaries for restic are built using a Docker container. You can
+find it on `Docker Hub <https://hub.docker.com/r/restic/builder>`__ as
+``restic/builder``, the ``Dockerfile`` and instructions on how to build the
+container can be found in the `GitHub repository
+<https://github.com/restic/builder>`__
+
+The container serves the following goals:
+ * Have a very controlled environment which is independent from the local system
+ * Make it easy to have the correct version of the Go compiler at the right path
+ * Make it easy to pass in the source code to build at a well-defined path
+
+The following steps are necessary to build the binaries:
+
+1. Either build the container (see the instructions in the `repository's README <https://github.com/restic/builder>`__). Alternatively, download the container from the hub:
+
+.. code::
+
+    docker pull restic/builder
+
+2. Extract the source code somewhere:
+
+.. code::
+
+    tar xvzf restic-0.9.3.tar.gz
+
+3. Create a directory to place the resulting binaries in:
+
+.. code::
+
+    mkdir output
+
+3. Mount the source code and the output directory in the container and run the default command, which starts ``helpers/build-release-binaries/main.go``:
+
+.. code::
+
+    docker run --rm \
+        --volume "$PWD/restic-0.9.3:/restic" \
+        --volume "$PWD/output:/output" \
+        restic/builder
+
+4. If anything goes wrong, you can enable debug output by specifying the call to ``helpers/build-release-binaries/main.go`` like this:
+
+.. code::
+
+    docker run --rm \
+        --volume "$PWD/restic-0.9.3:/restic" \
+        --volume "$PWD/output:/output" \
+        restic/builder \
+        go run -mod=vendor helpers/build-release-binaries/main.go --verbose
+
+Prepare a New Release
+*********************
+
+Publishing a new release of restic requires many different steps. We've
+automated this in the Go program ``helpers/prepare-release/main.go`` which also
+includes checking that e.g. the changelog is correctly generated. The only
+required argument is the new version number (in `Semantic Versioning
+<https://semver.org/>`__ format ``MAJOR.MINOR.PATCH``):
+
+.. code::
+
+    go run -mod=vendor helpers/prepare-release/main.go 0.9.3
+
+Checks can be skipped on demand via flags, please see ``--help`` for details.

doc/faq.rst

@@ -27,6 +27,54 @@ strictly necessary. With high probability this is duplicate data. In
 order to clean it up, the command ``restic prune`` can be used. The
 cause of this bug is not yet known.
 
+I ran a ``restic`` command but it is not working as intented, what do I do now?
+-------------------------------------------------------------------------------
+
+If you are running a restic command and it is not working as you hoped it would,
+there is an easy way of checking how your shell interpreted the command you are trying to run.
+
+Here is an example of a mistake in a backup command that results in the command not working as expected.
+A user wants to run the following ``restic backup`` command
+
+::
+
+$ restic backup --exclude "~/documents" ~
+
+.. important:: This command contains an intentional user error described in this paragraph.
+
+This command will result in a complete backup of the current logged in user's home directory and it won't exclude the folder ``~/documents/`` - which is not what the user wanted to achieve.
+The problem is how the path to ``~/documents`` is passed to restic.
+
+In order to spot an issue like this, you can make use of the following ruby command preceeding your restic command.
+
+::
+
+    $ ruby -e 'puts ARGV.inspect' restic backup --exclude "~/documents" ~
+    ["restic", "backup", "--exclude", "~/documents", "/home/john"]
+
+As you can see, the command outputs every argument you have passed to the shell. This is what restic sees when you run your command.
+The error here is that the tilde ``~`` in ``"~/documents"`` didn't get expanded as it is quoted.
+
+::
+
+    $ echo ~/documents
+    /home/john/documents
+
+    $ echo "~/documents"
+    ~/document
+
+    $ echo "$HOME/documents"
+    /home/john/documents
+
+Restic handles globbing and expansion in the following ways:
+
+-  Globbing is only expanded for lines read via ``--files-from``
+-  Environment variables are not expanded in the file read via ``--files-from``
+-  ``*`` is expanded for paths read via ``--files-from``
+-  E.g. For backup targets given to restic as arguments on the shell, neither glob expansion nor shell variable replacement is done. If restic is called as ``restic backup '*' '$HOME'``, it will try to backup the literal file(s)/dir(s) ``*`` and ``$HOME``
+-  Double-asterisk ``**`` only works in exclude patterns as this is a custom extension built into restic; the shell must not expand it
+
+
 How can I specify encryption passwords automatically?
 -----------------------------------------------------
 

doc/index.rst

@@ -19,3 +19,4 @@ Restic Documentation
    110_talks
    faq
    manual_rest
+   developer_information

doc/man/restic-backup.1

@@ -49,7 +49,7 @@ given as the arguments.
     help for backup
 
 .PP
-\fB\-\-hostname\fP=""
+\fB\-\-host\fP="H"
     set the \fB\fChostname\fR for the snapshot manually. To prevent an expensive rescan use the "parent" flag
 
 .PP
@@ -88,7 +88,7 @@ given as the arguments.
 
 .PP
 \fB\-\-cache\-dir\fP=""
-    set the cache directory
+    set the cache directory. (default: use system default cache directory)
 
 .PP
 \fB\-\-cleanup\-cache\fP[=false]

doc/man/restic-cache.1

@@ -31,6 +31,10 @@ The "cache" command allows listing and cleaning local cache directories.
 \fB\-\-max\-age\fP=30
     max age in \fB\fCdays\fR for cache directories to be considered old
 
+.PP
+\fB\-\-no\-size\fP[=false]
+    do not output the size of the cache directories
+
 
 .SH OPTIONS INHERITED FROM PARENT COMMANDS
 .PP
@@ -39,7 +43,7 @@ The "cache" command allows listing and cleaning local cache directories.
 
 .PP
 \fB\-\-cache\-dir\fP=""
-    set the cache directory
+    set the cache directory. (default: use system default cache directory)
 
 .PP
 \fB\-\-cleanup\-cache\fP[=false]

doc/man/restic-cat.1

@@ -31,7 +31,7 @@ The "cat" command is used to print internal objects to stdout.
 
 .PP
 \fB\-\-cache\-dir\fP=""
-    set the cache directory
+    set the cache directory. (default: use system default cache directory)
 
 .PP
 \fB\-\-cleanup\-cache\fP[=false]

doc/man/restic-check.1

@@ -52,7 +52,7 @@ repository and not use a local cache.
 
 .PP
 \fB\-\-cache\-dir\fP=""
-    set the cache directory
+    set the cache directory. (default: use system default cache directory)
 
 .PP
 \fB\-\-cleanup\-cache\fP[=false]

doc/man/restic-diff.1

@@ -21,11 +21,14 @@ directory:
 
 .RS
 .IP \(bu 2
-The item was added
++  The item was added
+.IP \(bu 2
+\-  The item was removed
 .IP \(bu 2
-The item was removed
 U  The metadata (access mode, timestamps, ...) for the item was updated
+.IP \(bu 2
 M  The file's content was modified
+.IP \(bu 2
 T  The type was changed, e.g. a file was made a symlink
 
 .RE
@@ -48,7 +51,7 @@ T  The type was changed, e.g. a file was made a symlink
 
 .PP
 \fB\-\-cache\-dir\fP=""
-    set the cache directory
+    set the cache directory. (default: use system default cache directory)
 
 .PP
 \fB\-\-cleanup\-cache\fP[=false]

doc/man/restic-dump.1

@@ -48,7 +48,7 @@ repository.
 
 .PP
 \fB\-\-cache\-dir\fP=""
-    set the cache directory
+    set the cache directory. (default: use system default cache directory)
 
 .PP
 \fB\-\-cleanup\-cache\fP[=false]

doc/man/restic-find.1

@@ -5,21 +5,26 @@
 
 .SH NAME
 .PP
-restic\-find \- Find a file or directory
+restic\-find \- Find a file, a directory or restic IDs
 
 
 .SH SYNOPSIS
 .PP
-\fBrestic find [flags] PATTERN\fP
+\fBrestic find [flags] PATTERN...\fP
 
 
 .SH DESCRIPTION
 .PP
 The "find" command searches for files or directories in snapshots stored in the
 repo.
+It can also be used to search for restic blobs or trees for troubleshooting.
 
 
 .SH OPTIONS
+.PP
+\fB\-\-blob\fP[=false]
+    pattern is a blob\-ID
+
 .PP
 \fB\-h\fP, \fB\-\-help\fP[=false]
     help for find
@@ -44,10 +49,18 @@ repo.
 \fB\-O\fP, \fB\-\-oldest\fP=""
     oldest modification date/time
 
+.PP
+\fB\-\-pack\fP[=false]
+    pattern is a pack\-ID
+
 .PP
 \fB\-\-path\fP=[]
     only consider snapshots which include this (absolute) \fB\fCpath\fR, when no snapshot\-ID is given
 
+.PP
+\fB\-\-show\-pack\-id\fP[=false]
+    display the pack\-ID the blobs belong to (with \-\-blob)
+
 .PP
 \fB\-s\fP, \fB\-\-snapshot\fP=[]
     snapshot \fB\fCid\fR to search in (can be given multiple times)
@@ -56,6 +69,10 @@ repo.
 \fB\-\-tag\fP=[]
     only consider snapshots which include this \fB\fCtaglist\fR, when no snapshot\-ID is given
 
+.PP
+\fB\-\-tree\fP[=false]
+    pattern is a tree\-ID
+
 
 .SH OPTIONS INHERITED FROM PARENT COMMANDS
 .PP
@@ -64,7 +81,7 @@ repo.
 
 .PP
 \fB\-\-cache\-dir\fP=""
-    set the cache directory
+    set the cache directory. (default: use system default cache directory)
 
 .PP
 \fB\-\-cleanup\-cache\fP[=false]
@@ -115,6 +132,22 @@ repo.
     be verbose (specify \-\-verbose multiple times or level \fB\fCn\fR)
 
 
+.SH EXAMPLE
+.PP
+.RS
+
+.nf
+restic find config.json
+restic find \-\-json "*.yml" "*.json"
+restic find \-\-json \-\-blob 420f620f b46ebe8a ddd38656
+restic find \-\-show\-pack\-id \-\-blob 420f620f
+restic find \-\-tree 577c2bc9 f81f2e22 a62827a9
+restic find \-\-pack 025c1d06
+
+.fi
+.RE
+
+
 .SH SEE ALSO
 .PP
 \fBrestic(1)\fP

doc/man/restic-forget.1

@@ -48,7 +48,7 @@ data after 'forget' was run successfully, see the 'prune' command.
 
 .PP
 \fB\-\-keep\-within\fP=
-    keep snapshots that were created within \fB\fCduration\fR before the newest (e.g. 1y5m7d)
+    keep snapshots that are older than \fB\fCduration\fR (eg. 1y5m7d) relative to the latest snapshot
 
 .PP
 \fB\-\-keep\-tag\fP=[]
@@ -58,10 +58,6 @@ data after 'forget' was run successfully, see the 'prune' command.
 \fB\-\-host\fP=""
     only consider snapshots with the given \fB\fChost\fR
 
-.PP
-\fB\-\-hostname\fP=""
-    only consider snapshots with the given \fB\fChostname\fR (deprecated)
-
 .PP
 \fB\-\-tag\fP=[]
     only consider snapshots which include this \fB\fCtaglist\fR in the format \fB\fCtag[,tag,...]\fR (can be specified multiple times)
@@ -98,7 +94,7 @@ data after 'forget' was run successfully, see the 'prune' command.
 
 .PP
 \fB\-\-cache\-dir\fP=""
-    set the cache directory
+    set the cache directory. (default: use system default cache directory)
 
 .PP
 \fB\-\-cleanup\-cache\fP[=false]

doc/man/restic-generate.1

@@ -44,7 +44,7 @@ and the auto\-completion files for bash and zsh).
 
 .PP
 \fB\-\-cache\-dir\fP=""
-    set the cache directory
+    set the cache directory. (default: use system default cache directory)
 
 .PP
 \fB\-\-cleanup\-cache\fP[=false]

doc/man/restic-init.1

@@ -31,7 +31,7 @@ The "init" command initializes a new repository.
 
 .PP
 \fB\-\-cache\-dir\fP=""
-    set the cache directory
+    set the cache directory. (default: use system default cache directory)
 
 .PP
 \fB\-\-cleanup\-cache\fP[=false]

doc/man/restic-key.1

@@ -35,7 +35,7 @@ The "key" command manages keys (passwords) for accessing the repository.
 
 .PP
 \fB\-\-cache\-dir\fP=""
-    set the cache directory
+    set the cache directory. (default: use system default cache directory)
 
 .PP
 \fB\-\-cleanup\-cache\fP[=false]

doc/man/restic-list.1

@@ -31,7 +31,7 @@ The "list" command allows listing objects in the repository based on type.
 
 .PP
 \fB\-\-cache\-dir\fP=""
-    set the cache directory
+    set the cache directory. (default: use system default cache directory)
 
 .PP
 \fB\-\-cleanup\-cache\fP[=false]

doc/man/restic-ls.1

@@ -10,15 +10,27 @@ restic\-ls \- List files in a snapshot
 
 .SH SYNOPSIS
 .PP
-\fBrestic ls [flags] [snapshot\-ID ...]\fP
+\fBrestic ls [flags] [snapshotID] [dir...]\fP
 
 
 .SH DESCRIPTION
 .PP
-The "ls" command allows listing files and directories in a snapshot.
+The "ls" command lists files and directories in a snapshot.
 
 .PP
-The special snapshot\-ID "latest" can be used to list files and directories of the latest snapshot in the repository.
+The special snapshot ID "latest" can be used to list files and
+directories of the latest snapshot in the repository. The
+\-\-host flag can be used in conjunction to select the latest
+snapshot originating from a certain host only.
+
+.PP
+File listings can optionally be filtered by directories. Any
+positional arguments after the snapshot ID are interpreted as
+absolute directory paths, and only files inside those directories
+will be listed. If the \-\-recursive flag is used, then the filter
+will allow traversing into matching directories' subfolders.
+Any directory paths specified must be absolute (starting with
+a path separator); paths use the forward slash '/' as separator.
 
 
 .SH OPTIONS
@@ -38,6 +50,10 @@ The special snapshot\-ID "latest" can be used to list files and directories of t
 \fB\-\-path\fP=[]
     only consider snapshots which include this (absolute) \fB\fCpath\fR, when no snapshot ID is given
 
+.PP
+\fB\-\-recursive\fP[=false]
+    include files in subfolders of the listed directories
+
 .PP
 \fB\-\-tag\fP=[]
     only consider snapshots which include this \fB\fCtaglist\fR, when no snapshot ID is given
@@ -50,7 +66,7 @@ The special snapshot\-ID "latest" can be used to list files and directories of t
 
 .PP
 \fB\-\-cache\-dir\fP=""
-    set the cache directory
+    set the cache directory. (default: use system default cache directory)
 
 .PP
 \fB\-\-cleanup\-cache\fP[=false]

doc/man/restic-migrate.1

@@ -36,7 +36,7 @@ name is explicitly given, a list of migrations that can be applied is printed.
 
 .PP
 \fB\-\-cache\-dir\fP=""
-    set the cache directory
+    set the cache directory. (default: use system default cache directory)
 
 .PP
 \fB\-\-cleanup\-cache\fP[=false]

doc/man/restic-mount.1

@@ -92,7 +92,7 @@ For details please see the documentation for time.Format() at:
 
 .PP
 \fB\-\-cache\-dir\fP=""
-    set the cache directory
+    set the cache directory. (default: use system default cache directory)
 
 .PP
 \fB\-\-cleanup\-cache\fP[=false]

doc/man/restic-prune.1

@@ -32,7 +32,7 @@ referenced and therefore not needed any more.
 
 .PP
 \fB\-\-cache\-dir\fP=""
-    set the cache directory
+    set the cache directory. (default: use system default cache directory)
 
 .PP
 \fB\-\-cleanup\-cache\fP[=false]

doc/man/restic-rebuild-index.1

@@ -32,7 +32,7 @@ repository.
 
 .PP
 \fB\-\-cache\-dir\fP=""
-    set the cache directory
+    set the cache directory. (default: use system default cache directory)
 
 .PP
 \fB\-\-cleanup\-cache\fP[=false]

doc/man/restic-restore.1

@@ -64,7 +64,7 @@ repository.
 
 .PP
 \fB\-\-cache\-dir\fP=""
-    set the cache directory
+    set the cache directory. (default: use system default cache directory)
 
 .PP
 \fB\-\-cleanup\-cache\fP[=false]

doc/man/restic-self-update.1

@@ -0,0 +1,94 @@
+.TH "restic backup" "1" "Jan 2017" "generated by `restic generate`" "" 
+.nh
+.ad l
+
+
+.SH NAME
+.PP
+restic\-self\-update \- Update the restic binary
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic self\-update [flags]\fP
+
+
+.SH DESCRIPTION
+.PP
+The command "update\-restic" downloads the latest stable release of restic from
+GitHub and replaces the currently running binary. After download, the
+authenticity of the binary is verified using the GPG signature on the release
+files.
+
+
+.SH OPTIONS
+.PP
+\fB\-h\fP, \fB\-\-help\fP[=false]
+    help for self\-update
+
+.PP
+\fB\-\-output\fP="./restic\-generate.temp"
+    Save the downloaded file as \fB\fCfilename\fR
+
+
+.SH OPTIONS INHERITED FROM PARENT COMMANDS
+.PP
+\fB\-\-cacert\fP=[]
+    \fB\fCfile\fR to load root certificates from (default: use system certificates)
+
+.PP
+\fB\-\-cache\-dir\fP=""
+    set the cache directory. (default: use system default cache directory)
+
+.PP
+\fB\-\-cleanup\-cache\fP[=false]
+    auto remove old cache directories
+
+.PP
+\fB\-\-json\fP[=false]
+    set output mode to JSON for commands that support it
+
+.PP
+\fB\-\-limit\-download\fP=0
+    limits downloads to a maximum rate in KiB/s. (default: unlimited)
+
+.PP
+\fB\-\-limit\-upload\fP=0
+    limits uploads to a maximum rate in KiB/s. (default: unlimited)
+
+.PP
+\fB\-\-no\-cache\fP[=false]
+    do not use a local cache
+
+.PP
+\fB\-\-no\-lock\fP[=false]
+    do not lock the repo, this allows some operations on read\-only repos
+
+.PP
+\fB\-o\fP, \fB\-\-option\fP=[]
+    set extended option (\fB\fCkey=value\fR, can be specified multiple times)
+
+.PP
+\fB\-p\fP, \fB\-\-password\-file\fP=""
+    read the repository password from a file (default: $RESTIC\_PASSWORD\_FILE)
+
+.PP
+\fB\-q\fP, \fB\-\-quiet\fP[=false]
+    do not output comprehensive progress report
+
+.PP
+\fB\-r\fP, \fB\-\-repo\fP=""
+    repository to backup to or restore from (default: $RESTIC\_REPOSITORY)
+
+.PP
+\fB\-\-tls\-client\-cert\fP=""
+    path to a file containing PEM encoded TLS client certificate and private key
+
+.PP
+\fB\-v\fP, \fB\-\-verbose\fP[=0]
+    be verbose (specify \-\-verbose multiple times or level \fB\fCn\fR)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic(1)\fP

doc/man/restic-snapshots.1

@@ -51,7 +51,7 @@ The "snapshots" command lists all snapshots stored in the repository.
 
 .PP
 \fB\-\-cache\-dir\fP=""
-    set the cache directory
+    set the cache directory. (default: use system default cache directory)
 
 .PP
 \fB\-\-cleanup\-cache\fP[=false]

doc/man/restic-stats.1

@@ -29,23 +29,22 @@ to calculate.
 .PP
 The modes are:
 
-.PP
+.RS
+.IP \(bu 2
 restore\-size: (default) Counts the size of the restored files.
-
-.PP
+.IP \(bu 2
 files\-by\-contents: Counts total size of files, where a file is
-                     considered unique if it has unique contents.
-
-.PP
-raw\-data: Counts the size of blobs in the repository, regardless
-            of how many files reference them.
-
-.PP
+considered unique if it has unique contents.
+.IP \(bu 2
+raw\-data: Counts the size of blobs in the repository, regardless of
+how many files reference them.
+.IP \(bu 2
 blobs\-per\-file: A combination of files\-by\-contents and raw\-data.
-
-.PP
+.IP \(bu 2
 Refer to the online manual for more details about each mode.
 
+.RE
+
 
 .SH OPTIONS
 .PP
@@ -53,7 +52,7 @@ Refer to the online manual for more details about each mode.
     help for stats
 
 .PP
-\fB\-\-host\fP=""
+\fB\-\-host\fP="H"
     filter latest snapshot by this hostname
 
 .PP
@@ -68,7 +67,7 @@ Refer to the online manual for more details about each mode.
 
 .PP
 \fB\-\-cache\-dir\fP=""
-    set the cache directory
+    set the cache directory. (default: use system default cache directory)
 
 .PP
 \fB\-\-cleanup\-cache\fP[=false]

doc/man/restic-tag.1

@@ -62,7 +62,7 @@ When no snapshot\-ID is given, all snapshots matching the host, tag and path fil
 
 .PP
 \fB\-\-cache\-dir\fP=""
-    set the cache directory
+    set the cache directory. (default: use system default cache directory)
 
 .PP
 \fB\-\-cleanup\-cache\fP[=false]

doc/man/restic-unlock.1

@@ -35,7 +35,7 @@ The "unlock" command removes stale locks that have been created by other restic
 
 .PP
 \fB\-\-cache\-dir\fP=""
-    set the cache directory
+    set the cache directory. (default: use system default cache directory)
 
 .PP
 \fB\-\-cleanup\-cache\fP[=false]

doc/man/restic-version.1

@@ -32,7 +32,7 @@ and the version of this software.
 
 .PP
 \fB\-\-cache\-dir\fP=""
-    set the cache directory
+    set the cache directory. (default: use system default cache directory)
 
 .PP
 \fB\-\-cleanup\-cache\fP[=false]

doc/man/restic.1

@@ -26,7 +26,7 @@ directories in an encrypted repository stored on different backends.
 
 .PP
 \fB\-\-cache\-dir\fP=""
-    set the cache directory
+    set the cache directory. (default: use system default cache directory)
 
 .PP
 \fB\-\-cleanup\-cache\fP[=false]
@@ -83,4 +83,4 @@ directories in an encrypted repository stored on different backends.
 
 .SH SEE ALSO
 .PP
-\fBrestic\-backup(1)\fP, \fBrestic\-cache(1)\fP, \fBrestic\-cat(1)\fP, \fBrestic\-check(1)\fP, \fBrestic\-diff(1)\fP, \fBrestic\-dump(1)\fP, \fBrestic\-find(1)\fP, \fBrestic\-forget(1)\fP, \fBrestic\-generate(1)\fP, \fBrestic\-init(1)\fP, \fBrestic\-key(1)\fP, \fBrestic\-list(1)\fP, \fBrestic\-ls(1)\fP, \fBrestic\-migrate(1)\fP, \fBrestic\-mount(1)\fP, \fBrestic\-prune(1)\fP, \fBrestic\-rebuild\-index(1)\fP, \fBrestic\-restore(1)\fP, \fBrestic\-snapshots(1)\fP, \fBrestic\-stats(1)\fP, \fBrestic\-tag(1)\fP, \fBrestic\-unlock(1)\fP, \fBrestic\-version(1)\fP
+\fBrestic\-backup(1)\fP, \fBrestic\-cache(1)\fP, \fBrestic\-cat(1)\fP, \fBrestic\-check(1)\fP, \fBrestic\-diff(1)\fP, \fBrestic\-dump(1)\fP, \fBrestic\-find(1)\fP, \fBrestic\-forget(1)\fP, \fBrestic\-generate(1)\fP, \fBrestic\-init(1)\fP, \fBrestic\-key(1)\fP, \fBrestic\-list(1)\fP, \fBrestic\-ls(1)\fP, \fBrestic\-migrate(1)\fP, \fBrestic\-mount(1)\fP, \fBrestic\-prune(1)\fP, \fBrestic\-rebuild\-index(1)\fP, \fBrestic\-restore(1)\fP, \fBrestic\-self\-update(1)\fP, \fBrestic\-snapshots(1)\fP, \fBrestic\-stats(1)\fP, \fBrestic\-tag(1)\fP, \fBrestic\-unlock(1)\fP, \fBrestic\-version(1)\fP

doc/manual_rest.rst

@@ -43,7 +43,7 @@ Usage help is available:
 
     Flags:
           --cacert file              file to load root certificates from (default: use system certificates)
-          --cache-dir string         set the cache directory
+          --cache-dir string         set the cache directory. (default: use system default cache directory)
           --cleanup-cache            auto remove old cache directories
       -h, --help                     help for restic
           --json                     set output mode to JSON for commands that support it
@@ -94,7 +94,7 @@ command:
 
     Global Flags:
           --cacert file              file to load root certificates from (default: use system certificates)
-          --cache-dir string         set the cache directory
+          --cache-dir string         set the cache directory. (default: use system default cache directory)
           --cleanup-cache            auto remove old cache directories
           --json                     set output mode to JSON for commands that support it
           --limit-download int       limits downloads to a maximum rate in KiB/s. (default: unlimited)
@@ -169,7 +169,7 @@ Browse repository objects
 
 Internally, a repository stores data of several different types
 described in the `design
-documentation <https://github.com/restic/restic/blob/master/doc/Design.rst>`__.
+documentation <https://github.com/restic/restic/blob/master/doc/design.rst>`__.
 You can ``list`` objects such as blobs, packs, index, snapshots, keys or
 locks with the following command:
 

doc/zsh-completion.zsh

@@ -7,7 +7,7 @@ case $state in
   level1)
     case $words[1] in
       restic)
-        _arguments '1: :(backup cache cat check diff dump find forget generate help init key list ls migrate mount options prune rebuild-index restore snapshots stats tag unlock version)'
+        _arguments '1: :(backup cache cat check diff dump find forget generate help init key list ls migrate mount options prune rebuild-index restore self-update snapshots stats tag unlock version)'
       ;;
       *)
         _arguments '*: :_files'

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.6
+FROM alpine:latest
 
 COPY restic /usr/bin
 

docker/build.sh

@@ -4,8 +4,8 @@ set -e
 
 echo "Build binary using golang docker image"
 docker run --rm -ti \
-    -v `pwd`:/go/src/github.com/restic/restic \
-    -w /go/src/github.com/restic/restic golang:1.8.3-alpine go run build.go
+    -v "`pwd`":/go/src/github.com/restic/restic \
+    -w /go/src/github.com/restic/restic golang:1.11.1-alpine go run build.go
 
 echo "Build docker image restic/restic:latest"
 docker build --rm -t restic/restic:latest -f docker/Dockerfile .

go.mod

@@ -0,0 +1,54 @@
+module github.com/restic/restic
+
+require (
+	bazil.org/fuse v0.0.0-20180421153158-65cc252bf669
+	cloud.google.com/go v0.27.0 // indirect
+	github.com/Azure/azure-sdk-for-go v20.1.0+incompatible
+	github.com/Azure/go-autorest v10.15.3+incompatible // indirect
+	github.com/cenkalti/backoff v2.0.0+incompatible
+	github.com/cpuguy83/go-md2man v1.0.8 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
+	github.com/dnaeon/go-vcr v0.0.0-20180814043457-aafff18a5cc2 // indirect
+	github.com/elithrar/simple-scrypt v1.3.0
+	github.com/go-ini/ini v1.38.2 // indirect
+	github.com/golang/protobuf v1.2.0 // indirect
+	github.com/google/go-cmp v0.2.0
+	github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jtolds/gls v4.2.1+incompatible // indirect
+	github.com/juju/ratelimit v1.0.1
+	github.com/kr/fs v0.1.0 // indirect
+	github.com/kr/pretty v0.1.0 // indirect
+	github.com/kurin/blazer v0.5.1
+	github.com/marstr/guid v1.1.0 // indirect
+	github.com/mattn/go-isatty v0.0.4
+	github.com/minio/minio-go v6.0.7+incompatible
+	github.com/mitchellh/go-homedir v1.0.0 // indirect
+	github.com/ncw/swift v1.0.41
+	github.com/pkg/errors v0.8.0
+	github.com/pkg/profile v1.2.1
+	github.com/pkg/sftp v1.8.2
+	github.com/pkg/xattr v0.3.1
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/restic/chunker v0.2.0
+	github.com/russross/blackfriday v1.5.1 // indirect
+	github.com/satori/go.uuid v1.2.0 // indirect
+	github.com/smartystreets/assertions v0.0.0-20180820201707-7c9eb446e3cf // indirect
+	github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a // indirect
+	github.com/spf13/cobra v0.0.3
+	github.com/spf13/pflag v1.0.2
+	github.com/stretchr/testify v1.2.2 // indirect
+	golang.org/x/crypto v0.0.0-20180904163835-0709b304e793
+	golang.org/x/net v0.0.0-20180906233101-161cd47e91fd
+	golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
+	golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f
+	golang.org/x/sys v0.0.0-20180907202204-917fdcba135d
+	golang.org/x/text v0.3.0
+	google.golang.org/api v0.0.0-20180907210053-b609d5e6b7ab
+	google.golang.org/appengine v1.1.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
+	gopkg.in/ini.v1 v1.38.2 // indirect
+	gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637
+	gopkg.in/yaml.v2 v2.2.1 // indirect
+)

go.sum

@@ -0,0 +1,106 @@
+bazil.org/fuse v0.0.0-20180421153158-65cc252bf669 h1:FNCRpXiquG1aoyqcIWVFmpTSKVcx2bQD38uZZeGtdlw=
+bazil.org/fuse v0.0.0-20180421153158-65cc252bf669/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8=
+cloud.google.com/go v0.27.0 h1:Xa8ZWro6QYKOwDKtxfKsiE0ea2jD39nx32RxtF5RjYE=
+cloud.google.com/go v0.27.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/Azure/azure-sdk-for-go v20.1.0+incompatible h1:b8OWFQuH5MPi2LYyAR2Ga+7KVH9ipwiSSSMga04/Urc=
+github.com/Azure/azure-sdk-for-go v20.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
+github.com/Azure/go-autorest v10.15.3+incompatible h1:nhKI/bvazIs3C3TFGoSqKY6hZ8f5od5mb5/UcS6HVIY=
+github.com/Azure/go-autorest v10.15.3+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
+github.com/cenkalti/backoff v2.0.0+incompatible h1:5IIPUHhlnUZbcHQsQou5k1Tn58nJkeJL9U+ig5CHJbY=
+github.com/cenkalti/backoff v2.0.0+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
+github.com/cpuguy83/go-md2man v1.0.8 h1:DwoNytLphI8hzS2Af4D0dfaEaiSq2bN05mEm4R6vf8M=
+github.com/cpuguy83/go-md2man v1.0.8/go.mod h1:N6JayAiVKtlHSnuTCeuLSQVs75hb8q+dYQLjr7cDsKY=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dnaeon/go-vcr v0.0.0-20180814043457-aafff18a5cc2 h1:G9/PqfhOrt8JXnw0DGTfVoOkKHDhOlEZqhE/cu+NvQM=
+github.com/dnaeon/go-vcr v0.0.0-20180814043457-aafff18a5cc2/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
+github.com/elithrar/simple-scrypt v1.3.0 h1:KIlOlxdoQf9JWKl5lMAJ28SY2URB0XTRDn2TckyzAZg=
+github.com/elithrar/simple-scrypt v1.3.0/go.mod h1:U2XQRI95XHY0St410VE3UjT7vuKb1qPwrl/EJwEqnZo=
+github.com/go-ini/ini v1.38.2 h1:6Hl/z3p3iFkA0dlDfzYxuFuUGD+kaweypF6btsR2/Q4=
+github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c h1:16eHWuMGvCjSfgRJKqIzapE78onvvTbdi1rMkU00lZw=
+github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
+github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/juju/ratelimit v1.0.1 h1:+7AIFJVQ0EQgq/K9+0Krm7m530Du7tIz0METWzN0RgY=
+github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk=
+github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kurin/blazer v0.5.1 h1:mBc4i1uhHJEqU0KvzOgpMHhkwf+EcXvxjWEUS7HG+eY=
+github.com/kurin/blazer v0.5.1/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU=
+github.com/marstr/guid v1.1.0 h1:/M4H/1G4avsieL6BbUwCOBzulmoeKVP5ux/3mQNnbyI=
+github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
+github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
+github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/minio/minio-go v6.0.7+incompatible h1:nWABqotkiT/3aLgFnG30doQiwFkDMM9xnGGQnS+Ao6M=
+github.com/minio/minio-go v6.0.7+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8=
+github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/ncw/swift v1.0.41 h1:kfoTVQKt1A4n0m1Q3YWku9OoXfpo06biqVfi73yseBs=
+github.com/ncw/swift v1.0.41/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
+github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.2.1 h1:F++O52m40owAmADcojzM+9gyjmMOY/T4oYJkgFDH8RE=
+github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
+github.com/pkg/sftp v1.8.2 h1:3upwlsK5/USEeM5gzIe9eWdzU4sV+kG3gKKg3RLBuWE=
+github.com/pkg/sftp v1.8.2/go.mod h1:NxmoDg/QLVWluQDUYG7XBZTLUpKeFa8e3aMf1BfjyHk=
+github.com/pkg/xattr v0.3.1 h1:6ceg5jxT3cH4lM5n8S2PmiNeOv61MK08yvvYJwyrPH0=
+github.com/pkg/xattr v0.3.1/go.mod h1:CBdxFOf0VLbaj6HKuP2ITOVV7NY6ycPKgIgnSx2ZNVs=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/restic/chunker v0.2.0 h1:GjvmvFuv2mx0iekZs+iAlrioo2UtgsGSSplvoXaVHDU=
+github.com/restic/chunker v0.2.0/go.mod h1:VdjruEj+7BU1ZZTW8Qqi1exxRx2Omf2JH0NsUEkQ29s=
+github.com/russross/blackfriday v1.5.1 h1:B8ZN6pD4PVofmlDCDUdELeYrbsVIDM/bpjW3v3zgcRc=
+github.com/russross/blackfriday v1.5.1/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
+github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
+github.com/smartystreets/assertions v0.0.0-20180820201707-7c9eb446e3cf h1:6V1qxN6Usn4jy8unvggSJz/NC790tefw8Zdy6OZS5co=
+github.com/smartystreets/assertions v0.0.0-20180820201707-7c9eb446e3cf/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a h1:JSvGDIbmil4Ui/dDdFBExb7/cmkNjyX5F97oglmvCDo=
+github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
+github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc=
+github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180525142821-c11f84a56e43/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180907202204-917fdcba135d h1:kWn1hlsqeUrk6JsLJO0ZFyz9bMg8u85voZlIuc68ZU4=
+golang.org/x/sys v0.0.0-20180907202204-917fdcba135d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+google.golang.org/api v0.0.0-20180907210053-b609d5e6b7ab h1:qNpJa8m9WofZ7RLj+7o15Ppapwm30+RweyIDSNpw8ps=
+google.golang.org/api v0.0.0-20180907210053-b609d5e6b7ab/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
+google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/ini.v1 v1.38.2 h1:dGcbywv4RufeGeiMycPT/plKB5FtmLKLnWKwBiLhUA4=
+gopkg.in/ini.v1 v1.38.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 h1:yiW+nvdHb9LVqSHQBXfZCieqV4fzYhNBql77zY0ykqs=
+gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637/go.mod h1:BHsqpu/nsuzkT5BpiH1EMZPLyqSMM8JbIavyFACoFNk=
+gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

helpers/.gitignore

@@ -0,0 +1,2 @@
+build-release-binaries/build-release-binaries
+prepare-release/prepare-release

helpers/build-release-binaries/main.go

@@ -0,0 +1,269 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/spf13/pflag"
+	"golang.org/x/sync/errgroup"
+)
+
+var opts = struct {
+	Verbose   bool
+	SourceDir string
+	OutputDir string
+	Version   string
+}{}
+
+func init() {
+	pflag.BoolVarP(&opts.Verbose, "verbose", "v", false, "be verbose")
+	pflag.StringVarP(&opts.SourceDir, "source", "s", "/restic", "path to the source code `directory`")
+	pflag.StringVarP(&opts.OutputDir, "output", "o", "/output", "path to the output `directory`")
+	pflag.StringVar(&opts.Version, "version", "", "use `x.y.z` as the version for output files")
+	pflag.Parse()
+}
+
+func die(f string, args ...interface{}) {
+	if !strings.HasSuffix(f, "\n") {
+		f += "\n"
+	}
+	f = "\x1b[31m" + f + "\x1b[0m"
+	fmt.Fprintf(os.Stderr, f, args...)
+	os.Exit(1)
+}
+
+func msg(f string, args ...interface{}) {
+	if !strings.HasSuffix(f, "\n") {
+		f += "\n"
+	}
+	f = "\x1b[32m" + f + "\x1b[0m"
+	fmt.Printf(f, args...)
+}
+
+func verbose(f string, args ...interface{}) {
+	if !opts.Verbose {
+		return
+	}
+	if !strings.HasSuffix(f, "\n") {
+		f += "\n"
+	}
+	f = "\x1b[32m" + f + "\x1b[0m"
+	fmt.Printf(f, args...)
+}
+
+func run(cmd string, args ...string) {
+	c := exec.Command(cmd, args...)
+	c.Stdout = os.Stdout
+	c.Stderr = os.Stderr
+	err := c.Run()
+	if err != nil {
+		die("error running %s %s: %v", cmd, args, err)
+	}
+}
+
+func rm(file string) {
+	err := os.Remove(file)
+
+	if os.IsNotExist(err) {
+		err = nil
+	}
+
+	if err != nil {
+		die("error removing %v: %v", file, err)
+	}
+}
+
+func rmdir(dir string) {
+	err := os.RemoveAll(dir)
+	if err != nil {
+		die("error removing %v: %v", dir, err)
+	}
+}
+
+func mkdir(dir string) {
+	err := os.MkdirAll(dir, 0755)
+	if err != nil {
+		die("mkdir %v: %v", dir, err)
+	}
+}
+
+func getwd() string {
+	pwd, err := os.Getwd()
+	if err != nil {
+		die("Getwd(): %v", err)
+	}
+	return pwd
+}
+
+func abs(dir string) string {
+	absDir, err := filepath.Abs(dir)
+	if err != nil {
+		die("unable to find absolute path for %v: %v", dir, err)
+	}
+	return absDir
+}
+
+func build(sourceDir, outputDir, goos, goarch string) (filename string) {
+	filename = fmt.Sprintf("%v_%v_%v", "restic", goos, goarch)
+
+	if opts.Version != "" {
+		filename = fmt.Sprintf("%v_%v_%v_%v", "restic", opts.Version, goos, goarch)
+	}
+
+	if goos == "windows" {
+		filename += ".exe"
+	}
+	outputFile := filepath.Join(outputDir, filename)
+
+	c := exec.Command("go", "build",
+		"-mod=vendor",
+		"-o", outputFile,
+		"-ldflags", "-s -w",
+		"-tags", "selfupdate",
+		"./cmd/restic",
+	)
+	c.Stdout = os.Stdout
+	c.Stderr = os.Stderr
+	c.Dir = sourceDir
+
+	verbose("run %v %v in %v", "go", c.Args, c.Dir)
+
+	c.Dir = sourceDir
+	c.Env = append(os.Environ(),
+		"CGO_ENABLED=0",
+		"GOPROXY=off",
+		"GOOS="+goos,
+		"GOARCH="+goarch,
+	)
+
+	err := c.Run()
+	if err != nil {
+		die("error building %v/%v: %v", goos, goarch, err)
+	}
+
+	return filename
+}
+
+func modTime(file string) time.Time {
+	fi, err := os.Lstat(file)
+	if err != nil {
+		die("unable to get modtime of %v: %v", file, err)
+	}
+
+	return fi.ModTime()
+}
+
+func touch(file string, t time.Time) {
+	err := os.Chtimes(file, t, t)
+	if err != nil {
+		die("unable to update timestamps for %v: %v", file, err)
+	}
+}
+
+func chmod(file string, mode os.FileMode) {
+	err := os.Chmod(file, mode)
+	if err != nil {
+		die("unable to chmod %v to %s: %v", file, mode, err)
+	}
+}
+
+func compress(goos, inputDir, filename string) (outputFile string) {
+	var c *exec.Cmd
+	switch goos {
+	case "windows":
+		outputFile = strings.TrimSuffix(filename, ".exe") + ".zip"
+		c = exec.Command("zip", "-q", "-X", outputFile, filename)
+		c.Dir = inputDir
+	default:
+		outputFile = filename + ".bz2"
+		c = exec.Command("bzip2", filename)
+		c.Dir = inputDir
+	}
+
+	rm(filepath.Join(inputDir, outputFile))
+
+	c.Stdout = os.Stdout
+	c.Stderr = os.Stderr
+	c.Dir = inputDir
+
+	verbose("run %v %v in %v", "go", c.Args, c.Dir)
+
+	err := c.Run()
+	if err != nil {
+		die("error compressing: %v", err)
+	}
+
+	rm(filepath.Join(inputDir, filename))
+
+	return outputFile
+}
+
+func buildForTarget(sourceDir, outputDir, goos, goarch string) (filename string) {
+	mtime := modTime(filepath.Join(sourceDir, "VERSION"))
+
+	filename = build(sourceDir, outputDir, goos, goarch)
+	touch(filepath.Join(outputDir, filename), mtime)
+	chmod(filepath.Join(outputDir, filename), 0755)
+	filename = compress(goos, outputDir, filename)
+	return filename
+}
+
+func buildTargets(sourceDir, outputDir string, targets map[string][]string) {
+	start := time.Now()
+	msg("building with %d workers", runtime.NumCPU())
+
+	type Job struct{ GOOS, GOARCH string }
+
+	var wg errgroup.Group
+	ch := make(chan Job)
+
+	for i := 0; i < runtime.NumCPU(); i++ {
+		wg.Go(func() error {
+			for job := range ch {
+				start := time.Now()
+				verbose("build %v/%v", job.GOOS, job.GOARCH)
+				buildForTarget(sourceDir, outputDir, job.GOOS, job.GOARCH)
+				msg("built %v/%v in %.3fs", job.GOOS, job.GOARCH, time.Since(start).Seconds())
+			}
+			return nil
+		})
+	}
+
+	wg.Go(func() error {
+		for goos, archs := range targets {
+			for _, goarch := range archs {
+				ch <- Job{goos, goarch}
+			}
+		}
+		close(ch)
+		return nil
+	})
+
+	_ = wg.Wait()
+	msg("build finished in %.3fs", time.Since(start).Seconds())
+}
+
+var defaultBuildTargets = map[string][]string{
+	"darwin":  []string{"386", "amd64"},
+	"freebsd": []string{"386", "amd64", "arm"},
+	"linux":   []string{"386", "amd64", "arm", "arm64"},
+	"openbsd": []string{"386", "amd64"},
+	"windows": []string{"386", "amd64"},
+}
+
+func main() {
+	if len(pflag.Args()) != 0 {
+		die("USAGE: build-release-binaries [OPTIONS]")
+	}
+
+	sourceDir := abs(opts.SourceDir)
+	outputDir := abs(opts.OutputDir)
+	mkdir(outputDir)
+
+	buildTargets(sourceDir, outputDir, defaultBuildTargets)
+}

helpers/prepare-release/main.go

@@ -1,5 +1,3 @@
-// +build ignore
-
 package main
 
 import (
@@ -11,7 +9,6 @@ import (
 	"os/exec"
 	"path/filepath"
 	"regexp"
-	"sort"
 	"strings"
 
 	"github.com/spf13/pflag"
@@ -27,8 +24,7 @@ var opts = struct {
 	IgnoreChangelogCurrent     bool
 	IgnoreDockerBuildGoVersion bool
 
-	tarFilename string
-	buildDir    string
+	OutputDir string
 }{}
 
 var versionRegex = regexp.MustCompile(`^\d+\.\d+\.\d+$`)
@@ -40,6 +36,9 @@ func init() {
 	pflag.BoolVar(&opts.IgnoreChangelogReleaseDate, "ignore-changelog-release-date", false, "ignore missing subdir with date in changelog/")
 	pflag.BoolVar(&opts.IgnoreChangelogCurrent, "ignore-changelog-current", false, "ignore check if CHANGELOG.md is up to date")
 	pflag.BoolVar(&opts.IgnoreDockerBuildGoVersion, "ignore-docker-build-go-version", false, "ignore check if docker builder go version is up to date")
+
+	pflag.StringVar(&opts.OutputDir, "output-dir", "", "use `dir` as output directory")
+
 	pflag.Parse()
 }
 
@@ -70,6 +69,21 @@ func run(cmd string, args ...string) {
 	}
 }
 
+func replace(filename, from, to string) {
+	reg := regexp.MustCompile(from)
+
+	buf, err := ioutil.ReadFile(filename)
+	if err != nil {
+		die("error reading file %v: %v", filename, err)
+	}
+
+	buf = reg.ReplaceAll(buf, []byte(to))
+	err = ioutil.WriteFile(filename, buf, 0644)
+	if err != nil {
+		die("error writing file %v: %v", filename, err)
+	}
+}
+
 func rm(file string) {
 	err := os.Remove(file)
 	if err != nil {
@@ -238,6 +252,7 @@ func preCheckDockerBuilderGoVersion() {
 	}
 	localVersion := strings.TrimSpace(string(buf))
 
+	msg("update docker container restic/builder")
 	run("docker", "pull", "restic/builder")
 	buf, err = exec.Command("docker", "run", "--rm", "restic/builder", "go", "version").Output()
 	if err != nil {
@@ -272,80 +287,127 @@ func generateFiles() {
 	}
 }
 
+var versionPattern = `var version = ".*"`
+
+const versionCodeFile = "cmd/restic/global.go"
+
 func updateVersion() {
 	err := ioutil.WriteFile("VERSION", []byte(opts.Version+"\n"), 0644)
 	if err != nil {
 		die("unable to write version to file: %v", err)
 	}
 
-	if len(uncommittedChanges("VERSION")) > 0 {
-		msg("committing file VERSION")
-		run("git", "commit", "-m", fmt.Sprintf("Add VERSION for %v", opts.Version), "VERSION")
+	newVersion := fmt.Sprintf("var version = %q", opts.Version)
+	replace(versionCodeFile, versionPattern, newVersion)
+
+	if len(uncommittedChanges("VERSION")) > 0 || len(uncommittedChanges(versionCodeFile)) > 0 {
+		msg("committing version files")
+		run("git", "commit", "-m", fmt.Sprintf("Add version for %v", opts.Version), "VERSION", versionCodeFile)
 	}
 }
 
+func updateVersionDev() {
+	newVersion := fmt.Sprintf(`var version = "%s-dev (compiled manually)"`, opts.Version)
+	replace(versionCodeFile, versionPattern, newVersion)
+
+	msg("committing cmd/restic/global.go with dev version")
+	run("git", "commit", "-m", fmt.Sprintf("Set development version for %v", opts.Version), "VERSION", versionCodeFile)
+}
+
 func addTag() {
 	tagname := "v" + opts.Version
 	msg("add tag %v", tagname)
 	run("git", "tag", "-a", "-s", "-m", tagname, tagname)
 }
 
-func exportTar() {
+func exportTar(version, tarFilename string) {
 	cmd := fmt.Sprintf("git archive --format=tar --prefix=restic-%s/ v%s | gzip -n > %s",
-		opts.Version, opts.Version, opts.tarFilename)
+		version, version, tarFilename)
 	run("sh", "-c", cmd)
-	msg("build restic-%s.tar.gz", opts.Version)
+	msg("build restic-%s.tar.gz", version)
 }
 
-func runBuild() {
-	msg("building binaries...")
-	run("docker", "run", "--rm", "--volume", getwd()+":/home/build", "restic/builder", "build.sh", opts.tarFilename)
-}
-
-func findBuildDir() string {
-	nameRegex := regexp.MustCompile(`restic-` + opts.Version + `-\d{8}-\d{6}`)
-
-	f, err := os.Open(".")
+func extractTar(filename, outputDir string) {
+	msg("extract tar into %v", outputDir)
+	c := exec.Command("tar", "xz", "--strip-components=1", "-f", filename)
+	c.Stdout = os.Stdout
+	c.Stderr = os.Stderr
+	c.Dir = outputDir
+	err := c.Run()
 	if err != nil {
-		die("Open(.): %v", err)
+		die("error extracting tar: %v", err)
+	}
+}
+
+func runBuild(sourceDir, outputDir, version string) {
+	msg("building binaries...")
+	run("docker", "run", "--rm",
+		"--volume", sourceDir+":/restic",
+		"--volume", outputDir+":/output",
+		"restic/builder",
+		"go", "run", "-mod=vendor", "helpers/build-release-binaries/main.go",
+		"--version", version)
+}
+
+func readdir(dir string) []string {
+	fis, err := ioutil.ReadDir(dir)
+	if err != nil {
+		die("readdir %v failed: %v", dir, err)
 	}
 
-	entries, err := f.Readdirnames(-1)
+	filenames := make([]string, 0, len(fis))
+	for _, fi := range fis {
+		filenames = append(filenames, fi.Name())
+	}
+	return filenames
+}
+
+func sha256sums(inputDir, outputFile string) {
+	msg("runnnig sha256sum in %v", inputDir)
+
+	filenames := readdir(inputDir)
+
+	f, err := os.Create(outputFile)
 	if err != nil {
-		die("Readdirnames(): %v", err)
+		die("unable to create %v: %v", outputFile, err)
+	}
+
+	c := exec.Command("sha256sum", filenames...)
+	c.Stdout = f
+	c.Stderr = os.Stderr
+	c.Dir = inputDir
+
+	err = c.Run()
+	if err != nil {
+		die("error running sha256sums: %v", err)
 	}
 
 	err = f.Close()
 	if err != nil {
-		die("Close(): %v", err)
+		die("close %v: %v", outputFile, err)
 	}
-
-	sort.Slice(entries, func(i, j int) bool {
-		return entries[j] < entries[i]
-	})
-
-	for _, entry := range entries {
-		if nameRegex.MatchString(entry) {
-			msg("found restic build dir: %v", entry)
-			return entry
-		}
-	}
-
-	die("restic build dir not found")
-	return ""
 }
 
-func signFiles() {
-	run("gpg", "--armor", "--detach-sign", filepath.Join(opts.buildDir, "SHA256SUMS"))
-	run("gpg", "--armor", "--detach-sign", filepath.Join(opts.buildDir, opts.tarFilename))
+func signFiles(filenames ...string) {
+	for _, filename := range filenames {
+		run("gpg", "--armor", "--detach-sign", filename)
+	}
 }
 
-func updateDocker() {
-	cmd := fmt.Sprintf("bzcat %s/restic_%s_linux_amd64.bz2 > restic", opts.buildDir, opts.Version)
+func updateDocker(outputDir, version string) {
+	cmd := fmt.Sprintf("bzcat %s/restic_%s_linux_amd64.bz2 > restic", outputDir, version)
 	run("sh", "-c", cmd)
 	run("chmod", "+x", "restic")
 	run("docker", "build", "--rm", "--tag", "restic/restic:latest", "-f", "docker/Dockerfile", ".")
-	run("docker", "tag", "restic/restic:latest", "restic/restic:"+opts.Version)
+	run("docker", "tag", "restic/restic:latest", "restic/restic:"+version)
+}
+
+func tempdir(prefix string) string {
+	dir, err := ioutil.TempDir(getwd(), prefix)
+	if err != nil {
+		die("unable to create temp dir %q: %v", prefix, err)
+	}
+	return dir
 }
 
 func main() {
@@ -358,8 +420,6 @@ func main() {
 		die("invalid new version")
 	}
 
-	opts.tarFilename = fmt.Sprintf("restic-%s.tar.gz", opts.Version)
-
 	preCheckBranchMaster()
 	preCheckUncommittedChanges()
 	preCheckVersionExists()
@@ -368,18 +428,33 @@ func main() {
 	preCheckChangelogCurrent()
 	preCheckChangelogVersion()
 
+	if opts.OutputDir == "" {
+		opts.OutputDir = tempdir("build-output-")
+	}
+	sourceDir := tempdir("source-")
+
+	msg("using output dir %v", opts.OutputDir)
+	msg("using source dir %v", sourceDir)
+
 	generateFiles()
 	updateVersion()
 	addTag()
+	updateVersionDev()
 
-	exportTar()
-	runBuild()
-	opts.buildDir = findBuildDir()
-	signFiles()
+	tarFilename := filepath.Join(opts.OutputDir, fmt.Sprintf("restic-%s.tar.gz", opts.Version))
+	exportTar(opts.Version, tarFilename)
 
-	updateDocker()
+	extractTar(tarFilename, sourceDir)
+	runBuild(sourceDir, opts.OutputDir, opts.Version)
+	rmdir(sourceDir)
 
-	msg("done, build dir is %v", opts.buildDir)
+	sha256sums(opts.OutputDir, filepath.Join(opts.OutputDir, "SHA256SUMS"))
+
+	signFiles(filepath.Join(opts.OutputDir, "SHA256SUMS"), tarFilename)
+
+	updateDocker(opts.OutputDir, opts.Version)
+
+	msg("done, output dir is %v", opts.OutputDir)
 
 	msg("now run:\n\ngit push --tags origin master\ndocker push restic/restic\n")
 }

internal/archiver/archiver.go

@@ -16,6 +16,10 @@ import (
 	tomb "gopkg.in/tomb.v2"
 )
 
+// SelectByNameFunc returns true for all items that should be included (files and
+// dirs). If false is returned, files are ignored and dirs are not even walked.
+type SelectByNameFunc func(item string) bool
+
 // SelectFunc returns true for all items that should be included (files and
 // dirs). If false is returned, files are ignored and dirs are not even walked.
 type SelectFunc func(item string, fi os.FileInfo) bool
@@ -43,10 +47,11 @@ func (s *ItemStats) Add(other ItemStats) {
 
 // Archiver saves a directory structure to the repo.
 type Archiver struct {
-	Repo    restic.Repository
-	Select  SelectFunc
-	FS      fs.FS
-	Options Options
+	Repo         restic.Repository
+	SelectByName SelectByNameFunc
+	Select       SelectFunc
+	FS           fs.FS
+	Options      Options
 
 	blobSaver *BlobSaver
 	fileSaver *FileSaver
@@ -119,10 +124,11 @@ func (o Options) ApplyDefaults() Options {
 // New initializes a new archiver.
 func New(repo restic.Repository, fs fs.FS, opts Options) *Archiver {
 	arch := &Archiver{
-		Repo:    repo,
-		Select:  func(string, os.FileInfo) bool { return true },
-		FS:      fs,
-		Options: opts.ApplyDefaults(),
+		Repo:         repo,
+		SelectByName: func(item string) bool { return true },
+		Select:       func(item string, fi os.FileInfo) bool { return true },
+		FS:           fs,
+		Options:      opts.ApplyDefaults(),
 
 		CompleteItem: func(string, *restic.Node, *restic.Node, ItemStats, time.Duration) {},
 		StartFile:    func(string) {},
@@ -294,10 +300,10 @@ func (fn *FutureNode) wait(ctx context.Context) {
 }
 
 // Save saves a target (file or directory) to the repo. If the item is
-// excluded,this function returns a nil node and error, with excluded set to
+// excluded, this function returns a nil node and error, with excluded set to
 // true.
 //
-// Errors and completion is needs to be handled by the caller.
+// Errors and completion needs to be handled by the caller.
 //
 // snPath is the path within the current snapshot.
 func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous *restic.Node) (fn FutureNode, excluded bool, err error) {
@@ -316,6 +322,13 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
 
 	fn.absTarget = abstarget
 
+	// exclude files by path before running Lstat to reduce number of lstat calls
+	if !arch.SelectByName(abstarget) {
+		debug.Log("%v is excluded by path", target)
+		return FutureNode{}, true, nil
+	}
+
+	// get file info and run remaining select functions that require file information
 	fi, err := arch.FS.Lstat(target)
 	if !arch.Select(abstarget, fi) {
 		debug.Log("%v is excluded", target)

internal/archiver/archiver_test.go

@@ -137,7 +137,7 @@ func TestArchiverSaveFile(t *testing.T) {
 			tempdir, repo, cleanup := prepareTempdirRepoSrc(t, TestDir{"file": testfile})
 			defer cleanup()
 
-			node, stats := saveFile(t, repo, filepath.Join(tempdir, "file"), fs.Track{fs.Local{}})
+			node, stats := saveFile(t, repo, filepath.Join(tempdir, "file"), fs.Track{FS: fs.Local{}})
 
 			TestEnsureFileContent(ctx, t, repo, "file", node, testfile)
 			if stats.DataSize != uint64(len(testfile.Content)) {
@@ -218,7 +218,7 @@ func TestArchiverSave(t *testing.T) {
 
 			var tmb tomb.Tomb
 
-			arch := New(repo, fs.Track{fs.Local{}}, Options{})
+			arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
 			arch.Error = func(item string, fi os.FileInfo, err error) error {
 				t.Errorf("archiver error for %v: %v", item, err)
 				return err
@@ -358,7 +358,7 @@ func BenchmarkArchiverSaveFileSmall(b *testing.B) {
 		tempdir, repo, cleanup := prepareTempdirRepoSrc(b, d)
 		b.StartTimer()
 
-		_, stats := saveFile(b, repo, filepath.Join(tempdir, "file"), fs.Track{fs.Local{}})
+		_, stats := saveFile(b, repo, filepath.Join(tempdir, "file"), fs.Track{FS: fs.Local{}})
 
 		b.StopTimer()
 		if stats.DataSize != fileSize {
@@ -391,7 +391,7 @@ func BenchmarkArchiverSaveFileLarge(b *testing.B) {
 		tempdir, repo, cleanup := prepareTempdirRepoSrc(b, d)
 		b.StartTimer()
 
-		_, stats := saveFile(b, repo, filepath.Join(tempdir, "file"), fs.Track{fs.Local{}})
+		_, stats := saveFile(b, repo, filepath.Join(tempdir, "file"), fs.Track{FS: fs.Local{}})
 
 		b.StopTimer()
 		if stats.DataSize != fileSize {
@@ -471,7 +471,7 @@ func TestArchiverSaveFileIncremental(t *testing.T) {
 
 	for i := 0; i < 3; i++ {
 		appendToFile(t, testfile, data)
-		node, _ := saveFile(t, repo, testfile, fs.Track{fs.Local{}})
+		node, _ := saveFile(t, repo, testfile, fs.Track{FS: fs.Local{}})
 
 		t.Logf("node blobs: %v", node.Content)
 
@@ -752,7 +752,7 @@ func TestArchiverSaveDir(t *testing.T) {
 			tempdir, repo, cleanup := prepareTempdirRepoSrc(t, test.src)
 			defer cleanup()
 
-			arch := New(repo, fs.Track{fs.Local{}}, Options{})
+			arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
 			arch.runWorkers(ctx, &tmb)
 
 			chdir := tempdir
@@ -842,7 +842,7 @@ func TestArchiverSaveDirIncremental(t *testing.T) {
 		var tmb tomb.Tomb
 		ctx := tmb.Context(context.Background())
 
-		arch := New(repo, fs.Track{fs.Local{}}, Options{})
+		arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
 		arch.runWorkers(ctx, &tmb)
 
 		fi, err := fs.Lstat(tempdir)
@@ -1002,7 +1002,7 @@ func TestArchiverSaveTree(t *testing.T) {
 			tempdir, repo, cleanup := prepareTempdirRepoSrc(t, test.src)
 			defer cleanup()
 
-			testFS := fs.Track{fs.Local{}}
+			testFS := fs.Track{FS: fs.Local{}}
 
 			arch := New(repo, testFS, Options{})
 			arch.runWorkers(ctx, &tmb)
@@ -1291,7 +1291,7 @@ func TestArchiverSnapshot(t *testing.T) {
 			tempdir, repo, cleanup := prepareTempdirRepoSrc(t, test.src)
 			defer cleanup()
 
-			arch := New(repo, fs.Track{fs.Local{}}, Options{})
+			arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
 
 			chdir := tempdir
 			if test.chdir != "" {
@@ -1455,7 +1455,7 @@ func TestArchiverSnapshotSelect(t *testing.T) {
 			tempdir, repo, cleanup := prepareTempdirRepoSrc(t, test.src)
 			defer cleanup()
 
-			arch := New(repo, fs.Track{fs.Local{}}, Options{})
+			arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
 			arch.Select = test.selFn
 
 			back := fs.TestChdir(t, tempdir)
@@ -1559,7 +1559,7 @@ func TestArchiverParent(t *testing.T) {
 			defer cleanup()
 
 			testFS := &MockFS{
-				FS:        fs.Track{fs.Local{}},
+				FS:        fs.Track{FS: fs.Local{}},
 				bytesRead: make(map[string]int),
 			}
 
@@ -1732,7 +1732,7 @@ func TestArchiverErrorReporting(t *testing.T) {
 				test.prepare(t)
 			}
 
-			arch := New(repo, fs.Track{fs.Local{}}, Options{})
+			arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
 			arch.Error = test.errFn
 
 			_, snapshotID, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
@@ -1867,7 +1867,7 @@ func TestArchiverAbortEarlyOnError(t *testing.T) {
 			defer back()
 
 			testFS := &TrackFS{
-				FS:     fs.Track{fs.Local{}},
+				FS:     fs.Track{FS: fs.Local{}},
 				opened: make(map[string]uint),
 			}
 

internal/archiver/scanner.go

@@ -12,23 +12,21 @@ import (
 // stats concerning the files and folders found. Select is used to decide which
 // items should be included. Error is called when an error occurs.
 type Scanner struct {
-	FS     fs.FS
-	Select SelectFunc
-	Error  ErrorFunc
-	Result func(item string, s ScanStats)
+	FS           fs.FS
+	SelectByName SelectByNameFunc
+	Select       SelectFunc
+	Error        ErrorFunc
+	Result       func(item string, s ScanStats)
 }
 
 // NewScanner initializes a new Scanner.
 func NewScanner(fs fs.FS) *Scanner {
 	return &Scanner{
-		FS: fs,
-		Select: func(item string, fi os.FileInfo) bool {
-			return true
-		},
-		Error: func(item string, fi os.FileInfo, err error) error {
-			return err
-		},
-		Result: func(item string, s ScanStats) {},
+		FS:           fs,
+		SelectByName: func(item string) bool { return true },
+		Select:       func(item string, fi os.FileInfo) bool { return true },
+		Error:        func(item string, fi os.FileInfo, err error) error { return err },
+		Result:       func(item string, s ScanStats) {},
 	}
 }
 
@@ -54,33 +52,31 @@ func (s *Scanner) Scan(ctx context.Context, targets []string) error {
 		}
 
 		if ctx.Err() != nil {
-			return ctx.Err()
+			return nil
 		}
 	}
 
-	if ctx.Err() != nil {
-		return ctx.Err()
-	}
 	s.Result("", stats)
 	return nil
 }
 
 func (s *Scanner) scan(ctx context.Context, stats ScanStats, target string) (ScanStats, error) {
 	if ctx.Err() != nil {
-		return stats, ctx.Err()
+		return stats, nil
 	}
 
+	// exclude files by path before running stat to reduce number of lstat calls
+	if !s.SelectByName(target) {
+		return stats, nil
+	}
+
+	// get file information
 	fi, err := s.FS.Lstat(target)
 	if err != nil {
-		// ignore error if the target is to be excluded anyway
-		if !s.Select(target, nil) {
-			return stats, nil
-		}
-
-		// else return filtered error
 		return stats, s.Error(target, fi, err)
 	}
 
+	// run remaining select functions that require file information
 	if !s.Select(target, fi) {
 		return stats, nil
 	}
@@ -90,10 +86,6 @@ func (s *Scanner) scan(ctx context.Context, stats ScanStats, target string) (Sca
 		stats.Files++
 		stats.Bytes += uint64(fi.Size())
 	case fi.Mode().IsDir():
-		if ctx.Err() != nil {
-			return stats, ctx.Err()
-		}
-
 		names, err := readdirnames(s.FS, target)
 		if err != nil {
 			return stats, s.Error(target, fi, err)
@@ -110,9 +102,6 @@ func (s *Scanner) scan(ctx context.Context, stats ScanStats, target string) (Sca
 		stats.Others++
 	}
 
-	if ctx.Err() != nil {
-		return stats, ctx.Err()
-	}
 	s.Result(target, stats)
 	return stats, nil
 }

internal/archiver/scanner_test.go

@@ -96,7 +96,7 @@ func TestScanner(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			sc := NewScanner(fs.Track{fs.Local{}})
+			sc := NewScanner(fs.Track{FS: fs.Local{}})
 			if test.selFn != nil {
 				sc.Select = test.selFn
 			}
@@ -237,7 +237,7 @@ func TestScannerError(t *testing.T) {
 				test.prepare(t)
 			}
 
-			sc := NewScanner(fs.Track{fs.Local{}})
+			sc := NewScanner(fs.Track{FS: fs.Local{}})
 			if test.selFn != nil {
 				sc.Select = test.selFn
 			}
@@ -289,7 +289,7 @@ func TestScannerCancel(t *testing.T) {
 		"other": TestFile{Content: "other"},
 	}
 
-	result := ScanStats{Files: 2, Bytes: 6}
+	result := ScanStats{Files: 2, Dirs: 1, Bytes: 6}
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -307,7 +307,7 @@ func TestScannerCancel(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	sc := NewScanner(fs.Track{fs.Local{}})
+	sc := NewScanner(fs.Track{FS: fs.Local{}})
 	var lastStats ScanStats
 	sc.Result = func(item string, s ScanStats) {
 		lastStats = s
@@ -319,12 +319,8 @@ func TestScannerCancel(t *testing.T) {
 	}
 
 	err = sc.Scan(ctx, []string{"."})
-	if err == nil {
-		t.Errorf("did not find expected error")
-	}
-
-	if err != context.Canceled {
-		t.Errorf("unexpected error found, want %v, got %v", context.Canceled, err)
+	if err != nil {
+		t.Errorf("unexpected error %v found", err)
 	}
 
 	if lastStats != result {

internal/archiver/testing_test.go

@@ -194,12 +194,12 @@ func TestTestWalkFiles(t *testing.T) {
 				},
 			},
 			want: map[string]string{
-				"foo":    "<File>",
-				"subdir": "<Dir>",
+				"foo":                                "<File>",
+				"subdir":                             "<Dir>",
 				filepath.FromSlash("subdir/subfile"): "<File>",
-				"x": "<Dir>",
-				filepath.FromSlash("x/y"):      "<Dir>",
-				filepath.FromSlash("x/y/link"): "<Symlink>",
+				"x":                                  "<Dir>",
+				filepath.FromSlash("x/y"):            "<Dir>",
+				filepath.FromSlash("x/y/link"):       "<Symlink>",
 			},
 		},
 	}
@@ -373,7 +373,7 @@ func TestTestEnsureSnapshot(t *testing.T) {
 	}{
 		{
 			files: map[string]interface{}{
-				"foo": TestFile{Content: "foo"},
+				"foo":                                TestFile{Content: "foo"},
 				filepath.FromSlash("subdir/subfile"): TestFile{Content: "bar"},
 				filepath.FromSlash("x/y/link"):       TestSymlink{Target: filepath.FromSlash("../../foo")},
 			},

internal/archiver/tree.go

@@ -232,7 +232,6 @@ func unrollTree(f fs.FS, t *Tree) error {
 				}
 
 				return errors.Errorf("tree unrollTree: collision on path, node %#v, path %q", node, f.Join(t.Path, entry))
-				continue
 			}
 			t.Nodes[entry] = Tree{Path: f.Join(t.Path, entry)}
 		}

internal/backend/b2/b2.go

@@ -5,7 +5,6 @@ import (
 	"io"
 	"net/http"
 	"path"
-	"strings"
 
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
@@ -257,71 +256,45 @@ func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error {
 	return errors.Wrap(obj.Delete(ctx), "Delete")
 }
 
-// List returns a channel that yields all names of blobs of type t. A
-// goroutine is started for this. If the channel done is closed, sending
-// stops.
+type semLocker struct {
+	*backend.Semaphore
+}
+
+func (sm semLocker) Lock()   { sm.GetToken() }
+func (sm semLocker) Unlock() { sm.ReleaseToken() }
+
+// List returns a channel that yields all names of blobs of type t.
 func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
 	debug.Log("List %v", t)
 
-	prefix, _ := be.Basedir(t)
-	cur := &b2.Cursor{Prefix: prefix}
-
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	for {
-		be.sem.GetToken()
-		objs, c, err := be.bucket.ListCurrentObjects(ctx, be.listMaxItems, cur)
-		be.sem.ReleaseToken()
+	prefix, _ := be.Basedir(t)
+	iter := be.bucket.List(ctx, b2.ListPrefix(prefix), b2.ListPageSize(be.listMaxItems), b2.ListLocker(semLocker{be.sem}))
 
-		if err != nil && err != io.EOF {
-			debug.Log("List: %v", err)
+	for iter.Next() {
+		obj := iter.Object()
+
+		attrs, err := obj.Attrs(ctx)
+		if err != nil {
 			return err
 		}
 
-		debug.Log("returned %v items", len(objs))
-		for _, obj := range objs {
-			// Skip objects returned that do not have the specified prefix.
-			if !strings.HasPrefix(obj.Name(), prefix) {
-				continue
-			}
-
-			m := path.Base(obj.Name())
-			if m == "" {
-				continue
-			}
-
-			if ctx.Err() != nil {
-				return ctx.Err()
-			}
-
-			attrs, err := obj.Attrs(ctx)
-			if err != nil {
-				return err
-			}
-
-			fi := restic.FileInfo{
-				Name: m,
-				Size: attrs.Size,
-			}
-
-			err = fn(fi)
-			if err != nil {
-				return err
-			}
-
-			if ctx.Err() != nil {
-				return ctx.Err()
-			}
+		fi := restic.FileInfo{
+			Name: path.Base(obj.Name()),
+			Size: attrs.Size,
 		}
 
-		if err == io.EOF {
-			return ctx.Err()
+		if err := fn(fi); err != nil {
+			return err
 		}
-		cur = c
 	}
-
-	return ctx.Err()
+	if err := iter.Err(); err != nil {
+		debug.Log("List: %v", err)
+		return err
+	}
+	return nil
 }
 
 // Remove keys for a specified backend type.

internal/cache/backend.go

@@ -106,12 +106,18 @@ func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error {
 		return nil
 	}
 
-	err := b.Backend.Load(ctx, h, 0, 0, func(rd io.Reader) error {
-		return b.Cache.Save(h, rd)
-	})
-	if err != nil {
-		// try to remove from the cache, ignore errors
-		_ = b.Cache.Remove(h)
+	// test again, maybe the file was cached in the meantime
+	if !b.Cache.Has(h) {
+
+		// nope, it's still not in the cache, pull it from the repo and save it
+
+		err := b.Backend.Load(ctx, h, 0, 0, func(rd io.Reader) error {
+			return b.Cache.Save(h, rd)
+		})
+		if err != nil {
+			// try to remove from the cache, ignore errors
+			_ = b.Cache.Remove(h)
+		}
 	}
 
 	// signal other waiting goroutines that the file may now be cached
@@ -122,7 +128,7 @@ func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error {
 	delete(b.inProgress, h)
 	b.inProgressMutex.Unlock()
 
-	return err
+	return nil
 }
 
 // loadFromCacheOrDelegate will try to load the file from the cache, and fall

internal/cache/cache.go

@@ -19,6 +19,7 @@ import (
 type Cache struct {
 	Path             string
 	Base             string
+	Created          bool
 	PerformReadahead func(restic.Handle) bool
 }
 
@@ -98,7 +99,7 @@ func New(id string, basedir string) (c *Cache, err error) {
 		}
 	}
 
-	err = mkdirCacheDir(basedir)
+	created, err := mkdirCacheDir(basedir)
 	if err != nil {
 		return nil, err
 	}
@@ -121,8 +122,13 @@ func New(id string, basedir string) (c *Cache, err error) {
 	}
 
 	// create the repo cache dir if it does not exist yet
-	if err = fs.MkdirAll(cachedir, dirMode); err != nil {
-		return nil, err
+	_, err = fs.Lstat(cachedir)
+	if os.IsNotExist(err) {
+		err = fs.MkdirAll(cachedir, dirMode)
+		if err != nil {
+			return nil, err
+		}
+		created = true
 	}
 
 	// update the timestamp so that we can detect old cache dirs
@@ -145,8 +151,9 @@ func New(id string, basedir string) (c *Cache, err error) {
 	}
 
 	c = &Cache{
-		Path: cachedir,
-		Base: basedir,
+		Path:    cachedir,
+		Base:    basedir,
+		Created: created,
 		PerformReadahead: func(restic.Handle) bool {
 			// do not perform readahead by default
 			return false

internal/cache/dir.go

@@ -68,25 +68,31 @@ func DefaultDir() (cachedir string, err error) {
 	return cachedir, nil
 }
 
-func mkdirCacheDir(cachedir string) error {
+// mkdirCacheDir ensures that the cache directory exists. It it didn't, created
+// is set to true.
+func mkdirCacheDir(cachedir string) (created bool, err error) {
+	var newCacheDir bool
+
 	fi, err := fs.Stat(cachedir)
 	if os.IsNotExist(errors.Cause(err)) {
 		err = fs.MkdirAll(cachedir, 0700)
 		if err != nil {
-			return errors.Wrap(err, "MkdirAll")
+			return true, errors.Wrap(err, "MkdirAll")
 		}
 
 		fi, err = fs.Stat(cachedir)
 		debug.Log("create cache dir %v", cachedir)
+
+		newCacheDir = true
 	}
 
 	if err != nil {
-		return errors.Wrap(err, "Stat")
+		return newCacheDir, errors.Wrap(err, "Stat")
 	}
 
 	if !fi.IsDir() {
-		return errors.Errorf("cache dir %v is not a directory", cachedir)
+		return newCacheDir, errors.Errorf("cache dir %v is not a directory", cachedir)
 	}
 
-	return nil
+	return newCacheDir, nil
 }

internal/cache/file.go

@@ -63,6 +63,12 @@ func (c *Cache) Load(h restic.Handle, length int, offset int64) (io.ReadCloser,
 		return nil, errors.Errorf("cached file %v is truncated, removing", h)
 	}
 
+	if fi.Size() < offset+int64(length) {
+		_ = f.Close()
+		_ = c.Remove(h)
+		return nil, errors.Errorf("cached file %v is too small, removing", h)
+	}
+
 	if offset > 0 {
 		if _, err = f.Seek(offset, io.SeekStart); err != nil {
 			_ = f.Close()

internal/cache/file_test.go

@@ -218,7 +218,7 @@ func TestFileLoad(t *testing.T) {
 		{32*1024 + 5, 0},
 		{0, 123},
 		{0, 64*1024 + 234},
-		{100, 5234142},
+		{100, 5234142 - 100},
 	}
 
 	for _, test := range tests {

internal/debug/hooks.go

@@ -1,4 +1,4 @@
-// +build !release
+// +build debug
 
 package debug
 

internal/debug/hooks_release.go

@@ -1,4 +1,4 @@
-// +build release
+// +build !debug
 
 package debug