Add VERSION file for 0.5.0

Print a sensible error for Go < 1.7 (the "context" package is missing
there)

.travis.yml

@@ -2,8 +2,8 @@ language: go
 sudo: false
 
 go:
-  - 1.6.4
-  - 1.7.4
+  - 1.7.5
+  - 1.8
   - tip
 
 os:
@@ -17,14 +17,14 @@ env:
 matrix:
   exclude:
     - os: osx
-      go: 1.6.4
+      go: 1.7.5
     - os: osx
       go: tip
     - os: linux
-      go: 1.7.4
+      go: 1.8
   include:
     - os: linux
-      go: 1.7.4
+      go: 1.8
       sudo: true
       env:
         RESTIC_TEST_FUSE=1
@@ -48,7 +48,6 @@ install:
   - export GOBIN="$GOPATH/bin"
   - export PATH="$PATH:$GOBIN"
   - go env
-  - ulimit -n 2048
 
 script:
   - go run run_integration_tests.go

CONTRIBUTING.md

@@ -77,7 +77,7 @@ Just clone the repository, `cd` to it and run `gb build` to build the binary:
     [...]
     $ bin/restic version
     restic compiled manually
-    compiled at unknown time with go1.6
+    compiled at unknown time with go1.7
 
 The following commands can be used to run all the tests:
 

Dockerfile

@@ -14,11 +14,11 @@
 #   docker run --rm -v $PWD:/home/travis/restic restic/test gb test -v ./backend
 #
 # build the image for an older version of Go:
-#   docker build --build-arg GOVERSION=1.3.3 -t restic/test:go1.3.3 .
+#   docker build --build-arg GOVERSION=1.6.4 -t restic/test:go1.6.4 .
 
 FROM ubuntu:14.04
 
-ARG GOVERSION=1.7
+ARG GOVERSION=1.7.5
 ARG GOARCH=amd64
 
 # install dependencies

README.md

@@ -19,10 +19,22 @@ The latest documentation can be viewed online at
 a menu that allows switching to the documentation and user manual for the
 latest released version.
 
+News
+====
+
+You can follow the restic project on Twitter
+[@resticbackup](https://twitter.com/resticbackup) or by subscribing to the
+[development blog](https://restic.github.io/blog/).
+
+Install restic
+==============
+
+You can download the latest pre-compiled binary from the [restic release page](https://github.com/restic/restic/releases/latest).
+
 Build restic
 ============
 
-Install Go/Golang (at least version 1.6), then run `go run build.go`,
+Install Go/Golang (at least version 1.7), then run `go run build.go`,
 afterwards you'll find the binary in the current directory:
 
     $ go run build.go
@@ -32,6 +44,16 @@ afterwards you'll find the binary in the current directory:
       restic [OPTIONS] <command>
     [...]
 
+You can easily cross-compile restic for all supported platforms, just supply
+the target OS and platform via the command-line options like this (for Windows
+and FreeBSD respectively):
+
+    $ go run build.go --goos windows --goarch amd64
+
+    $ go run build.go --goos freebsd --goarch 386
+
+The resulting binary is statically linked and does not require any libraries.
+
 More documentation can be found in the [user manual](doc/Manual.md).
 
 At the moment, the only tested compiler for restic is the official Go compiler.

VERSION

@@ -1 +1 @@
-0.4.0
+0.5.0

Vagrantfile

@@ -1,7 +1,7 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 
-GO_VERSION = '1.6'
+GO_VERSION = '1.7'
 
 def packages_freebsd
   return <<-EOF

appveyor.yml

@@ -17,8 +17,8 @@ init:
 
 install:
   - rmdir c:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.7.4.windows-amd64.msi
-  - msiexec /i go1.7.4.windows-amd64.msi /q
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.8.windows-amd64.msi
+  - msiexec /i go1.8.windows-amd64.msi /q
   - go version
   - go env
   - appveyor DownloadFile http://sourceforge.netcologne.de/project/gnuwin32/tar/1.13-1/tar-1.13-1-bin.zip -FileName tar.zip

build.go

@@ -291,6 +291,12 @@ func (cs Constants) LDFlags() string {
 }
 
 func main() {
+	ver := runtime.Version()
+	if strings.HasPrefix(ver, "go1") && ver < "go1.7" {
+		fmt.Fprintf(os.Stderr, "Go version %s detected, restic requires at least Go 1.7\n", ver)
+		os.Exit(1)
+	}
+
 	buildTags := []string{}
 
 	skipNext := false

build_release_binaries.sh

@@ -36,7 +36,7 @@ for R in       \
 
     echo $filename
 
-    go run ../build.go --goos $OS --goarch $ARCH --output ${filename}
+    go run build.go --goos $OS --goarch $ARCH --output ${filename}
     if [[ "$OS" == "windows" ]]; then
         zip ${filename%.exe}.zip ${filename}
         rm ${filename}
@@ -53,7 +53,7 @@ mv restic-$VERSION.tar.gz ${dir}
 
 echo "creating checksums"
 pushd ${dir}
-sha256sum restic_*.{zip,bz2} > SHA256SUMS
+sha256sum restic_*.{zip,bz2} restic-$VERSION.tar.gz > SHA256SUMS
 gpg --armor --detach-sign SHA256SUMS
 popd
 

doc/Design.md

@@ -285,7 +285,7 @@ This way, the password can be changed without having to re-encrypt all data.
 Snapshots
 ---------
 
-A snapshots represents a directory with all files and sub-directories at a
+A snapshot represents a directory with all files and sub-directories at a
 given point in time. For each backup that is made, a new snapshot is created. A
 snapshot is a JSON document that is stored in an encrypted file below the
 directory `snapshots` in the repository. The filename is the storage ID. This
@@ -294,6 +294,31 @@ string is unique and used within restic to uniquely identify a snapshot.
 The command `restic cat snapshot` can be used as follows to decrypt and
 pretty-print the contents of a snapshot file:
 
+```console
+$ restic -r /tmp/restic-repo cat snapshot 251c2e58
+enter password for repository:
+{
+  "time": "2015-01-02T18:10:50.895208559+01:00",
+  "tree": "2da81727b6585232894cfbb8f8bdab8d1eccd3d8f7c92bc934d62e62e618ffdf",
+  "dir": "/tmp/testdata",
+  "hostname": "kasimir",
+  "username": "fd0",
+  "uid": 1000,
+  "gid": 100,
+  "tags": [
+    "NL"
+  ]
+}
+```
+
+Here it can be seen that this snapshot represents the contents of the directory
+`/tmp/testdata`. The most important field is `tree`. When the meta data (e.g.
+the tags) of a snapshot change, the snapshot needs to be re-encrypted and saved.
+This will change the storage ID, so in order to relate these seemingly
+different snapshots, a field `original` is introduced which contains the ID of
+the original snapshot, e.g. after adding the tag `DE` to the snapshot above it
+becomes:
+
 ```console
 $ restic -r /tmp/restic-repo cat snapshot 22a5af1b
 enter password for repository:
@@ -304,12 +329,17 @@ enter password for repository:
   "hostname": "kasimir",
   "username": "fd0",
   "uid": 1000,
-  "gid": 100
+  "gid": 100,
+  "tags": [
+    "NL",
+    "DE"
+  ],
+  "original": "251c2e5841355f743f9d4ffd3260bee765acee40a6229857e32b60446991b837"
 }
 ```
 
-Here it can be seen that this snapshot represents the contents of the directory
-`/tmp/testdata`. The most important field is `tree`.
+Once introduced, the `original` field is not modified when the snapshot's meta
+data is changed again.
 
 All content within a restic repository is referenced according to its SHA-256
 hash. Before saving, each file is split into variable sized Blobs of data. The

doc/Manual.md

@@ -1,7 +1,13 @@
-Thanks for using restic. This document will give you an overview of the basic
+nhanks for using restic. This document will give you an overview of the basic
 functionality provided by restic.
 
-# Building/installing restic
+# Installing restic
+
+## from pre-compiled binary
+
+You can download the latest pre-compiled binary from the [restic release page](https://github.com/restic/restic/releases/latest).
+
+## Mac OS X
 
 If you are using Mac OS X, you can install restic using the
 [homebrew](http://brew.sh/) packet manager:
@@ -11,25 +17,19 @@ $ brew tap restic/restic
 $ brew install restic
 ```
 
+## archlinux
+
 On archlinux, there is a package called `restic-git` which can be installed from AUR, e.g. with `pacaur`:
 
 ```console
 $ pacaur -S restic-git
 ```
 
-At debian stable you can install 'go' directly from the repositories (as root):
+# Building restic
 
-```console
-$ apt-get install golang-go
-```
-
-after installation of 'go' go straight forward to 'git clone [...]'
-
-If you are using Linux, BSD or Windows, the only way to install restic on your
-system right now is to compile it from source. restic is written in the Go
-programming language and you need at least Go version 1.6. Building restic may
-also work with older versions of Go, but that's not supported. See the [Getting
-started](https://golang.org/doc/install) guide of the Go project for
+restic is written in the Go programming language and you need at least Go version 1.7.
+Building restic may also work with older versions of Go, but that's not supported.
+See the [Getting started](https://golang.org/doc/install) guide of the Go project for
 instructions how to install Go.
 
 In order to build restic from source, execute the following steps:
@@ -43,9 +43,21 @@ $ cd restic
 $ go run build.go
 ```
 
+You can easily cross-compile restic for all supported platforms, just supply
+the target OS and platform via the command-line options like this (for Windows
+and FreeBSD respectively):
+
+    $ go run build.go --goos windows --goarch amd64
+
+    $ go run build.go --goos freebsd --goarch 386
+
+The resulting binary is statically linked and does not require any libraries.
+
 At the moment, the only tested compiler for restic is the official Go compiler.
 Building restic with gccgo may work, but is not supported.
 
+# Usage help
+
 Usage help is available:
 
 ```console
@@ -71,10 +83,12 @@ Available Commands:
   rebuild-index build a new index file
   restore       extract the data from a snapshot
   snapshots     list all snapshots
+  tag           modifies tags on snapshots
   unlock        remove locks other processes created
   version       Print version information
 
 Flags:
+      --json                   set output mode to JSON for commands that support it
       --no-lock                do not lock the repo, this allows some operations on read-only repos
   -p, --password-file string   read the repository password from a file
   -q, --quiet                  do not output comprehensive progress report
@@ -108,6 +122,7 @@ Flags:
       --tag tag                 add a tag for the new snapshot (can be specified multiple times)
 
 Global Flags:
+      --json                   set output mode to JSON for commands that support it
       --no-lock                do not lock the repo, this allows some operations on read-only repos
   -p, --password-file string   read the repository password from a file
   -q, --quiet                  do not output comprehensive progress report
@@ -139,6 +154,8 @@ Please note that knowledge of your password is required to access the repository
 Losing your password means that your data is irrecoverably lost.
 ```
 
+Other backends like sftp and s3 are [described in a later section](#create-an-sftp-repository) of this document.
+
 Remembering your password is important! If you lose it, you won't be able to
 access data stored in the repository.
 
@@ -212,6 +229,15 @@ snapshot 31f7bd63 saved
 In fact several hosts may use the same repository to backup directories and
 files leading to a greater de-duplication.
 
+Please be aware that when you backup different directories (or the directories
+to be saved have a variable name component like a time/date), restic always
+needs to read all files and only afterwards can compute which parts of the
+files need to be saved. When you backup the same directory again (maybe with
+new or changed files) restic will find the old snapshot in the repo and by
+default only reads those files that are new or have been modified since the
+last snapshot. This is decided based on the modify date of the file in the
+file system.
+
 You can exclude folders and files by specifying exclude-patterns.
 Either specify them with multiple `--exclude`'s or one `--exclude-file`
 
@@ -226,7 +252,7 @@ $ restic -r /tmp/backup backup ~/work --exclude=*.c --exclude-file=exclude
 
 Patterns use [`filepath.Glob`](https://golang.org/pkg/path/filepath/#Glob) internally,
 see [`filepath.Match`](https://golang.org/pkg/path/filepath/#Match) for syntax.
-Additionally `**` exludes arbitrary subdirectories.
+Additionally `**` excludes arbitrary subdirectories.
 Environment-variables in exclude-files are expanded with [`os.ExpandEnv`](https://golang.org/pkg/os/#ExpandEnv).
 
 By specifying the option `--one-file-system` you can instruct restic to only
@@ -352,7 +378,6 @@ enter password for repository:
 restoring <Snapshot of [/home/art] at 2015-05-08 21:45:17.884408621 +0200 CEST> to /tmp/restore-work
 ```
 
-
 # Manage repository keys
 
 The `key` command allows you to set multiple access keys or passwords per
@@ -380,6 +405,45 @@ enter password for repository:
 *eb78040b    username    kasimir   2015-08-12 13:29:57
 ```
 
+# Manage tags
+
+Managing tags on snapshots is done with the `tag` command. The existing set of
+tags can be replaced completely, tags can be added to removed. The result is
+directly visible in the `snapshots` command.
+
+Let's say we want to tag snapshot `590c8fc8` with the tags `NL` and `CH` and
+remove all other tags that may be present, the following command does that:
+
+```console
+$ restic -r /tmp/backup tag --set NL,CH 590c8fc8
+Create exclusive lock for repository
+Modified tags on 1 snapshots
+```
+
+Note the snapshot ID has changed, so between each change we need to look up
+the new ID of the snapshot. But there is an even better way, the `tag` command
+accepts `--tag` for a filter, so we can filter snapshots based on the tag we
+just added.
+
+So we can add and remove tags incrementally like this:
+
+```console
+$ restic -r /tmp/backup tag --tag NL --remove CH
+Create exclusive lock for repository
+Modified tags on 1 snapshots
+
+$ restic -r /tmp/backup tag --tag NL --add UK
+Create exclusive lock for repository
+Modified tags on 1 snapshots
+
+$ restic -r /tmp/backup tag --tag NL --remove NL
+Create exclusive lock for repository
+Modified tags on 1 snapshots
+
+$ restic -r /tmp/backup tag --tag NL --add SOMETHING
+No snapshots were modified
+```
+
 # Check integrity and consistency
 
 Imagine your repository is saved on a server that has a faulty hard drive, or
@@ -425,6 +489,11 @@ Don't forget to umount after quitting!
 
 Mounting repositories via FUSE is not possible on Windows and OpenBSD.
 
+Restic supports storage and preservation of hard links. However, since hard links
+exist in the scope of a filesystem by definition, restoring hard links from a fuse
+mount should be done by a program that preserves hard links. A program that does so
+is rsync, used with the option --hard-links.
+
 # Create an SFTP repository
 
 In order to backup data via SFTP, you must first set up a server with SSH and
@@ -509,8 +578,8 @@ only available via HTTP, you can specify the URL to the server like this:
 
 ### Pre-Requisites
 
-* Download and Install [Minio Server](https://minio.io/download/).
-* You can also refer to [https://docs.minio.io](https://docs.minio.io) for step by step guidance on installation and getting started on Minio CLient and Minio Server.
+* Download and Install [Minio Server](https://minio.io/downloads/#minio-server).
+* You can also refer to [https://docs.minio.io](https://docs.minio.io) for step by step guidance on installation and getting started on Minio Client and Minio Server.
 
 You must first setup the following environment variables with the credentials of your running Minio Server.
 
@@ -537,7 +606,8 @@ be done either manually (by specifying a snapshot ID to remove) or by using a
 policy that describes which snapshots to forget. For all remove operations, two
 commands need to be called in sequence: `forget` to remove a snapshot and
 `prune` to actually remove the data that was referenced by the snapshot from
-the repository.
+the repository. This can be automated with the `--prune` option of the `forget`
+command, which runs `prune` automatically if snapshots have been removed.
 
 ## Remove a single snapshot
 
@@ -602,6 +672,41 @@ done
 
 Afterwards the repository is smaller.
 
+You can automate this two-step process by using the `--prune` switch to
+`forget`:
+
+```console
+$ restic forget --keep-last 1 --prune
+snapshots for host mopped, directories /home/user/work:
+
+keep 1 snapshots:
+ID        Date                 Host        Tags        Directory
+----------------------------------------------------------------------
+4bba301e  2017-02-21 10:49:18  mopped                  /home/user/work
+
+remove 1 snapshots:
+ID        Date                 Host        Tags        Directory
+----------------------------------------------------------------------
+8c02b94b  2017-02-21 10:48:33  mopped                  /home/user/work
+
+1 snapshots have been removed, running prune
+counting files in repo
+building new index for repo
+[0:00] 100.00%  37 / 37 packs
+repository contains 37 packs (5521 blobs) with 151.012 MiB bytes
+processed 5521 blobs: 0 duplicate blobs, 0B duplicate
+load all snapshots
+find data that is still in use for 1 snapshots
+[0:00] 100.00%  1 / 1 snapshots
+found 5323 of 5521 data blobs still in use, removing 198 blobs
+will delete 0 packs and rewrite 27 packs, this frees 22.106 MiB
+creating new index
+[0:00] 100.00%  30 / 30 packs
+saved new index as b49f3e68
+done
+```
+
+
 ## Removing snapshots according to a policy
 
 Removing snapshots manually is tedious and error-prone, therefore restic allows
@@ -733,3 +838,10 @@ enter password for repository:
   "gid": 20
 }
 ```
+
+# Scripting restic
+
+Restic supports the output of some commands in JSON format. The JSON flag ```--json``` is currently supported only by ```restic snapshots```.
+
+```console
+$ restic -r /tmp/backup snapshots --json```

src/cmds/restic/cmd_backup.go

@@ -3,14 +3,13 @@ package main
 import (
 	"bufio"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"restic"
 	"strings"
 	"time"
 
-	"golang.org/x/crypto/ssh/terminal"
-
 	"github.com/spf13/cobra"
 
 	"restic/archiver"
@@ -28,6 +27,10 @@ The "backup" command creates a new snapshot and saves the files and directories
 given as the arguments.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
+		if backupOptions.Stdin && backupOptions.FilesFrom == "-" {
+			return errors.Fatal("cannot use both `--stdin` and `--files-from -`")
+		}
+
 		if backupOptions.Stdin {
 			return readBackupFromStdin(backupOptions, globalOptions, args)
 		}
@@ -46,6 +49,7 @@ type BackupOptions struct {
 	Stdin          bool
 	StdinFilename  string
 	Tags           []string
+	Hostname       string
 	FilesFrom      string
 }
 
@@ -54,15 +58,22 @@ var backupOptions BackupOptions
 func init() {
 	cmdRoot.AddCommand(cmdBackup)
 
+	hostname, err := os.Hostname()
+	if err != nil {
+		debug.Log("os.Hostname() returned err: %v", err)
+		hostname = ""
+	}
+
 	f := cmdBackup.Flags()
 	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent snapshot (default: last snapshot in the repo that has the same target files/directories)")
-	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories. Overrides the "parent" flag`)
-	f.StringSliceVarP(&backupOptions.Excludes, "exclude", "e", []string{}, "exclude a `pattern` (can be specified multiple times)")
+	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
+	f.StringSliceVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
 	f.StringVar(&backupOptions.ExcludeFile, "exclude-file", "", "read exclude patterns from a file")
-	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "Exclude other file systems")
+	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems")
 	f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
-	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "", "file name to use when reading from stdin")
-	f.StringSliceVar(&backupOptions.Tags, "tag", []string{}, "add a `tag` for the new snapshot (can be specified multiple times)")
+	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "file name to use when reading from stdin")
+	f.StringSliceVar(&backupOptions.Tags, "tag", nil, "add a `tag` for the new snapshot (can be specified multiple times)")
+	f.StringVar(&backupOptions.Hostname, "hostname", hostname, "set the `hostname` for the snapshot manually")
 	f.StringVar(&backupOptions.FilesFrom, "files-from", "", "read the files to backup from file (can be combined with file args)")
 }
 
@@ -123,8 +134,7 @@ func newArchiveProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress
 			s.Errors)
 		status2 := fmt.Sprintf("ETA %s ", formatSeconds(eta))
 
-		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
-		if err == nil {
+		if w := stdoutTerminalWidth(); w > 0 {
 			maxlen := w - len(status2) - 1
 
 			if maxlen < 4 {
@@ -168,8 +178,7 @@ func newArchiveStdinProgress(gopts GlobalOptions) *restic.Progress {
 			formatBytes(s.Bytes),
 			formatBytes(bps))
 
-		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
-		if err == nil {
+		if w := stdoutTerminalWidth(); w > 0 {
 			maxlen := w - len(status1)
 
 			if maxlen < 4 {
@@ -232,7 +241,15 @@ func gatherDevices(items []string) (deviceMap map[uint64]struct{}, err error) {
 
 func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string) error {
 	if len(args) != 0 {
-		return errors.Fatalf("when reading from stdin, no additional files can be specified")
+		return errors.Fatal("when reading from stdin, no additional files can be specified")
+	}
+
+	if opts.StdinFilename == "" {
+		return errors.Fatal("filename for backup from stdin must not be empty")
+	}
+
+	if gopts.password == "" && gopts.PasswordFile == "" {
+		return errors.Fatal("unable to read password from stdin when data is to be read from stdin, use --password-file or $RESTIC_PASSWORD")
 	}
 
 	repo, err := OpenRepository(gopts)
@@ -251,7 +268,13 @@ func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string)
 		return err
 	}
 
-	_, id, err := archiver.ArchiveReader(repo, newArchiveStdinProgress(gopts), os.Stdin, opts.StdinFilename, opts.Tags)
+	r := &archiver.Reader{
+		Repository: repo,
+		Tags:       opts.Tags,
+		Hostname:   opts.Hostname,
+	}
+
+	_, id, err := r.Archive(opts.StdinFilename, os.Stdin, newArchiveStdinProgress(gopts))
 	if err != nil {
 		return err
 	}
@@ -262,23 +285,32 @@ func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string)
 
 // readFromFile will read all lines from the given filename and write them to a
 // string array, if filename is empty readFromFile returns and empty string
-// array
+// array. If filename is a dash (-), readFromFile will read the lines from
+// the standard input.
 func readLinesFromFile(filename string) ([]string, error) {
 	if filename == "" {
 		return nil, nil
 	}
 
-	file, err := os.Open(filename)
-	if err != nil {
-		return nil, err
+	var r io.Reader = os.Stdin
+	if filename != "-" {
+		f, err := os.Open(filename)
+		if err != nil {
+			return nil, err
+		}
+		defer f.Close()
+		r = f
 	}
-	defer file.Close()
 
 	var lines []string
 
-	scanner := bufio.NewScanner(file)
+	scanner := bufio.NewScanner(r)
 	for scanner.Scan() {
-		lines = append(lines, scanner.Text())
+		line := scanner.Text()
+		if line == "" {
+			continue
+		}
+		lines = append(lines, line)
 	}
 
 	if err := scanner.Err(); err != nil {
@@ -289,6 +321,10 @@ func readLinesFromFile(filename string) ([]string, error) {
 }
 
 func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
+	if opts.FilesFrom == "-" && gopts.password == "" && gopts.PasswordFile == "" {
+		return errors.Fatal("no password; either use `--password-file` option or put the password into the RESTIC_PASSWORD environment variable")
+	}
+
 	fromfile, err := readLinesFromFile(opts.FilesFrom)
 	if err != nil {
 		return err
@@ -299,7 +335,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 	// same time
 	args = append(args, fromfile...)
 	if len(args) == 0 {
-		return errors.Fatalf("wrong number of parameters")
+		return errors.Fatal("wrong number of parameters")
 	}
 
 	target := make([]string, 0, len(args))
@@ -355,13 +391,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 
 	// Find last snapshot to set it as parent, if not already set
 	if !opts.Force && parentSnapshotID == nil {
-		hostname, err := os.Hostname()
-		if err != nil {
-			debug.Log("os.Hostname() returned err: %v", err)
-			hostname = ""
-		}
-
-		id, err := restic.FindLatestSnapshot(repo, target, hostname)
+		id, err := restic.FindLatestSnapshot(repo, target, opts.Tags, opts.Hostname)
 		if err == nil {
 			parentSnapshotID = &id
 		} else if err != restic.ErrNoSnapshotFound {
@@ -437,7 +467,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 		Warnf("%s\rwarning for %s: %v\n", ClearLine(), dir, err)
 	}
 
-	_, id, err := arch.Snapshot(newArchiveProgress(gopts, stat), target, opts.Tags, parentSnapshotID)
+	_, id, err := arch.Snapshot(newArchiveProgress(gopts, stat), target, opts.Tags, opts.Hostname, parentSnapshotID)
 	if err != nil {
 		return err
 	}

src/cmds/restic/cmd_cat.go

@@ -30,7 +30,7 @@ func init() {
 
 func runCat(gopts GlobalOptions, args []string) error {
 	if len(args) < 1 || (args[0] != "masterkey" && args[0] != "config" && len(args) != 2) {
-		return errors.Fatalf("type or ID not specified")
+		return errors.Fatal("type or ID not specified")
 	}
 
 	repo, err := OpenRepository(gopts)

src/cmds/restic/cmd_check.go

@@ -7,8 +7,6 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"golang.org/x/crypto/ssh/terminal"
-
 	"restic"
 	"restic/checker"
 	"restic/errors"
@@ -26,7 +24,7 @@ finds. It can also be used to read all data and therefore simulate a restore.
 	},
 }
 
-// CheckOptions bundle all options for the 'check' command.
+// CheckOptions bundles all options for the 'check' command.
 type CheckOptions struct {
 	ReadData    bool
 	CheckUnused bool
@@ -38,8 +36,8 @@ func init() {
 	cmdRoot.AddCommand(cmdCheck)
 
 	f := cmdCheck.Flags()
-	f.BoolVar(&checkOptions.ReadData, "read-data", false, "Read all data blobs")
-	f.BoolVar(&checkOptions.CheckUnused, "check-unused", false, "Find unused blobs")
+	f.BoolVar(&checkOptions.ReadData, "read-data", false, "read all data blobs")
+	f.BoolVar(&checkOptions.CheckUnused, "check-unused", false, "find unused blobs")
 }
 
 func newReadProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress {
@@ -55,8 +53,7 @@ func newReadProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress {
 			formatPercent(s.Blobs, todo.Blobs),
 			s.Blobs, todo.Blobs)
 
-		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
-		if err == nil {
+		if w := stdoutTerminalWidth(); w > 0 {
 			if len(status) > w {
 				max := w - len(status) - 4
 				status = status[:max] + "... "

src/cmds/restic/cmd_dump.go

@@ -22,7 +22,7 @@ var cmdDump = &cobra.Command{
 	Use:   "dump [indexes|snapshots|trees|all|packs]",
 	Short: "dump data structures",
 	Long: `
-The "dump" command dumps data structures from a repository as JSON objects. It
+The "dump" command dumps data structures from the repository as JSON objects. It
 is used for debugging purposes only.`,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runDump(globalOptions, args)
@@ -168,7 +168,7 @@ func dumpIndexes(repo restic.Repository) error {
 
 func runDump(gopts GlobalOptions, args []string) error {
 	if len(args) != 1 {
-		return errors.Fatalf("type not specified")
+		return errors.Fatal("type not specified")
 	}
 
 	repo, err := OpenRepository(gopts)

src/cmds/restic/cmd_find.go

@@ -1,7 +1,9 @@
 package main
 
 import (
+	"context"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -23,11 +25,16 @@ repo. `,
 	},
 }
 
-// FindOptions bundle all options for the find command.
+// FindOptions bundles all options for the find command.
 type FindOptions struct {
-	Oldest   string
-	Newest   string
-	Snapshot string
+	Oldest          string
+	Newest          string
+	Snapshots       []string
+	CaseInsensitive bool
+	ListLong        bool
+	Host            string
+	Paths           []string
+	Tags            []string
 }
 
 var findOptions FindOptions
@@ -36,19 +43,21 @@ func init() {
 	cmdRoot.AddCommand(cmdFind)
 
 	f := cmdFind.Flags()
-	f.StringVarP(&findOptions.Oldest, "oldest", "o", "", "Oldest modification date/time")
-	f.StringVarP(&findOptions.Newest, "newest", "n", "", "Newest modification date/time")
-	f.StringVarP(&findOptions.Snapshot, "snapshot", "s", "", "Snapshot ID to search in")
+	f.StringVarP(&findOptions.Oldest, "oldest", "o", "", "oldest modification date/time")
+	f.StringVarP(&findOptions.Newest, "newest", "n", "", "newest modification date/time")
+	f.StringSliceVarP(&findOptions.Snapshots, "snapshot", "s", nil, "snapshot `id` to search in (can be given multiple times)")
+	f.BoolVarP(&findOptions.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern")
+	f.BoolVarP(&findOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
+
+	f.StringVarP(&findOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
+	f.StringSliceVar(&findOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot-ID is given")
+	f.StringSliceVar(&findOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
 }
 
 type findPattern struct {
 	oldest, newest time.Time
 	pattern        string
-}
-
-type findResult struct {
-	node *restic.Node
-	path string
+	ignoreCase     bool
 }
 
 var timeFormats = []string{
@@ -75,20 +84,25 @@ func parseTime(str string) (time.Time, error) {
 	return time.Time{}, errors.Fatalf("unable to parse time: %q", str)
 }
 
-func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, path string) ([]findResult, error) {
+func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, prefix string, snapshotID *string) error {
 	debug.Log("checking tree %v\n", id)
+
 	tree, err := repo.LoadTree(id)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
-	results := []findResult{}
 	for _, node := range tree.Nodes {
 		debug.Log("  testing entry %q\n", node.Name)
 
-		m, err := filepath.Match(pat.pattern, node.Name)
+		name := node.Name
+		if pat.ignoreCase {
+			name = strings.ToLower(name)
+		}
+
+		m, err := filepath.Match(pat.pattern, name)
 		if err != nil {
-			return nil, err
+			return err
 		}
 
 		if m {
@@ -103,69 +117,55 @@ func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, path
 				continue
 			}
 
-			results = append(results, findResult{node: node, path: path})
+			if snapshotID != nil {
+				Verbosef("Found matching entries in snapshot %s\n", *snapshotID)
+				snapshotID = nil
+			}
+			Printf(formatNode(prefix, node, findOptions.ListLong) + "\n")
 		} else {
 			debug.Log("    pattern does not match\n")
 		}
 
 		if node.Type == "dir" {
-			subdirResults, err := findInTree(repo, pat, *node.Subtree, filepath.Join(path, node.Name))
-			if err != nil {
-				return nil, err
+			if err := findInTree(repo, pat, *node.Subtree, filepath.Join(prefix, node.Name), snapshotID); err != nil {
+				return err
 			}
-
-			results = append(results, subdirResults...)
 		}
 	}
 
-	return results, nil
-}
-
-func findInSnapshot(repo *repository.Repository, pat findPattern, id restic.ID) error {
-	debug.Log("searching in snapshot %s\n  for entries within [%s %s]", id.Str(), pat.oldest, pat.newest)
-
-	sn, err := restic.LoadSnapshot(repo, id)
-	if err != nil {
-		return err
-	}
-
-	results, err := findInTree(repo, pat, *sn.Tree, "")
-	if err != nil {
-		return err
-	}
-
-	if len(results) == 0 {
-		return nil
-	}
-	Verbosef("found %d matching entries in snapshot %s\n", len(results), id)
-	for _, res := range results {
-		res.node.Name = filepath.Join(res.path, res.node.Name)
-		Printf("  %s\n", res.node)
-	}
-
+	return nil
+}
+
+func findInSnapshot(repo *repository.Repository, sn *restic.Snapshot, pat findPattern) error {
+	debug.Log("searching in snapshot %s\n  for entries within [%s %s]", sn.ID(), pat.oldest, pat.newest)
+
+	snapshotID := sn.ID().Str()
+	if err := findInTree(repo, pat, *sn.Tree, string(filepath.Separator), &snapshotID); err != nil {
+		return err
+	}
 	return nil
 }
 
 func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 	if len(args) != 1 {
-		return errors.Fatalf("wrong number of arguments")
+		return errors.Fatal("wrong number of arguments")
 	}
 
-	var (
-		err error
-		pat findPattern
-	)
+	var err error
+	pat := findPattern{pattern: args[0]}
+	if opts.CaseInsensitive {
+		pat.pattern = strings.ToLower(pat.pattern)
+		pat.ignoreCase = true
+	}
 
 	if opts.Oldest != "" {
-		pat.oldest, err = parseTime(opts.Oldest)
-		if err != nil {
+		if pat.oldest, err = parseTime(opts.Oldest); err != nil {
 			return err
 		}
 	}
 
 	if opts.Newest != "" {
-		pat.newest, err = parseTime(opts.Newest)
-		if err != nil {
+		if pat.newest, err = parseTime(opts.Newest); err != nil {
 			return err
 		}
 	}
@@ -183,28 +183,14 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	err = repo.LoadIndex()
-	if err != nil {
+	if err = repo.LoadIndex(); err != nil {
 		return err
 	}
 
-	pat.pattern = args[0]
-
-	if opts.Snapshot != "" {
-		snapshotID, err := restic.FindSnapshot(repo, opts.Snapshot)
-		if err != nil {
-			return errors.Fatalf("invalid id %q: %v", args[1], err)
-		}
-
-		return findInSnapshot(repo, pat, snapshotID)
-	}
-
-	done := make(chan struct{})
-	defer close(done)
-	for snapshotID := range repo.List(restic.SnapshotFile, done) {
-		err := findInSnapshot(repo, pat, snapshotID)
-
-		if err != nil {
+	ctx, cancel := context.WithCancel(gopts.ctx)
+	defer cancel()
+	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, opts.Snapshots) {
+		if err = findInSnapshot(repo, sn, pat); err != nil {
 			return err
 		}
 	}

src/cmds/restic/cmd_forget.go

@@ -1,10 +1,10 @@
 package main
 
 import (
-	"encoding/hex"
-	"fmt"
-	"io"
+	"context"
+	"encoding/json"
 	"restic"
+	"sort"
 	"strings"
 
 	"github.com/spf13/cobra"
@@ -25,19 +25,21 @@ data after 'forget' was run successfully, see the 'prune' command. `,
 
 // ForgetOptions collects all options for the forget command.
 type ForgetOptions struct {
-	Last    int
-	Hourly  int
-	Daily   int
-	Weekly  int
-	Monthly int
-	Yearly  int
-
+	Last     int
+	Hourly   int
+	Daily    int
+	Weekly   int
+	Monthly  int
+	Yearly   int
 	KeepTags []string
 
-	Hostname string
-	Tags     []string
+	Host  string
+	Tags  []string
+	Paths []string
 
-	DryRun bool
+	GroupByTags bool
+	DryRun      bool
+	Prune       bool
 }
 
 var forgetOptions ForgetOptions
@@ -53,51 +55,17 @@ func init() {
 	f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots")
 	f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots")
 
-	f.StringSliceVar(&forgetOptions.KeepTags, "keep-tag", []string{}, "always keep snapshots with this `tag` (can be specified multiple times)")
-	f.StringVar(&forgetOptions.Hostname, "hostname", "", "only forget snapshots for the given hostname")
-	f.StringSliceVar(&forgetOptions.Tags, "tag", []string{}, "only forget snapshots with the `tag` (can be specified multiple times)")
+	f.StringSliceVar(&forgetOptions.KeepTags, "keep-tag", []string{}, "keep snapshots with this `tag` (can be specified multiple times)")
+	f.BoolVarP(&forgetOptions.GroupByTags, "group-by-tags", "G", false, "Group by host,paths,tags instead of just host,paths")
+	// Sadly the commonly used shortcut `H` is already used.
+	f.StringVar(&forgetOptions.Host, "host", "", "only consider snapshots with the given `host`")
+	// Deprecated since 2017-03-07.
+	f.StringVar(&forgetOptions.Host, "hostname", "", "only consider snapshots with the given `hostname` (deprecated)")
+	f.StringSliceVar(&forgetOptions.Tags, "tag", nil, "only consider snapshots which include this `tag` (can be specified multiple times)")
+	f.StringSliceVar(&forgetOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` (can be specified multiple times)")
 
 	f.BoolVarP(&forgetOptions.DryRun, "dry-run", "n", false, "do not delete anything, just print what would be done")
-}
-
-func printSnapshots(w io.Writer, snapshots restic.Snapshots) {
-	tab := NewTable()
-	tab.Header = fmt.Sprintf("%-8s  %-19s  %-10s  %-10s  %s", "ID", "Date", "Host", "Tags", "Directory")
-	tab.RowFormat = "%-8s  %-19s  %-10s  %-10s  %s"
-
-	for _, sn := range snapshots {
-		if len(sn.Paths) == 0 {
-			continue
-		}
-
-		firstTag := ""
-		if len(sn.Tags) > 0 {
-			firstTag = sn.Tags[0]
-		}
-
-		tab.Rows = append(tab.Rows, []interface{}{sn.ID().Str(), sn.Time.Format(TimeFormat), sn.Hostname, firstTag, sn.Paths[0]})
-
-		rows := len(sn.Paths)
-		if len(sn.Tags) > rows {
-			rows = len(sn.Tags)
-		}
-
-		for i := 1; i < rows; i++ {
-			path := ""
-			if len(sn.Paths) > i {
-				path = sn.Paths[i]
-			}
-
-			tag := ""
-			if len(sn.Tags) > i {
-				tag = sn.Tags[i]
-			}
-
-			tab.Rows = append(tab.Rows, []interface{}{"", "", "", tag, path})
-		}
-	}
-
-	tab.Write(w)
+	f.BoolVar(&forgetOptions.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed")
 }
 
 func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
@@ -112,38 +80,45 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	// parse arguments as hex strings
-	var ids []string
-	for _, s := range args {
-		_, err := hex.DecodeString(s)
-		if err != nil {
-			Warnf("argument %q is not a snapshot ID, ignoring\n", s)
-			continue
-		}
-
-		ids = append(ids, s)
+	// group by hostname and dirs
+	type key struct {
+		Hostname string
+		Paths    []string
+		Tags     []string
 	}
+	snapshotGroups := make(map[string]restic.Snapshots)
 
-	// process all snapshot IDs given as arguments
-	for _, s := range ids {
-		id, err := restic.FindSnapshot(repo, s)
-		if err != nil {
-			Warnf("cound not find a snapshot for ID %q, ignoring\n", s)
-			continue
-		}
-
-		if !opts.DryRun {
-			h := restic.Handle{Type: restic.SnapshotFile, Name: id.String()}
-			err = repo.Backend().Remove(h)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
+		if len(args) > 0 {
+			// When explicit snapshots args are given, remove them immediately.
+			if !opts.DryRun {
+				h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
+				if err = repo.Backend().Remove(h); err != nil {
+					return err
+				}
+				Verbosef("removed snapshot %v\n", sn.ID().Str())
+			} else {
+				Verbosef("would have removed snapshot %v\n", sn.ID().Str())
+			}
+		} else {
+			var tags []string
+			if opts.GroupByTags {
+				tags = sn.Tags
+				sort.StringSlice(tags).Sort()
+			}
+			sort.StringSlice(sn.Paths).Sort()
+			k, err := json.Marshal(key{Hostname: sn.Hostname, Tags: tags, Paths: sn.Paths})
 			if err != nil {
 				return err
 			}
-
-			Verbosef("removed snapshot %v\n", id.Str())
-		} else {
-			Verbosef("would remove snapshot %v\n", id.Str())
+			snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn)
 		}
 	}
+	if len(args) > 0 {
+		return nil
+	}
 
 	policy := restic.ExpirePolicy{
 		Last:    opts.Last,
@@ -156,49 +131,36 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	if policy.Empty() {
+		Verbosef("no policy was specified, no snapshots will be removed\n")
 		return nil
 	}
 
-	// then, load all remaining snapshots
-	snapshots, err := restic.LoadAllSnapshots(repo)
-	if err != nil {
-		return err
-	}
-
-	// group by hostname and dirs
-	type key struct {
-		Hostname string
-		Dirs     string
-	}
-
-	snapshotGroups := make(map[key]restic.Snapshots)
-
-	for _, sn := range snapshots {
-		if opts.Hostname != "" && sn.Hostname != opts.Hostname {
-			continue
+	removeSnapshots := 0
+	for k, snapshotGroup := range snapshotGroups {
+		var key key
+		if json.Unmarshal([]byte(k), &key) != nil {
+			return err
 		}
-
-		if !sn.HasTags(opts.Tags) {
-			continue
+		if opts.GroupByTags {
+			Printf("snapshots for host %v, tags [%v], paths: [%v]:\n\n", key.Hostname, strings.Join(key.Tags, ", "), strings.Join(key.Paths, ", "))
+		} else {
+			Printf("snapshots for host %v, paths: [%v]:\n\n", key.Hostname, strings.Join(key.Paths, ", "))
 		}
-
-		k := key{Hostname: sn.Hostname, Dirs: strings.Join(sn.Paths, ":")}
-		list := snapshotGroups[k]
-		list = append(list, sn)
-		snapshotGroups[k] = list
-	}
-
-	for key, snapshotGroup := range snapshotGroups {
-		Printf("snapshots for host %v, directories %v:\n\n", key.Hostname, key.Dirs)
 		keep, remove := restic.ApplyPolicy(snapshotGroup, policy)
 
-		Printf("keep %d snapshots:\n", len(keep))
-		printSnapshots(globalOptions.stdout, keep)
-		Printf("\n")
+		if len(keep) != 0 {
+			Printf("keep %d snapshots:\n", len(keep))
+			PrintSnapshots(globalOptions.stdout, keep)
+			Printf("\n")
+		}
 
-		Printf("remove %d snapshots:\n", len(remove))
-		printSnapshots(globalOptions.stdout, remove)
-		Printf("\n")
+		if len(remove) != 0 {
+			Printf("remove %d snapshots:\n", len(remove))
+			PrintSnapshots(globalOptions.stdout, remove)
+			Printf("\n")
+		}
+
+		removeSnapshots += len(remove)
 
 		if !opts.DryRun {
 			for _, sn := range remove {
@@ -211,5 +173,12 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
+	if removeSnapshots > 0 && opts.Prune {
+		Printf("%d snapshots have been removed, running prune\n", removeSnapshots)
+		if !opts.DryRun {
+			return pruneRepository(gopts, repo)
+		}
+	}
+
 	return nil
 }

src/cmds/restic/cmd_key.go

@@ -1,20 +1,20 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"restic"
-
-	"github.com/spf13/cobra"
-
 	"restic/errors"
 	"restic/repository"
+
+	"github.com/spf13/cobra"
 )
 
 var cmdKey = &cobra.Command{
 	Use:   "key [list|add|rm|passwd] [ID]",
 	Short: "manage keys (passwords)",
 	Long: `
-The "key" command manages keys (passwords) for accessing a repository.
+The "key" command manages keys (passwords) for accessing the repository.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runKey(globalOptions, args)
@@ -25,15 +25,12 @@ func init() {
 	cmdRoot.AddCommand(cmdKey)
 }
 
-func listKeys(s *repository.Repository) error {
+func listKeys(ctx context.Context, s *repository.Repository) error {
 	tab := NewTable()
 	tab.Header = fmt.Sprintf(" %-10s  %-10s  %-10s  %s", "ID", "User", "Host", "Created")
 	tab.RowFormat = "%s%-10s  %-10s  %-10s  %s"
 
-	done := make(chan struct{})
-	defer close(done)
-
-	for id := range s.List(restic.KeyFile, done) {
+	for id := range s.List(restic.KeyFile, ctx.Done()) {
 		k, err := repository.LoadKey(s, id.String())
 		if err != nil {
 			Warnf("LoadKey() failed: %v\n", err)
@@ -120,10 +117,13 @@ func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
 }
 
 func runKey(gopts GlobalOptions, args []string) error {
-	if len(args) < 1 || (args[0] == "rm" && len(args) != 2) {
-		return errors.Fatalf("wrong number of arguments")
+	if len(args) < 1 || (args[0] == "rm" && len(args) != 2) || (args[0] != "rm" && len(args) != 1) {
+		return errors.Fatal("wrong number of arguments")
 	}
 
+	ctx, cancel := context.WithCancel(gopts.ctx)
+	defer cancel()
+
 	repo, err := OpenRepository(gopts)
 	if err != nil {
 		return err
@@ -137,7 +137,7 @@ func runKey(gopts GlobalOptions, args []string) error {
 			return err
 		}
 
-		return listKeys(repo)
+		return listKeys(ctx, repo)
 	case "add":
 		lock, err := lockRepo(repo)
 		defer unlockRepo(lock)

src/cmds/restic/cmd_list.go

@@ -11,9 +11,9 @@ import (
 
 var cmdList = &cobra.Command{
 	Use:   "list [blobs|packs|index|snapshots|keys|locks]",
-	Short: "list items in the repository",
+	Short: "list objects in the repository",
 	Long: `
-
+The "list" command allows listing objects in the repository based on type.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runList(globalOptions, args)
@@ -26,7 +26,7 @@ func init() {
 
 func runList(opts GlobalOptions, args []string) error {
 	if len(args) != 1 {
-		return errors.Fatalf("type not specified")
+		return errors.Fatal("type not specified")
 	}
 
 	repo, err := OpenRepository(opts)

src/cmds/restic/cmd_ls.go

@@ -1,8 +1,7 @@
 package main
 
 import (
-	"fmt"
-	"os"
+	"context"
 	"path/filepath"
 
 	"github.com/spf13/cobra"
@@ -13,7 +12,7 @@ import (
 )
 
 var cmdLs = &cobra.Command{
-	Use:   "ls [flags] snapshot-ID",
+	Use:   "ls [flags] [snapshot-ID ...]",
 	Short: "list files in a snapshot",
 	Long: `
 The "ls" command allows listing files and directories in a snapshot.
@@ -21,7 +20,7 @@ The "ls" command allows listing files and directories in a snapshot.
 The special snapshot-ID "latest" can be used to list files and directories of the latest snapshot in the repository.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runLs(globalOptions, args)
+		return runLs(lsOptions, globalOptions, args)
 	},
 }
 
@@ -29,6 +28,7 @@ The special snapshot-ID "latest" can be used to list files and directories of th
 type LsOptions struct {
 	ListLong bool
 	Host     string
+	Tags     []string
 	Paths    []string
 }
 
@@ -40,42 +40,22 @@ func init() {
 	flags := cmdLs.Flags()
 	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
 
-	flags.StringVarP(&lsOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`)
-	flags.StringSliceVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
+	flags.StringVarP(&lsOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
+	flags.StringSliceVar(&lsOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot ID is given")
+	flags.StringSliceVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
 }
 
-func printNode(prefix string, n *restic.Node) string {
-	if !lsOptions.ListLong {
-		return filepath.Join(prefix, n.Name)
-	}
-
-	switch n.Type {
-	case "file":
-		return fmt.Sprintf("%s %5d %5d %6d %s %s",
-			n.Mode, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name))
-	case "dir":
-		return fmt.Sprintf("%s %5d %5d %6d %s %s",
-			n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name))
-	case "symlink":
-		return fmt.Sprintf("%s %5d %5d %6d %s %s -> %s",
-			n.Mode|os.ModeSymlink, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name), n.LinkTarget)
-	default:
-		return fmt.Sprintf("<Node(%s) %s>", n.Type, n.Name)
-	}
-}
-
-func printTree(prefix string, repo *repository.Repository, id restic.ID) error {
-	tree, err := repo.LoadTree(id)
+func printTree(repo *repository.Repository, id *restic.ID, prefix string) error {
+	tree, err := repo.LoadTree(*id)
 	if err != nil {
 		return err
 	}
 
 	for _, entry := range tree.Nodes {
-		Printf(printNode(prefix, entry) + "\n")
+		Printf(formatNode(prefix, entry, lsOptions.ListLong) + "\n")
 
 		if entry.Type == "dir" && entry.Subtree != nil {
-			err = printTree(filepath.Join(prefix, entry.Name), repo, *entry.Subtree)
-			if err != nil {
+			if err = printTree(repo, entry.Subtree, filepath.Join(prefix, entry.Name)); err != nil {
 				return err
 			}
 		}
@@ -84,9 +64,9 @@ func printTree(prefix string, repo *repository.Repository, id restic.ID) error {
 	return nil
 }
 
-func runLs(gopts GlobalOptions, args []string) error {
-	if len(args) < 1 || len(args) > 2 {
-		return errors.Fatalf("no snapshot ID given")
+func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
+	if len(args) == 0 && opts.Host == "" && len(opts.Tags) == 0 && len(opts.Paths) == 0 {
+		return errors.Fatal("Invalid arguments, either give one or more snapshot IDs or set filters.")
 	}
 
 	repo, err := OpenRepository(gopts)
@@ -94,32 +74,18 @@ func runLs(gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	err = repo.LoadIndex()
-	if err != nil {
+	if err = repo.LoadIndex(); err != nil {
 		return err
 	}
 
-	snapshotIDString := args[0]
-	var id restic.ID
+	ctx, cancel := context.WithCancel(gopts.ctx)
+	defer cancel()
+	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
+		Verbosef("snapshot %s of %v at %s):\n", sn.ID().Str(), sn.Paths, sn.Time)
 
-	if snapshotIDString == "latest" {
-		id, err = restic.FindLatestSnapshot(repo, lsOptions.Paths, lsOptions.Host)
-		if err != nil {
-			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Host:%v", err, lsOptions.Paths, lsOptions.Host)
-		}
-	} else {
-		id, err = restic.FindSnapshot(repo, snapshotIDString)
-		if err != nil {
-			Exitf(1, "invalid id %q: %v", snapshotIDString, err)
+		if err = printTree(repo, sn.Tree, string(filepath.Separator)); err != nil {
+			return err
 		}
 	}
-
-	sn, err := restic.LoadSnapshot(repo, id)
-	if err != nil {
-		return err
-	}
-
-	Verbosef("snapshot of %v at %s:\n", sn.Paths, sn.Time)
-
-	return printTree("", repo, *sn.Tree)
+	return nil
 }

src/cmds/restic/cmd_mount.go

@@ -32,7 +32,12 @@ read-only mount.
 
 // MountOptions collects all options for the mount command.
 type MountOptions struct {
-	OwnerRoot bool
+	OwnerRoot  bool
+	AllowRoot  bool
+	AllowOther bool
+	Host       string
+	Tags       []string
+	Paths      []string
 }
 
 var mountOptions MountOptions
@@ -40,7 +45,14 @@ var mountOptions MountOptions
 func init() {
 	cmdRoot.AddCommand(cmdMount)
 
-	cmdMount.Flags().BoolVar(&mountOptions.OwnerRoot, "owner-root", false, "use 'root' as the owner of files and dirs")
+	mountFlags := cmdMount.Flags()
+	mountFlags.BoolVar(&mountOptions.OwnerRoot, "owner-root", false, "use 'root' as the owner of files and dirs")
+	mountFlags.BoolVar(&mountOptions.AllowRoot, "allow-root", false, "allow root user to access the data in the mounted directory")
+	mountFlags.BoolVar(&mountOptions.AllowOther, "allow-other", false, "allow other users to access the data in the mounted directory")
+
+	mountFlags.StringVarP(&mountOptions.Host, "host", "H", "", `only consider snapshots for this host`)
+	mountFlags.StringSliceVar(&mountOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`")
+	mountFlags.StringSliceVar(&mountOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`")
 }
 
 func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
@@ -64,11 +76,21 @@ func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
 			return err
 		}
 	}
-	c, err := systemFuse.Mount(
-		mountpoint,
+
+	mountOptions := []systemFuse.MountOption{
 		systemFuse.ReadOnly(),
 		systemFuse.FSName("restic"),
-	)
+	}
+
+	if opts.AllowRoot {
+		mountOptions = append(mountOptions, systemFuse.AllowRoot())
+	}
+
+	if opts.AllowOther {
+		mountOptions = append(mountOptions, systemFuse.AllowOther())
+	}
+
+	c, err := systemFuse.Mount(mountpoint, mountOptions...)
 	if err != nil {
 		return err
 	}
@@ -77,7 +99,7 @@ func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
 	Printf("Don't forget to umount after quitting!\n")
 
 	root := fs.Tree{}
-	root.Add("snapshots", fuse.NewSnapshotsDir(repo, opts.OwnerRoot))
+	root.Add("snapshots", fuse.NewSnapshotsDir(repo, opts.OwnerRoot, opts.Paths, opts.Tags, opts.Host))
 
 	debug.Log("serving mount at %v", mountpoint)
 	err = fs.Serve(c, &root)
@@ -95,7 +117,7 @@ func umount(mountpoint string) error {
 
 func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
 	if len(args) == 0 {
-		return errors.Fatalf("wrong number of parameters")
+		return errors.Fatal("wrong number of parameters")
 	}
 
 	mountpoint := args[0]

src/cmds/restic/cmd_prune.go

@@ -1,8 +1,8 @@
 package main
 
 import (
+	"context"
 	"fmt"
-	"os"
 	"restic"
 	"restic/debug"
 	"restic/errors"
@@ -11,8 +11,6 @@ import (
 	"time"
 
 	"github.com/spf13/cobra"
-
-	"golang.org/x/crypto/ssh/terminal"
 )
 
 var cmdPrune = &cobra.Command{
@@ -45,8 +43,7 @@ func newProgressMax(show bool, max uint64, description string) *restic.Progress
 			formatPercent(s.Blobs, max),
 			s.Blobs, max, description)
 
-		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
-		if err == nil {
+		if w := stdoutTerminalWidth(); w > 0 {
 			if len(status) > w {
 				max := w - len(status) - 4
 				status = status[:max] + "... "
@@ -75,13 +72,17 @@ func runPrune(gopts GlobalOptions) error {
 		return err
 	}
 
-	err = repo.LoadIndex()
+	return pruneRepository(gopts, repo)
+}
+
+func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
+	err := repo.LoadIndex()
 	if err != nil {
 		return err
 	}
 
-	done := make(chan struct{})
-	defer close(done)
+	ctx, cancel := context.WithCancel(gopts.ctx)
+	defer cancel()
 
 	var stats struct {
 		blobs     int
@@ -91,7 +92,7 @@ func runPrune(gopts GlobalOptions) error {
 	}
 
 	Verbosef("counting files in repo\n")
-	for _ = range repo.List(restic.DataFile, done) {
+	for _ = range repo.List(restic.DataFile, ctx.Done()) {
 		stats.packs++
 	}
 
@@ -180,7 +181,7 @@ func runPrune(gopts GlobalOptions) error {
 		}
 	}
 
-	removeBytes := 0
+	removeBytes := duplicateBytes
 
 	// find packs that are unneeded
 	removePacks := restic.NewIDSet()
@@ -213,48 +214,34 @@ func runPrune(gopts GlobalOptions) error {
 	Verbosef("will delete %d packs and rewrite %d packs, this frees %s\n",
 		len(removePacks), len(rewritePacks), formatBytes(uint64(removeBytes)))
 
-	err = repository.Repack(repo, rewritePacks, usedBlobs)
-	if err != nil {
-		return err
-	}
-
-	for packID := range removePacks {
-		h := restic.Handle{Type: restic.DataFile, Name: packID.String()}
-		err = repo.Backend().Remove(h)
+	if len(rewritePacks) != 0 {
+		bar = newProgressMax(!gopts.Quiet, uint64(len(rewritePacks)), "packs rewritten")
+		bar.Start()
+		err = repository.Repack(repo, rewritePacks, usedBlobs, bar)
 		if err != nil {
-			Warnf("unable to remove file %v from the repository\n", packID.Str())
+			return err
 		}
+		bar.Done()
 	}
 
-	Verbosef("creating new index\n")
-
-	stats.packs = 0
-	for _ = range repo.List(restic.DataFile, done) {
-		stats.packs++
+	if len(removePacks) != 0 {
+		bar = newProgressMax(!gopts.Quiet, uint64(len(removePacks)), "packs deleted")
+		bar.Start()
+		for packID := range removePacks {
+			h := restic.Handle{Type: restic.DataFile, Name: packID.String()}
+			err = repo.Backend().Remove(h)
+			if err != nil {
+				Warnf("unable to remove file %v from the repository\n", packID.Str())
+			}
+			bar.Report(restic.Stat{Blobs: 1})
+		}
+		bar.Done()
 	}
-	bar = newProgressMax(!gopts.Quiet, uint64(stats.packs), "packs")
-	idx, err = index.New(repo, bar)
-	if err != nil {
+
+	if err = rebuildIndex(ctx, repo); err != nil {
 		return err
 	}
 
-	var supersedes restic.IDs
-	for idxID := range repo.List(restic.IndexFile, done) {
-		h := restic.Handle{Type: restic.IndexFile, Name: idxID.String()}
-		err := repo.Backend().Remove(h)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "unable to remove index %v: %v\n", idxID.Str(), err)
-		}
-
-		supersedes = append(supersedes, idxID)
-	}
-
-	id, err := idx.Save(repo, supersedes)
-	if err != nil {
-		return err
-	}
-	Verbosef("saved new index as %v\n", id.Str())
-
 	Verbosef("done\n")
 	return nil
 }

src/cmds/restic/cmd_rebuild_index.go

@@ -1,7 +1,9 @@
 package main
 
 import (
-	"restic/repository"
+	"context"
+	"restic"
+	"restic/index"
 
 	"github.com/spf13/cobra"
 )
@@ -10,8 +12,8 @@ var cmdRebuildIndex = &cobra.Command{
 	Use:   "rebuild-index [flags]",
 	Short: "build a new index file",
 	Long: `
-The "rebuild-index" command creates a new index by combining the index files
-into a new one.
+The "rebuild-index" command creates a new index based on the pack files in the
+repository.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runRebuildIndex(globalOptions)
@@ -34,5 +36,49 @@ func runRebuildIndex(gopts GlobalOptions) error {
 		return err
 	}
 
-	return repository.RebuildIndex(repo)
+	ctx, cancel := context.WithCancel(gopts.ctx)
+	defer cancel()
+	return rebuildIndex(ctx, repo)
+}
+
+func rebuildIndex(ctx context.Context, repo restic.Repository) error {
+	Verbosef("counting files in repo\n")
+
+	var packs uint64
+	for _ = range repo.List(restic.DataFile, ctx.Done()) {
+		packs++
+	}
+
+	bar := newProgressMax(!globalOptions.Quiet, packs, "packs")
+	idx, err := index.New(repo, bar)
+	if err != nil {
+		return err
+	}
+
+	Verbosef("finding old index files\n")
+
+	var supersedes restic.IDs
+	for id := range repo.List(restic.IndexFile, ctx.Done()) {
+		supersedes = append(supersedes, id)
+	}
+
+	id, err := idx.Save(repo, supersedes)
+	if err != nil {
+		return err
+	}
+
+	Verbosef("saved new index as %v\n", id.Str())
+
+	Verbosef("remove %d old index files\n", len(supersedes))
+
+	for _, id := range supersedes {
+		if err := repo.Backend().Remove(restic.Handle{
+			Type: restic.IndexFile,
+			Name: id.String(),
+		}); err != nil {
+			Warnf("error removing old index %v: %v\n", id.Str(), err)
+		}
+	}
+
+	return nil
 }

src/cmds/restic/cmd_restore.go

@@ -31,6 +31,7 @@ type RestoreOptions struct {
 	Target  string
 	Host    string
 	Paths   []string
+	Tags    []string
 }
 
 var restoreOptions RestoreOptions
@@ -44,12 +45,13 @@ func init() {
 	flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to")
 
 	flags.StringVarP(&restoreOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`)
+	flags.StringSliceVar(&restoreOptions.Tags, "tag", nil, "only consider snapshots which include this `tag` for snapshot ID \"latest\"")
 	flags.StringSliceVar(&restoreOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
 }
 
 func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 	if len(args) != 1 {
-		return errors.Fatalf("no snapshot ID specified")
+		return errors.Fatal("no snapshot ID specified")
 	}
 
 	if opts.Target == "" {
@@ -85,7 +87,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 	var id restic.ID
 
 	if snapshotIDString == "latest" {
-		id, err = restic.FindLatestSnapshot(repo, opts.Paths, opts.Host)
+		id, err = restic.FindLatestSnapshot(repo, opts.Paths, opts.Tags, opts.Host)
 		if err != nil {
 			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Host:%v", err, opts.Paths, opts.Host)
 		}

src/cmds/restic/cmd_snapshots.go

@@ -1,9 +1,10 @@
 package main
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
-	"os"
-	"restic/errors"
+	"io"
 	"sort"
 
 	"github.com/spf13/cobra"
@@ -12,19 +13,20 @@ import (
 )
 
 var cmdSnapshots = &cobra.Command{
-	Use:   "snapshots",
+	Use:   "snapshots [snapshotID ...]",
 	Short: "list all snapshots",
 	Long: `
-The "snapshots" command lists all snapshots stored in a repository.
+The "snapshots" command lists all snapshots stored in the repository.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runSnapshots(snapshotOptions, globalOptions, args)
 	},
 }
 
-// SnapshotOptions bundle all options for the snapshots command.
+// SnapshotOptions bundles all options for the snapshots command.
 type SnapshotOptions struct {
 	Host  string
+	Tags  []string
 	Paths []string
 }
 
@@ -34,15 +36,12 @@ func init() {
 	cmdRoot.AddCommand(cmdSnapshots)
 
 	f := cmdSnapshots.Flags()
-	f.StringVar(&snapshotOptions.Host, "host", "", "only print snapshots for this host")
-	f.StringSliceVar(&snapshotOptions.Paths, "path", []string{}, "only print snapshots for this `path` (can be specified multiple times)")
+	f.StringVarP(&snapshotOptions.Host, "host", "H", "", "only consider snapshots for this `host`")
+	f.StringSliceVar(&snapshotOptions.Tags, "tag", nil, "only consider snapshots which include this `tag` (can be specified multiple times)")
+	f.StringSliceVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)")
 }
 
 func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error {
-	if len(args) != 0 {
-		return errors.Fatalf("wrong number of arguments")
-	}
-
 	repo, err := OpenRepository(gopts)
 	if err != nil {
 		return err
@@ -56,37 +55,47 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 		}
 	}
 
-	tab := NewTable()
-	tab.Header = fmt.Sprintf("%-8s  %-19s  %-10s  %-10s  %-3s %s", "ID", "Date", "Host", "Tags", "", "Directory")
-	tab.RowFormat = "%-8s  %-19s  %-10s  %-10s  %-3s %s"
+	ctx, cancel := context.WithCancel(gopts.ctx)
+	defer cancel()
 
-	done := make(chan struct{})
-	defer close(done)
+	var list restic.Snapshots
+	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
+		list = append(list, sn)
+	}
+	sort.Sort(sort.Reverse(list))
 
-	list := []*restic.Snapshot{}
-	for id := range repo.List(restic.SnapshotFile, done) {
-		sn, err := restic.LoadSnapshot(repo, id)
+	if gopts.JSON {
+		err := printSnapshotsJSON(gopts.stdout, list)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "error loading snapshot %s: %v\n", id, err)
-			continue
+			Warnf("error printing snapshot: %v\n", err)
 		}
+		return nil
+	}
+	PrintSnapshots(gopts.stdout, list)
 
-		if restic.SamePaths(sn.Paths, opts.Paths) && (opts.Host == "" || opts.Host == sn.Hostname) {
-			pos := sort.Search(len(list), func(i int) bool {
-				return list[i].Time.After(sn.Time)
-			})
+	return nil
+}
 
-			if pos < len(list) {
-				list = append(list, nil)
-				copy(list[pos+1:], list[pos:])
-				list[pos] = sn
-			} else {
-				list = append(list, sn)
+// PrintSnapshots prints a text table of the snapshots in list to stdout.
+func PrintSnapshots(stdout io.Writer, list restic.Snapshots) {
+
+	// Determine the max widths for host and tag.
+	maxHost, maxTag := 10, 6
+	for _, sn := range list {
+		if len(sn.Hostname) > maxHost {
+			maxHost = len(sn.Hostname)
+		}
+		for _, tag := range sn.Tags {
+			if len(tag) > maxTag {
+				maxTag = len(tag)
 			}
 		}
-
 	}
 
+	tab := NewTable()
+	tab.Header = fmt.Sprintf("%-8s  %-19s  %-*s  %-*s  %-3s %s", "ID", "Date", -maxHost, "Host", -maxTag, "Tags", "", "Directory")
+	tab.RowFormat = fmt.Sprintf("%%-8s  %%-19s  %%%ds  %%%ds  %%-3s %%s", -maxHost, -maxTag)
+
 	for _, sn := range list {
 		if len(sn.Paths) == 0 {
 			continue
@@ -98,6 +107,9 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 		}
 
 		rows := len(sn.Paths)
+		if rows < len(sn.Tags) {
+			rows = len(sn.Tags)
+		}
 
 		treeElement := "   "
 		if rows != 1 {
@@ -130,7 +142,29 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 		}
 	}
 
-	tab.Write(os.Stdout)
-
-	return nil
+	tab.Write(stdout)
+}
+
+// Snapshot helps to print Snaphots as JSON with their ID included.
+type Snapshot struct {
+	*restic.Snapshot
+
+	ID *restic.ID `json:"id"`
+}
+
+// printSnapshotsJSON writes the JSON representation of list to stdout.
+func printSnapshotsJSON(stdout io.Writer, list restic.Snapshots) error {
+
+	var snapshots []Snapshot
+
+	for _, sn := range list {
+
+		k := Snapshot{
+			Snapshot: sn,
+			ID:       sn.ID(),
+		}
+		snapshots = append(snapshots, k)
+	}
+
+	return json.NewEncoder(stdout).Encode(snapshots)
 }

src/cmds/restic/cmd_tag.go

@@ -0,0 +1,142 @@
+package main
+
+import (
+	"context"
+
+	"github.com/spf13/cobra"
+
+	"restic"
+	"restic/debug"
+	"restic/errors"
+	"restic/repository"
+)
+
+var cmdTag = &cobra.Command{
+	Use:   "tag [flags] [snapshot-ID ...]",
+	Short: "modifies tags on snapshots",
+	Long: `
+The "tag" command allows you to modify tags on exiting snapshots.
+
+You can either set/replace the entire set of tags on a snapshot, or
+add tags to/remove tags from the existing set.
+
+When no snapshot-ID is given, all snapshots matching the host, tag and path filter criteria are modified.
+`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runTag(tagOptions, globalOptions, args)
+	},
+}
+
+// TagOptions bundles all options for the 'tag' command.
+type TagOptions struct {
+	Host       string
+	Paths      []string
+	Tags       []string
+	SetTags    []string
+	AddTags    []string
+	RemoveTags []string
+}
+
+var tagOptions TagOptions
+
+func init() {
+	cmdRoot.AddCommand(cmdTag)
+
+	tagFlags := cmdTag.Flags()
+	tagFlags.StringSliceVar(&tagOptions.SetTags, "set", nil, "`tag` which will replace the existing tags (can be given multiple times)")
+	tagFlags.StringSliceVar(&tagOptions.AddTags, "add", nil, "`tag` which will be added to the existing tags (can be given multiple times)")
+	tagFlags.StringSliceVar(&tagOptions.RemoveTags, "remove", nil, "`tag` which will be removed from the existing tags (can be given multiple times)")
+
+	tagFlags.StringVarP(&tagOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
+	tagFlags.StringSliceVar(&tagOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot-ID is given")
+	tagFlags.StringSliceVar(&tagOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
+}
+
+func changeTags(repo *repository.Repository, sn *restic.Snapshot, setTags, addTags, removeTags []string) (bool, error) {
+	var changed bool
+
+	if len(setTags) != 0 {
+		// Setting the tag to an empty string really means no tags.
+		if len(setTags) == 1 && setTags[0] == "" {
+			setTags = nil
+		}
+		sn.Tags = setTags
+		changed = true
+	} else {
+		changed = sn.AddTags(addTags)
+		if sn.RemoveTags(removeTags) {
+			changed = true
+		}
+	}
+
+	if changed {
+		// Retain the original snapshot id over all tag changes.
+		if sn.Original == nil {
+			sn.Original = sn.ID()
+		}
+
+		// Save the new snapshot.
+		id, err := repo.SaveJSONUnpacked(restic.SnapshotFile, sn)
+		if err != nil {
+			return false, err
+		}
+
+		debug.Log("new snapshot saved as %v", id.Str())
+
+		if err = repo.Flush(); err != nil {
+			return false, err
+		}
+
+		// Remove the old snapshot.
+		h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
+		if err = repo.Backend().Remove(h); err != nil {
+			return false, err
+		}
+
+		debug.Log("old snapshot %v removed", sn.ID())
+	}
+	return changed, nil
+}
+
+func runTag(opts TagOptions, gopts GlobalOptions, args []string) error {
+	if len(opts.SetTags) == 0 && len(opts.AddTags) == 0 && len(opts.RemoveTags) == 0 {
+		return errors.Fatal("nothing to do!")
+	}
+	if len(opts.SetTags) != 0 && (len(opts.AddTags) != 0 || len(opts.RemoveTags) != 0) {
+		return errors.Fatal("--set and --add/--remove cannot be given at the same time")
+	}
+
+	repo, err := OpenRepository(gopts)
+	if err != nil {
+		return err
+	}
+
+	if !gopts.NoLock {
+		Verbosef("Create exclusive lock for repository\n")
+		lock, err := lockRepoExclusive(repo)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	}
+
+	changeCnt := 0
+	ctx, cancel := context.WithCancel(gopts.ctx)
+	defer cancel()
+	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
+		changed, err := changeTags(repo, sn, opts.SetTags, opts.AddTags, opts.RemoveTags)
+		if err != nil {
+			Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", sn.ID(), err)
+			continue
+		}
+		if changed {
+			changeCnt++
+		}
+	}
+	if changeCnt == 0 {
+		Verbosef("No snapshots were modified\n")
+	} else {
+		Verbosef("Modified tags on %v snapshots\n", changeCnt)
+	}
+	return nil
+}

src/cmds/restic/cmd_unlock.go

@@ -27,7 +27,7 @@ var unlockOptions UnlockOptions
 func init() {
 	cmdRoot.AddCommand(unlockCmd)
 
-	unlockCmd.Flags().BoolVar(&unlockOptions.RemoveAll, "remove-all", false, "Remove all locks, even non-stale ones")
+	unlockCmd.Flags().BoolVar(&unlockOptions.RemoveAll, "remove-all", false, "remove all locks, even non-stale ones")
 }
 
 func runUnlock(opts UnlockOptions, gopts GlobalOptions) error {

src/cmds/restic/cmd_version.go

@@ -9,7 +9,7 @@ import (
 
 var versionCmd = &cobra.Command{
 	Use:   "version",
-	Short: "Print version information",
+	Short: "print version information",
 	Long: `
 The "version" command prints detailed information about the build environment
 and the version of this software.

src/cmds/restic/find.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+	"context"
+
+	"restic"
+	"restic/repository"
+)
+
+// FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots.
+func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, host string, tags []string, paths []string, snapshotIDs []string) <-chan *restic.Snapshot {
+	out := make(chan *restic.Snapshot)
+	go func() {
+		defer close(out)
+		if len(snapshotIDs) != 0 {
+			var (
+				id         restic.ID
+				usedFilter bool
+				err        error
+			)
+			ids := make(restic.IDs, 0, len(snapshotIDs))
+			// Process all snapshot IDs given as arguments.
+			for _, s := range snapshotIDs {
+				if s == "latest" {
+					id, err = restic.FindLatestSnapshot(repo, paths, tags, host)
+					if err != nil {
+						Warnf("Ignoring %q, no snapshot matched given filter (Paths:%v Tags:%v Host:%v)\n", s, paths, tags, host)
+						usedFilter = true
+						continue
+					}
+				} else {
+					id, err = restic.FindSnapshot(repo, s)
+					if err != nil {
+						Warnf("Ignoring %q, it is not a snapshot id\n", s)
+						continue
+					}
+				}
+				ids = append(ids, id)
+			}
+
+			// Give the user some indication their filters are not used.
+			if !usedFilter && (host != "" || len(tags) != 0 || len(paths) != 0) {
+				Warnf("Ignoring filters as there are explicit snapshot ids given\n")
+			}
+
+			for _, id := range ids.Uniq() {
+				sn, err := restic.LoadSnapshot(repo, id)
+				if err != nil {
+					Warnf("Ignoring %q, could not load snapshot: %v\n", id, err)
+					continue
+				}
+				select {
+				case <-ctx.Done():
+					return
+				case out <- sn:
+				}
+			}
+			return
+		}
+
+		for id := range repo.List(restic.SnapshotFile, ctx.Done()) {
+			sn, err := restic.LoadSnapshot(repo, id)
+			if err != nil {
+				Warnf("Ignoring %q, could not load snapshot: %v\n", id, err)
+				continue
+			}
+			if (host != "" && host != sn.Hostname) || !sn.HasTags(tags) || !sn.HasPaths(paths) {
+				continue
+			}
+			select {
+			case <-ctx.Done():
+				return
+			case out <- sn:
+			}
+		}
+	}()
+	return out
+}

src/cmds/restic/format.go

@@ -2,7 +2,11 @@ package main
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
 	"time"
+
+	"restic"
 )
 
 func formatBytes(c uint64) string {
@@ -58,3 +62,23 @@ func formatDuration(d time.Duration) string {
 	sec := uint64(d / time.Second)
 	return formatSeconds(sec)
 }
+
+func formatNode(prefix string, n *restic.Node, long bool) string {
+	if !long {
+		return filepath.Join(prefix, n.Name)
+	}
+
+	switch n.Type {
+	case "file":
+		return fmt.Sprintf("%s %5d %5d %6d %s %s",
+			n.Mode, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name))
+	case "dir":
+		return fmt.Sprintf("%s %5d %5d %6d %s %s",
+			n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name))
+	case "symlink":
+		return fmt.Sprintf("%s %5d %5d %6d %s %s -> %s",
+			n.Mode|os.ModeSymlink, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name), n.LinkTarget)
+	default:
+		return fmt.Sprintf("<Node(%s) %s>", n.Type, n.Name)
+	}
+}

src/cmds/restic/global.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -31,7 +32,9 @@ type GlobalOptions struct {
 	PasswordFile string
 	Quiet        bool
 	NoLock       bool
+	JSON         bool
 
+	ctx      context.Context
 	password string
 	stdout   io.Writer
 	stderr   io.Writer
@@ -48,11 +51,19 @@ func init() {
 		globalOptions.password = pw
 	}
 
+	var cancel context.CancelFunc
+	globalOptions.ctx, cancel = context.WithCancel(context.Background())
+	AddCleanupHandler(func() error {
+		cancel()
+		return nil
+	})
+
 	f := cmdRoot.PersistentFlags()
 	f.StringVarP(&globalOptions.Repo, "repo", "r", os.Getenv("RESTIC_REPOSITORY"), "repository to backup to or restore from (default: $RESTIC_REPOSITORY)")
 	f.StringVarP(&globalOptions.PasswordFile, "password-file", "p", "", "read the repository password from a file")
 	f.BoolVarP(&globalOptions.Quiet, "quiet", "q", false, "do not output comprehensive progress report")
 	f.BoolVar(&globalOptions.NoLock, "no-lock", false, "do not lock the repo, this allows some operations on read-only repos")
+	f.BoolVarP(&globalOptions.JSON, "json", "", false, "set output mode to JSON for commands that support it")
 
 	restoreTerminal()
 }
@@ -80,6 +91,14 @@ func stdoutIsTerminal() bool {
 	return terminal.IsTerminal(int(os.Stdout.Fd()))
 }
 
+func stdoutTerminalWidth() int {
+	w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
+	if err != nil {
+		return 0
+	}
+	return w
+}
+
 // restoreTerminal installs a cleanup handler that restores the previous
 // terminal state on exit.
 func restoreTerminal() {
@@ -108,8 +127,7 @@ func restoreTerminal() {
 // current windows cmd shell.
 func ClearLine() string {
 	if runtime.GOOS == "windows" {
-		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
-		if err == nil {
+		if w := stdoutTerminalWidth(); w > 0 {
 			return strings.Repeat(" ", w-1) + "\r"
 		}
 		return ""

src/cmds/restic/integration_helpers_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -15,6 +16,7 @@ import (
 type dirEntry struct {
 	path string
 	fi   os.FileInfo
+	link uint64
 }
 
 func walkDir(dir string) <-chan *dirEntry {
@@ -36,6 +38,7 @@ func walkDir(dir string) <-chan *dirEntry {
 			ch <- &dirEntry{
 				path: name,
 				fi:   info,
+				link: nlink(info),
 			}
 
 			return nil
@@ -192,6 +195,7 @@ func withTestEnvironment(t testing.TB, f func(*testEnvironment, GlobalOptions))
 	gopts := GlobalOptions{
 		Repo:     env.repo,
 		Quiet:    true,
+		ctx:      context.Background(),
 		password: TestPassword,
 		stdout:   os.Stdout,
 		stderr:   os.Stderr,

src/cmds/restic/integration_helpers_unix_test.go

@@ -4,7 +4,9 @@ package main
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"syscall"
 )
 
@@ -37,5 +39,37 @@ func (e *dirEntry) equals(other *dirEntry) bool {
 		return false
 	}
 
+	if stat.Nlink != stat2.Nlink {
+		fmt.Fprintf(os.Stderr, "%v: Number of links do not match (%v != %v)\n", e.path, stat.Nlink, stat2.Nlink)
+		return false
+	}
+
 	return true
 }
+
+func nlink(info os.FileInfo) uint64 {
+	stat, _ := info.Sys().(*syscall.Stat_t)
+	return uint64(stat.Nlink)
+}
+
+func inode(info os.FileInfo) uint64 {
+	stat, _ := info.Sys().(*syscall.Stat_t)
+	return uint64(stat.Ino)
+}
+
+func createFileSetPerHardlink(dir string) map[uint64][]string {
+	var stat syscall.Stat_t
+	linkTests := make(map[uint64][]string)
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		return nil
+	}
+	for _, f := range files {
+
+		if err := syscall.Stat(filepath.Join(dir, f.Name()), &stat); err != nil {
+			return nil
+		}
+		linkTests[uint64(stat.Ino)] = append(linkTests[uint64(stat.Ino)], f.Name())
+	}
+	return linkTests
+}

src/cmds/restic/integration_helpers_windows_test.go

@@ -4,6 +4,7 @@ package main
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
 )
 
@@ -25,3 +26,24 @@ func (e *dirEntry) equals(other *dirEntry) bool {
 
 	return true
 }
+
+func nlink(info os.FileInfo) uint64 {
+	return 1
+}
+
+func inode(info os.FileInfo) uint64 {
+	return uint64(0)
+}
+
+func createFileSetPerHardlink(dir string) map[uint64][]string {
+	linkTests := make(map[uint64][]string)
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		return nil
+	}
+	for i, f := range files {
+		linkTests[uint64(i)] = append(linkTests[uint64(i)], f.Name())
+		i++
+	}
+	return linkTests
+}

src/cmds/restic/integration_test.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"bytes"
 	"crypto/rand"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -141,7 +142,9 @@ func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
 		globalOptions.Quiet = quiet
 	}()
 
-	OK(t, runLs(gopts, []string{snapshotID}))
+	opts := LsOptions{}
+
+	OK(t, runLs(opts, gopts, []string{snapshotID}))
 
 	return strings.Split(string(buf.Bytes()), "\n")
 }
@@ -160,6 +163,32 @@ func testRunFind(t testing.TB, gopts GlobalOptions, pattern string) []string {
 	return strings.Split(string(buf.Bytes()), "\n")
 }
 
+func testRunSnapshots(t testing.TB, gopts GlobalOptions) (newest *Snapshot, snapmap map[restic.ID]Snapshot) {
+	buf := bytes.NewBuffer(nil)
+	globalOptions.stdout = buf
+	globalOptions.JSON = true
+	defer func() {
+		globalOptions.stdout = os.Stdout
+		globalOptions.JSON = gopts.JSON
+	}()
+
+	opts := SnapshotOptions{}
+
+	OK(t, runSnapshots(opts, globalOptions, []string{}))
+
+	snapshots := []Snapshot{}
+	OK(t, json.Unmarshal(buf.Bytes(), &snapshots))
+
+	snapmap = make(map[restic.ID]Snapshot, len(snapshots))
+	for _, sn := range snapshots {
+		snapmap[*sn.ID] = sn
+		if newest == nil || sn.Time.After(newest.Time) {
+			newest = &sn
+		}
+	}
+	return
+}
+
 func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
 	opts := ForgetOptions{}
 	OK(t, runForget(opts, gopts, args))
@@ -516,23 +545,23 @@ func TestBackupExclude(t *testing.T) {
 		testRunBackup(t, []string{datadir}, opts, gopts)
 		snapshots, snapshotID := lastSnapshot(snapshots, loadSnapshotMap(t, gopts))
 		files := testRunLs(t, gopts, snapshotID)
-		Assert(t, includes(files, filepath.Join("testdata", "foo.tar.gz")),
+		Assert(t, includes(files, filepath.Join(string(filepath.Separator), "testdata", "foo.tar.gz")),
 			"expected file %q in first snapshot, but it's not included", "foo.tar.gz")
 
 		opts.Excludes = []string{"*.tar.gz"}
 		testRunBackup(t, []string{datadir}, opts, gopts)
 		snapshots, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, gopts))
 		files = testRunLs(t, gopts, snapshotID)
-		Assert(t, !includes(files, filepath.Join("testdata", "foo.tar.gz")),
+		Assert(t, !includes(files, filepath.Join(string(filepath.Separator), "testdata", "foo.tar.gz")),
 			"expected file %q not in first snapshot, but it's included", "foo.tar.gz")
 
 		opts.Excludes = []string{"*.tar.gz", "private/secret"}
 		testRunBackup(t, []string{datadir}, opts, gopts)
 		snapshots, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, gopts))
 		files = testRunLs(t, gopts, snapshotID)
-		Assert(t, !includes(files, filepath.Join("testdata", "foo.tar.gz")),
+		Assert(t, !includes(files, filepath.Join(string(filepath.Separator), "testdata", "foo.tar.gz")),
 			"expected file %q not in first snapshot, but it's included", "foo.tar.gz")
-		Assert(t, !includes(files, filepath.Join("testdata", "private", "secret", "passwords.txt")),
+		Assert(t, !includes(files, filepath.Join(string(filepath.Separator), "testdata", "private", "secret", "passwords.txt")),
 			"expected file %q not in first snapshot, but it's included", "passwords.txt")
 	})
 }
@@ -602,6 +631,105 @@ func TestIncrementalBackup(t *testing.T) {
 	})
 }
 
+func TestBackupTags(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		testRunInit(t, gopts)
+		SetupTarTestFixture(t, env.testdata, datafile)
+
+		opts := BackupOptions{}
+
+		testRunBackup(t, []string{env.testdata}, opts, gopts)
+		testRunCheck(t, gopts)
+		newest, _ := testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 0,
+			"expected no tags, got %v", newest.Tags)
+
+		opts.Tags = []string{"NL"}
+		testRunBackup(t, []string{env.testdata}, opts, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "NL",
+			"expected one NL tag, got %v", newest.Tags)
+	})
+}
+
+func testRunTag(t testing.TB, opts TagOptions, gopts GlobalOptions) {
+	OK(t, runTag(opts, gopts, []string{}))
+}
+
+func TestTag(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		testRunInit(t, gopts)
+		SetupTarTestFixture(t, env.testdata, datafile)
+
+		testRunBackup(t, []string{env.testdata}, BackupOptions{}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ := testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 0,
+			"expected no tags, got %v", newest.Tags)
+		Assert(t, newest.Original == nil,
+			"expected original ID to be nil, got %v", newest.Original)
+		originalID := *newest.ID
+
+		testRunTag(t, TagOptions{SetTags: []string{"NL"}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "NL",
+			"set failed, expected one NL tag, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
+
+		testRunTag(t, TagOptions{AddTags: []string{"CH"}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 2 && newest.Tags[0] == "NL" && newest.Tags[1] == "CH",
+			"add failed, expected CH,NL tags, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
+
+		testRunTag(t, TagOptions{RemoveTags: []string{"NL"}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "CH",
+			"remove failed, expected one CH tag, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
+
+		testRunTag(t, TagOptions{AddTags: []string{"US", "RU"}}, gopts)
+		testRunTag(t, TagOptions{RemoveTags: []string{"CH", "US", "RU"}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 0,
+			"expected no tags, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
+
+		// Check special case of removing all tags.
+		testRunTag(t, TagOptions{SetTags: []string{""}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 0,
+			"expected no tags, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
+	})
+}
+
 func testRunKeyListOtherIDs(t testing.TB, gopts GlobalOptions) []string {
 	buf := bytes.NewBuffer(nil)
 
@@ -836,7 +964,7 @@ func TestRestoreWithPermissionFailure(t *testing.T) {
 
 		testRunRestore(t, gopts, filepath.Join(env.base, "restore"), snapshots[0])
 
-		// make sure that all files have been restored, regardeless of any
+		// make sure that all files have been restored, regardless of any
 		// permission errors
 		files := testRunLs(t, gopts, snapshots[0].String())
 		for _, filename := range files {
@@ -935,7 +1063,7 @@ func TestRebuildIndex(t *testing.T) {
 		}
 
 		if !strings.Contains(out, "restic rebuild-index") {
-			t.Fatalf("did not find hint for rebuild-index comman")
+			t.Fatalf("did not find hint for rebuild-index command")
 		}
 
 		testRunRebuildIndex(t, gopts)
@@ -1011,3 +1139,100 @@ func TestPrune(t *testing.T) {
 		testRunCheck(t, gopts)
 	})
 }
+
+func TestHardLink(t *testing.T) {
+	// this test assumes a test set with a single directory containing hard linked files
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		datafile := filepath.Join("testdata", "test.hl.tar.gz")
+		fd, err := os.Open(datafile)
+		if os.IsNotExist(errors.Cause(err)) {
+			t.Skipf("unable to find data file %q, skipping", datafile)
+			return
+		}
+		OK(t, err)
+		OK(t, fd.Close())
+
+		testRunInit(t, gopts)
+
+		SetupTarTestFixture(t, env.testdata, datafile)
+
+		linkTests := createFileSetPerHardlink(env.testdata)
+
+		opts := BackupOptions{}
+
+		// first backup
+		testRunBackup(t, []string{env.testdata}, opts, gopts)
+		snapshotIDs := testRunList(t, "snapshots", gopts)
+		Assert(t, len(snapshotIDs) == 1,
+			"expected one snapshot, got %v", snapshotIDs)
+
+		testRunCheck(t, gopts)
+
+		// restore all backups and compare
+		for i, snapshotID := range snapshotIDs {
+			restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
+			t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir)
+			testRunRestore(t, gopts, restoredir, snapshotIDs[0])
+			Assert(t, directoriesEqualContents(env.testdata, filepath.Join(restoredir, "testdata")),
+				"directories are not equal")
+
+			linkResults := createFileSetPerHardlink(filepath.Join(restoredir, "testdata"))
+			Assert(t, linksEqual(linkTests, linkResults),
+				"links are not equal")
+		}
+
+		testRunCheck(t, gopts)
+	})
+}
+
+func linksEqual(source, dest map[uint64][]string) bool {
+	for _, vs := range source {
+		found := false
+		for kd, vd := range dest {
+			if linkEqual(vs, vd) {
+				delete(dest, kd)
+				found = true
+				break
+			}
+		}
+		if !found {
+			return false
+		}
+	}
+
+	if len(dest) != 0 {
+		return false
+	}
+
+	return true
+}
+
+func linkEqual(source, dest []string) bool {
+	// equal if sliced are equal without considering order
+	if source == nil && dest == nil {
+		return true
+	}
+
+	if source == nil || dest == nil {
+		return false
+	}
+
+	if len(source) != len(dest) {
+		return false
+	}
+
+	for i := range source {
+		found := false
+		for j := range dest {
+			if source[i] == dest[j] {
+				found = true
+				break
+			}
+		}
+		if !found {
+			return false
+		}
+	}
+
+	return true
+}

src/restic/archiver/archive_reader.go

@@ -11,11 +11,22 @@ import (
 	"github.com/restic/chunker"
 )
 
-// ArchiveReader reads from the reader and archives the data. Returned is the
-// resulting snapshot and its ID.
-func ArchiveReader(repo restic.Repository, p *restic.Progress, rd io.Reader, name string, tags []string) (*restic.Snapshot, restic.ID, error) {
+// Reader allows saving a stream of data to the repository.
+type Reader struct {
+	restic.Repository
+
+	Tags     []string
+	Hostname string
+}
+
+// Archive reads data from the reader and saves it to the repo.
+func (r *Reader) Archive(name string, rd io.Reader, p *restic.Progress) (*restic.Snapshot, restic.ID, error) {
+	if name == "" {
+		return nil, restic.ID{}, errors.New("no filename given")
+	}
+
 	debug.Log("start archiving %s", name)
-	sn, err := restic.NewSnapshot([]string{name}, tags)
+	sn, err := restic.NewSnapshot([]string{name}, r.Tags, r.Hostname)
 	if err != nil {
 		return nil, restic.ID{}, err
 	}
@@ -23,9 +34,10 @@ func ArchiveReader(repo restic.Repository, p *restic.Progress, rd io.Reader, nam
 	p.Start()
 	defer p.Done()
 
+	repo := r.Repository
 	chnker := chunker.New(rd, repo.Config().ChunkerPolynomial)
 
-	var ids restic.IDs
+	ids := restic.IDs{}
 	var fileSize uint64
 
 	for {

src/restic/archiver/archive_reader_test.go

@@ -2,9 +2,11 @@ package archiver
 
 import (
 	"bytes"
+	"errors"
 	"io"
 	"math/rand"
 	"restic"
+	"restic/checker"
 	"restic/repository"
 	"testing"
 )
@@ -77,7 +79,13 @@ func TestArchiveReader(t *testing.T) {
 
 	f := fakeFile(t, seed, size)
 
-	sn, id, err := ArchiveReader(repo, nil, f, "fakefile", []string{"test"})
+	r := &Reader{
+		Repository: repo,
+		Hostname:   "localhost",
+		Tags:       []string{"test"},
+	}
+
+	sn, id, err := r.Archive("fakefile", f, nil)
 	if err != nil {
 		t.Fatalf("ArchiveReader() returned error %v", err)
 	}
@@ -89,6 +97,80 @@ func TestArchiveReader(t *testing.T) {
 	t.Logf("snapshot saved as %v, tree is %v", id.Str(), sn.Tree.Str())
 
 	checkSavedFile(t, repo, *sn.Tree, "fakefile", fakeFile(t, seed, size))
+
+	checker.TestCheckRepo(t, repo)
+}
+
+func TestArchiveReaderNull(t *testing.T) {
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
+
+	r := &Reader{
+		Repository: repo,
+		Hostname:   "localhost",
+		Tags:       []string{"test"},
+	}
+
+	sn, id, err := r.Archive("fakefile", bytes.NewReader(nil), nil)
+	if err != nil {
+		t.Fatalf("ArchiveReader() returned error %v", err)
+	}
+
+	if id.IsNull() {
+		t.Fatalf("ArchiveReader() returned null ID")
+	}
+
+	t.Logf("snapshot saved as %v, tree is %v", id.Str(), sn.Tree.Str())
+
+	checker.TestCheckRepo(t, repo)
+}
+
+type errReader string
+
+func (e errReader) Read([]byte) (int, error) {
+	return 0, errors.New(string(e))
+}
+
+func countSnapshots(t testing.TB, repo restic.Repository) int {
+	done := make(chan struct{})
+	defer close(done)
+
+	snapshots := 0
+	for range repo.List(restic.SnapshotFile, done) {
+		snapshots++
+	}
+	return snapshots
+}
+
+func TestArchiveReaderError(t *testing.T) {
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
+
+	r := &Reader{
+		Repository: repo,
+		Hostname:   "localhost",
+		Tags:       []string{"test"},
+	}
+
+	sn, id, err := r.Archive("fakefile", errReader("error returned by reading stdin"), nil)
+	if err == nil {
+		t.Errorf("expected error not returned")
+	}
+
+	if sn != nil {
+		t.Errorf("Snapshot should be nil, but isn't")
+	}
+
+	if !id.IsNull() {
+		t.Errorf("id should be null, but %v returned", id.Str())
+	}
+
+	n := countSnapshots(t, repo)
+	if n > 0 {
+		t.Errorf("expected zero snapshots, but got %d", n)
+	}
+
+	checker.TestCheckRepo(t, repo)
 }
 
 func BenchmarkArchiveReader(t *testing.B) {
@@ -103,11 +185,17 @@ func BenchmarkArchiveReader(t *testing.B) {
 		t.Fatal(err)
 	}
 
+	r := &Reader{
+		Repository: repo,
+		Hostname:   "localhost",
+		Tags:       []string{"test"},
+	}
+
 	t.SetBytes(size)
 	t.ResetTimer()
 
 	for i := 0; i < t.N; i++ {
-		_, _, err := ArchiveReader(repo, nil, bytes.NewReader(buf), "fakefile", []string{"test"})
+		_, _, err := r.Archive("fakefile", bytes.NewReader(buf), nil)
 		if err != nil {
 			t.Fatal(err)
 		}

src/restic/archiver/archiver.go

@@ -142,7 +142,7 @@ func (arch *Archiver) reloadFileIfChanged(node *restic.Node, file fs.File) (*res
 	node, err = restic.NodeFromFileInfo(node.Path, fi)
 	if err != nil {
 		debug.Log("restic.NodeFromFileInfo returned error for %v: %v", node.Path, err)
-		return nil, err
+		arch.Warn(node.Path, fi, err)
 	}
 
 	return node, nil
@@ -275,11 +275,8 @@ func (arch *Archiver) fileWorker(wg *sync.WaitGroup, p *restic.Progress, done <-
 
 			node, err := restic.NodeFromFileInfo(e.Fullpath(), e.Info())
 			if err != nil {
-				// TODO: integrate error reporting
 				debug.Log("restic.NodeFromFileInfo returned error for %v: %v", node.Path, err)
-				e.Result() <- nil
-				p.Report(restic.Stat{Errors: 1})
-				continue
+				arch.Warn(e.Fullpath(), e.Info(), err)
 			}
 
 			// try to use old node, if present
@@ -307,11 +304,11 @@ func (arch *Archiver) fileWorker(wg *sync.WaitGroup, p *restic.Progress, done <-
 
 			// otherwise read file normally
 			if node.Type == "file" && len(node.Content) == 0 {
-				debug.Log("   read and save %v, content: %v", e.Path(), node.Content)
+				debug.Log("   read and save %v", e.Path())
 				node, err = arch.SaveFile(p, node)
 				if err != nil {
-					// TODO: integrate error reporting
 					fmt.Fprintf(os.Stderr, "error for %v: %v\n", node.Path, err)
+					arch.Warn(e.Path(), nil, err)
 					// ignore this file
 					e.Result() <- nil
 					p.Report(restic.Stat{Errors: 1})
@@ -371,25 +368,28 @@ func (arch *Archiver) dirWorker(wg *sync.WaitGroup, p *restic.Progress, done <-c
 
 				// else insert node
 				node := res.(*restic.Node)
-				tree.Insert(node)
 
 				if node.Type == "dir" {
 					debug.Log("got tree node for %s: %v", node.Path, node.Subtree)
 
+					if node.Subtree == nil {
+						debug.Log("subtree is nil for node %v", node.Path)
+						continue
+					}
+
 					if node.Subtree.IsNull() {
 						panic("invalid null subtree restic.ID")
 					}
 				}
+				tree.Insert(node)
 			}
 
 			node := &restic.Node{}
 
 			if dir.Path() != "" && dir.Info() != nil {
-				n, err := restic.NodeFromFileInfo(dir.Path(), dir.Info())
+				n, err := restic.NodeFromFileInfo(dir.Fullpath(), dir.Info())
 				if err != nil {
-					n.Error = err.Error()
-					dir.Result() <- n
-					continue
+					arch.Warn(dir.Path(), dir.Info(), err)
 				}
 				node = n
 			}
@@ -634,7 +634,7 @@ func (p baseNameSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 // Snapshot creates a snapshot of the given paths. If parentrestic.ID is set, this is
 // used to compare the files to the ones archived at the time this snapshot was
 // taken.
-func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, parentID *restic.ID) (*restic.Snapshot, restic.ID, error) {
+func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, hostname string, parentID *restic.ID) (*restic.Snapshot, restic.ID, error) {
 	paths = unique(paths)
 	sort.Sort(baseNameSlice(paths))
 
@@ -650,7 +650,7 @@ func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, parentI
 	defer p.Done()
 
 	// create new snapshot
-	sn, err := restic.NewSnapshot(paths, tags)
+	sn, err := restic.NewSnapshot(paths, tags, hostname)
 	if err != nil {
 		return nil, restic.ID{}, err
 	}
@@ -734,6 +734,21 @@ func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, parentI
 		return nil, restic.ID{}, err
 	}
 
+	// receive the top-level tree
+	root := (<-resCh).(*restic.Node)
+	debug.Log("root node received: %v", root.Subtree.Str())
+	sn.Tree = root.Subtree
+
+	// load top-level tree again to see if it is empty
+	toptree, err := arch.repo.LoadTree(*root.Subtree)
+	if err != nil {
+		return nil, restic.ID{}, err
+	}
+
+	if len(toptree.Nodes) == 0 {
+		return nil, restic.ID{}, errors.Fatal("no files/dirs saved, refusing to create empty snapshot")
+	}
+
 	// save index
 	err = arch.repo.SaveIndex()
 	if err != nil {
@@ -743,11 +758,6 @@ func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, parentI
 
 	debug.Log("saved indexes")
 
-	// receive the top-level tree
-	root := (<-resCh).(*restic.Node)
-	debug.Log("root node received: %v", root.Subtree.Str())
-	sn.Tree = root.Subtree
-
 	// save snapshot
 	id, err := arch.repo.SaveJSONUnpacked(restic.SnapshotFile, sn)
 	if err != nil {

src/restic/archiver/archiver_test.go

@@ -104,7 +104,7 @@ func archiveDirectory(b testing.TB) {
 
 	arch := archiver.New(repo)
 
-	_, id, err := arch.Snapshot(nil, []string{BenchArchiveDirectory}, nil, nil)
+	_, id, err := arch.Snapshot(nil, []string{BenchArchiveDirectory}, nil, "localhost", nil)
 	OK(b, err)
 
 	b.Logf("snapshot archived as %v", id)
@@ -220,7 +220,7 @@ func testParallelSaveWithDuplication(t *testing.T, seed int) {
 
 	errChannels := [](<-chan error){}
 
-	// interweaved processing of subsequent chunks
+	// interwoven processing of subsequent chunks
 	maxParallel := 2*duplication - 1
 	barrier := make(chan struct{}, maxParallel)
 
@@ -294,3 +294,23 @@ func assertNoUnreferencedPacks(t *testing.T, chkr *checker.Checker) {
 		OK(t, err)
 	}
 }
+
+func TestArchiveEmptySnapshot(t *testing.T) {
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
+
+	arch := archiver.New(repo)
+
+	sn, id, err := arch.Snapshot(nil, []string{"file-does-not-exist-123123213123", "file2-does-not-exist-too-123123123"}, nil, "localhost", nil)
+	if err == nil {
+		t.Errorf("expected error for empty snapshot, got nil")
+	}
+
+	if !id.IsNull() {
+		t.Errorf("expected null ID for empty snapshot, got %v", id.Str())
+	}
+
+	if sn != nil {
+		t.Errorf("expected null snapshot for empty snapshot, got %v", sn)
+	}
+}

src/restic/archiver/testing.go

@@ -8,7 +8,7 @@ import (
 // TestSnapshot creates a new snapshot of path.
 func TestSnapshot(t testing.TB, repo restic.Repository, path string, parent *restic.ID) *restic.Snapshot {
 	arch := New(repo)
-	sn, _, err := arch.Snapshot(nil, []string{path}, []string{"test"}, parent)
+	sn, _, err := arch.Snapshot(nil, []string{path}, []string{"test"}, "localhost", parent)
 	if err != nil {
 		t.Fatal(err)
 	}

src/restic/backend/rest/rest.go

@@ -17,7 +17,7 @@ import (
 	"restic/backend"
 )
 
-const connLimit = 10
+const connLimit = 40
 
 // make sure the rest backend implements restic.Backend
 var _ restic.Backend = &restBackend{}

src/restic/backend/s3/s3.go

@@ -3,6 +3,7 @@ package s3
 import (
 	"bytes"
 	"io"
+	"net/http"
 	"path"
 	"restic"
 	"strings"
@@ -15,7 +16,7 @@ import (
 	"restic/debug"
 )
 
-const connLimit = 10
+const connLimit = 40
 
 // s3 is a backend which stores the data on an S3 endpoint.
 type s3 struct {
@@ -36,6 +37,10 @@ func Open(cfg Config) (restic.Backend, error) {
 	}
 
 	be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix}
+
+	tr := &http.Transport{MaxIdleConnsPerHost: connLimit}
+	client.SetCustomTransport(tr)
+
 	be.createConnections()
 
 	found, err := client.BucketExists(cfg.Bucket)
@@ -104,6 +109,18 @@ func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) {
 	return errors.Wrap(err, "client.PutObject")
 }
 
+// wrapReader wraps an io.ReadCloser to run an additional function on Close.
+type wrapReader struct {
+	io.ReadCloser
+	f func()
+}
+
+func (wr wrapReader) Close() error {
+	err := wr.ReadCloser.Close()
+	wr.f()
+	return err
+}
+
 // Load returns a reader that yields the contents of the file at h at the
 // given offset. If length is nonzero, only a portion of the file is
 // returned. rd must be closed after use.
@@ -125,29 +142,49 @@ func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, er
 
 	objName := be.s3path(h)
 
+	// get token for connection
 	<-be.connChan
-	defer func() {
-		be.connChan <- struct{}{}
-	}()
 
 	obj, err := be.client.GetObject(be.bucketname, objName)
 	if err != nil {
 		debug.Log("  err %v", err)
+
+		// return token
+		be.connChan <- struct{}{}
+
 		return nil, errors.Wrap(err, "client.GetObject")
 	}
 
 	// if we're going to read the whole object, just pass it on.
 	if length == 0 {
 		debug.Log("Load %v: pass on object", h)
+
 		_, err = obj.Seek(offset, 0)
 		if err != nil {
 			_ = obj.Close()
+
+			// return token
+			be.connChan <- struct{}{}
+
 			return nil, errors.Wrap(err, "obj.Seek")
 		}
 
-		return obj, nil
+		rd := wrapReader{
+			ReadCloser: obj,
+			f: func() {
+				debug.Log("Close()")
+				// return token
+				be.connChan <- struct{}{}
+			},
+		}
+		return rd, nil
 	}
 
+	defer func() {
+		// return token
+		be.connChan <- struct{}{}
+	}()
+
 	// otherwise use a buffer with ReadAt
 	info, err := obj.Stat()
 	if err != nil {
@@ -157,7 +194,7 @@ func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, er
 
 	if offset > info.Size {
 		_ = obj.Close()
-		return nil, errors.Errorf("offset larger than file size")
+		return nil, errors.New("offset larger than file size")
 	}
 
 	l := int64(length)

src/restic/backend/sftp/sftp.go

@@ -153,7 +153,7 @@ func buildSSHCommand(cfg Config) []string {
 }
 
 // OpenWithConfig opens an sftp backend as described by the config by running
-// "ssh" with the appropiate arguments.
+// "ssh" with the appropriate arguments.
 func OpenWithConfig(cfg Config) (*SFTP, error) {
 	debug.Log("open with config %v", cfg)
 	return Open(cfg.Dir, "ssh", buildSSHCommand(cfg)...)
@@ -193,7 +193,7 @@ func Create(dir string, program string, args ...string) (*SFTP, error) {
 }
 
 // CreateWithConfig creates an sftp backend as described by the config by running
-// "ssh" with the appropiate arguments.
+// "ssh" with the appropriate arguments.
 func CreateWithConfig(cfg Config) (*SFTP, error) {
 	debug.Log("config %v", cfg)
 	return Create(cfg.Dir, "ssh", buildSSHCommand(cfg)...)

src/restic/backend/test/tests.go

@@ -245,21 +245,25 @@ func TestLoad(t testing.TB) {
 		buf, err := ioutil.ReadAll(rd)
 		if err != nil {
 			t.Errorf("Load(%d, %d) ReadAll() returned unexpected error: %v", l, o, err)
+			rd.Close()
 			continue
 		}
 
 		if l <= len(d) && len(buf) != l {
 			t.Errorf("Load(%d, %d) wrong number of bytes read: want %d, got %d", l, o, l, len(buf))
+			rd.Close()
 			continue
 		}
 
 		if l > len(d) && len(buf) != len(d) {
 			t.Errorf("Load(%d, %d) wrong number of bytes read for overlong read: want %d, got %d", l, o, l, len(buf))
+			rd.Close()
 			continue
 		}
 
 		if !bytes.Equal(buf, d) {
 			t.Errorf("Load(%d, %d) returned wrong bytes", l, o)
+			rd.Close()
 			continue
 		}
 

src/restic/blob.go

@@ -14,6 +14,11 @@ type Blob struct {
 	Offset uint
 }
 
+func (b Blob) String() string {
+	return fmt.Sprintf("<Blob (%v) %v, offset %v, length %v>",
+		b.Type, b.ID.Str(), b.Offset, b.Length)
+}
+
 // PackedBlob is a blob stored within a file.
 type PackedBlob struct {
 	Blob

src/restic/checker/checker.go

@@ -1,14 +1,17 @@
 package checker
 
 import (
-	"bytes"
+	"crypto/sha256"
 	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
 	"sync"
 
 	"restic/errors"
+	"restic/hashing"
 
 	"restic"
-	"restic/backend"
 	"restic/crypto"
 	"restic/debug"
 	"restic/pack"
@@ -77,6 +80,7 @@ func (c *Checker) LoadIndex() (hints []error, errs []error) {
 	debug.Log("Start")
 	type indexRes struct {
 		Index *repository.Index
+		err   error
 		ID    string
 	}
 
@@ -92,39 +96,40 @@ func (c *Checker) LoadIndex() (hints []error, errs []error) {
 			idx, err = repository.LoadIndexWithDecoder(c.repo, id, repository.DecodeOldIndex)
 		}
 
-		if err != nil {
-			return err
-		}
+		err = errors.Wrapf(err, "error loading index %v", id.Str())
 
 		select {
-		case indexCh <- indexRes{Index: idx, ID: id.String()}:
+		case indexCh <- indexRes{Index: idx, ID: id.String(), err: err}:
 		case <-done:
 		}
 
 		return nil
 	}
 
-	var perr error
 	go func() {
 		defer close(indexCh)
 		debug.Log("start loading indexes in parallel")
-		perr = repository.FilesInParallel(c.repo.Backend(), restic.IndexFile, defaultParallelism,
+		err := repository.FilesInParallel(c.repo.Backend(), restic.IndexFile, defaultParallelism,
 			repository.ParallelWorkFuncParseID(worker))
-		debug.Log("loading indexes finished, error: %v", perr)
+		debug.Log("loading indexes finished, error: %v", err)
+		if err != nil {
+			panic(err)
+		}
 	}()
 
 	done := make(chan struct{})
 	defer close(done)
 
-	if perr != nil {
-		errs = append(errs, perr)
-		return hints, errs
-	}
-
 	packToIndex := make(map[restic.ID]restic.IDSet)
 
 	for res := range indexCh {
-		debug.Log("process index %v", res.ID)
+		debug.Log("process index %v, err %v", res.ID, res.err)
+
+		if res.err != nil {
+			errs = append(errs, res.err)
+			continue
+		}
+
 		idxID, err := restic.ParseID(res.ID)
 		if err != nil {
 			errs = append(errs, errors.Errorf("unable to parse as index ID: %v", res.ID))
@@ -151,8 +156,6 @@ func (c *Checker) LoadIndex() (hints []error, errs []error) {
 		debug.Log("%d blobs processed", cnt)
 	}
 
-	debug.Log("done, error %v", perr)
-
 	debug.Log("checking for duplicate packs")
 	for packID := range c.packs {
 		debug.Log("  check pack %v: contained in %d indexes", packID.Str(), len(packToIndex[packID]))
@@ -659,36 +662,77 @@ func (c *Checker) CountPacks() uint64 {
 func checkPack(r restic.Repository, id restic.ID) error {
 	debug.Log("checking pack %v", id.Str())
 	h := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	buf, err := backend.LoadAll(r.Backend(), h)
+
+	rd, err := r.Backend().Load(h, 0, 0)
 	if err != nil {
 		return err
 	}
 
-	hash := restic.Hash(buf)
+	packfile, err := ioutil.TempFile("", "restic-temp-check-")
+	if err != nil {
+		return errors.Wrap(err, "TempFile")
+	}
+
+	defer func() {
+		packfile.Close()
+		os.Remove(packfile.Name())
+	}()
+
+	hrd := hashing.NewReader(rd, sha256.New())
+	size, err := io.Copy(packfile, hrd)
+	if err != nil {
+		return errors.Wrap(err, "Copy")
+	}
+
+	if err = rd.Close(); err != nil {
+		return err
+	}
+
+	hash := restic.IDFromHash(hrd.Sum(nil))
+	debug.Log("hash for pack %v is %v", id.Str(), hash.Str())
+
 	if !hash.Equal(id) {
 		debug.Log("Pack ID does not match, want %v, got %v", id.Str(), hash.Str())
 		return errors.Errorf("Pack ID does not match, want %v, got %v", id.Str(), hash.Str())
 	}
 
-	blobs, err := pack.List(r.Key(), bytes.NewReader(buf), int64(len(buf)))
+	blobs, err := pack.List(r.Key(), packfile, size)
 	if err != nil {
 		return err
 	}
 
 	var errs []error
+	var buf []byte
 	for i, blob := range blobs {
-		debug.Log("  check blob %d: %v", i, blob.ID.Str())
+		debug.Log("  check blob %d: %v", i, blob)
 
-		plainBuf := make([]byte, blob.Length)
-		n, err := crypto.Decrypt(r.Key(), plainBuf, buf[blob.Offset:blob.Offset+blob.Length])
+		buf = buf[:cap(buf)]
+		if uint(len(buf)) < blob.Length {
+			buf = make([]byte, blob.Length)
+		}
+		buf = buf[:blob.Length]
+
+		_, err := packfile.Seek(int64(blob.Offset), 0)
+		if err != nil {
+			return errors.Errorf("Seek(%v): %v", blob.Offset, err)
+		}
+
+		_, err = io.ReadFull(packfile, buf)
+		if err != nil {
+			debug.Log("  error loading blob %v: %v", blob.ID.Str(), err)
+			errs = append(errs, errors.Errorf("blob %v: %v", i, err))
+			continue
+		}
+
+		n, err := crypto.Decrypt(r.Key(), buf, buf)
 		if err != nil {
 			debug.Log("  error decrypting blob %v: %v", blob.ID.Str(), err)
 			errs = append(errs, errors.Errorf("blob %v: %v", i, err))
 			continue
 		}
-		plainBuf = plainBuf[:n]
+		buf = buf[:n]
 
-		hash := restic.Hash(plainBuf)
+		hash := restic.Hash(buf)
 		if !hash.Equal(blob.ID) {
 			debug.Log("  Blob ID does not match, want %v, got %v", blob.ID.Str(), hash.Str())
 			errs = append(errs, errors.Errorf("Blob ID does not match, want %v, got %v", blob.ID.Str(), hash.Str()))

src/restic/checker/checker_test.go

@@ -179,6 +179,48 @@ func TestUnreferencedBlobs(t *testing.T) {
 	test.Equals(t, unusedBlobsBySnapshot, blobs)
 }
 
+func TestModifiedIndex(t *testing.T) {
+	repodir, cleanup := test.Env(t, checkerTestData)
+	defer cleanup()
+
+	repo := repository.TestOpenLocal(t, repodir)
+
+	done := make(chan struct{})
+	defer close(done)
+
+	h := restic.Handle{
+		Type: restic.IndexFile,
+		Name: "90f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
+	}
+	f, err := repo.Backend().Load(h, 0, 0)
+	test.OK(t, err)
+
+	// save the index again with a modified name so that the hash doesn't match
+	// the content any more
+	h2 := restic.Handle{
+		Type: restic.IndexFile,
+		Name: "80f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
+	}
+	err = repo.Backend().Save(h2, f)
+	test.OK(t, err)
+
+	test.OK(t, f.Close())
+
+	chkr := checker.New(repo)
+	hints, errs := chkr.LoadIndex()
+	if len(errs) == 0 {
+		t.Fatalf("expected errors not found")
+	}
+
+	for _, err := range errs {
+		t.Logf("found expected error %v", err)
+	}
+
+	if len(hints) > 0 {
+		t.Errorf("expected no hints, got %v: %v", len(hints), hints)
+	}
+}
+
 var checkerDuplicateIndexTestData = filepath.Join("testdata", "duplicate-packs-in-index-test-repo.tar.gz")
 
 func TestDuplicatePacksInIndex(t *testing.T) {
@@ -261,7 +303,7 @@ func TestCheckerModifiedData(t *testing.T) {
 	defer cleanup()
 
 	arch := archiver.New(repo)
-	_, id, err := arch.Snapshot(nil, []string{"."}, nil, nil)
+	_, id, err := arch.Snapshot(nil, []string{"."}, nil, "localhost", nil)
 	test.OK(t, err)
 	t.Logf("archived as %v", id.Str())
 
@@ -299,3 +341,28 @@ func TestCheckerModifiedData(t *testing.T) {
 		t.Fatal("no error found, checker is broken")
 	}
 }
+
+func BenchmarkChecker(t *testing.B) {
+	repodir, cleanup := test.Env(t, checkerTestData)
+	defer cleanup()
+
+	repo := repository.TestOpenLocal(t, repodir)
+
+	chkr := checker.New(repo)
+	hints, errs := chkr.LoadIndex()
+	if len(errs) > 0 {
+		t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
+	}
+
+	if len(hints) > 0 {
+		t.Errorf("expected no hints, got %v: %v", len(hints), hints)
+	}
+
+	t.ResetTimer()
+
+	for i := 0; i < t.N; i++ {
+		test.OKs(t, checkPacks(chkr))
+		test.OKs(t, checkStruct(chkr))
+		test.OKs(t, checkData(chkr))
+	}
+}

src/restic/errors/wrap.go

@@ -18,3 +18,7 @@ var Errorf = errors.Errorf
 // Wrap wraps an error retrieved from outside of restic. Wrapped so that this
 // package does not appear in the stack trace.
 var Wrap = errors.Wrap
+
+// Wrapf returns an error annotating err with the format specifier. If err is
+// nil, Wrapf returns nil.
+var Wrapf = errors.Wrapf

src/restic/fs/doc.go

@@ -1,3 +1,3 @@
-// Package fs implements an OS independend abstraction of a file system
+// Package fs implements an OS independent abstraction of a file system
 // suitable for backup purposes.
 package fs

src/restic/fs/file.go

@@ -102,6 +102,12 @@ func Symlink(oldname, newname string) error {
 	return os.Symlink(fixpath(oldname), fixpath(newname))
 }
 
+// Link creates newname as a hard link to oldname.
+// If there is an error, it will be of type *LinkError.
+func Link(oldname, newname string) error {
+	return os.Link(fixpath(oldname), fixpath(newname))
+}
+
 // Stat returns a FileInfo structure describing the named file.
 // If there is an error, it will be of type *PathError.
 func Stat(name string) (os.FileInfo, error) {

src/restic/fuse/dir.go

@@ -114,9 +114,25 @@ func (d *dir) Attr(ctx context.Context, a *fuse.Attr) error {
 	a.Atime = d.node.AccessTime
 	a.Ctime = d.node.ChangeTime
 	a.Mtime = d.node.ModTime
+
+	a.Nlink = d.calcNumberOfLinks()
+
 	return nil
 }
 
+func (d *dir) calcNumberOfLinks() uint32 {
+	// a directory d has 2 hardlinks + the number
+	// of directories contained by d
+	var count uint32
+	count = 2
+	for _, node := range d.items {
+		if node.Type == "dir" {
+			count++
+		}
+	}
+	return count
+}
+
 func (d *dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
 	debug.Log("called")
 	ret := make([]fuse.Dirent, 0, len(d.items))
@@ -161,3 +177,21 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
 		return nil, fuse.ENOENT
 	}
 }
+
+func (d *dir) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error {
+	debug.Log("Listxattr(%v, %v)", d.node.Name, req.Size)
+	for _, attr := range d.node.ExtendedAttributes {
+		resp.Append(attr.Name)
+	}
+	return nil
+}
+
+func (d *dir) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
+	debug.Log("Getxattr(%v, %v, %v)", d.node.Name, req.Name, req.Size)
+	attrval := d.node.GetExtendedAttribute(req.Name)
+	if attrval != nil {
+		resp.Xattr = attrval
+		return nil
+	}
+	return fuse.ErrNoXattr
+}

src/restic/fuse/file.go

@@ -74,6 +74,7 @@ func (f *file) Attr(ctx context.Context, a *fuse.Attr) error {
 	a.Size = f.node.Size
 	a.Blocks = (f.node.Size / blockSize) + 1
 	a.BlockSize = blockSize
+	a.Nlink = uint32(f.node.Links)
 
 	if !f.ownerIsRoot {
 		a.Uid = f.node.UID
@@ -82,7 +83,9 @@ func (f *file) Attr(ctx context.Context, a *fuse.Attr) error {
 	a.Atime = f.node.AccessTime
 	a.Ctime = f.node.ChangeTime
 	a.Mtime = f.node.ModTime
+
 	return nil
+
 }
 
 func (f *file) getBlobAt(i int) (blob []byte, err error) {
@@ -161,3 +164,21 @@ func (f *file) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
 	}
 	return nil
 }
+
+func (f *file) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error {
+	debug.Log("Listxattr(%v, %v)", f.node.Name, req.Size)
+	for _, attr := range f.node.ExtendedAttributes {
+		resp.Append(attr.Name)
+	}
+	return nil
+}
+
+func (f *file) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
+	debug.Log("Getxattr(%v, %v, %v)", f.node.Name, req.Name, req.Size)
+	attrval := f.node.GetExtendedAttribute(req.Name)
+	if attrval != nil {
+		resp.Xattr = attrval
+		return nil
+	}
+	return fuse.ErrNoXattr
+}

src/restic/fuse/link.go

@@ -38,5 +38,8 @@ func (l *link) Attr(ctx context.Context, a *fuse.Attr) error {
 	a.Atime = l.node.AccessTime
 	a.Ctime = l.node.ChangeTime
 	a.Mtime = l.node.ModTime
+
+	a.Nlink = uint32(l.node.Links)
+
 	return nil
 }

src/restic/fuse/snapshot.go

@@ -32,6 +32,9 @@ var _ = fs.NodeStringLookuper(&SnapshotsDir{})
 type SnapshotsDir struct {
 	repo        restic.Repository
 	ownerIsRoot bool
+	paths       []string
+	tags        []string
+	host        string
 
 	// knownSnapshots maps snapshot timestamp to the snapshot
 	sync.RWMutex
@@ -40,12 +43,15 @@ type SnapshotsDir struct {
 }
 
 // NewSnapshotsDir returns a new dir object for the snapshots.
-func NewSnapshotsDir(repo restic.Repository, ownerIsRoot bool) *SnapshotsDir {
+func NewSnapshotsDir(repo restic.Repository, ownerIsRoot bool, paths []string, tags []string, host string) *SnapshotsDir {
 	debug.Log("fuse mount initiated")
 	return &SnapshotsDir{
 		repo:           repo,
-		knownSnapshots: make(map[string]SnapshotWithId),
 		ownerIsRoot:    ownerIsRoot,
+		paths:          paths,
+		tags:           tags,
+		host:           host,
+		knownSnapshots: make(map[string]SnapshotWithId),
 		processed:      restic.NewIDSet(),
 	}
 }
@@ -79,6 +85,13 @@ func (sn *SnapshotsDir) updateCache(ctx context.Context) error {
 			return err
 		}
 
+		// Filter snapshots we don't care for.
+		if (sn.host != "" && sn.host != snapshot.Hostname) ||
+			!snapshot.HasTags(sn.tags) ||
+			!snapshot.HasPaths(sn.paths) {
+			continue
+		}
+
 		timestamp := snapshot.Time.Format(time.RFC3339)
 		for i := 1; ; i++ {
 			if _, ok := sn.knownSnapshots[timestamp]; !ok {

src/restic/hardlinks_index.go

@@ -0,0 +1,57 @@
+package restic
+
+import (
+	"sync"
+)
+
+// HardlinkKey is a composed key for finding inodes on a specific device.
+type HardlinkKey struct {
+	Inode, Device uint64
+}
+
+// HardlinkIndex contains a list of inodes, devices these inodes are one, and associated file names.
+type HardlinkIndex struct {
+	m     sync.Mutex
+	Index map[HardlinkKey]string
+}
+
+// NewHardlinkIndex create a new index for hard links
+func NewHardlinkIndex() *HardlinkIndex {
+	return &HardlinkIndex{
+		Index: make(map[HardlinkKey]string),
+	}
+}
+
+// Has checks wether the link already exist in the index.
+func (idx *HardlinkIndex) Has(inode uint64, device uint64) bool {
+	idx.m.Lock()
+	defer idx.m.Unlock()
+	_, ok := idx.Index[HardlinkKey{inode, device}]
+
+	return ok
+}
+
+// Add adds a link to the index.
+func (idx *HardlinkIndex) Add(inode uint64, device uint64, name string) {
+	idx.m.Lock()
+	defer idx.m.Unlock()
+	_, ok := idx.Index[HardlinkKey{inode, device}]
+
+	if !ok {
+		idx.Index[HardlinkKey{inode, device}] = name
+	}
+}
+
+// GetFilename obtains the filename from the index.
+func (idx *HardlinkIndex) GetFilename(inode uint64, device uint64) string {
+	idx.m.Lock()
+	defer idx.m.Unlock()
+	return idx.Index[HardlinkKey{inode, device}]
+}
+
+// Remove removes a link from the index.
+func (idx *HardlinkIndex) Remove(inode uint64, device uint64) {
+	idx.m.Lock()
+	defer idx.m.Unlock()
+	delete(idx.Index, HardlinkKey{inode, device})
+}

src/restic/hardlinks_index_test.go

@@ -0,0 +1,35 @@
+package restic_test
+
+import (
+	"testing"
+
+	"restic"
+	. "restic/test"
+)
+
+// TestHardLinks contains various tests for HardlinkIndex.
+func TestHardLinks(t *testing.T) {
+
+	idx := restic.NewHardlinkIndex()
+
+	idx.Add(1, 2, "inode1-file1-on-device2")
+	idx.Add(2, 3, "inode2-file2-on-device3")
+
+	var sresult string
+	sresult = idx.GetFilename(1, 2)
+	Equals(t, sresult, "inode1-file1-on-device2")
+
+	sresult = idx.GetFilename(2, 3)
+	Equals(t, sresult, "inode2-file2-on-device3")
+
+	var bresult bool
+	bresult = idx.Has(1, 2)
+	Equals(t, bresult, true)
+
+	bresult = idx.Has(1, 3)
+	Equals(t, bresult, false)
+
+	idx.Remove(1, 2)
+	bresult = idx.Has(1, 2)
+	Equals(t, bresult, false)
+}

src/restic/id.go

@@ -43,7 +43,7 @@ func (id ID) String() string {
 	return hex.EncodeToString(id[:])
 }
 
-// NewRandomID retuns a randomly generated ID. When reading from rand fails,
+// NewRandomID returns a randomly generated ID. When reading from rand fails,
 // the function panics.
 func NewRandomID() ID {
 	id := ID{}

src/restic/list/list.go

@@ -25,7 +25,7 @@ func (l Result) PackID() restic.ID {
 	return l.packID
 }
 
-// Size ruturns the size of the pack.
+// Size returns the size of the pack.
 func (l Result) Size() int64 {
 	return l.size
 }

src/restic/lock.go

@@ -203,7 +203,7 @@ func (l *Lock) Stale() bool {
 
 	hn, err := os.Hostname()
 	if err != nil {
-		debug.Log("unable to find current hostnanme: %v", err)
+		debug.Log("unable to find current hostname: %v", err)
 		// since we cannot find the current hostname, assume that the lock is
 		// not stale.
 		return false

src/restic/node.go

@@ -12,31 +12,38 @@ import (
 
 	"restic/errors"
 
-	"runtime"
-
+	"bytes"
 	"restic/debug"
 	"restic/fs"
+	"runtime"
 )
 
+// ExtendedAttribute is a tuple storing the xattr name and value.
+type ExtendedAttribute struct {
+	Name  string `json:"name"`
+	Value []byte `json:"value"`
+}
+
 // Node is a file, directory or other item in a backup.
 type Node struct {
-	Name       string      `json:"name"`
-	Type       string      `json:"type"`
-	Mode       os.FileMode `json:"mode,omitempty"`
-	ModTime    time.Time   `json:"mtime,omitempty"`
-	AccessTime time.Time   `json:"atime,omitempty"`
-	ChangeTime time.Time   `json:"ctime,omitempty"`
-	UID        uint32      `json:"uid"`
-	GID        uint32      `json:"gid"`
-	User       string      `json:"user,omitempty"`
-	Group      string      `json:"group,omitempty"`
-	Inode      uint64      `json:"inode,omitempty"`
-	Size       uint64      `json:"size,omitempty"`
-	Links      uint64      `json:"links,omitempty"`
-	LinkTarget string      `json:"linktarget,omitempty"`
-	Device     uint64      `json:"device,omitempty"`
-	Content    IDs         `json:"content"`
-	Subtree    *ID         `json:"subtree,omitempty"`
+	Name               string              `json:"name"`
+	Type               string              `json:"type"`
+	Mode               os.FileMode         `json:"mode,omitempty"`
+	ModTime            time.Time           `json:"mtime,omitempty"`
+	AccessTime         time.Time           `json:"atime,omitempty"`
+	ChangeTime         time.Time           `json:"ctime,omitempty"`
+	UID                uint32              `json:"uid"`
+	GID                uint32              `json:"gid"`
+	User               string              `json:"user,omitempty"`
+	Group              string              `json:"group,omitempty"`
+	Inode              uint64              `json:"inode,omitempty"`
+	Size               uint64              `json:"size,omitempty"`
+	Links              uint64              `json:"links,omitempty"`
+	LinkTarget         string              `json:"linktarget,omitempty"`
+	ExtendedAttributes []ExtendedAttribute `json:"extended_attributes,omitempty"`
+	Device             uint64              `json:"device,omitempty"`
+	Content            IDs                 `json:"content"`
+	Subtree            *ID                 `json:"subtree,omitempty"`
 
 	Error string `json:"error,omitempty"`
 
@@ -56,7 +63,8 @@ func (node Node) String() string {
 	return fmt.Sprintf("<Node(%s) %s>", node.Type, node.Name)
 }
 
-// NodeFromFileInfo returns a new node from the given path and FileInfo.
+// NodeFromFileInfo returns a new node from the given path and FileInfo. It
+// returns the first error that is encountered, together with a node.
 func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) {
 	mask := os.ModePerm | os.ModeType | os.ModeSetuid | os.ModeSetgid | os.ModeSticky
 	node := &Node{
@@ -96,8 +104,18 @@ func nodeTypeFromFileInfo(fi os.FileInfo) string {
 	return ""
 }
 
+// GetExtendedAttribute gets the extended attribute.
+func (node Node) GetExtendedAttribute(a string) []byte {
+	for _, attr := range node.ExtendedAttributes {
+		if attr.Name == a {
+			return attr.Value
+		}
+	}
+	return nil
+}
+
 // CreateAt creates the node at the given path and restores all the meta data.
-func (node *Node) CreateAt(path string, repo Repository) error {
+func (node *Node) CreateAt(path string, repo Repository, idx *HardlinkIndex) error {
 	debug.Log("create node %v at %v", node.Name, path)
 
 	switch node.Type {
@@ -106,7 +124,7 @@ func (node *Node) CreateAt(path string, repo Repository) error {
 			return err
 		}
 	case "file":
-		if err := node.createFileAt(path, repo); err != nil {
+		if err := node.createFileAt(path, repo, idx); err != nil {
 			return err
 		}
 	case "symlink":
@@ -162,6 +180,22 @@ func (node Node) restoreMetadata(path string) error {
 		}
 	}
 
+	err = node.restoreExtendedAttributes(path)
+	if err != nil {
+		debug.Log("error restoring extended attributes for %v: %v", path, err)
+		return err
+	}
+
+	return nil
+}
+
+func (node Node) restoreExtendedAttributes(path string) error {
+	for _, attr := range node.ExtendedAttributes {
+		err := Setxattr(path, attr.Name, attr.Value)
+		if err != nil {
+			return err
+		}
+	}
 	return nil
 }
 
@@ -191,7 +225,15 @@ func (node Node) createDirAt(path string) error {
 	return nil
 }
 
-func (node Node) createFileAt(path string, repo Repository) error {
+func (node Node) createFileAt(path string, repo Repository, idx *HardlinkIndex) error {
+	if node.Links > 1 && idx.Has(node.Inode, node.Device) {
+		err := fs.Link(idx.GetFilename(node.Inode, node.Device), path)
+		if err != nil {
+			return errors.Wrap(err, "CreateHardlink")
+		}
+		return nil
+	}
+
 	f, err := fs.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0600)
 	defer f.Close()
 
@@ -207,7 +249,7 @@ func (node Node) createFileAt(path string, repo Repository) error {
 		}
 
 		buf = buf[:cap(buf)]
-		if uint(len(buf)) < size {
+		if len(buf) < CiphertextLength(int(size)) {
 			buf = NewBlobBuffer(int(size))
 		}
 
@@ -223,6 +265,8 @@ func (node Node) createFileAt(path string, repo Repository) error {
 		}
 	}
 
+	idx.Add(node.Inode, node.Device, path)
+
 	return nil
 }
 
@@ -340,6 +384,9 @@ func (node Node) Equals(other Node) bool {
 	if !node.sameContent(other) {
 		return false
 	}
+	if !node.sameExtendedAttributes(other) {
+		return false
+	}
 	if node.Subtree != nil {
 		if other.Subtree == nil {
 			return false
@@ -378,6 +425,51 @@ func (node Node) sameContent(other Node) bool {
 			return false
 		}
 	}
+	return true
+}
+
+func (node Node) sameExtendedAttributes(other Node) bool {
+	if len(node.ExtendedAttributes) != len(other.ExtendedAttributes) {
+		return false
+	}
+
+	// build a set of all attributes that node has
+	type mapvalue struct {
+		value   []byte
+		present bool
+	}
+	attributes := make(map[string]mapvalue)
+	for _, attr := range node.ExtendedAttributes {
+		attributes[attr.Name] = mapvalue{value: attr.Value}
+	}
+
+	for _, attr := range other.ExtendedAttributes {
+		v, ok := attributes[attr.Name]
+		if !ok {
+			// extended attribute is not set for node
+			debug.Log("other node has attribute %v, which is not present in node", attr.Name)
+			return false
+
+		}
+
+		if !bytes.Equal(v.value, attr.Value) {
+			// attribute has different value
+			debug.Log("attribute %v has different value", attr.Name)
+			return false
+		}
+
+		// remember that this attribute is present in other.
+		v.present = true
+		attributes[attr.Name] = v
+	}
+
+	// check for attributes that are not present in other
+	for name, v := range attributes {
+		if !v.present {
+			debug.Log("attribute %v not present in other node", name)
+			return false
+		}
+	}
 
 	return true
 }
@@ -485,18 +577,56 @@ func (node *Node) fillExtra(path string, fi os.FileInfo) error {
 	case "dir":
 	case "symlink":
 		node.LinkTarget, err = fs.Readlink(path)
-		err = errors.Wrap(err, "Readlink")
+		node.Links = uint64(stat.nlink())
+		if err != nil {
+			return errors.Wrap(err, "Readlink")
+		}
 	case "dev":
 		node.Device = uint64(stat.rdev())
+		node.Links = uint64(stat.nlink())
 	case "chardev":
 		node.Device = uint64(stat.rdev())
+		node.Links = uint64(stat.nlink())
 	case "fifo":
 	case "socket":
 	default:
-		err = errors.Errorf("invalid node type %q", node.Type)
+		return errors.Errorf("invalid node type %q", node.Type)
 	}
 
-	return err
+	if err = node.fillExtendedAttributes(path); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (node *Node) fillExtendedAttributes(path string) error {
+	if node.Type == "symlink" {
+		return nil
+	}
+
+	xattrs, err := Listxattr(path)
+	debug.Log("fillExtendedAttributes(%v) %v %v", path, xattrs, err)
+	if err != nil {
+		return err
+	}
+
+	node.ExtendedAttributes = make([]ExtendedAttribute, 0, len(xattrs))
+	for _, attr := range xattrs {
+		attrVal, err := Getxattr(path, attr)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "can not obtain extended attribute %v for %v:\n", attr, path)
+			continue
+		}
+		attr := ExtendedAttribute{
+			Name:  attr,
+			Value: attrVal,
+		}
+
+		node.ExtendedAttributes = append(node.ExtendedAttributes, attr)
+	}
+
+	return nil
 }
 
 type statT interface {

src/restic/node_openbsd.go

@@ -9,3 +9,19 @@ func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespe
 func (s statUnix) atim() syscall.Timespec { return s.Atim }
 func (s statUnix) mtim() syscall.Timespec { return s.Mtim }
 func (s statUnix) ctim() syscall.Timespec { return s.Ctim }
+
+// Getxattr retrieves extended attribute data associated with path.
+func Getxattr(path, name string) ([]byte, error) {
+	return nil, nil
+}
+
+// Listxattr retrieves a list of names of extended attributes associated with the
+// given path in the file system.
+func Listxattr(path string) ([]string, error) {
+	return nil, nil
+}
+
+// Setxattr associates name and data together as an attribute of path.
+func Setxattr(path, name string, data []byte) error {
+	return nil
+}

src/restic/node_test.go

@@ -176,9 +176,11 @@ func TestNodeRestoreAt(t *testing.T) {
 		}
 	}()
 
+	idx := restic.NewHardlinkIndex()
+
 	for _, test := range nodeTests {
 		nodePath := filepath.Join(tempdir, test.Name)
-		OK(t, test.CreateAt(nodePath, nil))
+		OK(t, test.CreateAt(nodePath, nil, idx))
 
 		if test.Type == "symlink" && runtime.GOOS == "windows" {
 			continue

src/restic/node_windows.go

@@ -22,8 +22,25 @@ func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespe
 	return nil
 }
 
+// Getxattr retrieves extended attribute data associated with path.
+func Getxattr(path, name string) ([]byte, error) {
+	return nil, nil
+}
+
+// Listxattr retrieves a list of names of extended attributes associated with the
+// given path in the file system.
+func Listxattr(path string) ([]string, error) {
+	return nil, nil
+}
+
+// Setxattr associates name and data together as an attribute of path.
+func Setxattr(path, name string, data []byte) error {
+	return nil
+}
+
 type statWin syscall.Win32FileAttributeData
 
+//ToStatT call the Windows system call Win32FileAttributeData.
 func toStatT(i interface{}) (statT, bool) {
 	if i == nil {
 		return nil, false

src/restic/node_xattr.go

@@ -0,0 +1,39 @@
+// +build !openbsd
+// +build !windows
+
+package restic
+
+import (
+	"restic/errors"
+	"syscall"
+
+	"github.com/pkg/xattr"
+)
+
+// Getxattr retrieves extended attribute data associated with path.
+func Getxattr(path, name string) ([]byte, error) {
+	b, e := xattr.Getxattr(path, name)
+	if err, ok := e.(*xattr.XAttrError); ok && err.Err == syscall.ENOTSUP {
+		return nil, nil
+	}
+	return b, errors.Wrap(e, "Getxattr")
+}
+
+// Listxattr retrieves a list of names of extended attributes associated with the
+// given path in the file system.
+func Listxattr(path string) ([]string, error) {
+	s, e := xattr.Listxattr(path)
+	if err, ok := e.(*xattr.XAttrError); ok && err.Err == syscall.ENOTSUP {
+		return nil, nil
+	}
+	return s, errors.Wrap(e, "Listxattr")
+}
+
+// Setxattr associates name and data together as an attribute of path.
+func Setxattr(path, name string, data []byte) error {
+	e := xattr.Setxattr(path, name, data)
+	if err, ok := e.(*xattr.XAttrError); ok && err.Err == syscall.ENOTSUP {
+		return nil
+	}
+	return errors.Wrap(e, "Setxattr")
+}

src/restic/repository/index.go

@@ -392,7 +392,7 @@ func (idx *Index) SetID(id restic.ID) error {
 	defer idx.m.Unlock()
 
 	if !idx.final {
-		return errors.New("indexs is not final")
+		return errors.New("index is not final")
 	}
 
 	if !idx.id.IsNull() {

src/restic/repository/index_rebuild.go

@@ -1,66 +0,0 @@
-package repository
-
-import (
-	"fmt"
-	"os"
-	"restic"
-	"restic/debug"
-	"restic/list"
-	"restic/worker"
-)
-
-// RebuildIndex lists all packs in the repo, writes a new index and removes all
-// old indexes. This operation should only be done with an exclusive lock in
-// place.
-func RebuildIndex(repo restic.Repository) error {
-	debug.Log("start rebuilding index")
-
-	done := make(chan struct{})
-	defer close(done)
-
-	ch := make(chan worker.Job)
-	go list.AllPacks(repo, ch, done)
-
-	idx := NewIndex()
-	for job := range ch {
-		id := job.Data.(restic.ID)
-
-		if job.Error != nil {
-			fmt.Fprintf(os.Stderr, "error for pack %v: %v\n", id, job.Error)
-			continue
-		}
-
-		res := job.Result.(list.Result)
-
-		for _, entry := range res.Entries() {
-			pb := restic.PackedBlob{
-				Blob:   entry,
-				PackID: res.PackID(),
-			}
-			idx.Store(pb)
-		}
-	}
-
-	oldIndexes := restic.NewIDSet()
-	for id := range repo.List(restic.IndexFile, done) {
-		idx.AddToSupersedes(id)
-		oldIndexes.Insert(id)
-	}
-
-	id, err := SaveIndex(repo, idx)
-	if err != nil {
-		debug.Log("error saving index: %v", err)
-		return err
-	}
-	debug.Log("new index saved as %v", id.Str())
-
-	for indexID := range oldIndexes {
-		h := restic.Handle{Type: restic.IndexFile, Name: indexID.String()}
-		err := repo.Backend().Remove(h)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "unable to remove index %v: %v\n", indexID.Str(), err)
-		}
-	}
-
-	return nil
-}

src/restic/repository/repack.go

@@ -18,7 +18,7 @@ import (
 // these packs. Each pack is loaded and the blobs listed in keepBlobs is saved
 // into a new pack. Afterwards, the packs are removed. This operation requires
 // an exclusive lock on the repo.
-func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet) (err error) {
+func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet, p *restic.Progress) (err error) {
 	debug.Log("repacking %d packs while keeping %d blobs", len(packs), len(keepBlobs))
 
 	for packID := range packs {
@@ -35,14 +35,16 @@ func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet
 			return err
 		}
 
-		defer beRd.Close()
-
 		hrd := hashing.NewReader(beRd, sha256.New())
 		packLength, err := io.Copy(tempfile, hrd)
 		if err != nil {
 			return errors.Wrap(err, "Copy")
 		}
 
+		if err = beRd.Close(); err != nil {
+			return errors.Wrap(err, "Close")
+		}
+
 		hash := restic.IDFromHash(hrd.Sum(nil))
 		debug.Log("pack %v loaded (%d bytes), hash %v", packID.Str(), packLength, hash.Str())
 
@@ -116,6 +118,9 @@ func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet
 		if err = os.Remove(tempfile.Name()); err != nil {
 			return errors.Wrap(err, "Remove")
 		}
+		if p != nil {
+			p.Report(restic.Stat{Blobs: 1})
+		}
 	}
 
 	if err := repo.Flush(); err != nil {

src/restic/repository/repack_test.go

@@ -4,6 +4,7 @@ import (
 	"io"
 	"math/rand"
 	"restic"
+	"restic/index"
 	"restic/repository"
 	"testing"
 )
@@ -131,7 +132,7 @@ func findPacksForBlobs(t *testing.T, repo restic.Repository, blobs restic.BlobSe
 }
 
 func repack(t *testing.T, repo restic.Repository, packs restic.IDSet, blobs restic.BlobSet) {
-	err := repository.Repack(repo, packs, blobs)
+	err := repository.Repack(repo, packs, blobs, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -144,8 +145,24 @@ func saveIndex(t *testing.T, repo restic.Repository) {
 }
 
 func rebuildIndex(t *testing.T, repo restic.Repository) {
-	if err := repository.RebuildIndex(repo); err != nil {
-		t.Fatalf("error rebuilding index: %v", err)
+	idx, err := index.New(repo, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for id := range repo.List(restic.IndexFile, nil) {
+		err = repo.Backend().Remove(restic.Handle{
+			Type: restic.IndexFile,
+			Name: id.String(),
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	_, err = idx.Save(repo, nil)
+	if err != nil {
+		t.Fatal(err)
 	}
 }
 

src/restic/repository/repository.go

@@ -56,12 +56,12 @@ func (r *Repository) LoadAndDecrypt(t restic.FileType, id restic.ID) ([]byte, er
 	h := restic.Handle{Type: t, Name: id.String()}
 	buf, err := backend.LoadAll(r.be, h)
 	if err != nil {
-		debug.Log("error loading %v: %v", id.Str(), err)
+		debug.Log("error loading %v: %v", h, err)
 		return nil, err
 	}
 
 	if t != restic.ConfigFile && !restic.Hash(buf).Equal(id) {
-		return nil, errors.New("invalid data returned")
+		return nil, errors.Errorf("load %v: invalid data returned", h)
 	}
 
 	// decrypt
@@ -442,50 +442,22 @@ func (r *Repository) KeyName() string {
 	return r.keyName
 }
 
-func (r *Repository) list(t restic.FileType, done <-chan struct{}, out chan<- restic.ID) {
-	defer close(out)
-	in := r.be.List(t, done)
-
-	var (
-		// disable sending on the outCh until we received a job
-		outCh chan<- restic.ID
-		// enable receiving from in
-		inCh = in
-		id   restic.ID
-		err  error
-	)
-
-	for {
-		select {
-		case <-done:
-			return
-		case strID, ok := <-inCh:
-			if !ok {
-				// input channel closed, we're done
-				return
-			}
-			id, err = restic.ParseID(strID)
-			if err != nil {
-				// ignore invalid IDs
-				continue
-			}
-
-			inCh = nil
-			outCh = out
-		case outCh <- id:
-			outCh = nil
-			inCh = in
-		}
-	}
-}
-
 // List returns a channel that yields all IDs of type t in the backend.
 func (r *Repository) List(t restic.FileType, done <-chan struct{}) <-chan restic.ID {
-	outCh := make(chan restic.ID)
-
-	go r.list(t, done, outCh)
-
-	return outCh
+	out := make(chan restic.ID)
+	go func() {
+		defer close(out)
+		for strID := range r.be.List(t, done) {
+			if id, err := restic.ParseID(strID); err == nil {
+				select {
+				case out <- id:
+				case <-done:
+					return
+				}
+			}
+		}
+	}()
+	return out
 }
 
 // ListPack returns the list of blobs saved in the pack id and the length of

src/restic/restorer.go

@@ -38,7 +38,7 @@ func NewRestorer(repo Repository, id ID) (*Restorer, error) {
 	return r, nil
 }
 
-func (res *Restorer) restoreTo(dst string, dir string, treeID ID) error {
+func (res *Restorer) restoreTo(dst string, dir string, treeID ID, idx *HardlinkIndex) error {
 	tree, err := res.repo.LoadTree(treeID)
 	if err != nil {
 		return res.Error(dir, nil, err)
@@ -50,7 +50,7 @@ func (res *Restorer) restoreTo(dst string, dir string, treeID ID) error {
 		debug.Log("SelectForRestore returned %v", selectedForRestore)
 
 		if selectedForRestore {
-			err := res.restoreNodeTo(node, dir, dst)
+			err := res.restoreNodeTo(node, dir, dst, idx)
 			if err != nil {
 				return err
 			}
@@ -62,7 +62,7 @@ func (res *Restorer) restoreTo(dst string, dir string, treeID ID) error {
 			}
 
 			subp := filepath.Join(dir, node.Name)
-			err = res.restoreTo(dst, subp, *node.Subtree)
+			err = res.restoreTo(dst, subp, *node.Subtree, idx)
 			if err != nil {
 				err = res.Error(subp, node, err)
 				if err != nil {
@@ -83,11 +83,11 @@ func (res *Restorer) restoreTo(dst string, dir string, treeID ID) error {
 	return nil
 }
 
-func (res *Restorer) restoreNodeTo(node *Node, dir string, dst string) error {
+func (res *Restorer) restoreNodeTo(node *Node, dir string, dst string, idx *HardlinkIndex) error {
 	debug.Log("node %v, dir %v, dst %v", node.Name, dir, dst)
 	dstPath := filepath.Join(dst, dir, node.Name)
 
-	err := node.CreateAt(dstPath, res.repo)
+	err := node.CreateAt(dstPath, res.repo, idx)
 	if err != nil {
 		debug.Log("node.CreateAt(%s) error %v", dstPath, err)
 	}
@@ -99,7 +99,7 @@ func (res *Restorer) restoreNodeTo(node *Node, dir string, dst string) error {
 		// Create parent directories and retry
 		err = fs.MkdirAll(filepath.Dir(dstPath), 0700)
 		if err == nil || os.IsExist(errors.Cause(err)) {
-			err = node.CreateAt(dstPath, res.repo)
+			err = node.CreateAt(dstPath, res.repo, idx)
 		}
 	}
 
@@ -116,10 +116,11 @@ func (res *Restorer) restoreNodeTo(node *Node, dir string, dst string) error {
 	return nil
 }
 
-// RestoreTo creates the directories and files in the snapshot below dir.
+// RestoreTo creates the directories and files in the snapshot below dst.
 // Before an item is created, res.Filter is called.
-func (res *Restorer) RestoreTo(dir string) error {
-	return res.restoreTo(dir, "", *res.sn.Tree)
+func (res *Restorer) RestoreTo(dst string) error {
+	idx := NewHardlinkIndex()
+	return res.restoreTo(dst, string(filepath.Separator), *res.sn.Tree, idx)
 }
 
 // Snapshot returns the snapshot this restorer is configured to use.

src/restic/snapshot.go

@@ -2,7 +2,6 @@ package restic
 
 import (
 	"fmt"
-	"os"
 	"os/user"
 	"path/filepath"
 	"time"
@@ -22,13 +21,14 @@ type Snapshot struct {
 	GID      uint32    `json:"gid,omitempty"`
 	Excludes []string  `json:"excludes,omitempty"`
 	Tags     []string  `json:"tags,omitempty"`
+	Original *ID       `json:"original,omitempty"`
 
 	id *ID // plaintext ID, used during restore
 }
 
 // NewSnapshot returns an initialized snapshot struct for the current user and
 // time.
-func NewSnapshot(paths []string, tags []string) (*Snapshot, error) {
+func NewSnapshot(paths []string, tags []string, hostname string) (*Snapshot, error) {
 	for i, path := range paths {
 		if p, err := filepath.Abs(path); err != nil {
 			paths[i] = p
@@ -36,17 +36,13 @@ func NewSnapshot(paths []string, tags []string) (*Snapshot, error) {
 	}
 
 	sn := &Snapshot{
-		Paths: paths,
-		Time:  time.Now(),
-		Tags:  tags,
+		Paths:    paths,
+		Time:     time.Now(),
+		Tags:     tags,
+		Hostname: hostname,
 	}
 
-	hn, err := os.Hostname()
-	if err == nil {
-		sn.Hostname = hn
-	}
-
-	err = sn.fillUserInfo()
+	err := sn.fillUserInfo()
 	if err != nil {
 		return nil, err
 	}
@@ -78,8 +74,7 @@ func LoadAllSnapshots(repo Repository) (snapshots []*Snapshot, err error) {
 
 		snapshots = append(snapshots, sn)
 	}
-
-	return snapshots, nil
+	return
 }
 
 func (sn Snapshot) String() string {
@@ -87,7 +82,7 @@ func (sn Snapshot) String() string {
 		sn.id.Str(), sn.Paths, sn.Time, sn.Username, sn.Hostname)
 }
 
-// ID retuns the snapshot's ID.
+// ID returns the snapshot's ID.
 func (sn Snapshot) ID() *ID {
 	return sn.id
 }
@@ -104,7 +99,42 @@ func (sn *Snapshot) fillUserInfo() error {
 	return err
 }
 
-// HasTags returns true if the snapshot has all the tags.
+// AddTags adds the given tags to the snapshots tags, preventing duplicates.
+// It returns true if any changes were made.
+func (sn *Snapshot) AddTags(addTags []string) (changed bool) {
+nextTag:
+	for _, add := range addTags {
+		for _, tag := range sn.Tags {
+			if tag == add {
+				continue nextTag
+			}
+		}
+		sn.Tags = append(sn.Tags, add)
+		changed = true
+	}
+	return
+}
+
+// RemoveTags removes the given tags from the snapshots tags and
+// returns true if any changes were made.
+func (sn *Snapshot) RemoveTags(removeTags []string) (changed bool) {
+	for _, remove := range removeTags {
+		for i, tag := range sn.Tags {
+			if tag == remove {
+				// https://github.com/golang/go/wiki/SliceTricks
+				sn.Tags[i] = sn.Tags[len(sn.Tags)-1]
+				sn.Tags[len(sn.Tags)-1] = ""
+				sn.Tags = sn.Tags[:len(sn.Tags)-1]
+
+				changed = true
+				break
+			}
+		}
+	}
+	return
+}
+
+// HasTags returns true if the snapshot has at least all of tags.
 func (sn *Snapshot) HasTags(tags []string) bool {
 nextTag:
 	for _, tag := range tags {
@@ -120,33 +150,35 @@ nextTag:
 	return true
 }
 
-// SamePaths compares the Snapshot's paths and provided paths are exactly the same
-func SamePaths(expected, actual []string) bool {
-	if len(expected) == 0 || len(actual) == 0 {
-		return true
-	}
-
-	for i := range expected {
-		found := false
-		for j := range actual {
-			if expected[i] == actual[j] {
-				found = true
-				break
+// HasPaths returns true if the snapshot has at least all of paths.
+func (sn *Snapshot) HasPaths(paths []string) bool {
+nextPath:
+	for _, path := range paths {
+		for _, snPath := range sn.Paths {
+			if path == snPath {
+				continue nextPath
 			}
 		}
-		if !found {
-			return false
-		}
+
+		return false
 	}
 
 	return true
 }
 
+// SamePaths returns true if the snapshot matches the entire paths set
+func (sn *Snapshot) SamePaths(paths []string) bool {
+	if len(sn.Paths) != len(paths) {
+		return false
+	}
+	return sn.HasPaths(paths)
+}
+
 // ErrNoSnapshotFound is returned when no snapshot for the given criteria could be found.
 var ErrNoSnapshotFound = errors.New("no snapshot found")
 
-// FindLatestSnapshot finds latest snapshot with optional target/directory and hostname filters.
-func FindLatestSnapshot(repo Repository, targets []string, hostname string) (ID, error) {
+// FindLatestSnapshot finds latest snapshot with optional target/directory, tags and hostname filters.
+func FindLatestSnapshot(repo Repository, targets []string, tags []string, hostname string) (ID, error) {
 	var (
 		latest   time.Time
 		latestID ID
@@ -158,7 +190,7 @@ func FindLatestSnapshot(repo Repository, targets []string, hostname string) (ID,
 		if err != nil {
 			return ID{}, errors.Errorf("Error listing snapshot: %v", err)
 		}
-		if snapshot.Time.After(latest) && SamePaths(snapshot.Paths, targets) && (hostname == "" || hostname == snapshot.Hostname) {
+		if snapshot.Time.After(latest) && (hostname == "" || hostname == snapshot.Hostname) && snapshot.HasTags(tags) && snapshot.HasPaths(targets) {
 			latest = snapshot.Time
 			latestID = snapshotID
 			found = true

src/restic/snapshot_test.go

@@ -10,6 +10,6 @@ import (
 func TestNewSnapshot(t *testing.T) {
 	paths := []string{"/home/foobar"}
 
-	_, err := restic.NewSnapshot(paths, nil)
+	_, err := restic.NewSnapshot(paths, nil, "foo")
 	OK(t, err)
 }

src/restic/testing.go

@@ -163,7 +163,7 @@ func TestCreateSnapshot(t testing.TB, repo Repository, at time.Time, depth int,
 	t.Logf("create fake snapshot at %s with seed %d", at, seed)
 
 	fakedir := fmt.Sprintf("fakedir-at-%v", at.Format("2006-01-02 15:04:05"))
-	snapshot, err := NewSnapshot([]string{fakedir}, []string{"test"})
+	snapshot, err := NewSnapshot([]string{fakedir}, []string{"test"}, "foo")
 	if err != nil {
 		t.Fatal(err)
 	}

src/restic/tree_test.go

@@ -82,7 +82,7 @@ func TestNodeComparison(t *testing.T) {
 	fi, err := os.Lstat("tree_test.go")
 	OK(t, err)
 
-	node, err := restic.NodeFromFileInfo("foo", fi)
+	node, err := restic.NodeFromFileInfo("tree_test.go", fi)
 	OK(t, err)
 
 	n2 := *node

src/restic/walk/walk_test.go

@@ -24,7 +24,7 @@ func TestWalkTree(t *testing.T) {
 
 	// archive a few files
 	arch := archiver.New(repo)
-	sn, _, err := arch.Snapshot(nil, dirs, nil, nil)
+	sn, _, err := arch.Snapshot(nil, dirs, nil, "localhost", nil)
 	OK(t, err)
 
 	// flush repo, write all packs

vendor/manifest

@@ -49,6 +49,12 @@
 			"revision": "8197a2e580736b78d704be0fc47b2324c0591a32",
 			"branch": "master"
 		},
+		{
+			"importpath": "github.com/pkg/xattr",
+			"repository": "https://github.com/pkg/xattr",
+			"revision": "b867675798fa7708a444945602b452ca493f2272",
+			"branch": "master"
+		},
 		{
 			"importpath": "github.com/restic/chunker",
 			"repository": "https://github.com/restic/chunker",

vendor/src/github.com/pkg/xattr/LICENSE

@@ -0,0 +1,25 @@
+Copyright (c) 2012 Dave Cheney. All rights reserved.
+Copyright (c) 2014 Kuba Podgórski. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/src/github.com/pkg/xattr/README.md

@@ -0,0 +1,25 @@
+[![GoDoc](https://godoc.org/github.com/pkg/xattr?status.svg)](http://godoc.org/github.com/pkg/xattr)
+[![Go Report Card](https://goreportcard.com/badge/github.com/pkg/xattr)](https://goreportcard.com/report/github.com/pkg/xattr)
+[![Build Status](https://travis-ci.org/pkg/xattr.svg?branch=master)](https://travis-ci.org/pkg/xattr)
+
+xattr
+=====
+Extended attribute support for Go (linux + darwin + freebsd).
+
+"Extended attributes are name:value pairs associated permanently with files and directories, similar to the environment strings associated with a process. An attribute may be defined or undefined. If it is defined, its value may be empty or non-empty." [See more...](https://en.wikipedia.org/wiki/Extended_file_attributes)
+
+
+### Example
+```
+  const path = "/tmp/myfile"
+  const prefix = "user."
+
+  if err := xattr.Setxattr(path, prefix+"test", []byte("test-attr-value")); err != nil {
+    log.Fatal(err)
+  }
+
+  var data []byte
+  data, err = xattr.Getxattr(path, prefix+"test"); err != nil {
+    log.Fatal(err)
+  }
+```

vendor/src/github.com/pkg/xattr/syscall_darwin.go

@@ -0,0 +1,39 @@
+// +build darwin
+
+package xattr
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+func getxattr(path string, name string, value *byte, size int, pos int, options int) (int, error) {
+
+	r0, _, e1 := syscall.Syscall6(syscall.SYS_GETXATTR, uintptr(unsafe.Pointer(syscall.StringBytePtr(path))), uintptr(unsafe.Pointer(syscall.StringBytePtr(name))), uintptr(unsafe.Pointer(value)), uintptr(size), uintptr(pos), uintptr(options))
+	if e1 != syscall.Errno(0) {
+		return int(r0), e1
+	}
+	return int(r0), nil
+}
+
+func listxattr(path string, namebuf *byte, size int, options int) (int, error) {
+	r0, _, e1 := syscall.Syscall6(syscall.SYS_LISTXATTR, uintptr(unsafe.Pointer(syscall.StringBytePtr(path))), uintptr(unsafe.Pointer(namebuf)), uintptr(size), uintptr(options), 0, 0)
+	if e1 != syscall.Errno(0) {
+		return int(r0), e1
+	}
+	return int(r0), nil
+}
+
+func setxattr(path string, name string, value *byte, size int, pos int, options int) error {
+	if _, _, e1 := syscall.Syscall6(syscall.SYS_SETXATTR, uintptr(unsafe.Pointer(syscall.StringBytePtr(path))), uintptr(unsafe.Pointer(syscall.StringBytePtr(name))), uintptr(unsafe.Pointer(value)), uintptr(size), uintptr(pos), uintptr(options)); e1 != syscall.Errno(0) {
+		return e1
+	}
+	return nil
+}
+
+func removexattr(path string, name string, options int) error {
+	if _, _, e1 := syscall.Syscall(syscall.SYS_REMOVEXATTR, uintptr(unsafe.Pointer(syscall.StringBytePtr(path))), uintptr(unsafe.Pointer(syscall.StringBytePtr(name))), uintptr(options)); e1 != syscall.Errno(0) {
+		return e1
+	}
+	return nil
+}

vendor/src/github.com/pkg/xattr/syscall_freebsd.go

@@ -0,0 +1,91 @@
+// +build freebsd
+
+package xattr
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+/*
+   ssize_t
+   extattr_get_file(const char *path, int attrnamespace,
+       const char *attrname, void *data, size_t nbytes);
+
+   ssize_t
+   extattr_set_file(const char *path, int attrnamespace,
+       const char *attrname, const void *data, size_t nbytes);
+
+   int
+   extattr_delete_file(const char *path, int attrnamespace,
+       const char *attrname);
+
+   ssize_t
+   extattr_list_file(const char *path, int attrnamespace, void *data,
+       size_t nbytes);
+*/
+
+func extattr_get_file(path string, attrnamespace int, attrname string, data *byte, nbytes int) (int, error) {
+	r, _, e := syscall.Syscall6(
+		syscall.SYS_EXTATTR_GET_FILE,
+		uintptr(unsafe.Pointer(syscall.StringBytePtr(path))),
+		uintptr(attrnamespace),
+		uintptr(unsafe.Pointer(syscall.StringBytePtr(attrname))),
+		uintptr(unsafe.Pointer(data)),
+		uintptr(nbytes),
+		0,
+	)
+	var err error
+	if e != 0 {
+		err = e
+	}
+	return int(r), err
+}
+
+func extattr_set_file(path string, attrnamespace int, attrname string, data *byte, nbytes int) (int, error) {
+	r, _, e := syscall.Syscall6(
+		syscall.SYS_EXTATTR_SET_FILE,
+		uintptr(unsafe.Pointer(syscall.StringBytePtr(path))),
+		uintptr(attrnamespace),
+		uintptr(unsafe.Pointer(syscall.StringBytePtr(attrname))),
+		uintptr(unsafe.Pointer(data)),
+		uintptr(nbytes),
+		0,
+	)
+	var err error
+	if e != 0 {
+		err = e
+	}
+	return int(r), err
+}
+
+func extattr_delete_file(path string, attrnamespace int, attrname string) error {
+	_, _, e := syscall.Syscall(
+		syscall.SYS_EXTATTR_DELETE_FILE,
+		uintptr(unsafe.Pointer(syscall.StringBytePtr(path))),
+		uintptr(attrnamespace),
+		uintptr(unsafe.Pointer(syscall.StringBytePtr(attrname))),
+	)
+	var err error
+	if e != 0 {
+		err = e
+	}
+	return err
+}
+
+func extattr_list_file(path string, attrnamespace int, data *byte, nbytes int) (int, error) {
+	r, _, e := syscall.Syscall6(
+		syscall.SYS_EXTATTR_LIST_FILE,
+		uintptr(unsafe.Pointer(syscall.StringBytePtr(path))),
+		uintptr(attrnamespace),
+		uintptr(unsafe.Pointer(data)),
+		uintptr(nbytes),
+		0,
+		0,
+	)
+	var err error
+	if e != 0 {
+		err = e
+	}
+	return int(r), err
+}

vendor/src/github.com/pkg/xattr/xattr.go

@@ -0,0 +1,32 @@
+/*
+Package xattr provides support for extended attributes on linux, darwin and freebsd.
+Extended attributes are name:value pairs associated permanently with files and directories, 
+similar to the environment strings associated with a process. 
+An attribute may be defined or undefined. If it is defined, its value may be empty or non-empty.
+More details you can find here: https://en.wikipedia.org/wiki/Extended_file_attributes
+*/
+package xattr
+
+// XAttrError records an error and the operation, file path and attribute that caused it.
+type XAttrError struct {
+	Op   string
+	Path string
+	Name string
+	Err  error
+}
+
+func (e *XAttrError) Error() string {
+	return e.Op + " " + e.Path + " " + e.Name + ": " + e.Err.Error()
+}
+
+// nullTermToStrings converts an array of NULL terminated UTF-8 strings to a []string.
+func nullTermToStrings(buf []byte) (result []string) {
+	offset := 0
+	for index, b := range buf {
+		if b == 0 {
+			result = append(result, string(buf[offset:index]))
+			offset = index + 1
+		}
+	}
+	return
+}

vendor/src/github.com/pkg/xattr/xattr_darwin.go

@@ -0,0 +1,59 @@
+// +build darwin
+
+package xattr
+
+// Getxattr retrieves extended attribute data associated with path.
+func Getxattr(path, name string) ([]byte, error) {
+	// find size.
+	size, err := getxattr(path, name, nil, 0, 0, 0)
+	if err != nil {
+		return nil, &XAttrError{"getxattr", path, name, err}
+	}
+	if size > 0 {
+		buf := make([]byte, size)
+		// Read into buffer of that size.
+		read, err := getxattr(path, name, &buf[0], size, 0, 0)
+		if err != nil {
+			return nil, &XAttrError{"getxattr", path, name, err}
+		}
+		return buf[:read], nil
+	}
+	return []byte{}, nil
+}
+
+// Listxattr retrieves a list of names of extended attributes associated
+// with the given path in the file system.
+func Listxattr(path string) ([]string, error) {
+	// find size.
+	size, err := listxattr(path, nil, 0, 0)
+	if err != nil {
+		return nil, &XAttrError{"listxattr", path, "", err}
+	}
+	if size > 0 {
+
+		buf := make([]byte, size)
+		// Read into buffer of that size.
+		read, err := listxattr(path, &buf[0], size, 0)
+		if err != nil {
+			return nil, &XAttrError{"listxattr", path, "", err}
+		}
+		return nullTermToStrings(buf[:read]), nil
+	}
+	return []string{}, nil
+}
+
+// Setxattr associates name and data together as an attribute of path.
+func Setxattr(path, name string, data []byte) error {
+	if err := setxattr(path, name, &data[0], len(data), 0, 0); err != nil {
+		return &XAttrError{"setxattr", path, name, err}
+	}
+	return nil
+}
+
+// Removexattr removes the attribute associated with the given path.
+func Removexattr(path, name string) error {
+	if err := removexattr(path, name, 0); err != nil {
+		return &XAttrError{"removexattr", path, name, err}
+	}
+	return nil
+}

vendor/src/github.com/pkg/xattr/xattr_freebsd.go

@@ -0,0 +1,85 @@
+// +build freebsd
+
+package xattr
+
+import (
+	"syscall"
+)
+
+const (
+	EXTATTR_NAMESPACE_USER = 1
+)
+
+// Getxattr retrieves extended attribute data associated with path.
+func Getxattr(path, name string) ([]byte, error) {
+	// find size.
+	size, err := extattr_get_file(path, EXTATTR_NAMESPACE_USER, name, nil, 0)
+	if err != nil {
+		return nil, &XAttrError{"extattr_get_file", path, name, err}
+	}
+	if size > 0 {
+		buf := make([]byte, size)
+		// Read into buffer of that size.
+		read, err := extattr_get_file(path, EXTATTR_NAMESPACE_USER, name, &buf[0], size)
+		if err != nil {
+			return nil, &XAttrError{"extattr_get_file", path, name, err}
+		}
+		return buf[:read], nil
+	}
+	return []byte{}, nil
+}
+
+// Listxattr retrieves a list of names of extended attributes associated
+// with the given path in the file system.
+func Listxattr(path string) ([]string, error) {
+	// find size.
+	size, err := extattr_list_file(path, EXTATTR_NAMESPACE_USER, nil, 0)
+	if err != nil {
+		return nil, &XAttrError{"extattr_list_file", path, "", err}
+	}
+	if size > 0 {
+		buf := make([]byte, size)
+		// Read into buffer of that size.
+		read, err := extattr_list_file(path, EXTATTR_NAMESPACE_USER, &buf[0], size)
+		if err != nil {
+			return nil, &XAttrError{"extattr_list_file", path, "", err}
+		}
+		return attrListToStrings(buf[:read]), nil
+	}
+	return []string{}, nil
+}
+
+// Setxattr associates name and data together as an attribute of path.
+func Setxattr(path, name string, data []byte) error {
+	written, err := extattr_set_file(path, EXTATTR_NAMESPACE_USER, name, &data[0], len(data))
+	if err != nil {
+		return &XAttrError{"extattr_set_file", path, name, err}
+	}
+	if written != len(data) {
+		return &XAttrError{"extattr_set_file", path, name, syscall.E2BIG}
+	}
+	return nil
+}
+
+// Removexattr removes the attribute associated with the given path.
+func Removexattr(path, name string) error {
+	if err := extattr_delete_file(path, EXTATTR_NAMESPACE_USER, name); err != nil {
+		return &XAttrError{"extattr_delete_file", path, name, err}
+	}
+	return nil
+}
+
+// attrListToStrings converts a sequnce of attribute name entries to a []string.
+// Each entry consists of a single byte containing the length
+// of the attribute name, followed by the attribute name.
+// The name is _not_ terminated by NUL.
+func attrListToStrings(buf []byte) []string {
+	var result []string
+	index := 0
+	for index < len(buf) {
+		next := index + 1 + int(buf[index])
+		result = append(result, string(buf[index+1:next]))
+		index = next
+	}
+	return result
+}

vendor/src/github.com/pkg/xattr/xattr_linux.go

@@ -0,0 +1,61 @@
+// +build linux
+
+package xattr
+
+import "syscall"
+
+// Getxattr retrieves extended attribute data associated with path.
+func Getxattr(path, name string) ([]byte, error) {
+	// find size.
+	size, err := syscall.Getxattr(path, name, nil)
+	if err != nil {
+		return nil, &XAttrError{"getxattr", path, name, err}
+	}
+	if size > 0 {
+		data := make([]byte, size)
+		// Read into buffer of that size.
+		read, err := syscall.Getxattr(path, name, data)
+		if err != nil {
+			return nil, &XAttrError{"getxattr", path, name, err}
+		}
+		return data[:read], nil
+	}
+	return []byte{}, nil
+}
+
+// Listxattr retrieves a list of names of extended attributes associated
+// with the given path in the file system.
+func Listxattr(path string) ([]string, error) {
+	// find size.
+	size, err := syscall.Listxattr(path, nil)
+	if err != nil {
+		return nil, &XAttrError{"listxattr", path, "", err}
+	}
+	if size > 0 {
+		buf := make([]byte, size)
+		// Read into buffer of that size.
+		read, err := syscall.Listxattr(path, buf)
+		if err != nil {
+			return nil, &XAttrError{"listxattr", path, "", err}
+		}
+		return nullTermToStrings(buf[:read]), nil
+	}
+	return []string{}, nil
+}
+
+// Setxattr associates name and data together as an attribute of path.
+func Setxattr(path, name string, data []byte) error {
+	if err := syscall.Setxattr(path, name, data, 0); err != nil {
+		return &XAttrError{"setxattr", path, name, err}
+	}
+	return nil
+}
+
+// Removexattr removes the attribute associated
+// with the given path.
+func Removexattr(path, name string) error {
+	if err := syscall.Removexattr(path, name); err != nil {
+		return &XAttrError{"removexattr", path, name, err}
+	}
+	return nil
+}

vendor/src/github.com/pkg/xattr/xattr_test.go

@@ -0,0 +1,57 @@
+// +build linux darwin freebsd
+
+package xattr
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+const UserPrefix = "user."
+
+func Test_setxattr(t *testing.T) {
+	tmp, err := ioutil.TempFile("", "")
+
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmp.Name())
+
+	err = Setxattr(tmp.Name(), UserPrefix+"test", []byte("test-attr-value"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	list, err := Listxattr(tmp.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	found := false
+	for _, name := range list {
+		if name == UserPrefix+"test" {
+			found = true
+		}
+	}
+
+	if !found {
+		t.Fatal("Listxattr did not return test attribute")
+	}
+
+	var data []byte
+	data, err = Getxattr(tmp.Name(), UserPrefix+"test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	value := string(data)
+	t.Log(value)
+	if "test-attr-value" != value {
+		t.Fail()
+	}
+
+	err = Removexattr(tmp.Name(), UserPrefix+"test")
+	if err != nil {
+		t.Fatal(err)
+	}
+}