Add VERSION file for 0.4.0

This was a temporary fix for Go earlier than 1.5 to run code on all
avaialble cores. We don't need that any more since we require at least
Go 1.6.

.travis.yml

@@ -2,8 +2,9 @@ language: go
 sudo: false
 
 go:
-  - 1.6.3
-  - 1.7.1
+  - 1.6.4
+  - 1.7.4
+  - tip
 
 os:
   - linux
@@ -16,16 +17,23 @@ env:
 matrix:
   exclude:
     - os: osx
-      go: 1.6.3
+      go: 1.6.4
+    - os: osx
+      go: tip
     - os: linux
-      go: 1.7.1
+      go: 1.7.4
   include:
     - os: linux
-      go: 1.7.1
+      go: 1.7.4
       sudo: true
       env:
         RESTIC_TEST_FUSE=1
+  allow_failures:
+    - go: tip
 
+branches:
+  only:
+    - master
 
 notifications:
   irc:
@@ -46,5 +54,4 @@ script:
   - go run run_integration_tests.go
 
 after_success:
-  - GOPATH=$PWD:$PWD/vendor goveralls -coverprofile=all.cov -service=travis-ci -repotoken "$COVERALLS_TOKEN"
   - bash <(curl -s https://codecov.io/bash) -f all.cov

CONTRIBUTING.md

@@ -3,7 +3,10 @@ This document describes the way you can contribute to the restic project.
 Ways to Help Out
 ================
 
-Thank you for your contribution!
+Thank you for your contribution! Please **open an issue first** (or add a
+comment to an existing issue) if you plan to work on any code or add a new
+feature. This way, duplicate work is prevented and we can discuss your ideas
+and design first.
 
 There are several ways you can help us out. First of all code contributions and
 bug fixes are most welcome. However even "minor" details as fixing spelling
@@ -83,7 +86,7 @@ The following commands can be used to run all the tests:
     [...]
 
 If you want to run your tests on Linux, OpenBSD or FreeBSD, you can use
-[vagrant](https://www.vagrantup.com/) with the proveded `Vagrantfile` to
+[vagrant](https://www.vagrantup.com/) with the provided `Vagrantfile` to
 quickly set up VMs and run the tests, e.g.:
 
     $ vagrant up freebsd

README.md

@@ -11,7 +11,8 @@ restic is a backup program that is fast, efficient and secure. Detailed
 information can be found in [the documentation](doc/index.md) and [the user
 manual](doc/Manual.md). The [design document](doc/Design.md) lists the
 technical background and gives detailed information about the structure of the
-repository and the data saved therein.
+repository and the data saved therein. The file [FAQ.md](doc/FAQ.md) lists the
+most frequently asked questions.
 
 The latest documentation can be viewed online at
 <https://restic.readthedocs.io/en/latest>. On the bottom left corner there is
@@ -39,10 +40,15 @@ Building restic with gccgo may work, but is not supported.
 Contribute and Documentation
 ============================
 
-Contributions are welcome! More information and a description of the
-development environment can be found in [`CONTRIBUTING.md`](CONTRIBUTING.md). A
-document describing the design of restic and the data structures stored on the
-back end is contained in [`doc/Design.md`](doc/Design.md).
+Contributions are welcome! Please **open an issue first** (or add a comment to
+an existing issue) if you plan to work on any code or add a new feature. This
+way, duplicate work is prevented and we can discuss your ideas and design
+first.
+
+More information and a description of the development environment can be found
+in [`CONTRIBUTING.md`](CONTRIBUTING.md). A document describing the design of
+restic and the data structures stored on the back end is contained in
+[`doc/Design.md`](doc/Design.md).
 
 If you'd like to start contributing to restic, but don't know exactly what do
 to, have a look at this great article by Dave Cheney:

VERSION

@@ -1 +1 @@
-0.3.1
+0.4.0

appveyor.yml

@@ -3,6 +3,10 @@ clone_folder: c:\restic
 environment:
   GOPATH: c:\gopath
 
+branches:
+  only:
+    - master
+
 init:
   - ps: >-
       $app = Get-WmiObject -Class Win32_Product -Filter "Vendor = 'http://golang.org'"
@@ -13,8 +17,8 @@ init:
 
 install:
   - rmdir c:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.7.windows-amd64.msi
-  - msiexec /i go1.7.windows-amd64.msi /q
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.7.4.windows-amd64.msi
+  - msiexec /i go1.7.4.windows-amd64.msi /q
   - go version
   - go env
   - appveyor DownloadFile http://sourceforge.netcologne.de/project/gnuwin32/tar/1.13-1/tar-1.13-1-bin.zip -FileName tar.zip

build.go

@@ -12,7 +12,6 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
-	"time"
 )
 
 var (
@@ -33,8 +32,6 @@ var config = struct {
 	Tests:     []string{"restic/...", "cmds/..."}, // tests to run
 }
 
-const timeFormat = "2006-01-02 15:04:05"
-
 // specialDir returns true if the file begins with a special character ('.' or '_').
 func specialDir(name string) bool {
 	if name == "." {
@@ -177,6 +174,7 @@ func showUsage(output io.Writer) {
 	fmt.Fprintf(output, "  -t     --tags          specify additional build tags\n")
 	fmt.Fprintf(output, "  -k     --keep-gopath   do not remove the GOPATH after build\n")
 	fmt.Fprintf(output, "  -T     --test          run tests\n")
+	fmt.Fprintf(output, "  -o     --output        set output file name\n")
 	fmt.Fprintf(output, "         --goos value    set GOOS for cross-compilation\n")
 	fmt.Fprintf(output, "         --goarch value  set GOARCH for cross-compilation\n")
 }
@@ -207,7 +205,7 @@ func cleanEnv() (env []string) {
 func build(cwd, goos, goarch, gopath string, args ...string) error {
 	args = append([]string{"build"}, args...)
 	cmd := exec.Command("go", args...)
-	cmd.Env = append(cleanEnv(), "GOPATH="+gopath, "GOARCH="+goarch, "GOOS="+goos)
+	cmd.Env = append(cleanEnv(), "GOPATH="+gopath, "GOARCH="+goarch, "GOOS="+goos, "CGO_ENABLED=0")
 	cmd.Dir = cwd
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
@@ -301,6 +299,8 @@ func main() {
 	targetGOOS := runtime.GOOS
 	targetGOARCH := runtime.GOARCH
 
+	var outputFilename string
+
 	for i, arg := range params {
 		if skipNext {
 			skipNext = false
@@ -318,6 +318,9 @@ func main() {
 			}
 			skipNext = true
 			buildTags = strings.Split(params[i+1], " ")
+		case "-o", "--output":
+			skipNext = true
+			outputFilename = params[i+1]
 		case "-T", "--test":
 			runTests = true
 		case "--goos":
@@ -380,9 +383,11 @@ func main() {
 		}
 	}()
 
-	outputFilename := config.Name
-	if targetGOOS == "windows" {
-		outputFilename += ".exe"
+	if outputFilename == "" {
+		outputFilename = config.Name
+		if targetGOOS == "windows" {
+			outputFilename += ".exe"
+		}
 	}
 
 	cwd, err := os.Getwd()
@@ -392,8 +397,7 @@ func main() {
 	output := filepath.Join(cwd, outputFilename)
 
 	version := getVersion()
-	compileTime := time.Now().Format(timeFormat)
-	constants := Constants{`main.compiledAt`: compileTime}
+	constants := Constants{}
 	if version != "" {
 		constants["main.version"] = version
 	}

build_release_binaries.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+set -e
+
+if [[ -z "$VERSION" ]]; then
+    echo '$VERSION unset'
+    exit 1
+fi
+
+dir=$(mktemp -d --tmpdir restic-release-XXXXXX)
+echo "path is ${dir}"
+
+for R in       \
+    darwin/386     \
+    darwin/amd64   \
+    freebsd/386    \
+    freebsd/amd64  \
+    freebsd/arm    \
+    linux/386      \
+    linux/amd64    \
+    linux/arm      \
+    linux/arm64    \
+    openbsd/386   \
+    openbsd/amd64 \
+    windows/386    \
+    windows/amd64  \
+    ; do \
+
+    OS=$(dirname $R)
+    ARCH=$(basename $R)
+    filename=restic_${VERSION}_${OS}_${ARCH}
+
+    if [[ "$OS" == "windows" ]]; then
+        filename="${filename}.exe"
+    fi
+
+    echo $filename
+
+    go run ../build.go --goos $OS --goarch $ARCH --output ${filename}
+    if [[ "$OS" == "windows" ]]; then
+        zip ${filename%.exe}.zip ${filename}
+        rm ${filename}
+        mv ${filename%.exe}.zip ${dir}
+    else
+        bzip2 ${filename}
+        mv ${filename}.bz2 ${dir}
+    fi
+done
+
+echo "packing sources"
+git archive --format=tar --prefix=restic-$VERSION/ v$VERSION | gzip -n > restic-$VERSION.tar.gz
+mv restic-$VERSION.tar.gz ${dir}
+
+echo "creating checksums"
+pushd ${dir}
+sha256sum restic_*.{zip,bz2} > SHA256SUMS
+gpg --armor --detach-sign SHA256SUMS
+popd
+
+echo "creating source signature file"
+gpg --armor --detach-sign ${dir}/restic-$VERSION.tar.gz
+
+echo
+echo "done, path is ${dir}"

doc/Design.md

@@ -327,10 +327,11 @@ A snapshot references a tree by the SHA-256 hash of the JSON string
 representation of its contents. Trees and data are saved in pack files in a
 subdirectory of the directory `data`.
 
-The command `restic cat tree` can be used to inspect the tree referenced above:
+The command `restic cat blob` can be used to inspect the tree referenced above
+(piping the output of the command to `jq .` so that the JSON is indented):
 
 ```console
-$ restic -r /tmp/restic-repo cat tree b8138ab08a4722596ac89c917827358da4672eac68e3c03a8115b88dbf4bfb59
+$ restic -r /tmp/restic-repo cat blob b8138ab08a4722596ac89c917827358da4672eac68e3c03a8115b88dbf4bfb59 | jq .
 enter password for repository:
 {
   "nodes": [
@@ -356,11 +357,11 @@ A tree contains a list of entries (in the field `nodes`) which contain meta
 data like a name and timestamps. When the entry references a directory, the
 field `subtree` contains the plain text ID of another tree object.
 
-When the command `restic cat tree` is used, the storage hash is needed to print
+When the command `restic cat blob` is used, the plaintext ID is needed to print
 a tree. The tree referenced above can be dumped as follows:
 
 ```console
-$ restic -r /tmp/restic-repo cat tree 8b238c8811cc362693e91a857460c78d3acf7d9edb2f111048691976803cf16e
+$ restic -r /tmp/restic-repo cat blob 8b238c8811cc362693e91a857460c78d3acf7d9edb2f111048691976803cf16e
 enter password for repository:
 {
   "nodes": [
@@ -389,8 +390,8 @@ enter password for repository:
 This tree contains a file entry. This time, the `subtree` field is not present
 and the `content` field contains a list with one plain text SHA-256 hash.
 
-The command `restic cat data` can be used to extract and decrypt data given a
-plaintext ID, e.g. for the data mentioned above:
+The command `restic cat blob` can also be used to extract and decrypt data
+given a plaintext ID, e.g. for the data mentioned above:
 
 ```console
 $ restic -r /tmp/restic-repo cat blob 50f77b3b4291e8411a027b9f9b9e64658181cc676ce6ba9958b95f268cb1109d | sha256sum

doc/FAQ.md

@@ -0,0 +1,20 @@
+FAQ
+===
+
+This is the list of Frequently Asked Questions for restic.
+
+`restic check` reports packs that aren't referenced in any index, is my repository broken?
+------------------------------------------------------------------------------------------
+
+When `restic check` reports that there are pack files in the repository that are not referenced in any index, that's (in contrast to what restic reports at the moment) not a source for concern. The output looks like this:
+
+    $ restic check
+    Create exclusive lock for repository
+    Load indexes
+    Check all packs
+    pack 819a9a52e4f51230afa89aefbf90df37fb70996337ae57e6f7a822959206a85e: not referenced in any index
+    pack de299e69fb075354a3775b6b045d152387201f1cdc229c31d1caa34c3b340141: not referenced in any index
+    Check snapshots, trees and blobs
+    Fatal: repository contains errors
+
+The message means that there is more data stored in the repo than strictly necessary. With high probability this is duplicate data. In order to clean it up, the command `restic prune` can be used. The cause of this bug is not yet known.

doc/Manual.md

@@ -77,7 +77,7 @@ Available Commands:
 Flags:
       --no-lock                do not lock the repo, this allows some operations on read-only repos
   -p, --password-file string   read the repository password from a file
-  -q, --quiet                  do not outputcomprehensive progress report
+  -q, --quiet                  do not output comprehensive progress report
   -r, --repo string            repository to backup to or restore from (default: $RESTIC_REPOSITORY)
 
 Use "restic [command] --help" for more information about a command.
@@ -110,7 +110,7 @@ Flags:
 Global Flags:
       --no-lock                do not lock the repo, this allows some operations on read-only repos
   -p, --password-file string   read the repository password from a file
-  -q, --quiet                  do not outputcomprehensive progress report
+  -q, --quiet                  do not output comprehensive progress report
   -r, --repo string            repository to backup to or restore from (default: $RESTIC_REPOSITORY)
 ```
 
@@ -448,6 +448,35 @@ You can also specify a relative (read: no slash (`/`) character at the
 beginning) directory, in this case the dir is relative to the remote user's
 home directory.
 
+# Create a REST server repository
+
+In order to backup data to the remote server via HTTP or HTTPS protocol,
+you must first set up a remote [REST server](https://github.com/restic/rest-server)
+instance.  Once the server is configured, accessing it is achieved by changing the
+URL scheme like this:
+
+```console
+$ restic -r rest:http://host:8000/
+```
+
+Depending on your REST server setup, you can use HTTPS protocol, password
+protection, or multiple repositories.  Or any combination of those features, as
+you see fit. TCP/IP port is also configurable. Here are some more examples:
+
+```console
+$ restic -r rest:https://host:8000/
+$ restic -r rest:https://user:pass@host:8000/
+$ restic -r rest:https://user:pass@host:8000/my_backup_repo/
+```
+
+If you use TLS, make sure your certificates are signed, 'cause restic client
+will refuse to communicate otherwise.  It's easy to obtain such certificates
+today, thanks to free certificate authorities like [Let’s
+Encrypt](https://letsencrypt.org/).
+
+REST server uses exactly the same directory structure as local backend, so you
+should be able to access it both locally and via HTTP, even simultaneously.
+
 # Create an Amazon S3 repository
 
 Restic can backup data to any Amazon S3 bucket. However, in this case, changing the URL scheme is not enough since Amazon uses special security credentials to sign HTTP requests. By consequence, you must first setup the following environment variables with the credentials you obtained while creating the bucket.

doc/REST_backend.md

@@ -6,6 +6,19 @@ following values are valid for `{type}`: `data`, `keys`, `locks`, `snapshots`,
 `index`, `config`. `{path}` is a path to the repository, so that multiple
 different repositories can be accessed. The default path is `/`.
 
+## POST {path}?create=true
+
+This request is used to initially create a new repository. The server responds
+with "200 OK" if the repository structure was created successfully or already
+exists, otherwise an error is returned.
+
+## DELETE {path}
+
+Deletes the repository on the server side. The server responds with "200 OK" if
+the repository was successfully removed. If this function is not implemented
+the server returns "501 Not Implemented", if this it is denied by the server it
+returns "403 Forbidden".
+
 ## HEAD {path}/config
 
 Returns "200 OK" if the repository has a configuration,

run_integration_tests.go

@@ -176,7 +176,7 @@ func (env *TravisEnvironment) Prepare() error {
 				"windows/386", "windows/amd64",
 				"darwin/386", "darwin/amd64",
 				"freebsd/386", "freebsd/amd64",
-				"opendbsd/386", "opendbsd/amd64",
+				"openbsd/386", "openbsd/amd64",
 			}
 			if !strings.HasPrefix(runtime.Version(), "go1.3") {
 				env.goxOSArch = append(env.goxOSArch,

src/cmds/restic-server/.gitignore

@@ -1,24 +0,0 @@
-# Compiled Object files, Static and Dynamic libs (Shared Objects)
-*.o
-*.a
-*.so
-
-# Folders
-_obj
-_test
-
-# Architecture specific extensions/prefixes
-*.[568vq]
-[568vq].out
-
-*.cgo1.go
-*.cgo2.c
-_cgo_defun.c
-_cgo_gotypes.go
-_cgo_export.*
-
-_testmain.go
-
-*.exe
-*.test
-*.prof

src/cmds/restic-server/LICENSE

@@ -1,24 +0,0 @@
-Copyright (c) 2015, Bertil Chapuis
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-* Redistributions of source code must retain the above copyright notice, this
-  list of conditions and the following disclaimer.
-
-* Redistributions in binary form must reproduce the above copyright notice,
-  this list of conditions and the following disclaimer in the documentation
-  and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-

src/cmds/restic-server/README.md

@@ -1,29 +0,0 @@
-# Restic Server
-
-Restic Server is a sample server that implement restic's rest backend api.
-It has been developed for demonstration purpose and is not intented to be used in production.
-
-## Getting started
-
-By default the server persists backup data in `/tmp/restic`. 
-Build and start the server with a custom persistence directory:
-
-```
-go build
-./restic-server -path /user/home/backup
-```
-
-The server use an `.htpasswd` file to specify users. You can create such a file at the root of the persistence directory by executing the following command. In order to append new user to the file, just omit the `-c` argument.
-
-```
-htpasswd -s -c .htpasswd username
-```
-
-By default the server uses http. This is not very secure since with Basic Authentication, username and passwords will be present in every request. In order to enable TLS support just add the `-tls` argument and add a private and public key at the root of your persistence directory. 
-
-Signed certificate are required by the restic backend but if you just want to test the feature you can generate unsigned keys with the following commands:
-
-```
-openssl genrsa -out private_key 2048
-openssl req -new -x509 -key private_key -out public_key -days 365
-```

src/cmds/restic-server/handlers.go

@@ -1,194 +0,0 @@
-// +build go1.4
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"restic/fs"
-)
-
-// Context contains repository meta-data.
-type Context struct {
-	path string
-}
-
-// AuthHandler wraps h with a http.HandlerFunc that performs basic
-// authentication against the user/passwords pairs stored in f and returns the
-// http.HandlerFunc.
-func AuthHandler(f *HtpasswdFile, h http.Handler) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		username, password, ok := r.BasicAuth()
-		if !ok {
-			http.Error(w, "401 unauthorized", 401)
-			return
-		}
-		if !f.Validate(username, password) {
-			http.Error(w, "401 unauthorized", 401)
-			return
-		}
-		h.ServeHTTP(w, r)
-	}
-}
-
-// CheckConfig returns a http.HandlerFunc that checks whether
-// a configuration exists.
-func CheckConfig(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		config := filepath.Join(c.path, "config")
-		st, err := os.Stat(config)
-		if err != nil {
-			http.Error(w, "404 not found", 404)
-			return
-		}
-		w.Header().Add("Content-Length", fmt.Sprint(st.Size()))
-	}
-}
-
-// GetConfig returns a http.HandlerFunc that allows for a
-// config to be retrieved.
-func GetConfig(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		config := filepath.Join(c.path, "config")
-		bytes, err := ioutil.ReadFile(config)
-		if err != nil {
-			http.Error(w, "404 not found", 404)
-			return
-		}
-		w.Write(bytes)
-	}
-}
-
-// SaveConfig returns a http.HandlerFunc that allows for a
-// config to be saved.
-func SaveConfig(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		config := filepath.Join(c.path, "config")
-		bytes, err := ioutil.ReadAll(r.Body)
-		if err != nil {
-			http.Error(w, "400 bad request", 400)
-			return
-		}
-		errw := ioutil.WriteFile(config, bytes, 0600)
-		if errw != nil {
-			http.Error(w, "500 internal server error", 500)
-			return
-		}
-		w.Write([]byte("200 ok"))
-	}
-}
-
-// ListBlobs returns a http.HandlerFunc that lists
-// all blobs of a given type in an arbitrary order.
-func ListBlobs(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		vars := strings.Split(r.RequestURI, "/")
-		dir := vars[1]
-		path := filepath.Join(c.path, dir)
-		files, err := ioutil.ReadDir(path)
-		if err != nil {
-			http.Error(w, "404 not found", 404)
-			return
-		}
-		names := make([]string, len(files))
-		for i, f := range files {
-			names[i] = f.Name()
-		}
-		data, err := json.Marshal(names)
-		if err != nil {
-			http.Error(w, "500 internal server error", 500)
-			return
-		}
-		w.Write(data)
-	}
-}
-
-// CheckBlob reutrns a http.HandlerFunc that tests whether a blob exists
-// and returns 200, if it does, or 404 otherwise.
-func CheckBlob(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		vars := strings.Split(r.RequestURI, "/")
-		dir := vars[1]
-		name := vars[2]
-		path := filepath.Join(c.path, dir, name)
-		st, err := os.Stat(path)
-		if err != nil {
-			http.Error(w, "404 not found", 404)
-			return
-		}
-		w.Header().Add("Content-Length", fmt.Sprint(st.Size()))
-	}
-}
-
-// GetBlob returns a http.HandlerFunc that retrieves a blob
-// from the repository.
-func GetBlob(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		vars := strings.Split(r.RequestURI, "/")
-		dir := vars[1]
-		name := vars[2]
-		path := filepath.Join(c.path, dir, name)
-		file, err := fs.Open(path)
-		if err != nil {
-			http.Error(w, "404 not found", 404)
-			return
-		}
-		defer file.Close()
-		http.ServeContent(w, r, "", time.Unix(0, 0), file)
-	}
-}
-
-// SaveBlob returns a http.HandlerFunc that saves a blob to the repository.
-func SaveBlob(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		vars := strings.Split(r.RequestURI, "/")
-		dir := vars[1]
-		name := vars[2]
-		path := filepath.Join(c.path, dir, name)
-		tmp := path + "_tmp"
-		tf, err := fs.OpenFile(tmp, os.O_CREATE|os.O_WRONLY, 0600)
-		if err != nil {
-			http.Error(w, "500 internal server error", 500)
-			return
-		}
-		if _, err := io.Copy(tf, r.Body); err != nil {
-			http.Error(w, "400 bad request", 400)
-			tf.Close()
-			os.Remove(tmp)
-			return
-		}
-		if err := tf.Close(); err != nil {
-			http.Error(w, "500 internal server error", 500)
-		}
-		if err := os.Rename(tmp, path); err != nil {
-			http.Error(w, "500 internal server error", 500)
-			return
-		}
-		w.Write([]byte("200 ok"))
-	}
-}
-
-// DeleteBlob returns a http.HandlerFunc that deletes a blob from the
-// repository.
-func DeleteBlob(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		vars := strings.Split(r.RequestURI, "/")
-		dir := vars[1]
-		name := vars[2]
-		path := filepath.Join(c.path, dir, name)
-		err := os.Remove(path)
-		if err != nil {
-			http.Error(w, "500 internal server error", 500)
-			return
-		}
-		w.Write([]byte("200 ok"))
-	}
-}

src/cmds/restic-server/htpasswd.go

@@ -1,97 +0,0 @@
-// +build go1.4
-
-package main
-
-/*
-Copied from: github.com/bitly/oauth2_proxy
-
-MIT License
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-
-import (
-	"crypto/sha1"
-	"encoding/base64"
-	"encoding/csv"
-	"io"
-	"log"
-
-	"restic/fs"
-)
-
-// lookup passwords in a htpasswd file
-// The entries must have been created with -s for SHA encryption
-
-// HtpasswdFile is a map for usernames to passwords.
-type HtpasswdFile struct {
-	Users map[string]string
-}
-
-// NewHtpasswdFromFile reads the users and passwords from a htpasswd
-// file and returns them. If an error is encountered, it is returned, together
-// with a nil-Pointer for the HtpasswdFile.
-func NewHtpasswdFromFile(path string) (*HtpasswdFile, error) {
-	r, err := fs.Open(path)
-	if err != nil {
-		return nil, err
-	}
-	defer r.Close()
-	return NewHtpasswd(r)
-}
-
-// NewHtpasswd reads the users and passwords from a htpasswd
-// datastream in file and returns them. If an error is encountered,
-// it is returned, together with a nil-Pointer for the HtpasswdFile.
-func NewHtpasswd(file io.Reader) (*HtpasswdFile, error) {
-	cr := csv.NewReader(file)
-	cr.Comma = ':'
-	cr.Comment = '#'
-	cr.TrimLeadingSpace = true
-
-	records, err := cr.ReadAll()
-	if err != nil {
-		return nil, err
-	}
-	h := &HtpasswdFile{Users: make(map[string]string)}
-	for _, record := range records {
-		h.Users[record[0]] = record[1]
-	}
-	return h, nil
-}
-
-// Validate returns true if password matches the stored password
-// for user. If no password for user is stored, or the password
-// is wrong, false is returned.
-func (h *HtpasswdFile) Validate(user string, password string) bool {
-	realPassword, exists := h.Users[user]
-	if !exists {
-		return false
-	}
-	if realPassword[:5] == "{SHA}" {
-		d := sha1.New()
-		d.Write([]byte(password))
-		if realPassword[5:] == base64.StdEncoding.EncodeToString(d.Sum(nil)) {
-			return true
-		}
-	} else {
-		log.Printf("Invalid htpasswd entry for %s. Must be a SHA entry.", user)
-	}
-	return false
-}

src/cmds/restic-server/router.go

@@ -1,137 +0,0 @@
-// +build go1.4
-
-package main
-
-import (
-	"log"
-	"net/http"
-	"strings"
-)
-
-// Route is a handler for a path that was already split.
-type Route struct {
-	path    []string
-	handler http.Handler
-}
-
-// Router maps HTTP methods to a slice of Route handlers.
-type Router struct {
-	routes map[string][]Route
-}
-
-// NewRouter creates a new Router and returns a pointer to it.
-func NewRouter() *Router {
-	return &Router{make(map[string][]Route)}
-}
-
-// Options registers handler for path with method "OPTIONS".
-func (router *Router) Options(path string, handler http.Handler) {
-	router.Handle("OPTIONS", path, handler)
-}
-
-// OptionsFunc registers handler for path with method "OPTIONS".
-func (router *Router) OptionsFunc(path string, handler http.HandlerFunc) {
-	router.Handle("OPTIONS", path, handler)
-}
-
-// Get registers handler for path with method "GET".
-func (router *Router) Get(path string, handler http.Handler) {
-	router.Handle("GET", path, handler)
-}
-
-// GetFunc registers handler for path with method "GET".
-func (router *Router) GetFunc(path string, handler http.HandlerFunc) {
-	router.Handle("GET", path, handler)
-}
-
-// Head registers handler for path with method "HEAD".
-func (router *Router) Head(path string, handler http.Handler) {
-	router.Handle("HEAD", path, handler)
-}
-
-// HeadFunc registers handler for path with method "HEAD".
-func (router *Router) HeadFunc(path string, handler http.HandlerFunc) {
-	router.Handle("HEAD", path, handler)
-}
-
-// Post registers handler for path with method "POST".
-func (router *Router) Post(path string, handler http.Handler) {
-	router.Handle("POST", path, handler)
-}
-
-// PostFunc registers handler for path with method "POST".
-func (router *Router) PostFunc(path string, handler http.HandlerFunc) {
-	router.Handle("POST", path, handler)
-}
-
-// Put registers handler for path with method "PUT".
-func (router *Router) Put(path string, handler http.Handler) {
-	router.Handle("PUT", path, handler)
-}
-
-// PutFunc registers handler for path with method "PUT".
-func (router *Router) PutFunc(path string, handler http.HandlerFunc) {
-	router.Handle("PUT", path, handler)
-}
-
-// Delete registers handler for path with method "DELETE".
-func (router *Router) Delete(path string, handler http.Handler) {
-	router.Handle("DELETE", path, handler)
-}
-
-// DeleteFunc registers handler for path with method "DELETE".
-func (router *Router) DeleteFunc(path string, handler http.HandlerFunc) {
-	router.Handle("DELETE", path, handler)
-}
-
-// Trace registers handler for path with method "TRACE".
-func (router *Router) Trace(path string, handler http.Handler) {
-	router.Handle("TRACE", path, handler)
-}
-
-// TraceFunc registers handler for path with method "TRACE".
-func (router *Router) TraceFunc(path string, handler http.HandlerFunc) {
-	router.Handle("TRACE", path, handler)
-}
-
-// Connect registers handler for path with method "Connect".
-func (router *Router) Connect(path string, handler http.Handler) {
-	router.Handle("Connect", path, handler)
-}
-
-// ConnectFunc registers handler for path with method "Connect".
-func (router *Router) ConnectFunc(path string, handler http.HandlerFunc) {
-	router.Handle("Connect", path, handler)
-}
-
-// Handle registers a http.Handler for method and uri
-func (router *Router) Handle(method string, uri string, handler http.Handler) {
-	routes := router.routes[method]
-	path := strings.Split(uri, "/")
-	routes = append(routes, Route{path, handler})
-	router.routes[method] = routes
-}
-
-func (router *Router) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	method := r.Method
-	uri := r.RequestURI
-	path := strings.Split(uri, "/")
-
-	log.Printf("%s %s", method, uri)
-
-ROUTE:
-	for _, route := range router.routes[method] {
-		if len(route.path) != len(path) {
-			continue
-		}
-		for i := 0; i < len(route.path); i++ {
-			if !strings.HasPrefix(route.path[i], ":") && route.path[i] != path[i] {
-				continue ROUTE
-			}
-		}
-		route.handler.ServeHTTP(w, r)
-		return
-	}
-
-	http.Error(w, "404 not found", 404)
-}

src/cmds/restic-server/router_test.go

@@ -1,74 +0,0 @@
-// +build go1.4
-
-package main
-
-import (
-	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-)
-
-func TestRouter(t *testing.T) {
-	router := NewRouter()
-
-	getConfig := []byte("GET /config")
-	router.GetFunc("/config", func(w http.ResponseWriter, r *http.Request) {
-		w.Write(getConfig)
-	})
-
-	postConfig := []byte("POST /config")
-	router.PostFunc("/config", func(w http.ResponseWriter, r *http.Request) {
-		w.Write(postConfig)
-	})
-
-	getBlobs := []byte("GET /blobs/")
-	router.GetFunc("/blobs/", func(w http.ResponseWriter, r *http.Request) {
-		w.Write(getBlobs)
-	})
-
-	getBlob := []byte("GET /blobs/:sha")
-	router.GetFunc("/blobs/:sha", func(w http.ResponseWriter, r *http.Request) {
-		w.Write(getBlob)
-	})
-
-	server := httptest.NewServer(router)
-	defer server.Close()
-
-	getConfigResp, _ := http.Get(server.URL + "/config")
-	getConfigBody, _ := ioutil.ReadAll(getConfigResp.Body)
-	if getConfigResp.StatusCode != 200 {
-		t.Fatalf("Wanted HTTP Status 200, got %d", getConfigResp.StatusCode)
-	}
-	if string(getConfig) != string(getConfigBody) {
-		t.Fatalf("Config wrong:\nWanted '%s'\nGot: '%s'", string(getConfig), string(getConfigBody))
-	}
-
-	postConfigResp, _ := http.Post(server.URL+"/config", "binary/octet-stream", strings.NewReader("post test"))
-	postConfigBody, _ := ioutil.ReadAll(postConfigResp.Body)
-	if postConfigResp.StatusCode != 200 {
-		t.Fatalf("Wanted HTTP Status 200, got %d", postConfigResp.StatusCode)
-	}
-	if string(postConfig) != string(postConfigBody) {
-		t.Fatalf("Config wrong:\nWanted '%s'\nGot: '%s'", string(postConfig), string(postConfigBody))
-	}
-
-	getBlobsResp, _ := http.Get(server.URL + "/blobs/")
-	getBlobsBody, _ := ioutil.ReadAll(getBlobsResp.Body)
-	if getBlobsResp.StatusCode != 200 {
-		t.Fatalf("Wanted HTTP Status 200, got %d", getBlobsResp.StatusCode)
-	}
-	if string(getBlobs) != string(getBlobsBody) {
-		t.Fatalf("Config wrong:\nWanted '%s'\nGot: '%s'", string(getBlobs), string(getBlobsBody))
-	}
-
-	getBlobResp, _ := http.Get(server.URL + "/blobs/test")
-	getBlobBody, _ := ioutil.ReadAll(getBlobResp.Body)
-	if getBlobResp.StatusCode != 200 {
-		t.Fatalf("Wanted HTTP Status 200, got %d", getBlobResp.StatusCode)
-	}
-	if string(getBlob) != string(getBlobBody) {
-		t.Fatalf("Config wrong:\nWanted '%s'\nGot: '%s'", string(getBlob), string(getBlobBody))
-	}
-}

src/cmds/restic-server/server.go

@@ -1,73 +0,0 @@
-// +build go1.4
-
-package main
-
-import (
-	"flag"
-	"log"
-	"net/http"
-	"os"
-	"path/filepath"
-)
-
-const (
-	defaultHTTPPort  = ":8000"
-	defaultHTTPSPort = ":8443"
-)
-
-func main() {
-	// Parse command-line args
-	var path = flag.String("path", "/tmp/restic", "specifies the path of the data directory")
-	var tls = flag.Bool("tls", false, "turns on tls support")
-	flag.Parse()
-
-	// Create the missing directories
-	dirs := []string{
-		"data",
-		"snapshots",
-		"index",
-		"locks",
-		"keys",
-		"tmp",
-	}
-	for _, d := range dirs {
-		os.MkdirAll(filepath.Join(*path, d), 0700)
-	}
-
-	// Define the routes
-	context := &Context{*path}
-	router := NewRouter()
-	router.HeadFunc("/config", CheckConfig(context))
-	router.GetFunc("/config", GetConfig(context))
-	router.PostFunc("/config", SaveConfig(context))
-	router.GetFunc("/:dir/", ListBlobs(context))
-	router.HeadFunc("/:dir/:name", CheckBlob(context))
-	router.GetFunc("/:type/:name", GetBlob(context))
-	router.PostFunc("/:type/:name", SaveBlob(context))
-	router.DeleteFunc("/:type/:name", DeleteBlob(context))
-
-	// Check for a password file
-	var handler http.Handler
-	htpasswdFile, err := NewHtpasswdFromFile(filepath.Join(*path, ".htpasswd"))
-	if err != nil {
-		log.Println("Authentication disabled")
-		handler = router
-	} else {
-		log.Println("Authentication enabled")
-		handler = AuthHandler(htpasswdFile, router)
-	}
-
-	// start the server
-	if !*tls {
-		log.Printf("start server on port %s\n", defaultHTTPPort)
-		http.ListenAndServe(defaultHTTPPort, handler)
-	} else {
-		privateKey := filepath.Join(*path, "private_key")
-		publicKey := filepath.Join(*path, "public_key")
-		log.Println("TLS enabled")
-		log.Printf("private key: %s", privateKey)
-		log.Printf("public key: %s", publicKey)
-		log.Printf("start server on port %s\n", defaultHTTPSPort)
-		http.ListenAndServeTLS(defaultHTTPSPort, publicKey, privateKey, handler)
-	}
-}

src/cmds/restic/cleanup.go

@@ -62,8 +62,13 @@ func CleanupHandler(c <-chan os.Signal) {
 	for s := range c {
 		debug.Log("signal %v received, cleaning up", s)
 		fmt.Printf("%sInterrupt received, cleaning up\n", ClearLine())
-		RunCleanupHandlers()
-		fmt.Println("exiting")
-		os.Exit(0)
+		Exit(0)
 	}
 }
+
+// Exit runs the cleanup handlers and then terminates the process with the
+// given exit code.
+func Exit(code int) {
+	RunCleanupHandlers()
+	os.Exit(code)
+}

src/cmds/restic/cmd_backup.go

@@ -289,7 +289,7 @@ func readLinesFromFile(filename string) ([]string, error) {
 }
 
 func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
-	target, err := readLinesFromFile(opts.FilesFrom)
+	fromfile, err := readLinesFromFile(opts.FilesFrom)
 	if err != nil {
 		return err
 	}
@@ -297,11 +297,12 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 	// merge files from files-from into normal args so we can reuse the normal
 	// args checks and have the ability to use both files-from and args at the
 	// same time
-	args = append(args, target...)
+	args = append(args, fromfile...)
 	if len(args) == 0 {
 		return errors.Fatalf("wrong number of parameters")
 	}
 
+	target := make([]string, 0, len(args))
 	for _, d := range args {
 		if a, err := filepath.Abs(d); err == nil {
 			d = a

src/cmds/restic/cmd_cat.go

@@ -9,13 +9,12 @@ import (
 
 	"restic"
 	"restic/backend"
-	"restic/debug"
 	"restic/errors"
 	"restic/repository"
 )
 
 var cmdCat = &cobra.Command{
-	Use:   "cat [flags] [pack|blob|tree|snapshot|key|masterkey|config|lock] ID",
+	Use:   "cat [flags] [pack|blob|snapshot|index|key|masterkey|config|lock] ID",
 	Short: "print internal objects to stdout",
 	Long: `
 The "cat" command is used to print internal objects to stdout.
@@ -99,7 +98,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 		return nil
 	case "key":
 		h := restic.Handle{Type: restic.KeyFile, Name: id.String()}
-		buf, err := backend.LoadAll(repo.Backend(), h, nil)
+		buf, err := backend.LoadAll(repo.Backend(), h)
 		if err != nil {
 			return err
 		}
@@ -150,7 +149,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 	switch tpe {
 	case "pack":
 		h := restic.Handle{Type: restic.DataFile, Name: id.String()}
-		buf, err := backend.LoadAll(repo.Backend(), h, nil)
+		buf, err := backend.LoadAll(repo.Backend(), h)
 		if err != nil {
 			return err
 		}
@@ -172,7 +171,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 			blob := list[0]
 
 			buf := make([]byte, blob.Length)
-			n, err := repo.LoadBlob(restic.DataBlob, id, buf)
+			n, err := repo.LoadBlob(t, id, buf)
 			if err != nil {
 				return err
 			}
@@ -184,23 +183,6 @@ func runCat(gopts GlobalOptions, args []string) error {
 
 		return errors.Fatal("blob not found")
 
-	case "tree":
-		debug.Log("cat tree %v", id.Str())
-		tree, err := repo.LoadTree(id)
-		if err != nil {
-			debug.Log("unable to load tree %v: %v", id.Str(), err)
-			return err
-		}
-
-		buf, err := json.MarshalIndent(&tree, "", "  ")
-		if err != nil {
-			debug.Log("error json.MarshalIndent(): %v", err)
-			return err
-		}
-
-		_, err = os.Stdout.Write(append(buf, '\n'))
-		return nil
-
 	default:
 		return errors.Fatal("invalid type")
 	}

src/cmds/restic/cmd_forget.go

@@ -133,7 +133,8 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		}
 
 		if !opts.DryRun {
-			err = repo.Backend().Remove(restic.SnapshotFile, id.String())
+			h := restic.Handle{Type: restic.SnapshotFile, Name: id.String()}
+			err = repo.Backend().Remove(h)
 			if err != nil {
 				return err
 			}
@@ -201,7 +202,8 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 
 		if !opts.DryRun {
 			for _, sn := range remove {
-				err = repo.Backend().Remove(restic.SnapshotFile, sn.ID().String())
+				h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
+				err = repo.Backend().Remove(h)
 				if err != nil {
 					return err
 				}

src/cmds/restic/cmd_key.go

@@ -87,7 +87,8 @@ func deleteKey(repo *repository.Repository, name string) error {
 		return errors.Fatal("refusing to remove key currently used to access repository")
 	}
 
-	err := repo.Backend().Remove(restic.KeyFile, name)
+	h := restic.Handle{Type: restic.KeyFile, Name: name}
+	err := repo.Backend().Remove(h)
 	if err != nil {
 		return err
 	}
@@ -107,7 +108,8 @@ func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
 		return errors.Fatalf("creating new key failed: %v\n", err)
 	}
 
-	err = repo.Backend().Remove(restic.KeyFile, repo.KeyName())
+	h := restic.Handle{Type: restic.KeyFile, Name: repo.KeyName()}
+	err = repo.Backend().Remove(h)
 	if err != nil {
 		return err
 	}

src/cmds/restic/cmd_list.go

@@ -1,8 +1,10 @@
 package main
 
 import (
+	"fmt"
 	"restic"
 	"restic/errors"
+	"restic/index"
 
 	"github.com/spf13/cobra"
 )
@@ -52,6 +54,19 @@ func runList(opts GlobalOptions, args []string) error {
 		t = restic.KeyFile
 	case "locks":
 		t = restic.LockFile
+	case "blobs":
+		idx, err := index.Load(repo, nil)
+		if err != nil {
+			return err
+		}
+
+		for _, pack := range idx.Packs {
+			for _, entry := range pack.Entries {
+				fmt.Printf("%v %v\n", entry.Type, entry.ID)
+			}
+		}
+
+		return nil
 	default:
 		return errors.Fatal("invalid type")
 	}

src/cmds/restic/cmd_ls.go

@@ -17,22 +17,35 @@ var cmdLs = &cobra.Command{
 	Short: "list files in a snapshot",
 	Long: `
 The "ls" command allows listing files and directories in a snapshot.
+
+The special snapshot-ID "latest" can be used to list files and directories of the latest snapshot in the repository.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runLs(globalOptions, args)
 	},
 }
 
-var listLong bool
+// LsOptions collects all options for the ls command.
+type LsOptions struct {
+	ListLong bool
+	Host     string
+	Paths    []string
+}
+
+var lsOptions LsOptions
 
 func init() {
 	cmdRoot.AddCommand(cmdLs)
 
-	cmdLs.Flags().BoolVarP(&listLong, "long", "l", false, "use a long listing format showing size and mode")
+	flags := cmdLs.Flags()
+	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
+
+	flags.StringVarP(&lsOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`)
+	flags.StringSliceVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
 }
 
 func printNode(prefix string, n *restic.Node) string {
-	if !listLong {
+	if !lsOptions.ListLong {
 		return filepath.Join(prefix, n.Name)
 	}
 
@@ -86,9 +99,19 @@ func runLs(gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	id, err := restic.FindSnapshot(repo, args[0])
-	if err != nil {
-		return err
+	snapshotIDString := args[0]
+	var id restic.ID
+
+	if snapshotIDString == "latest" {
+		id, err = restic.FindLatestSnapshot(repo, lsOptions.Paths, lsOptions.Host)
+		if err != nil {
+			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Host:%v", err, lsOptions.Paths, lsOptions.Host)
+		}
+	} else {
+		id, err = restic.FindSnapshot(repo, snapshotIDString)
+		if err != nil {
+			Exitf(1, "invalid id %q: %v", snapshotIDString, err)
+		}
 	}
 
 	sn, err := restic.LoadSnapshot(repo, id)

src/cmds/restic/cmd_prune.go

@@ -103,11 +103,13 @@ func runPrune(gopts GlobalOptions) error {
 		return err
 	}
 
+	blobs := 0
 	for _, pack := range idx.Packs {
 		stats.bytes += pack.Size
+		blobs += len(pack.Entries)
 	}
 	Verbosef("repository contains %v packs (%v blobs) with %v bytes\n",
-		len(idx.Packs), len(idx.Blobs), formatBytes(uint64(stats.bytes)))
+		len(idx.Packs), blobs, formatBytes(uint64(stats.bytes)))
 
 	blobCount := make(map[restic.BlobHandle]int)
 	duplicateBlobs := 0
@@ -164,14 +166,17 @@ func runPrune(gopts GlobalOptions) error {
 
 	// find packs that need a rewrite
 	rewritePacks := restic.NewIDSet()
-	for h, blob := range idx.Blobs {
-		if !usedBlobs.Has(h) {
-			rewritePacks.Merge(blob.Packs)
-			continue
-		}
+	for _, pack := range idx.Packs {
+		for _, blob := range pack.Entries {
+			h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
+			if !usedBlobs.Has(h) {
+				rewritePacks.Insert(pack.ID)
+				continue
+			}
 
-		if blobCount[h] > 1 {
-			rewritePacks.Merge(blob.Packs)
+			if blobCount[h] > 1 {
+				rewritePacks.Insert(pack.ID)
+			}
 		}
 	}
 
@@ -214,7 +219,8 @@ func runPrune(gopts GlobalOptions) error {
 	}
 
 	for packID := range removePacks {
-		err = repo.Backend().Remove(restic.DataFile, packID.String())
+		h := restic.Handle{Type: restic.DataFile, Name: packID.String()}
+		err = repo.Backend().Remove(h)
 		if err != nil {
 			Warnf("unable to remove file %v from the repository\n", packID.Str())
 		}
@@ -234,7 +240,8 @@ func runPrune(gopts GlobalOptions) error {
 
 	var supersedes restic.IDs
 	for idxID := range repo.List(restic.IndexFile, done) {
-		err := repo.Backend().Remove(restic.IndexFile, idxID.String())
+		h := restic.Handle{Type: restic.IndexFile, Name: idxID.String()}
+		err := repo.Backend().Remove(h)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "unable to remove index %v: %v\n", idxID.Str(), err)
 		}

src/cmds/restic/cmd_snapshots.go

@@ -57,8 +57,8 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 	}
 
 	tab := NewTable()
-	tab.Header = fmt.Sprintf("%-8s  %-19s  %-10s  %-10s  %s", "ID", "Date", "Host", "Tags", "Directory")
-	tab.RowFormat = "%-8s  %-19s  %-10s  %-10s  %s"
+	tab.Header = fmt.Sprintf("%-8s  %-19s  %-10s  %-10s  %-3s %s", "ID", "Date", "Host", "Tags", "", "Directory")
+	tab.RowFormat = "%-8s  %-19s  %-10s  %-10s  %-3s %s"
 
 	done := make(chan struct{})
 	defer close(done)
@@ -97,9 +97,15 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 			firstTag = sn.Tags[0]
 		}
 
-		tab.Rows = append(tab.Rows, []interface{}{sn.ID().Str(), sn.Time.Format(TimeFormat), sn.Hostname, firstTag, sn.Paths[0]})
-
 		rows := len(sn.Paths)
+
+		treeElement := "   "
+		if rows != 1 {
+			treeElement = "┌──"
+		}
+
+		tab.Rows = append(tab.Rows, []interface{}{sn.ID().Str(), sn.Time.Format(TimeFormat), sn.Hostname, firstTag, treeElement, sn.Paths[0]})
+
 		if len(sn.Tags) > rows {
 			rows = len(sn.Tags)
 		}
@@ -115,7 +121,12 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 				tag = sn.Tags[i]
 			}
 
-			tab.Rows = append(tab.Rows, []interface{}{"", "", "", tag, path})
+			treeElement := "│"
+			if i == (rows - 1) {
+				treeElement = "└──"
+			}
+
+			tab.Rows = append(tab.Rows, []interface{}{"", "", "", tag, treeElement, path})
 		}
 	}
 

src/cmds/restic/cmd_version.go

@@ -15,8 +15,8 @@ The "version" command prints detailed information about the build environment
 and the version of this software.
 `,
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Printf("restic %s\ncompiled at %s with %v on %v/%v\n",
-			version, compiledAt, runtime.Version(), runtime.GOOS, runtime.GOARCH)
+		fmt.Printf("restic %s\ncompiled with %v on %v/%v\n",
+			version, runtime.Version(), runtime.GOOS, runtime.GOARCH)
 	},
 }
 

src/cmds/restic/global.go

@@ -10,8 +10,6 @@ import (
 	"strings"
 	"syscall"
 
-	"github.com/spf13/cobra"
-
 	"restic/backend/local"
 	"restic/backend/rest"
 	"restic/backend/s3"
@@ -26,19 +24,6 @@ import (
 )
 
 var version = "compiled manually"
-var compiledAt = "unknown time"
-
-func parseEnvironment(cmd *cobra.Command, args []string) {
-	repo := os.Getenv("RESTIC_REPOSITORY")
-	if repo != "" {
-		globalOptions.Repo = repo
-	}
-
-	pw := os.Getenv("RESTIC_PASSWORD")
-	if pw != "" {
-		globalOptions.password = pw
-	}
-}
 
 // GlobalOptions hold all global options for restic.
 type GlobalOptions struct {
@@ -58,10 +43,15 @@ var globalOptions = GlobalOptions{
 }
 
 func init() {
+	pw := os.Getenv("RESTIC_PASSWORD")
+	if pw != "" {
+		globalOptions.password = pw
+	}
+
 	f := cmdRoot.PersistentFlags()
-	f.StringVarP(&globalOptions.Repo, "repo", "r", "", "repository to backup to or restore from (default: $RESTIC_REPOSITORY)")
+	f.StringVarP(&globalOptions.Repo, "repo", "r", os.Getenv("RESTIC_REPOSITORY"), "repository to backup to or restore from (default: $RESTIC_REPOSITORY)")
 	f.StringVarP(&globalOptions.PasswordFile, "password-file", "p", "", "read the repository password from a file")
-	f.BoolVarP(&globalOptions.Quiet, "quiet", "q", false, "do not outputcomprehensive progress report")
+	f.BoolVarP(&globalOptions.Quiet, "quiet", "q", false, "do not output comprehensive progress report")
 	f.BoolVar(&globalOptions.NoLock, "no-lock", false, "do not lock the repo, this allows some operations on read-only repos")
 
 	restoreTerminal()
@@ -132,7 +122,7 @@ func Printf(format string, args ...interface{}) {
 	_, err := fmt.Fprintf(globalOptions.stdout, format, args...)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to write to stdout: %v\n", err)
-		os.Exit(100)
+		Exit(100)
 	}
 }
 
@@ -175,18 +165,19 @@ func Warnf(format string, args ...interface{}) {
 	_, err := fmt.Fprintf(globalOptions.stderr, format, args...)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to write to stderr: %v\n", err)
-		os.Exit(100)
+		Exit(100)
 	}
 }
 
-// Exitf uses Warnf to write the message and then calls os.Exit(exitcode).
+// Exitf uses Warnf to write the message and then terminates the process with
+// the given exit code.
 func Exitf(exitcode int, format string, args ...interface{}) {
 	if format[len(format)-1] != '\n' {
 		format += "\n"
 	}
 
 	Warnf(format, args...)
-	os.Exit(exitcode)
+	Exit(exitcode)
 }
 
 // readPassword reads the password from the given reader directly.

src/cmds/restic/global_debug.go

@@ -0,0 +1,60 @@
+// +build debug
+
+package main
+
+import (
+	"fmt"
+	"net/http"
+	_ "net/http/pprof"
+	"os"
+	"restic/errors"
+
+	"github.com/pkg/profile"
+)
+
+var (
+	listenMemoryProfile string
+	memProfilePath      string
+	cpuProfilePath      string
+
+	prof interface {
+		Stop()
+	}
+)
+
+func init() {
+	f := cmdRoot.PersistentFlags()
+	f.StringVar(&listenMemoryProfile, "listen-profile", "", "listen on this `address:port` for memory profiling")
+	f.StringVar(&memProfilePath, "mem-profile", "", "write memory profile to `dir`")
+	f.StringVar(&cpuProfilePath, "cpu-profile", "", "write cpu profile to `dir`")
+}
+
+func runDebug() error {
+	if listenMemoryProfile != "" {
+		fmt.Fprintf(os.Stderr, "running memory profile HTTP server on %v\n", listenMemoryProfile)
+		go func() {
+			err := http.ListenAndServe(listenMemoryProfile, nil)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "memory profile listen failed: %v\n", err)
+			}
+		}()
+	}
+
+	if memProfilePath != "" && cpuProfilePath != "" {
+		return errors.Fatal("only one profile (memory or CPU) may be activated at the same time")
+	}
+
+	if memProfilePath != "" {
+		prof = profile.Start(profile.Quiet, profile.MemProfile, profile.ProfilePath(memProfilePath))
+	} else if memProfilePath != "" {
+		prof = profile.Start(profile.Quiet, profile.CPUProfile, profile.ProfilePath(memProfilePath))
+	}
+
+	return nil
+}
+
+func shutdownDebug() {
+	if prof != nil {
+		prof.Stop()
+	}
+}

src/cmds/restic/global_release.go

@@ -0,0 +1,9 @@
+// +build !debug
+
+package main
+
+// runDebug is a noop without the debug tag.
+func runDebug() error { return nil }
+
+// shutdownDebug is a noop without the debug tag.
+func shutdownDebug() {}

src/cmds/restic/integration_test.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	mrand "math/rand"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -159,6 +160,15 @@ func testRunFind(t testing.TB, gopts GlobalOptions, pattern string) []string {
 	return strings.Split(string(buf.Bytes()), "\n")
 }
 
+func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
+	opts := ForgetOptions{}
+	OK(t, runForget(opts, gopts, args))
+}
+
+func testRunPrune(t testing.TB, gopts GlobalOptions) {
+	OK(t, runPrune(gopts))
+}
+
 func TestBackup(t *testing.T) {
 	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
 		datafile := filepath.Join("testdata", "backup-data.tar.gz")
@@ -730,6 +740,30 @@ func TestRestoreFilter(t *testing.T) {
 	})
 }
 
+func TestRestore(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		testRunInit(t, gopts)
+
+		for i := 0; i < 10; i++ {
+			p := filepath.Join(env.testdata, fmt.Sprintf("foo/bar/testfile%v", i))
+			OK(t, os.MkdirAll(filepath.Dir(p), 0755))
+			OK(t, appendRandomData(p, uint(mrand.Intn(5<<21))))
+		}
+
+		opts := BackupOptions{}
+
+		testRunBackup(t, []string{env.testdata}, opts, gopts)
+		testRunCheck(t, gopts)
+
+		// Restore latest without any filters
+		restoredir := filepath.Join(env.base, "restore")
+		testRunRestoreLatest(t, gopts, restoredir, nil, "")
+
+		Assert(t, directoriesEqualContents(env.testdata, filepath.Join(restoredir, filepath.Base(env.testdata))),
+			"directories are not equal")
+	})
+}
+
 func TestRestoreLatest(t *testing.T) {
 
 	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
@@ -947,3 +981,33 @@ func TestCheckRestoreNoLock(t *testing.T) {
 		testRunRestore(t, gopts, filepath.Join(env.base, "restore"), snapshotIDs[0])
 	})
 }
+
+func TestPrune(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		fd, err := os.Open(datafile)
+		if os.IsNotExist(errors.Cause(err)) {
+			t.Skipf("unable to find data file %q, skipping", datafile)
+			return
+		}
+		OK(t, err)
+		OK(t, fd.Close())
+
+		testRunInit(t, gopts)
+
+		SetupTarTestFixture(t, env.testdata, datafile)
+		opts := BackupOptions{}
+
+		testRunBackup(t, []string{filepath.Join(env.testdata, "0", "0", "1")}, opts, gopts)
+		testRunBackup(t, []string{filepath.Join(env.testdata, "0", "0", "2")}, opts, gopts)
+		testRunBackup(t, []string{filepath.Join(env.testdata, "0", "0", "3")}, opts, gopts)
+
+		snapshotIDs := testRunList(t, "snapshots", gopts)
+		Assert(t, len(snapshotIDs) == 3,
+			"expected one snapshot, got %v", snapshotIDs)
+
+		testRunForget(t, gopts, snapshotIDs[0].String())
+		testRunPrune(t, gopts)
+		testRunCheck(t, gopts)
+	})
+}

src/cmds/restic/lock.go

@@ -36,6 +36,7 @@ func lockRepository(repo *repository.Repository, exclusive bool) (*restic.Lock,
 	if err != nil {
 		return nil, err
 	}
+	debug.Log("create lock %p (exclusive %v)", lock, exclusive)
 
 	globalLocks.Lock()
 	if globalLocks.cancelRefresh == nil {
@@ -88,7 +89,7 @@ func unlockRepo(lock *restic.Lock) error {
 	globalLocks.Lock()
 	defer globalLocks.Unlock()
 
-	debug.Log("unlocking repository")
+	debug.Log("unlocking repository with lock %p", lock)
 	if err := lock.Unlock(); err != nil {
 		debug.Log("error while unlocking: %v", err)
 		return err

src/cmds/restic/main.go

@@ -5,7 +5,6 @@ import (
 	"os"
 	"restic"
 	"restic/debug"
-	"runtime"
 
 	"github.com/spf13/cobra"
 
@@ -20,20 +19,17 @@ var cmdRoot = &cobra.Command{
 restic is a backup program which allows saving multiple revisions of files and
 directories in an encrypted repository stored on different backends.
 `,
-	SilenceErrors:    true,
-	SilenceUsage:     true,
-	PersistentPreRun: parseEnvironment,
-}
+	SilenceErrors: true,
+	SilenceUsage:  true,
 
-func init() {
-	// set GOMAXPROCS to number of CPUs
-	if runtime.Version() < "go1.5" {
-		gomaxprocs := os.Getenv("GOMAXPROCS")
-		debug.Log("read GOMAXPROCS from env variable, value: %s", gomaxprocs)
-		if gomaxprocs == "" {
-			runtime.GOMAXPROCS(runtime.NumCPU())
-		}
-	}
+	// run the debug functions for all subcommands (if build tag "debug" is
+	// enabled)
+	PersistentPreRunE: func(*cobra.Command, []string) error {
+		return runDebug()
+	},
+	PersistentPostRun: func(*cobra.Command, []string) {
+		shutdownDebug()
+	},
 }
 
 func main() {
@@ -49,9 +45,10 @@ func main() {
 		fmt.Fprintf(os.Stderr, "%+v\n", err)
 	}
 
-	RunCleanupHandlers()
-
+	var exitCode int
 	if err != nil {
-		os.Exit(1)
+		exitCode = 1
 	}
+
+	Exit(exitCode)
 }

src/restic/archiver/archive_reader_test.go

@@ -44,7 +44,7 @@ func checkSavedFile(t *testing.T, repo restic.Repository, treeID restic.ID, name
 			t.Fatal(err)
 		}
 
-		buf := make([]byte, int(size))
+		buf := restic.NewBlobBuffer(int(size))
 		n := loadBlob(t, repo, id, buf)
 		if n != len(buf) {
 			t.Errorf("wrong number of bytes read, want %d, got %d", len(buf), n)

src/restic/archiver/archiver.go

@@ -210,14 +210,14 @@ func (arch *Archiver) SaveFile(p *restic.Progress, node *restic.Node) (*restic.N
 	file, err := fs.Open(node.Path)
 	defer file.Close()
 	if err != nil {
-		return nil, errors.Wrap(err, "Open")
+		return node, errors.Wrap(err, "Open")
 	}
 
 	debug.RunHook("archiver.SaveFile", node.Path)
 
 	node, err = arch.reloadFileIfChanged(node, file)
 	if err != nil {
-		return nil, err
+		return node, err
 	}
 
 	chnker := chunker.New(file, arch.repo.Config().ChunkerPolynomial)
@@ -230,7 +230,7 @@ func (arch *Archiver) SaveFile(p *restic.Progress, node *restic.Node) (*restic.N
 		}
 
 		if err != nil {
-			return nil, errors.Wrap(err, "chunker.Next")
+			return node, errors.Wrap(err, "chunker.Next")
 		}
 
 		resCh := make(chan saveResult, 1)
@@ -240,7 +240,7 @@ func (arch *Archiver) SaveFile(p *restic.Progress, node *restic.Node) (*restic.N
 
 	results, err := waitForResults(resultChannels)
 	if err != nil {
-		return nil, err
+		return node, err
 	}
 	err = updateNodeContent(node, results)
 

src/restic/archiver/archiver_duplication_test.go

@@ -39,15 +39,15 @@ func randomID() restic.ID {
 func forgetfulBackend() restic.Backend {
 	be := &mock.Backend{}
 
-	be.TestFn = func(t restic.FileType, name string) (bool, error) {
+	be.TestFn = func(h restic.Handle) (bool, error) {
 		return false, nil
 	}
 
-	be.LoadFn = func(h restic.Handle, p []byte, off int64) (int, error) {
-		return 0, errors.New("not found")
+	be.LoadFn = func(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+		return nil, errors.New("not found")
 	}
 
-	be.SaveFn = func(h restic.Handle, p []byte) error {
+	be.SaveFn = func(h restic.Handle, rd io.Reader) error {
 		return nil
 	}
 
@@ -55,7 +55,7 @@ func forgetfulBackend() restic.Backend {
 		return restic.FileInfo{}, errors.New("not found")
 	}
 
-	be.RemoveFn = func(t restic.FileType, name string) error {
+	be.RemoveFn = func(h restic.Handle) error {
 		return nil
 	}
 
@@ -142,6 +142,11 @@ func testArchiverDuplication(t *testing.T) {
 	close(done)
 
 	wg.Wait()
+
+	err = repo.Flush()
+	if err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestArchiverDuplication(t *testing.T) {

src/restic/backend.go

@@ -1,5 +1,7 @@
 package restic
 
+import "io"
+
 // Backend is used to store and access data.
 type Backend interface {
 	// Location returns a string that describes the type and location of the
@@ -7,22 +9,22 @@ type Backend interface {
 	Location() string
 
 	// Test a boolean value whether a File with the name and type exists.
-	Test(t FileType, name string) (bool, error)
+	Test(h Handle) (bool, error)
 
 	// Remove removes a File with type t and name.
-	Remove(t FileType, name string) error
+	Remove(h Handle) error
 
 	// Close the backend
 	Close() error
 
-	// Load returns the data stored in the backend for h at the given offset
-	// and saves it in p. Load has the same semantics as io.ReaderAt, except
-	// that a negative offset is also allowed. In this case it references a
-	// position relative to the end of the file (similar to Seek()).
-	Load(h Handle, p []byte, off int64) (int, error)
-
 	// Save stores the data in the backend under the given handle.
-	Save(h Handle, p []byte) error
+	Save(h Handle, rd io.Reader) error
+
+	// Load returns a reader that yields the contents of the file at h at the
+	// given offset. If length is larger than zero, only a portion of the file
+	// is returned. rd must be closed after use. If an error is returned, the
+	// ReadCloser must be nil.
+	Load(h Handle, length int, offset int64) (io.ReadCloser, error)
 
 	// Stat returns information about the File identified by h.
 	Stat(h Handle) (FileInfo, error)

src/restic/backend/local/backend_test.go

@@ -51,13 +51,6 @@ func TestLocalBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestLocalBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestLocalBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/local/local.go

@@ -101,58 +101,18 @@ func dirname(base string, t restic.FileType, name string) string {
 	return filepath.Join(base, n)
 }
 
-// Load returns the data stored in the backend for h at the given offset and
-// saves it in p. Load has the same semantics as io.ReaderAt, with one
-// exception: when off is lower than zero, it is treated as an offset relative
-// to the end of the file.
-func (b *Local) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
-	debug.Log("Load %v, length %v at %v", h, len(p), off)
-	if err := h.Valid(); err != nil {
-		return 0, err
-	}
-
-	f, err := fs.Open(filename(b.p, h.Type, h.Name))
-	if err != nil {
-		return 0, errors.Wrap(err, "Open")
-	}
-
-	defer func() {
-		e := f.Close()
-		if err == nil {
-			err = errors.Wrap(e, "Close")
-		}
-	}()
-
-	switch {
-	case off > 0:
-		_, err = f.Seek(off, 0)
-	case off < 0:
-		_, err = f.Seek(off, 2)
-	}
-
-	if err != nil {
-		return 0, errors.Wrap(err, "Seek")
-	}
-
-	return io.ReadFull(f, p)
-}
-
-// writeToTempfile saves p into a tempfile in tempdir.
-func writeToTempfile(tempdir string, p []byte) (filename string, err error) {
+// copyToTempfile saves p into a tempfile in tempdir.
+func copyToTempfile(tempdir string, rd io.Reader) (filename string, err error) {
 	tmpfile, err := ioutil.TempFile(tempdir, "temp-")
 	if err != nil {
 		return "", errors.Wrap(err, "TempFile")
 	}
 
-	n, err := tmpfile.Write(p)
+	_, err = io.Copy(tmpfile, rd)
 	if err != nil {
 		return "", errors.Wrap(err, "Write")
 	}
 
-	if n != len(p) {
-		return "", errors.New("not all bytes writen")
-	}
-
 	if err = tmpfile.Sync(); err != nil {
 		return "", errors.Wrap(err, "Syncn")
 	}
@@ -166,14 +126,14 @@ func writeToTempfile(tempdir string, p []byte) (filename string, err error) {
 }
 
 // Save stores data in the backend at the handle.
-func (b *Local) Save(h restic.Handle, p []byte) (err error) {
-	debug.Log("Save %v, length %v", h, len(p))
+func (b *Local) Save(h restic.Handle, rd io.Reader) (err error) {
+	debug.Log("Save %v", h)
 	if err := h.Valid(); err != nil {
 		return err
 	}
 
-	tmpfile, err := writeToTempfile(filepath.Join(b.p, backend.Paths.Temp), p)
-	debug.Log("saved %v (%d bytes) to %v", h, len(p), tmpfile)
+	tmpfile, err := copyToTempfile(filepath.Join(b.p, backend.Paths.Temp), rd)
+	debug.Log("saved %v to %v", h, tmpfile)
 	if err != nil {
 		return err
 	}
@@ -210,6 +170,39 @@ func (b *Local) Save(h restic.Handle, p []byte) (err error) {
 	return setNewFileMode(filename, fi)
 }
 
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (b *Local) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	f, err := os.Open(filename(b.p, h.Type, h.Name))
+	if err != nil {
+		return nil, err
+	}
+
+	if offset > 0 {
+		_, err = f.Seek(offset, 0)
+		if err != nil {
+			f.Close()
+			return nil, err
+		}
+	}
+
+	if length > 0 {
+		return backend.LimitReadCloser(f, int64(length)), nil
+	}
+
+	return f, nil
+}
+
 // Stat returns information about a blob.
 func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) {
 	debug.Log("Stat %v", h)
@@ -226,9 +219,9 @@ func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) {
 }
 
 // Test returns true if a blob of the given type and name exists in the backend.
-func (b *Local) Test(t restic.FileType, name string) (bool, error) {
-	debug.Log("Test %v %v", t, name)
-	_, err := fs.Stat(filename(b.p, t, name))
+func (b *Local) Test(h restic.Handle) (bool, error) {
+	debug.Log("Test %v", h)
+	_, err := fs.Stat(filename(b.p, h.Type, h.Name))
 	if err != nil {
 		if os.IsNotExist(errors.Cause(err)) {
 			return false, nil
@@ -240,9 +233,9 @@ func (b *Local) Test(t restic.FileType, name string) (bool, error) {
 }
 
 // Remove removes the blob with the given name and type.
-func (b *Local) Remove(t restic.FileType, name string) error {
-	debug.Log("Remove %v %v", t, name)
-	fn := filename(b.p, t, name)
+func (b *Local) Remove(h restic.Handle) error {
+	debug.Log("Remove %v", h)
+	fn := filename(b.p, h.Type, h.Name)
 
 	// reset read-only flag
 	err := fs.Chmod(fn, 0666)

src/restic/backend/mem/backend_test.go

@@ -51,13 +51,6 @@ func TestMemBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestMemBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestMemBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/mem/mem_backend.go

@@ -1,21 +1,19 @@
 package mem
 
 import (
+	"bytes"
 	"io"
+	"io/ioutil"
 	"restic"
 	"sync"
 
+	"restic/backend"
 	"restic/errors"
 
 	"restic/debug"
 )
 
-type entry struct {
-	Type restic.FileType
-	Name string
-}
-
-type memMap map[entry][]byte
+type memMap map[restic.Handle][]byte
 
 // make sure that MemoryBackend implements backend.Backend
 var _ restic.Backend = &MemoryBackend{}
@@ -39,61 +37,21 @@ func New() *MemoryBackend {
 }
 
 // Test returns whether a file exists.
-func (be *MemoryBackend) Test(t restic.FileType, name string) (bool, error) {
+func (be *MemoryBackend) Test(h restic.Handle) (bool, error) {
 	be.m.Lock()
 	defer be.m.Unlock()
 
-	debug.Log("test %v %v", t, name)
+	debug.Log("Test %v", h)
 
-	if _, ok := be.data[entry{t, name}]; ok {
+	if _, ok := be.data[h]; ok {
 		return true, nil
 	}
 
 	return false, nil
 }
 
-// Load reads data from the backend.
-func (be *MemoryBackend) Load(h restic.Handle, p []byte, off int64) (int, error) {
-	if err := h.Valid(); err != nil {
-		return 0, err
-	}
-
-	be.m.Lock()
-	defer be.m.Unlock()
-
-	if h.Type == restic.ConfigFile {
-		h.Name = ""
-	}
-
-	debug.Log("get %v offset %v len %v", h, off, len(p))
-
-	if _, ok := be.data[entry{h.Type, h.Name}]; !ok {
-		return 0, errors.New("no such data")
-	}
-
-	buf := be.data[entry{h.Type, h.Name}]
-	switch {
-	case off > int64(len(buf)):
-		return 0, errors.New("offset beyond end of file")
-	case off < -int64(len(buf)):
-		off = 0
-	case off < 0:
-		off = int64(len(buf)) + off
-	}
-
-	buf = buf[off:]
-
-	n := copy(p, buf)
-
-	if len(p) > len(buf) {
-		return n, io.ErrUnexpectedEOF
-	}
-
-	return n, nil
-}
-
 // Save adds new Data to the backend.
-func (be *MemoryBackend) Save(h restic.Handle, p []byte) error {
+func (be *MemoryBackend) Save(h restic.Handle, rd io.Reader) error {
 	if err := h.Valid(); err != nil {
 		return err
 	}
@@ -105,18 +63,59 @@ func (be *MemoryBackend) Save(h restic.Handle, p []byte) error {
 		h.Name = ""
 	}
 
-	if _, ok := be.data[entry{h.Type, h.Name}]; ok {
+	if _, ok := be.data[h]; ok {
 		return errors.New("file already exists")
 	}
 
-	debug.Log("save %v bytes at %v", len(p), h)
-	buf := make([]byte, len(p))
-	copy(buf, p)
-	be.data[entry{h.Type, h.Name}] = buf
+	buf, err := ioutil.ReadAll(rd)
+	if err != nil {
+		return err
+	}
+
+	be.data[h] = buf
+	debug.Log("saved %v bytes at %v", len(buf), h)
 
 	return nil
 }
 
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (be *MemoryBackend) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	be.m.Lock()
+	defer be.m.Unlock()
+
+	if h.Type == restic.ConfigFile {
+		h.Name = ""
+	}
+
+	debug.Log("Load %v offset %v len %v", h, offset, length)
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if _, ok := be.data[h]; !ok {
+		return nil, errors.New("no such data")
+	}
+
+	buf := be.data[h]
+	if offset > int64(len(buf)) {
+		return nil, errors.New("offset beyond end of file")
+	}
+
+	buf = buf[offset:]
+	if length > 0 && len(buf) > length {
+		buf = buf[:length]
+	}
+
+	return backend.Closer{Reader: bytes.NewReader(buf)}, nil
+}
+
 // Stat returns information about a file in the backend.
 func (be *MemoryBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 	be.m.Lock()
@@ -132,7 +131,7 @@ func (be *MemoryBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 
 	debug.Log("stat %v", h)
 
-	e, ok := be.data[entry{h.Type, h.Name}]
+	e, ok := be.data[h]
 	if !ok {
 		return restic.FileInfo{}, errors.New("no such data")
 	}
@@ -141,17 +140,17 @@ func (be *MemoryBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 }
 
 // Remove deletes a file from the backend.
-func (be *MemoryBackend) Remove(t restic.FileType, name string) error {
+func (be *MemoryBackend) Remove(h restic.Handle) error {
 	be.m.Lock()
 	defer be.m.Unlock()
 
-	debug.Log("get %v %v", t, name)
+	debug.Log("Remove %v", h)
 
-	if _, ok := be.data[entry{t, name}]; !ok {
+	if _, ok := be.data[h]; !ok {
 		return errors.New("no such data")
 	}
 
-	delete(be.data, entry{t, name})
+	delete(be.data, h)
 
 	return nil
 }

src/restic/backend/rest/backend_test.go

@@ -51,13 +51,6 @@ func TestRestBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestRestBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestRestBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/rest/rest.go

@@ -1,7 +1,6 @@
 package rest
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -12,6 +11,7 @@ import (
 	"restic"
 	"strings"
 
+	"restic/debug"
 	"restic/errors"
 
 	"restic/backend"
@@ -19,6 +19,9 @@ import (
 
 const connLimit = 10
 
+// make sure the rest backend implements restic.Backend
+var _ restic.Backend = &restBackend{}
+
 // restPath returns the path to the given resource.
 func restPath(url *url.URL, h restic.Handle) string {
 	u := *url
@@ -71,65 +74,18 @@ func (b *restBackend) Location() string {
 	return b.url.String()
 }
 
-// Load returns the data stored in the backend for h at the given offset
-// and saves it in p. Load has the same semantics as io.ReaderAt.
-func (b *restBackend) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
-	if err := h.Valid(); err != nil {
-		return 0, err
-	}
-
-	// invert offset
-	if off < 0 {
-		info, err := b.Stat(h)
-		if err != nil {
-			return 0, errors.Wrap(err, "Stat")
-		}
-
-		if -off > info.Size {
-			off = 0
-		} else {
-			off = info.Size + off
-		}
-	}
-
-	req, err := http.NewRequest("GET", restPath(b.url, h), nil)
-	if err != nil {
-		return 0, errors.Wrap(err, "http.NewRequest")
-	}
-	req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p))))
-	<-b.connChan
-	resp, err := b.client.Do(req)
-	b.connChan <- struct{}{}
-
-	if resp != nil {
-		defer func() {
-			io.Copy(ioutil.Discard, resp.Body)
-			e := resp.Body.Close()
-
-			if err == nil {
-				err = errors.Wrap(e, "Close")
-			}
-		}()
-	}
-
-	if err != nil {
-		return 0, errors.Wrap(err, "client.Do")
-	}
-	if resp.StatusCode != 200 && resp.StatusCode != 206 {
-		return 0, errors.Errorf("unexpected HTTP response code %v", resp.StatusCode)
-	}
-
-	return io.ReadFull(resp.Body, p)
-}
-
 // Save stores data in the backend at the handle.
-func (b *restBackend) Save(h restic.Handle, p []byte) (err error) {
+func (b *restBackend) Save(h restic.Handle, rd io.Reader) (err error) {
 	if err := h.Valid(); err != nil {
 		return err
 	}
 
+	// make sure that client.Post() cannot close the reader by wrapping it in
+	// backend.Closer, which has a noop method.
+	rd = backend.Closer{Reader: rd}
+
 	<-b.connChan
-	resp, err := b.client.Post(restPath(b.url, h), "binary/octet-stream", bytes.NewReader(p))
+	resp, err := b.client.Post(restPath(b.url, h), "binary/octet-stream", rd)
 	b.connChan <- struct{}{}
 
 	if resp != nil {
@@ -154,6 +110,56 @@ func (b *restBackend) Save(h restic.Handle, p []byte) (err error) {
 	return nil
 }
 
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (b *restBackend) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if length < 0 {
+		return nil, errors.Errorf("invalid length %d", length)
+	}
+
+	req, err := http.NewRequest("GET", restPath(b.url, h), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "http.NewRequest")
+	}
+
+	byteRange := fmt.Sprintf("bytes=%d-", offset)
+	if length > 0 {
+		byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1)
+	}
+	req.Header.Add("Range", byteRange)
+	debug.Log("Load(%v) send range %v", h, byteRange)
+
+	<-b.connChan
+	resp, err := b.client.Do(req)
+	b.connChan <- struct{}{}
+
+	if err != nil {
+		if resp != nil {
+			io.Copy(ioutil.Discard, resp.Body)
+			resp.Body.Close()
+		}
+		return nil, errors.Wrap(err, "client.Do")
+	}
+
+	if resp.StatusCode != 200 && resp.StatusCode != 206 {
+		io.Copy(ioutil.Discard, resp.Body)
+		resp.Body.Close()
+		return nil, errors.Errorf("unexpected HTTP response code %v", resp.StatusCode)
+	}
+
+	return resp.Body, nil
+}
+
 // Stat returns information about a blob.
 func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 	if err := h.Valid(); err != nil {
@@ -188,8 +194,8 @@ func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 }
 
 // Test returns true if a blob of the given type and name exists in the backend.
-func (b *restBackend) Test(t restic.FileType, name string) (bool, error) {
-	_, err := b.Stat(restic.Handle{Type: t, Name: name})
+func (b *restBackend) Test(h restic.Handle) (bool, error) {
+	_, err := b.Stat(h)
 	if err != nil {
 		return false, nil
 	}
@@ -198,8 +204,7 @@ func (b *restBackend) Test(t restic.FileType, name string) (bool, error) {
 }
 
 // Remove removes the blob with the given name and type.
-func (b *restBackend) Remove(t restic.FileType, name string) error {
-	h := restic.Handle{Type: t, Name: name}
+func (b *restBackend) Remove(h restic.Handle) error {
 	if err := h.Valid(); err != nil {
 		return err
 	}

src/restic/backend/rest/rest_test.go

@@ -37,7 +37,7 @@ func init() {
 			return nil, err
 		}
 
-		exists, err := be.Test(restic.ConfigFile, "")
+		exists, err := be.Test(restic.Handle{Type: restic.ConfigFile, Name: ""})
 		if err != nil {
 			return nil, err
 		}

src/restic/backend/s3/backend_test.go

@@ -51,13 +51,6 @@ func TestS3BackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestS3BackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestS3BackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/s3/s3.go

@@ -7,6 +7,7 @@ import (
 	"restic"
 	"strings"
 
+	"restic/backend"
 	"restic/errors"
 
 	"github.com/minio/minio-go"
@@ -54,11 +55,11 @@ func Open(cfg Config) (restic.Backend, error) {
 	return be, nil
 }
 
-func (be *s3) s3path(t restic.FileType, name string) string {
-	if t == restic.ConfigFile {
-		return path.Join(be.prefix, string(t))
+func (be *s3) s3path(h restic.Handle) string {
+	if h.Type == restic.ConfigFile {
+		return path.Join(be.prefix, string(h.Type))
 	}
-	return path.Join(be.prefix, string(t), name)
+	return path.Join(be.prefix, string(h.Type), h.Name)
 }
 
 func (be *s3) createConnections() {
@@ -73,88 +74,15 @@ func (be *s3) Location() string {
 	return be.bucketname
 }
 
-// Load returns the data stored in the backend for h at the given offset
-// and saves it in p. Load has the same semantics as io.ReaderAt.
-func (be s3) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
-	var obj *minio.Object
-
-	debug.Log("%v, offset %v, len %v", h, off, len(p))
-	objName := be.s3path(h.Type, h.Name)
-
-	<-be.connChan
-	defer func() {
-		be.connChan <- struct{}{}
-	}()
-
-	obj, err = be.client.GetObject(be.bucketname, objName)
-	if err != nil {
-		debug.Log("  err %v", err)
-		return 0, errors.Wrap(err, "client.GetObject")
-	}
-
-	// make sure that the object is closed properly.
-	defer func() {
-		e := obj.Close()
-		if err == nil {
-			err = errors.Wrap(e, "Close")
-		}
-	}()
-
-	info, err := obj.Stat()
-	if err != nil {
-		return 0, errors.Wrap(err, "obj.Stat")
-	}
-
-	// handle negative offsets
-	if off < 0 {
-		// if the negative offset is larger than the object itself, read from
-		// the beginning.
-		if -off > info.Size {
-			off = 0
-		} else {
-			// otherwise compute the offset from the end of the file.
-			off = info.Size + off
-		}
-	}
-
-	// return an error if the offset is beyond the end of the file
-	if off > info.Size {
-		return 0, errors.Wrap(io.EOF, "")
-	}
-
-	var nextError error
-
-	// manually create an io.ErrUnexpectedEOF
-	if off+int64(len(p)) > info.Size {
-		newlen := info.Size - off
-		p = p[:newlen]
-
-		nextError = io.ErrUnexpectedEOF
-
-		debug.Log("    capped buffer to %v byte", len(p))
-	}
-
-	n, err = obj.ReadAt(p, off)
-	if int64(n) == info.Size-off && errors.Cause(err) == io.EOF {
-		err = nil
-	}
-
-	if err == nil {
-		err = nextError
-	}
-
-	return n, err
-}
-
 // Save stores data in the backend at the handle.
-func (be s3) Save(h restic.Handle, p []byte) (err error) {
+func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) {
 	if err := h.Valid(); err != nil {
 		return err
 	}
 
-	debug.Log("%v with %d bytes", h, len(p))
+	debug.Log("Save %v", h)
 
-	objName := be.s3path(h.Type, h.Name)
+	objName := be.s3path(h)
 
 	// Check key does not already exist
 	_, err = be.client.StatObject(be.bucketname, objName)
@@ -168,19 +96,97 @@ func (be s3) Save(h restic.Handle, p []byte) (err error) {
 		be.connChan <- struct{}{}
 	}()
 
-	debug.Log("PutObject(%v, %v, %v, %v)",
-		be.bucketname, objName, int64(len(p)), "binary/octet-stream")
-	n, err := be.client.PutObject(be.bucketname, objName, bytes.NewReader(p), "binary/octet-stream")
+	debug.Log("PutObject(%v, %v)",
+		be.bucketname, objName)
+	n, err := be.client.PutObject(be.bucketname, objName, rd, "binary/octet-stream")
 	debug.Log("%v -> %v bytes, err %#v", objName, n, err)
 
 	return errors.Wrap(err, "client.PutObject")
 }
 
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if length < 0 {
+		return nil, errors.Errorf("invalid length %d", length)
+	}
+
+	var obj *minio.Object
+
+	objName := be.s3path(h)
+
+	<-be.connChan
+	defer func() {
+		be.connChan <- struct{}{}
+	}()
+
+	obj, err := be.client.GetObject(be.bucketname, objName)
+	if err != nil {
+		debug.Log("  err %v", err)
+		return nil, errors.Wrap(err, "client.GetObject")
+	}
+
+	// if we're going to read the whole object, just pass it on.
+	if length == 0 {
+		debug.Log("Load %v: pass on object", h)
+		_, err = obj.Seek(offset, 0)
+		if err != nil {
+			_ = obj.Close()
+			return nil, errors.Wrap(err, "obj.Seek")
+		}
+
+		return obj, nil
+	}
+
+	// otherwise use a buffer with ReadAt
+	info, err := obj.Stat()
+	if err != nil {
+		_ = obj.Close()
+		return nil, errors.Wrap(err, "obj.Stat")
+	}
+
+	if offset > info.Size {
+		_ = obj.Close()
+		return nil, errors.Errorf("offset larger than file size")
+	}
+
+	l := int64(length)
+	if offset+l > info.Size {
+		l = info.Size - offset
+	}
+
+	buf := make([]byte, l)
+	n, err := obj.ReadAt(buf, offset)
+	debug.Log("Load %v: use buffer with ReadAt: %v, %v", h, n, err)
+	if err == io.EOF {
+		debug.Log("Load %v: shorten buffer %v -> %v", h, len(buf), n)
+		buf = buf[:n]
+		err = nil
+	}
+
+	if err != nil {
+		_ = obj.Close()
+		return nil, errors.Wrap(err, "obj.ReadAt")
+	}
+
+	return backend.Closer{Reader: bytes.NewReader(buf)}, nil
+}
+
 // Stat returns information about a blob.
-func (be s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
+func (be *s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
 	debug.Log("%v", h)
 
-	objName := be.s3path(h.Type, h.Name)
+	objName := be.s3path(h)
 	var obj *minio.Object
 
 	obj, err = be.client.GetObject(be.bucketname, objName)
@@ -207,9 +213,9 @@ func (be s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
 }
 
 // Test returns true if a blob of the given type and name exists in the backend.
-func (be *s3) Test(t restic.FileType, name string) (bool, error) {
+func (be *s3) Test(h restic.Handle) (bool, error) {
 	found := false
-	objName := be.s3path(t, name)
+	objName := be.s3path(h)
 	_, err := be.client.StatObject(be.bucketname, objName)
 	if err == nil {
 		found = true
@@ -220,10 +226,10 @@ func (be *s3) Test(t restic.FileType, name string) (bool, error) {
 }
 
 // Remove removes the blob with the given name and type.
-func (be *s3) Remove(t restic.FileType, name string) error {
-	objName := be.s3path(t, name)
+func (be *s3) Remove(h restic.Handle) error {
+	objName := be.s3path(h)
 	err := be.client.RemoveObject(be.bucketname, objName)
-	debug.Log("%v %v -> err %v", t, name, err)
+	debug.Log("Remove(%v) -> err %v", h, err)
 	return errors.Wrap(err, "client.RemoveObject")
 }
 
@@ -234,7 +240,7 @@ func (be *s3) List(t restic.FileType, done <-chan struct{}) <-chan string {
 	debug.Log("listing %v", t)
 	ch := make(chan string)
 
-	prefix := be.s3path(t, "") + "/"
+	prefix := be.s3path(restic.Handle{Type: t}) + "/"
 
 	listresp := be.client.ListObjects(be.bucketname, prefix, true, done)
 
@@ -262,7 +268,7 @@ func (be *s3) removeKeys(t restic.FileType) error {
 	done := make(chan struct{})
 	defer close(done)
 	for key := range be.List(restic.DataFile, done) {
-		err := be.Remove(restic.DataFile, key)
+		err := be.Remove(restic.Handle{Type: restic.DataFile, Name: key})
 		if err != nil {
 			return err
 		}
@@ -287,7 +293,7 @@ func (be *s3) Delete() error {
 		}
 	}
 
-	return be.Remove(restic.ConfigFile, "")
+	return be.Remove(restic.Handle{Type: restic.ConfigFile})
 }
 
 // Close does nothing

src/restic/backend/s3/s3_test.go

@@ -44,7 +44,7 @@ func init() {
 			return nil, err
 		}
 
-		exists, err := be.Test(restic.ConfigFile, "")
+		exists, err := be.Test(restic.Handle{Type: restic.ConfigFile})
 		if err != nil {
 			return nil, err
 		}

src/restic/backend/sftp/backend_test.go

@@ -51,13 +51,6 @@ func TestSftpBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestSftpBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestSftpBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/sftp/sftp.go

@@ -259,11 +259,11 @@ func (r *SFTP) mkdirAll(dir string, mode os.FileMode) error {
 }
 
 // Rename temp file to final name according to type and name.
-func (r *SFTP) renameFile(oldname string, t restic.FileType, name string) error {
-	filename := r.filename(t, name)
+func (r *SFTP) renameFile(oldname string, h restic.Handle) error {
+	filename := r.filename(h)
 
 	// create directories if necessary
-	if t == restic.DataFile {
+	if h.Type == restic.DataFile {
 		err := r.mkdirAll(path.Dir(filename), backend.Modes.Dir)
 		if err != nil {
 			return err
@@ -297,22 +297,22 @@ func Join(parts ...string) string {
 }
 
 // Construct path for given restic.Type and name.
-func (r *SFTP) filename(t restic.FileType, name string) string {
-	if t == restic.ConfigFile {
+func (r *SFTP) filename(h restic.Handle) string {
+	if h.Type == restic.ConfigFile {
 		return Join(r.p, "config")
 	}
 
-	return Join(r.dirname(t, name), name)
+	return Join(r.dirname(h), h.Name)
 }
 
 // Construct directory for given backend.Type.
-func (r *SFTP) dirname(t restic.FileType, name string) string {
+func (r *SFTP) dirname(h restic.Handle) string {
 	var n string
-	switch t {
+	switch h.Type {
 	case restic.DataFile:
 		n = backend.Paths.Data
-		if len(name) > 2 {
-			n = Join(n, name[:2])
+		if len(h.Name) > 2 {
+			n = Join(n, h.Name[:2])
 		}
 	case restic.SnapshotFile:
 		n = backend.Paths.Snapshots
@@ -326,47 +326,9 @@ func (r *SFTP) dirname(t restic.FileType, name string) string {
 	return Join(r.p, n)
 }
 
-// Load returns the data stored in the backend for h at the given offset
-// and saves it in p. Load has the same semantics as io.ReaderAt.
-func (r *SFTP) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
-	debug.Log("load %v, %d bytes, offset %v", h, len(p), off)
-	if err := r.clientError(); err != nil {
-		return 0, err
-	}
-
-	if err := h.Valid(); err != nil {
-		return 0, err
-	}
-
-	f, err := r.c.Open(r.filename(h.Type, h.Name))
-	if err != nil {
-		return 0, errors.Wrap(err, "Open")
-	}
-
-	defer func() {
-		e := f.Close()
-		if err == nil {
-			err = errors.Wrap(e, "Close")
-		}
-	}()
-
-	switch {
-	case off > 0:
-		_, err = f.Seek(off, 0)
-	case off < 0:
-		_, err = f.Seek(off, 2)
-	}
-
-	if err != nil {
-		return 0, errors.Wrap(err, "Seek")
-	}
-
-	return io.ReadFull(f, p)
-}
-
 // Save stores data in the backend at the handle.
-func (r *SFTP) Save(h restic.Handle, p []byte) (err error) {
-	debug.Log("save %v bytes to %v", h, len(p))
+func (r *SFTP) Save(h restic.Handle, rd io.Reader) (err error) {
+	debug.Log("save to %v", h)
 	if err := r.clientError(); err != nil {
 		return err
 	}
@@ -380,31 +342,60 @@ func (r *SFTP) Save(h restic.Handle, p []byte) (err error) {
 		return err
 	}
 
-	debug.Log("save %v (%d bytes) to %v", h, len(p), filename)
-
-	n, err := tmpfile.Write(p)
+	n, err := io.Copy(tmpfile, rd)
 	if err != nil {
 		return errors.Wrap(err, "Write")
 	}
 
-	if n != len(p) {
-		return errors.New("not all bytes writen")
-	}
+	debug.Log("saved %v (%d bytes) to %v", h, n, filename)
 
 	err = tmpfile.Close()
 	if err != nil {
 		return errors.Wrap(err, "Close")
 	}
 
-	err = r.renameFile(filename, h.Type, h.Name)
+	err = r.renameFile(filename, h)
 	debug.Log("save %v: rename %v: %v",
 		h, path.Base(filename), err)
 	return err
 }
 
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (r *SFTP) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	f, err := r.c.Open(r.filename(h))
+	if err != nil {
+		return nil, err
+	}
+
+	if offset > 0 {
+		_, err = f.Seek(offset, 0)
+		if err != nil {
+			_ = f.Close()
+			return nil, err
+		}
+	}
+
+	if length > 0 {
+		return backend.LimitReadCloser(f, int64(length)), nil
+	}
+
+	return f, nil
+}
+
 // Stat returns information about a blob.
 func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) {
-	debug.Log("stat %v", h)
+	debug.Log("Stat(%v)", h)
 	if err := r.clientError(); err != nil {
 		return restic.FileInfo{}, err
 	}
@@ -413,7 +404,7 @@ func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) {
 		return restic.FileInfo{}, err
 	}
 
-	fi, err := r.c.Lstat(r.filename(h.Type, h.Name))
+	fi, err := r.c.Lstat(r.filename(h))
 	if err != nil {
 		return restic.FileInfo{}, errors.Wrap(err, "Lstat")
 	}
@@ -422,13 +413,13 @@ func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) {
 }
 
 // Test returns true if a blob of the given type and name exists in the backend.
-func (r *SFTP) Test(t restic.FileType, name string) (bool, error) {
-	debug.Log("type %v, name %v", t, name)
+func (r *SFTP) Test(h restic.Handle) (bool, error) {
+	debug.Log("Test(%v)", h)
 	if err := r.clientError(); err != nil {
 		return false, err
 	}
 
-	_, err := r.c.Lstat(r.filename(t, name))
+	_, err := r.c.Lstat(r.filename(h))
 	if os.IsNotExist(errors.Cause(err)) {
 		return false, nil
 	}
@@ -441,13 +432,13 @@ func (r *SFTP) Test(t restic.FileType, name string) (bool, error) {
 }
 
 // Remove removes the content stored at name.
-func (r *SFTP) Remove(t restic.FileType, name string) error {
-	debug.Log("type %v, name %v", t, name)
+func (r *SFTP) Remove(h restic.Handle) error {
+	debug.Log("Remove(%v)", h)
 	if err := r.clientError(); err != nil {
 		return err
 	}
 
-	return r.c.Remove(r.filename(t, name))
+	return r.c.Remove(r.filename(h))
 }
 
 // List returns a channel that yields all names of blobs of type t. A
@@ -462,7 +453,7 @@ func (r *SFTP) List(t restic.FileType, done <-chan struct{}) <-chan string {
 
 		if t == restic.DataFile {
 			// read first level
-			basedir := r.dirname(t, "")
+			basedir := r.dirname(restic.Handle{Type: t})
 
 			list1, err := r.c.ReadDir(basedir)
 			if err != nil {
@@ -495,7 +486,7 @@ func (r *SFTP) List(t restic.FileType, done <-chan struct{}) <-chan string {
 				}
 			}
 		} else {
-			entries, err := r.c.ReadDir(r.dirname(t, ""))
+			entries, err := r.c.ReadDir(r.dirname(restic.Handle{Type: t}))
 			if err != nil {
 				return
 			}

src/restic/backend/test/backend_test.go

@@ -51,13 +51,6 @@ func TestTestBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestTestBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestTestBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/test/tests.go

@@ -4,13 +4,16 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"math/rand"
+	"os"
 	"reflect"
 	"restic"
+	"restic/errors"
 	"sort"
+	"strings"
 	"testing"
 
-	"restic/errors"
 	"restic/test"
 
 	"restic/backend"
@@ -127,7 +130,7 @@ func TestCreateWithConfig(t testing.TB) {
 	}
 
 	// remove config
-	err = b.Remove(restic.ConfigFile, "")
+	err = b.Remove(restic.Handle{Type: restic.ConfigFile, Name: ""})
 	if err != nil {
 		t.Fatalf("unexpected error removing config: %v", err)
 	}
@@ -152,12 +155,12 @@ func TestConfig(t testing.TB) {
 	var testString = "Config"
 
 	// create config and read it back
-	_, err := backend.LoadAll(b, restic.Handle{Type: restic.ConfigFile}, nil)
+	_, err := backend.LoadAll(b, restic.Handle{Type: restic.ConfigFile})
 	if err == nil {
 		t.Fatalf("did not get expected error for non-existing config")
 	}
 
-	err = b.Save(restic.Handle{Type: restic.ConfigFile}, []byte(testString))
+	err = b.Save(restic.Handle{Type: restic.ConfigFile}, strings.NewReader(testString))
 	if err != nil {
 		t.Fatalf("Save() error: %v", err)
 	}
@@ -166,7 +169,7 @@ func TestConfig(t testing.TB) {
 	// same config
 	for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} {
 		h := restic.Handle{Type: restic.ConfigFile, Name: name}
-		buf, err := backend.LoadAll(b, h, nil)
+		buf, err := backend.LoadAll(b, h)
 		if err != nil {
 			t.Fatalf("unable to read config with name %q: %v", name, err)
 		}
@@ -182,12 +185,12 @@ func TestLoad(t testing.TB) {
 	b := open(t)
 	defer close(t)
 
-	_, err := b.Load(restic.Handle{}, nil, 0)
+	_, err := b.Load(restic.Handle{}, 0, 0)
 	if err == nil {
 		t.Fatalf("Load() did not return an error for invalid handle")
 	}
 
-	_, err = b.Load(restic.Handle{Type: restic.DataFile, Name: "foobar"}, nil, 0)
+	_, err = b.Load(restic.Handle{Type: restic.DataFile, Name: "foobar"}, 0, 0)
 	if err == nil {
 		t.Fatalf("Load() did not return an error for non-existing blob")
 	}
@@ -198,11 +201,20 @@ func TestLoad(t testing.TB) {
 	id := restic.Hash(data)
 
 	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	err = b.Save(handle, data)
+	err = b.Save(handle, bytes.NewReader(data))
 	if err != nil {
 		t.Fatalf("Save() error: %v", err)
 	}
 
+	rd, err := b.Load(handle, 100, -1)
+	if err == nil {
+		t.Fatalf("Load() returned no error for negative offset!")
+	}
+
+	if rd != nil {
+		t.Fatalf("Load() returned a non-nil reader for negative offset!")
+	}
+
 	for i := 0; i < 50; i++ {
 		l := rand.Intn(length + 2000)
 		o := rand.Intn(length + 2000)
@@ -215,157 +227,60 @@ func TestLoad(t testing.TB) {
 			d = d[:0]
 		}
 
-		if l > 0 && l < len(d) {
-			d = d[:l]
-		}
-
-		buf := make([]byte, l)
-		n, err := b.Load(handle, buf, int64(o))
-
-		// if we requested data beyond the end of the file, require
-		// ErrUnexpectedEOF error
-		if l > len(d) {
-			if errors.Cause(err) != io.ErrUnexpectedEOF {
-				t.Errorf("Load(%d, %d) did not return io.ErrUnexpectedEOF", len(buf), int64(o))
-			}
-			err = nil
-			buf = buf[:len(d)]
-		}
-
-		if err != nil {
-			t.Errorf("Load(%d, %d): unexpected error: %v", len(buf), int64(o), err)
-			continue
-		}
-
-		if n != len(buf) {
-			t.Errorf("Load(%d, %d): wrong length returned, want %d, got %d",
-				len(buf), int64(o), len(buf), n)
-			continue
-		}
-
-		buf = buf[:n]
-		if !bytes.Equal(buf, d) {
-			t.Errorf("Load(%d, %d) returned wrong bytes", len(buf), int64(o))
-			continue
-		}
-	}
-
-	// test with negative offset
-	for i := 0; i < 50; i++ {
-		l := rand.Intn(length + 2000)
-		o := rand.Intn(length + 2000)
-
-		d := data
-		if o < len(d) {
-			d = d[len(d)-o:]
-		} else {
-			o = 0
+		getlen := l
+		if l >= len(d) && rand.Float32() >= 0.5 {
+			getlen = 0
 		}
 
 		if l > 0 && l < len(d) {
 			d = d[:l]
 		}
 
-		buf := make([]byte, l)
-		n, err := b.Load(handle, buf, -int64(o))
-
-		// if we requested data beyond the end of the file, require
-		// ErrUnexpectedEOF error
-		if l > len(d) {
-			if errors.Cause(err) != io.ErrUnexpectedEOF {
-				t.Errorf("Load(%d, %d) did not return io.ErrUnexpectedEOF", len(buf), int64(o))
-				continue
-			}
-			err = nil
-			buf = buf[:len(d)]
-		}
-
+		rd, err := b.Load(handle, getlen, int64(o))
 		if err != nil {
-			t.Errorf("Load(%d, %d): unexpected error: %v", len(buf), int64(o), err)
+			t.Errorf("Load(%d, %d) returned unexpected error: %v", l, o, err)
 			continue
 		}
 
-		if n != len(buf) {
-			t.Errorf("Load(%d, %d): wrong length returned, want %d, got %d",
-				len(buf), int64(o), len(buf), n)
+		buf, err := ioutil.ReadAll(rd)
+		if err != nil {
+			t.Errorf("Load(%d, %d) ReadAll() returned unexpected error: %v", l, o, err)
+			continue
+		}
+
+		if l <= len(d) && len(buf) != l {
+			t.Errorf("Load(%d, %d) wrong number of bytes read: want %d, got %d", l, o, l, len(buf))
+			continue
+		}
+
+		if l > len(d) && len(buf) != len(d) {
+			t.Errorf("Load(%d, %d) wrong number of bytes read for overlong read: want %d, got %d", l, o, l, len(buf))
 			continue
 		}
 
-		buf = buf[:n]
 		if !bytes.Equal(buf, d) {
-			t.Errorf("Load(%d, %d) returned wrong bytes", len(buf), int64(o))
+			t.Errorf("Load(%d, %d) returned wrong bytes", l, o)
+			continue
+		}
+
+		err = rd.Close()
+		if err != nil {
+			t.Errorf("Load(%d, %d) rd.Close() returned unexpected error: %v", l, o, err)
 			continue
 		}
 	}
 
-	// load with a too-large buffer, this should return io.ErrUnexpectedEOF
-	buf := make([]byte, length+100)
-	n, err := b.Load(handle, buf, 0)
-	if n != length {
-		t.Errorf("wrong length for larger buffer returned, want %d, got %d", length, n)
-	}
-
-	if errors.Cause(err) != io.ErrUnexpectedEOF {
-		t.Errorf("wrong error returned for larger buffer: want io.ErrUnexpectedEOF, got %#v", err)
-	}
-
-	test.OK(t, b.Remove(restic.DataFile, id.String()))
+	test.OK(t, b.Remove(handle))
 }
 
-// TestLoadNegativeOffset tests the backend's Load function with negative offsets.
-func TestLoadNegativeOffset(t testing.TB) {
-	b := open(t)
-	defer close(t)
+type errorCloser struct {
+	io.Reader
+	t testing.TB
+}
 
-	length := rand.Intn(1<<24) + 2000
-
-	data := test.Random(23, length)
-	id := restic.Hash(data)
-
-	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	err := b.Save(handle, data)
-	if err != nil {
-		t.Fatalf("Save() error: %v", err)
-	}
-
-	// test normal reads
-	for i := 0; i < 50; i++ {
-		l := rand.Intn(length + 2000)
-		o := -rand.Intn(length + 2000)
-
-		buf := make([]byte, l)
-		n, err := b.Load(handle, buf, int64(o))
-
-		// if we requested data beyond the end of the file, require
-		// ErrUnexpectedEOF error
-		if len(buf) > -o {
-			if errors.Cause(err) != io.ErrUnexpectedEOF {
-				t.Errorf("Load(%d, %d) did not return io.ErrUnexpectedEOF", len(buf), o)
-				continue
-			}
-			err = nil
-			buf = buf[:-o]
-		}
-
-		if err != nil {
-			t.Errorf("Load(%d, %d) returned error: %v", len(buf), o, err)
-			continue
-		}
-
-		if n != len(buf) {
-			t.Errorf("Load(%d, %d) returned short read, only got %d bytes", len(buf), o, n)
-			continue
-		}
-
-		p := len(data) + o
-		if !bytes.Equal(buf, data[p:p+len(buf)]) {
-			t.Errorf("Load(%d, %d) returned wrong bytes", len(buf), o)
-			continue
-		}
-
-	}
-
-	test.OK(t, b.Remove(restic.DataFile, id.String()))
+func (ec errorCloser) Close() error {
+	ec.t.Error("forbidden method close was called")
+	return errors.New("forbidden method close was called")
 }
 
 // TestSave tests saving data in the backend.
@@ -384,10 +299,10 @@ func TestSave(t testing.TB) {
 			Type: restic.DataFile,
 			Name: fmt.Sprintf("%s-%d", id, i),
 		}
-		err := b.Save(h, data)
+		err := b.Save(h, bytes.NewReader(data))
 		test.OK(t, err)
 
-		buf, err := backend.LoadAll(b, h, nil)
+		buf, err := backend.LoadAll(b, h)
 		test.OK(t, err)
 		if len(buf) != len(data) {
 			t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf))
@@ -404,11 +319,51 @@ func TestSave(t testing.TB) {
 			t.Fatalf("Stat() returned different size, want %q, got %d", len(data), fi.Size)
 		}
 
-		err = b.Remove(h.Type, h.Name)
+		err = b.Remove(h)
 		if err != nil {
 			t.Fatalf("error removing item: %v", err)
 		}
 	}
+
+	// test saving from a tempfile
+	tmpfile, err := ioutil.TempFile("", "restic-backend-save-test-")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	length := rand.Intn(1<<23) + 200000
+	data := test.Random(23, length)
+	copy(id[:], data)
+
+	if _, err = tmpfile.Write(data); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err = tmpfile.Seek(0, 0); err != nil {
+		t.Fatal(err)
+	}
+
+	h := restic.Handle{Type: restic.DataFile, Name: id.String()}
+
+	// wrap the tempfile in an errorCloser, so we can detect if the backend
+	// closes the reader
+	err = b.Save(h, errorCloser{t: t, Reader: tmpfile})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err = tmpfile.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	if err = os.Remove(tmpfile.Name()); err != nil {
+		t.Fatal(err)
+	}
+
+	err = b.Remove(h)
+	if err != nil {
+		t.Fatalf("error removing item: %v", err)
+	}
 }
 
 var filenameTests = []struct {
@@ -430,13 +385,13 @@ func TestSaveFilenames(t testing.TB) {
 
 	for i, test := range filenameTests {
 		h := restic.Handle{Name: test.name, Type: restic.DataFile}
-		err := b.Save(h, []byte(test.data))
+		err := b.Save(h, strings.NewReader(test.data))
 		if err != nil {
 			t.Errorf("test %d failed: Save() returned %v", i, err)
 			continue
 		}
 
-		buf, err := backend.LoadAll(b, h, nil)
+		buf, err := backend.LoadAll(b, h)
 		if err != nil {
 			t.Errorf("test %d failed: Load() returned %v", i, err)
 			continue
@@ -446,7 +401,7 @@ func TestSaveFilenames(t testing.TB) {
 			t.Errorf("test %d: returned wrong bytes", i)
 		}
 
-		err = b.Remove(h.Type, h.Name)
+		err = b.Remove(h)
 		if err != nil {
 			t.Errorf("test %d failed: Remove() returned %v", i, err)
 			continue
@@ -464,10 +419,12 @@ var testStrings = []struct {
 	{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"},
 }
 
-func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) {
+func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) restic.Handle {
 	id := restic.Hash(data)
-	err := b.Save(restic.Handle{Name: id.String(), Type: tpe}, data)
+	h := restic.Handle{Name: id.String(), Type: tpe}
+	err := b.Save(h, bytes.NewReader(data))
 	test.OK(t, err)
+	return h
 }
 
 // TestBackend tests all functions of the backend.
@@ -485,21 +442,21 @@ func TestBackend(t testing.TB) {
 			test.OK(t, err)
 
 			// test if blob is already in repository
-			ret, err := b.Test(tpe, id.String())
+			h := restic.Handle{Type: tpe, Name: id.String()}
+			ret, err := b.Test(h)
 			test.OK(t, err)
 			test.Assert(t, !ret, "blob was found to exist before creating")
 
 			// try to stat a not existing blob
-			h := restic.Handle{Type: tpe, Name: id.String()}
 			_, err = b.Stat(h)
 			test.Assert(t, err != nil, "blob data could be extracted before creation")
 
 			// try to read not existing blob
-			_, err = b.Load(h, nil, 0)
+			_, err = b.Load(h, 0, 0)
 			test.Assert(t, err != nil, "blob reader could be obtained before creation")
 
 			// try to get string out, should fail
-			ret, err = b.Test(tpe, id.String())
+			ret, err = b.Test(h)
 			test.OK(t, err)
 			test.Assert(t, !ret, "id %q was found (but should not have)", ts.id)
 		}
@@ -510,7 +467,7 @@ func TestBackend(t testing.TB) {
 
 			// test Load()
 			h := restic.Handle{Type: tpe, Name: ts.id}
-			buf, err := backend.LoadAll(b, h, nil)
+			buf, err := backend.LoadAll(b, h)
 			test.OK(t, err)
 			test.Equals(t, ts.data, string(buf))
 
@@ -520,9 +477,18 @@ func TestBackend(t testing.TB) {
 			length := end - start
 
 			buf2 := make([]byte, length)
-			n, err := b.Load(h, buf2, int64(start))
+			rd, err := b.Load(h, len(buf2), int64(start))
 			test.OK(t, err)
-			test.Equals(t, length, n)
+			n, err := io.ReadFull(rd, buf2)
+			test.OK(t, err)
+			test.Equals(t, len(buf2), n)
+
+			remaining, err := io.Copy(ioutil.Discard, rd)
+			test.OK(t, err)
+			test.Equals(t, int64(0), remaining)
+
+			test.OK(t, rd.Close())
+
 			test.Equals(t, ts.data[start:end], string(buf2))
 		}
 
@@ -530,20 +496,21 @@ func TestBackend(t testing.TB) {
 		ts := testStrings[0]
 
 		// create blob
-		err := b.Save(restic.Handle{Type: tpe, Name: ts.id}, []byte(ts.data))
+		err := b.Save(restic.Handle{Type: tpe, Name: ts.id}, strings.NewReader(ts.data))
 		test.Assert(t, err != nil, "expected error, got %v", err)
 
 		// remove and recreate
-		err = b.Remove(tpe, ts.id)
+		h := restic.Handle{Type: tpe, Name: ts.id}
+		err = b.Remove(h)
 		test.OK(t, err)
 
 		// test that the blob is gone
-		ok, err := b.Test(tpe, ts.id)
+		ok, err := b.Test(h)
 		test.OK(t, err)
 		test.Assert(t, ok == false, "removed blob still present")
 
 		// create blob
-		err = b.Save(restic.Handle{Type: tpe, Name: ts.id}, []byte(ts.data))
+		err = b.Save(h, strings.NewReader(ts.data))
 		test.OK(t, err)
 
 		// list items
@@ -578,12 +545,14 @@ func TestBackend(t testing.TB) {
 				id, err := restic.ParseID(ts.id)
 				test.OK(t, err)
 
-				found, err := b.Test(tpe, id.String())
+				h := restic.Handle{Type: tpe, Name: id.String()}
+
+				found, err := b.Test(h)
 				test.OK(t, err)
 
-				test.OK(t, b.Remove(tpe, id.String()))
+				test.OK(t, b.Remove(h))
 
-				found, err = b.Test(tpe, id.String())
+				found, err = b.Test(h)
 				test.OK(t, err)
 				test.Assert(t, !found, fmt.Sprintf("id %q not found after removal", id))
 			}

src/restic/backend/utils.go

@@ -2,29 +2,51 @@ package backend
 
 import (
 	"io"
+	"io/ioutil"
 	"restic"
-
-	"restic/errors"
 )
 
-// LoadAll reads all data stored in the backend for the handle. The buffer buf
-// is resized to accomodate all data in the blob. Errors returned by be.Load()
-// are passed on, except io.ErrUnexpectedEOF is silenced and nil returned
-// instead, since it means this function is working properly.
-func LoadAll(be restic.Backend, h restic.Handle, buf []byte) ([]byte, error) {
-	fi, err := be.Stat(h)
+// LoadAll reads all data stored in the backend for the handle.
+func LoadAll(be restic.Backend, h restic.Handle) (buf []byte, err error) {
+	rd, err := be.Load(h, 0, 0)
 	if err != nil {
-		return nil, errors.Wrap(err, "Stat")
+		return nil, err
 	}
 
-	if fi.Size > int64(len(buf)) {
-		buf = make([]byte, int(fi.Size))
-	}
+	defer func() {
+		io.Copy(ioutil.Discard, rd)
+		e := rd.Close()
+		if err == nil {
+			err = e
+		}
+	}()
 
-	n, err := be.Load(h, buf, 0)
-	if errors.Cause(err) == io.ErrUnexpectedEOF {
-		err = nil
-	}
-	buf = buf[:n]
-	return buf, err
+	return ioutil.ReadAll(rd)
+}
+
+// Closer wraps an io.Reader and adds a Close() method that does nothing.
+type Closer struct {
+	io.Reader
+}
+
+// Close is a no-op.
+func (c Closer) Close() error {
+	return nil
+}
+
+// LimitedReadCloser wraps io.LimitedReader and exposes the Close() method.
+type LimitedReadCloser struct {
+	io.ReadCloser
+	io.Reader
+}
+
+// Read reads data from the limited reader.
+func (l *LimitedReadCloser) Read(p []byte) (int, error) {
+	return l.Reader.Read(p)
+}
+
+// LimitReadCloser returns a new reader wraps r in an io.LimitReader, but also
+// exposes the Close() method.
+func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser {
+	return &LimitedReadCloser{ReadCloser: r, Reader: io.LimitReader(r, n)}
 }

src/restic/backend/utils_test.go

@@ -21,10 +21,10 @@ func TestLoadAll(t *testing.T) {
 		data := Random(23+i, rand.Intn(MiB)+500*KiB)
 
 		id := restic.Hash(data)
-		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, data)
+		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
 		OK(t, err)
 
-		buf, err := backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}, nil)
+		buf, err := backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()})
 		OK(t, err)
 
 		if len(buf) != len(data) {
@@ -46,11 +46,11 @@ func TestLoadSmallBuffer(t *testing.T) {
 		data := Random(23+i, rand.Intn(MiB)+500*KiB)
 
 		id := restic.Hash(data)
-		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, data)
+		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
 		OK(t, err)
 
 		buf := make([]byte, len(data)-23)
-		buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}, buf)
+		buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()})
 		OK(t, err)
 
 		if len(buf) != len(data) {
@@ -72,11 +72,11 @@ func TestLoadLargeBuffer(t *testing.T) {
 		data := Random(23+i, rand.Intn(MiB)+500*KiB)
 
 		id := restic.Hash(data)
-		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, data)
+		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
 		OK(t, err)
 
 		buf := make([]byte, len(data)+100)
-		buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}, buf)
+		buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()})
 		OK(t, err)
 
 		if len(buf) != len(data) {

src/restic/buffer.go

@@ -0,0 +1,21 @@
+package restic
+
+import "restic/crypto"
+
+// NewBlobBuffer returns a buffer that is large enough to hold a blob of size
+// plaintext bytes, including the crypto overhead.
+func NewBlobBuffer(size int) []byte {
+	return make([]byte, size, size+crypto.Extension)
+}
+
+// PlaintextLength returns the plaintext length of a blob with ciphertextSize
+// bytes.
+func PlaintextLength(ciphertextSize int) int {
+	return ciphertextSize - crypto.Extension
+}
+
+// CiphertextLength returns the encrypted length of a blob with plaintextSize
+// bytes.
+func CiphertextLength(plaintextSize int) int {
+	return plaintextSize + crypto.Extension
+}

src/restic/checker/checker.go

@@ -187,7 +187,8 @@ func packIDTester(repo restic.Repository, inChan <-chan restic.ID, errChan chan<
 	defer wg.Done()
 
 	for id := range inChan {
-		ok, err := repo.Backend().Test(restic.DataFile, id.String())
+		h := restic.Handle{Type: restic.DataFile, Name: id.String()}
+		ok, err := repo.Backend().Test(h)
 		if err != nil {
 			err = PackError{ID: id, Err: err}
 		} else {
@@ -658,7 +659,7 @@ func (c *Checker) CountPacks() uint64 {
 func checkPack(r restic.Repository, id restic.ID) error {
 	debug.Log("checking pack %v", id.Str())
 	h := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	buf, err := backend.LoadAll(r.Backend(), h, nil)
+	buf, err := backend.LoadAll(r.Backend(), h)
 	if err != nil {
 		return err
 	}

src/restic/checker/checker_test.go

@@ -1,6 +1,7 @@
 package checker_test
 
 import (
+	"io"
 	"math/rand"
 	"path/filepath"
 	"sort"
@@ -8,7 +9,6 @@ import (
 
 	"restic"
 	"restic/archiver"
-	"restic/backend/mem"
 	"restic/checker"
 	"restic/repository"
 	"restic/test"
@@ -73,8 +73,11 @@ func TestMissingPack(t *testing.T) {
 
 	repo := repository.TestOpenLocal(t, repodir)
 
-	packID := "657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6"
-	test.OK(t, repo.Backend().Remove(restic.DataFile, packID))
+	packHandle := restic.Handle{
+		Type: restic.DataFile,
+		Name: "657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6",
+	}
+	test.OK(t, repo.Backend().Remove(packHandle))
 
 	chkr := checker.New(repo)
 	hints, errs := chkr.LoadIndex()
@@ -92,7 +95,7 @@ func TestMissingPack(t *testing.T) {
 		"expected exactly one error, got %v", len(errs))
 
 	if err, ok := errs[0].(checker.PackError); ok {
-		test.Equals(t, packID, err.ID.String())
+		test.Equals(t, packHandle.Name, err.ID.String())
 	} else {
 		t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err)
 	}
@@ -105,9 +108,12 @@ func TestUnreferencedPack(t *testing.T) {
 	repo := repository.TestOpenLocal(t, repodir)
 
 	// index 3f1a only references pack 60e0
-	indexID := "3f1abfcb79c6f7d0a3be517d2c83c8562fba64ef2c8e9a3544b4edaf8b5e3b44"
 	packID := "60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e"
-	test.OK(t, repo.Backend().Remove(restic.IndexFile, indexID))
+	indexHandle := restic.Handle{
+		Type: restic.IndexFile,
+		Name: "3f1abfcb79c6f7d0a3be517d2c83c8562fba64ef2c8e9a3544b4edaf8b5e3b44",
+	}
+	test.OK(t, repo.Backend().Remove(indexHandle))
 
 	chkr := checker.New(repo)
 	hints, errs := chkr.LoadIndex()
@@ -137,8 +143,11 @@ func TestUnreferencedBlobs(t *testing.T) {
 
 	repo := repository.TestOpenLocal(t, repodir)
 
-	snID := "51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02"
-	test.OK(t, repo.Backend().Remove(restic.SnapshotFile, snID))
+	snapshotHandle := restic.Handle{
+		Type: restic.SnapshotFile,
+		Name: "51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02",
+	}
+	test.OK(t, repo.Backend().Remove(snapshotHandle))
 
 	unusedBlobsBySnapshot := restic.IDs{
 		restic.TestParseID("58c748bbe2929fdf30c73262bd8313fe828f8925b05d1d4a87fe109082acb849"),
@@ -208,15 +217,35 @@ type errorBackend struct {
 	ProduceErrors bool
 }
 
-func (b errorBackend) Load(h restic.Handle, p []byte, off int64) (int, error) {
-	n, err := b.Backend.Load(h, p, off)
+func (b errorBackend) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	rd, err := b.Backend.Load(h, length, offset)
+	if err != nil {
+		return rd, err
+	}
 
 	if b.ProduceErrors {
-		induceError(p)
+		return errorReadCloser{rd}, err
+	}
+
+	return rd, nil
+}
+
+type errorReadCloser struct {
+	io.ReadCloser
+}
+
+func (erd errorReadCloser) Read(p []byte) (int, error) {
+	n, err := erd.ReadCloser.Read(p)
+	if n > 0 {
+		induceError(p[:n])
 	}
 	return n, err
 }
 
+func (erd errorReadCloser) Close() error {
+	return erd.ReadCloser.Close()
+}
+
 // induceError flips a bit in the slice.
 func induceError(data []byte) {
 	if rand.Float32() < 0.2 {
@@ -228,19 +257,15 @@ func induceError(data []byte) {
 }
 
 func TestCheckerModifiedData(t *testing.T) {
-	be := mem.New()
-
-	repository.TestUseLowSecurityKDFParameters(t)
-
-	repo := repository.New(be)
-	test.OK(t, repo.Init(test.TestPassword))
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
 
 	arch := archiver.New(repo)
 	_, id, err := arch.Snapshot(nil, []string{"."}, nil, nil)
 	test.OK(t, err)
 	t.Logf("archived as %v", id.Str())
 
-	beError := &errorBackend{Backend: be}
+	beError := &errorBackend{Backend: repo.Backend()}
 	checkRepo := repository.New(beError)
 	test.OK(t, checkRepo.SearchKey(test.TestPassword, 5))
 
@@ -266,7 +291,7 @@ func TestCheckerModifiedData(t *testing.T) {
 	}
 
 	for _, err := range checkData(chkr) {
-		t.Logf("struct error: %v", err)
+		t.Logf("data error: %v", err)
 		errFound = true
 	}
 

src/restic/fuse/file.go

@@ -4,8 +4,6 @@
 package fuse
 
 import (
-	"sync"
-
 	"restic/errors"
 
 	"restic"
@@ -35,29 +33,23 @@ type file struct {
 	node        *restic.Node
 	ownerIsRoot bool
 
-	sizes []uint
+	sizes []int
 	blobs [][]byte
 }
 
 const defaultBlobSize = 128 * 1024
 
-var blobPool = sync.Pool{
-	New: func() interface{} {
-		return make([]byte, defaultBlobSize)
-	},
-}
-
 func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool) (*file, error) {
 	debug.Log("create new file for %v with %d blobs", node.Name, len(node.Content))
 	var bytes uint64
-	sizes := make([]uint, len(node.Content))
+	sizes := make([]int, len(node.Content))
 	for i, id := range node.Content {
 		size, err := repo.LookupBlobSize(id, restic.DataBlob)
 		if err != nil {
 			return nil, err
 		}
 
-		sizes[i] = size
+		sizes[i] = int(size)
 		bytes += uint64(size)
 	}
 
@@ -99,16 +91,12 @@ func (f *file) getBlobAt(i int) (blob []byte, err error) {
 		return f.blobs[i], nil
 	}
 
-	buf := blobPool.Get().([]byte)
-	buf = buf[:cap(buf)]
-
-	if uint(len(buf)) < f.sizes[i] {
-		if len(buf) > defaultBlobSize {
-			blobPool.Put(buf)
-		}
-		buf = make([]byte, f.sizes[i])
+	// release earlier blobs
+	for j := 0; j < i; j++ {
+		f.blobs[j] = nil
 	}
 
+	buf := restic.NewBlobBuffer(f.sizes[i])
 	n, err := f.repo.LoadBlob(restic.DataBlob, f.node.Content[i], buf)
 	if err != nil {
 		debug.Log("LoadBlob(%v, %v) failed: %v", f.node.Name, f.node.Content[i], err)
@@ -169,10 +157,7 @@ func (f *file) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadR
 
 func (f *file) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
 	for i := range f.blobs {
-		if f.blobs[i] != nil {
-			blobPool.Put(f.blobs[i])
-			f.blobs[i] = nil
-		}
+		f.blobs[i] = nil
 	}
 	return nil
 }

src/restic/fuse/file_test.go

@@ -9,7 +9,9 @@ import (
 	"testing"
 	"time"
 
-	"restic/errors"
+	"golang.org/x/net/context"
+
+	"restic/repository"
 
 	"bazil.org/fuse"
 
@@ -17,108 +19,96 @@ import (
 	. "restic/test"
 )
 
-type MockRepo struct {
-	blobs map[restic.ID][]byte
-}
-
-func NewMockRepo(content map[restic.ID][]byte) *MockRepo {
-	return &MockRepo{blobs: content}
-}
-
-func (m *MockRepo) LookupBlobSize(id restic.ID, t restic.BlobType) (uint, error) {
-	buf, ok := m.blobs[id]
-	if !ok {
-		return 0, errors.New("blob not found")
-	}
-
-	return uint(len(buf)), nil
-}
-
-func (m *MockRepo) LoadBlob(t restic.BlobType, id restic.ID, buf []byte) (int, error) {
-	size, err := m.LookupBlobSize(id, t)
-	if err != nil {
-		return 0, err
-	}
-
-	if uint(len(buf)) < size {
-		return 0, errors.New("buffer too small")
-	}
-
-	buf = buf[:size]
-	copy(buf, m.blobs[id])
-	return int(size), nil
-}
-
-type MockContext struct{}
-
-func (m MockContext) Deadline() (time.Time, bool)       { return time.Now(), false }
-func (m MockContext) Done() <-chan struct{}             { return nil }
-func (m MockContext) Err() error                        { return nil }
-func (m MockContext) Value(key interface{}) interface{} { return nil }
-
-var testContent = genTestContent()
-var testContentLengths = []uint{
-	4646 * 1024,
-	655 * 1024,
-	378 * 1024,
-	8108 * 1024,
-	558 * 1024,
-}
-var testMaxFileSize uint
-
-func genTestContent() map[restic.ID][]byte {
-	m := make(map[restic.ID][]byte)
-
-	for _, length := range testContentLengths {
-		buf := Random(int(length), int(length))
-		id := restic.Hash(buf)
-		m[id] = buf
-		testMaxFileSize += length
-	}
-
-	return m
-}
-
-const maxBufSize = 20 * 1024 * 1024
-
-func testRead(t *testing.T, f *file, offset, length int, data []byte) {
-	ctx := MockContext{}
+func testRead(t testing.TB, f *file, offset, length int, data []byte) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
 
 	req := &fuse.ReadRequest{
 		Offset: int64(offset),
 		Size:   length,
 	}
 	resp := &fuse.ReadResponse{
-		Data: make([]byte, length),
+		Data: data,
 	}
 	OK(t, f.Read(ctx, req, resp))
 }
 
-var offsetReadsTests = []struct {
-	offset, length int
-}{
-	{0, 5 * 1024 * 1024},
-	{4000 * 1024, 1000 * 1024},
+func firstSnapshotID(t testing.TB, repo restic.Repository) (first restic.ID) {
+	done := make(chan struct{})
+	defer close(done)
+	for id := range repo.List(restic.SnapshotFile, done) {
+		if first.IsNull() {
+			first = id
+		}
+	}
+	return first
+}
+
+func loadFirstSnapshot(t testing.TB, repo restic.Repository) *restic.Snapshot {
+	id := firstSnapshotID(t, repo)
+	sn, err := restic.LoadSnapshot(repo, id)
+	OK(t, err)
+	return sn
+}
+
+func loadTree(t testing.TB, repo restic.Repository, id restic.ID) *restic.Tree {
+	tree, err := repo.LoadTree(id)
+	OK(t, err)
+	return tree
 }
 
 func TestFuseFile(t *testing.T) {
-	repo := NewMockRepo(testContent)
-	ctx := MockContext{}
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
 
-	memfile := make([]byte, 0, maxBufSize)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	timestamp, err := time.Parse(time.RFC3339, "2017-01-24T10:42:56+01:00")
+	OK(t, err)
+	restic.TestCreateSnapshot(t, repo, timestamp, 2, 0.1)
+
+	sn := loadFirstSnapshot(t, repo)
+	tree := loadTree(t, repo, *sn.Tree)
+
+	var content restic.IDs
+	for _, node := range tree.Nodes {
+		content = append(content, node.Content...)
+	}
+	t.Logf("tree loaded, content: %v", content)
+
+	var (
+		filesize uint64
+		memfile  []byte
+	)
+	for _, id := range content {
+		size, err := repo.LookupBlobSize(id, restic.DataBlob)
+		OK(t, err)
+		filesize += uint64(size)
+
+		buf := restic.NewBlobBuffer(int(size))
+		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
+		OK(t, err)
+
+		if uint(n) != size {
+			t.Fatalf("not enough bytes read for id %v: want %v, got %v", id.Str(), size, n)
+		}
+
+		if uint(len(buf)) != size {
+			t.Fatalf("buffer has wrong length for id %v: want %v, got %v", id.Str(), size, len(buf))
+		}
 
-	var ids restic.IDs
-	for id, buf := range repo.blobs {
-		ids = append(ids, id)
 		memfile = append(memfile, buf...)
 	}
 
+	t.Logf("filesize is %v, memfile has size %v", filesize, len(memfile))
+
 	node := &restic.Node{
 		Name:    "foo",
 		Inode:   23,
 		Mode:    0742,
-		Size:    42,
-		Content: ids,
+		Size:    filesize,
+		Content: content,
 	}
 	f, err := newFile(repo, node, false)
 	OK(t, err)
@@ -131,28 +121,19 @@ func TestFuseFile(t *testing.T) {
 	Equals(t, node.Size, attr.Size)
 	Equals(t, (node.Size/uint64(attr.BlockSize))+1, attr.Blocks)
 
-	for i, test := range offsetReadsTests {
-		b := memfile[test.offset : test.offset+test.length]
-		buf := make([]byte, test.length)
-		testRead(t, f, test.offset, test.length, buf)
-		if !bytes.Equal(b, buf) {
-			t.Errorf("test %d failed, wrong data returned", i)
-		}
-	}
-
 	for i := 0; i < 200; i++ {
-		length := rand.Intn(int(testMaxFileSize) / 2)
-		offset := rand.Intn(int(testMaxFileSize))
-		if length+offset > int(testMaxFileSize) {
-			diff := length + offset - int(testMaxFileSize)
-			length -= diff
-		}
+		offset := rand.Intn(int(filesize))
+		length := rand.Intn(int(filesize)-offset) + 100
 
 		b := memfile[offset : offset+length]
+
 		buf := make([]byte, length)
+
 		testRead(t, f, offset, length, buf)
 		if !bytes.Equal(b, buf) {
-			t.Errorf("test %d failed (offset %d, length %d), wrong data returned", i, offset, length)
+			t.Errorf("test %d failed, wrong data returned (offset %v, length %v)", i, offset, length)
 		}
 	}
+
+	OK(t, f.Release(ctx, nil))
 }

src/restic/hashing/reader.go

@@ -0,0 +1,29 @@
+package hashing
+
+import (
+	"hash"
+	"io"
+)
+
+// Reader hashes all data read from the underlying reader.
+type Reader struct {
+	r io.Reader
+	h hash.Hash
+}
+
+// NewReader returns a new Reader that uses the hash h.
+func NewReader(r io.Reader, h hash.Hash) *Reader {
+	return &Reader{
+		h: h,
+		r: io.TeeReader(r, h),
+	}
+}
+
+func (h *Reader) Read(p []byte) (int, error) {
+	return h.r.Read(p)
+}
+
+// Sum returns the hash of the data read so far.
+func (h *Reader) Sum(d []byte) []byte {
+	return h.h.Sum(d)
+}

src/restic/hashing/reader_test.go

@@ -0,0 +1,73 @@
+package hashing
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/sha256"
+	"io"
+	"io/ioutil"
+	"testing"
+)
+
+func TestReader(t *testing.T) {
+	tests := []int{5, 23, 2<<18 + 23, 1 << 20}
+
+	for _, size := range tests {
+		data := make([]byte, size)
+		_, err := io.ReadFull(rand.Reader, data)
+		if err != nil {
+			t.Fatalf("ReadFull: %v", err)
+		}
+
+		expectedHash := sha256.Sum256(data)
+
+		rd := NewReader(bytes.NewReader(data), sha256.New())
+		n, err := io.Copy(ioutil.Discard, rd)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if n != int64(size) {
+			t.Errorf("Reader: invalid number of bytes written: got %d, expected %d",
+				n, size)
+		}
+
+		resultingHash := rd.Sum(nil)
+
+		if !bytes.Equal(expectedHash[:], resultingHash) {
+			t.Errorf("Reader: hashes do not match: expected %02x, got %02x",
+				expectedHash, resultingHash)
+		}
+	}
+}
+
+func BenchmarkReader(b *testing.B) {
+	buf := make([]byte, 1<<22)
+	_, err := io.ReadFull(rand.Reader, buf)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	expectedHash := sha256.Sum256(buf)
+
+	b.SetBytes(int64(len(buf)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		rd := NewReader(bytes.NewReader(buf), sha256.New())
+		n, err := io.Copy(ioutil.Discard, rd)
+		if err != nil {
+			b.Fatal(err)
+		}
+
+		if n != int64(len(buf)) {
+			b.Errorf("Reader: invalid number of bytes written: got %d, expected %d",
+				n, len(buf))
+		}
+
+		resultingHash := rd.Sum(nil)
+		if !bytes.Equal(expectedHash[:], resultingHash) {
+			b.Errorf("Reader: hashes do not match: expected %02x, got %02x",
+				expectedHash, resultingHash)
+		}
+	}
+}

src/restic/hashing/writer.go

@@ -0,0 +1,31 @@
+package hashing
+
+import (
+	"hash"
+	"io"
+)
+
+// Writer transparently hashes all data while writing it to the underlying writer.
+type Writer struct {
+	w io.Writer
+	h hash.Hash
+}
+
+// NewWriter wraps the writer w and feeds all data written to the hash h.
+func NewWriter(w io.Writer, h hash.Hash) *Writer {
+	return &Writer{
+		h: h,
+		w: io.MultiWriter(w, h),
+	}
+}
+
+// Write wraps the write method of the underlying writer and also hashes all data.
+func (h *Writer) Write(p []byte) (int, error) {
+	n, err := h.w.Write(p)
+	return n, err
+}
+
+// Sum returns the hash of all data written so far.
+func (h *Writer) Sum(d []byte) []byte {
+	return h.h.Sum(d)
+}

src/restic/hashing/writer_test.go

@@ -0,0 +1,74 @@
+package hashing
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/sha256"
+	"io"
+	"io/ioutil"
+	"testing"
+)
+
+func TestWriter(t *testing.T) {
+	tests := []int{5, 23, 2<<18 + 23, 1 << 20}
+
+	for _, size := range tests {
+		data := make([]byte, size)
+		_, err := io.ReadFull(rand.Reader, data)
+		if err != nil {
+			t.Fatalf("ReadFull: %v", err)
+		}
+
+		expectedHash := sha256.Sum256(data)
+
+		wr := NewWriter(ioutil.Discard, sha256.New())
+
+		n, err := io.Copy(wr, bytes.NewReader(data))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if n != int64(size) {
+			t.Errorf("Writer: invalid number of bytes written: got %d, expected %d",
+				n, size)
+		}
+
+		resultingHash := wr.Sum(nil)
+
+		if !bytes.Equal(expectedHash[:], resultingHash) {
+			t.Errorf("Writer: hashes do not match: expected %02x, got %02x",
+				expectedHash, resultingHash)
+		}
+	}
+}
+
+func BenchmarkWriter(b *testing.B) {
+	buf := make([]byte, 1<<22)
+	_, err := io.ReadFull(rand.Reader, buf)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	expectedHash := sha256.Sum256(buf)
+
+	b.SetBytes(int64(len(buf)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		wr := NewWriter(ioutil.Discard, sha256.New())
+		n, err := io.Copy(wr, bytes.NewReader(buf))
+		if err != nil {
+			b.Fatal(err)
+		}
+
+		if n != int64(len(buf)) {
+			b.Errorf("Writer: invalid number of bytes written: got %d, expected %d",
+				n, len(buf))
+		}
+
+		resultingHash := wr.Sum(nil)
+		if !bytes.Equal(expectedHash[:], resultingHash) {
+			b.Errorf("Writer: hashes do not match: expected %02x, got %02x",
+				expectedHash, resultingHash)
+		}
+	}
+}

src/restic/id.go

@@ -114,3 +114,13 @@ func (id *ID) UnmarshalJSON(b []byte) error {
 
 	return nil
 }
+
+// IDFromHash returns the ID for the hash.
+func IDFromHash(hash []byte) (id ID) {
+	if len(hash) != idSize {
+		panic("invalid hash type, not enough/too many bytes")
+	}
+
+	copy(id[:], hash)
+	return id
+}

src/restic/index/index.go

@@ -14,27 +14,20 @@ import (
 
 // Pack contains information about the contents of a pack.
 type Pack struct {
+	ID      restic.ID
 	Size    int64
 	Entries []restic.Blob
 }
 
-// Blob contains information about a blob.
-type Blob struct {
-	Size  int64
-	Packs restic.IDSet
-}
-
 // Index contains information about blobs and packs stored in a repo.
 type Index struct {
 	Packs    map[restic.ID]Pack
-	Blobs    map[restic.BlobHandle]Blob
 	IndexIDs restic.IDSet
 }
 
 func newIndex() *Index {
 	return &Index{
 		Packs:    make(map[restic.ID]Pack),
-		Blobs:    make(map[restic.BlobHandle]Blob),
 		IndexIDs: restic.NewIDSet(),
 	}
 }
@@ -69,9 +62,6 @@ func New(repo restic.Repository, p *restic.Progress) (*Index, error) {
 		if err != nil {
 			return nil, err
 		}
-
-		p := Pack{Entries: j.Entries(), Size: j.Size()}
-		idx.Packs[packID] = p
 	}
 
 	return idx, nil
@@ -179,19 +169,7 @@ func (idx *Index) AddPack(id restic.ID, size int64, entries []restic.Blob) error
 		return errors.Errorf("pack %v already present in the index", id.Str())
 	}
 
-	idx.Packs[id] = Pack{Size: size, Entries: entries}
-
-	for _, entry := range entries {
-		h := restic.BlobHandle{ID: entry.ID, Type: entry.Type}
-		if _, ok := idx.Blobs[h]; !ok {
-			idx.Blobs[h] = Blob{
-				Size:  int64(entry.Length),
-				Packs: restic.NewIDSet(),
-			}
-		}
-
-		idx.Blobs[h].Packs.Insert(id)
-	}
+	idx.Packs[id] = Pack{ID: id, Size: size, Entries: entries}
 
 	return nil
 }
@@ -202,15 +180,6 @@ func (idx *Index) RemovePack(id restic.ID) error {
 		return errors.Errorf("pack %v not found in the index", id.Str())
 	}
 
-	for _, blob := range idx.Packs[id].Entries {
-		h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
-		idx.Blobs[h].Packs.Delete(id)
-
-		if len(idx.Blobs[h].Packs) == 0 {
-			delete(idx.Blobs, h)
-		}
-	}
-
 	delete(idx.Packs, id)
 
 	return nil
@@ -239,14 +208,11 @@ func (idx *Index) DuplicateBlobs() (dups restic.BlobSet) {
 func (idx *Index) PacksForBlobs(blobs restic.BlobSet) (packs restic.IDSet) {
 	packs = restic.NewIDSet()
 
-	for h := range blobs {
-		blob, ok := idx.Blobs[h]
-		if !ok {
-			continue
-		}
-
-		for id := range blob.Packs {
-			packs.Insert(id)
+	for id, p := range idx.Packs {
+		for _, entry := range p.Entries {
+			if blobs.Has(restic.BlobHandle{ID: entry.ID, Type: entry.Type}) {
+				packs.Insert(id)
+			}
 		}
 	}
 
@@ -264,31 +230,20 @@ type Location struct {
 var ErrBlobNotFound = errors.New("blob not found in index")
 
 // FindBlob returns a list of packs and positions the blob can be found in.
-func (idx *Index) FindBlob(h restic.BlobHandle) ([]Location, error) {
-	blob, ok := idx.Blobs[h]
-	if !ok {
-		return nil, ErrBlobNotFound
+func (idx *Index) FindBlob(h restic.BlobHandle) (result []Location, err error) {
+	for id, p := range idx.Packs {
+		for _, entry := range p.Entries {
+			if entry.ID.Equal(h.ID) && entry.Type == h.Type {
+				result = append(result, Location{
+					PackID: id,
+					Blob:   entry,
+				})
+			}
+		}
 	}
 
-	result := make([]Location, 0, len(blob.Packs))
-	for packID := range blob.Packs {
-		pack, ok := idx.Packs[packID]
-		if !ok {
-			return nil, errors.Errorf("pack %v not found in index", packID.Str())
-		}
-
-		for _, entry := range pack.Entries {
-			if entry.Type != h.Type {
-				continue
-			}
-
-			if !entry.ID.Equal(h.ID) {
-				continue
-			}
-
-			loc := Location{PackID: packID, Blob: entry}
-			result = append(result, loc)
-		}
+	if len(result) == 0 {
+		return nil, ErrBlobNotFound
 	}
 
 	return result, nil

src/restic/index/index_test.go

@@ -3,7 +3,9 @@ package index
 import (
 	"math/rand"
 	"restic"
+	"restic/checker"
 	"restic/repository"
+	"restic/test"
 	"testing"
 	"time"
 )
@@ -25,9 +27,14 @@ func createFilledRepo(t testing.TB, snapshots int, dup float32) (restic.Reposito
 
 func validateIndex(t testing.TB, repo restic.Repository, idx *Index) {
 	for id := range repo.List(restic.DataFile, nil) {
-		if _, ok := idx.Packs[id]; !ok {
+		p, ok := idx.Packs[id]
+		if !ok {
 			t.Errorf("pack %v missing from index", id.Str())
 		}
+
+		if !p.ID.Equal(id) {
+			t.Errorf("pack %v has invalid ID: want %v, got %v", id.Str(), id, p.ID)
+		}
 	}
 }
 
@@ -135,6 +142,40 @@ func BenchmarkIndexNew(b *testing.B) {
 		if idx == nil {
 			b.Fatalf("New() returned nil index")
 		}
+		b.Logf("idx %v packs", len(idx.Packs))
+	}
+}
+
+func BenchmarkIndexSave(b *testing.B) {
+	repo, cleanup := repository.TestRepository(b)
+	defer cleanup()
+
+	idx, err := New(repo, nil)
+	test.OK(b, err)
+
+	for i := 0; i < 8000; i++ {
+		entries := make([]restic.Blob, 0, 200)
+		for j := 0; j < cap(entries); j++ {
+			entries = append(entries, restic.Blob{
+				ID:     restic.NewRandomID(),
+				Length: 1000,
+				Offset: 5,
+				Type:   restic.DataBlob,
+			})
+		}
+
+		idx.AddPack(restic.NewRandomID(), 10000, entries)
+	}
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		id, err := idx.Save(repo, nil)
+		if err != nil {
+			b.Fatalf("New() returned error %v", err)
+		}
+
+		b.Logf("saved as %v", id.Str())
 	}
 }
 
@@ -151,7 +192,7 @@ func TestIndexDuplicateBlobs(t *testing.T) {
 	if len(dups) == 0 {
 		t.Errorf("no duplicate blobs found")
 	}
-	t.Logf("%d packs, %d unique blobs", len(idx.Packs), len(idx.Blobs))
+	t.Logf("%d packs, %d duplicate blobs", len(idx.Packs), len(dups))
 
 	packs := idx.PacksForBlobs(dups)
 	if len(packs) == 0 {
@@ -169,7 +210,7 @@ func loadIndex(t testing.TB, repo restic.Repository) *Index {
 	return idx
 }
 
-func TestIndexSave(t *testing.T) {
+func TestSave(t *testing.T) {
 	repo, cleanup := createFilledRepo(t, 3, 0)
 	defer cleanup()
 
@@ -193,7 +234,8 @@ func TestIndexSave(t *testing.T) {
 
 	for id := range idx.IndexIDs {
 		t.Logf("remove index %v", id.Str())
-		err = repo.Backend().Remove(restic.IndexFile, id.String())
+		h := restic.Handle{Type: restic.IndexFile, Name: id.String()}
+		err = repo.Backend().Remove(h)
 		if err != nil {
 			t.Errorf("error removing index %v: %v", id, err)
 		}
@@ -219,6 +261,42 @@ func TestIndexSave(t *testing.T) {
 	}
 }
 
+func TestIndexSave(t *testing.T) {
+	repo, cleanup := createFilledRepo(t, 3, 0)
+	defer cleanup()
+
+	idx := loadIndex(t, repo)
+
+	id, err := idx.Save(repo, idx.IndexIDs.List())
+	if err != nil {
+		t.Fatalf("unable to save new index: %v", err)
+	}
+
+	t.Logf("new index saved as %v", id.Str())
+
+	for id := range idx.IndexIDs {
+		t.Logf("remove index %v", id.Str())
+		h := restic.Handle{Type: restic.IndexFile, Name: id.String()}
+		err = repo.Backend().Remove(h)
+		if err != nil {
+			t.Errorf("error removing index %v: %v", id, err)
+		}
+	}
+
+	idx2 := loadIndex(t, repo)
+	t.Logf("load new index with %d packs", len(idx2.Packs))
+
+	checker := checker.New(repo)
+	hints, errs := checker.LoadIndex()
+	for _, h := range hints {
+		t.Logf("hint: %v\n", h)
+	}
+
+	for _, err := range errs {
+		t.Errorf("checker found error: %v", err)
+	}
+}
+
 func TestIndexAddRemovePack(t *testing.T) {
 	repo, cleanup := createFilledRepo(t, 3, 0)
 	defer cleanup()
@@ -249,12 +327,7 @@ func TestIndexAddRemovePack(t *testing.T) {
 		if err == nil {
 			t.Errorf("removed blob %v found in index", h)
 		}
-
-		if _, ok := idx.Blobs[h]; ok {
-			t.Errorf("removed blob %v found in index.Blobs", h)
-		}
 	}
-
 }
 
 // example index serialization from doc/Design.md

src/restic/lock.go

@@ -186,7 +186,7 @@ func (l *Lock) Unlock() error {
 		return nil
 	}
 
-	return l.repo.Backend().Remove(LockFile, l.lockID.String())
+	return l.repo.Backend().Remove(Handle{Type: LockFile, Name: l.lockID.String()})
 }
 
 var staleTimeout = 30 * time.Minute
@@ -234,7 +234,7 @@ func (l *Lock) Refresh() error {
 		return err
 	}
 
-	err = l.repo.Backend().Remove(LockFile, l.lockID.String())
+	err = l.repo.Backend().Remove(Handle{Type: LockFile, Name: l.lockID.String()})
 	if err != nil {
 		return err
 	}
@@ -289,7 +289,7 @@ func RemoveStaleLocks(repo Repository) error {
 		}
 
 		if lock.Stale() {
-			return repo.Backend().Remove(LockFile, id.String())
+			return repo.Backend().Remove(Handle{Type: LockFile, Name: id.String()})
 		}
 
 		return nil
@@ -299,6 +299,6 @@ func RemoveStaleLocks(repo Repository) error {
 // RemoveAllLocks removes all locks forcefully.
 func RemoveAllLocks(repo Repository) error {
 	return eachLock(repo, func(id ID, lock *Lock, err error) error {
-		return repo.Backend().Remove(LockFile, id.String())
+		return repo.Backend().Remove(Handle{Type: LockFile, Name: id.String()})
 	})
 }

src/restic/lock_test.go

@@ -102,7 +102,8 @@ func createFakeLock(repo restic.Repository, t time.Time, pid int) (restic.ID, er
 }
 
 func removeLock(repo restic.Repository, id restic.ID) error {
-	return repo.Backend().Remove(restic.LockFile, id.String())
+	h := restic.Handle{Type: restic.LockFile, Name: id.String()}
+	return repo.Backend().Remove(h)
 }
 
 var staleLockTests = []struct {
@@ -162,7 +163,8 @@ func TestLockStale(t *testing.T) {
 }
 
 func lockExists(repo restic.Repository, t testing.TB, id restic.ID) bool {
-	exists, err := repo.Backend().Test(restic.LockFile, id.String())
+	h := restic.Handle{Type: restic.LockFile, Name: id.String()}
+	exists, err := repo.Backend().Test(h)
 	OK(t, err)
 
 	return exists

src/restic/mock/backend.go

@@ -1,6 +1,7 @@
 package mock
 
 import (
+	"io"
 	"restic"
 
 	"restic/errors"
@@ -9,12 +10,12 @@ import (
 // Backend implements a mock backend.
 type Backend struct {
 	CloseFn    func() error
-	LoadFn     func(h restic.Handle, p []byte, off int64) (int, error)
-	SaveFn     func(h restic.Handle, p []byte) error
+	SaveFn     func(h restic.Handle, rd io.Reader) error
+	LoadFn     func(h restic.Handle, length int, offset int64) (io.ReadCloser, error)
 	StatFn     func(h restic.Handle) (restic.FileInfo, error)
 	ListFn     func(restic.FileType, <-chan struct{}) <-chan string
-	RemoveFn   func(restic.FileType, string) error
-	TestFn     func(restic.FileType, string) (bool, error)
+	RemoveFn   func(h restic.Handle) error
+	TestFn     func(h restic.Handle) (bool, error)
 	DeleteFn   func() error
 	LocationFn func() string
 }
@@ -37,22 +38,22 @@ func (m *Backend) Location() string {
 	return m.LocationFn()
 }
 
-// Load loads data from the backend.
-func (m *Backend) Load(h restic.Handle, p []byte, off int64) (int, error) {
-	if m.LoadFn == nil {
-		return 0, errors.New("not implemented")
-	}
-
-	return m.LoadFn(h, p, off)
-}
-
 // Save data in the backend.
-func (m *Backend) Save(h restic.Handle, p []byte) error {
+func (m *Backend) Save(h restic.Handle, rd io.Reader) error {
 	if m.SaveFn == nil {
 		return errors.New("not implemented")
 	}
 
-	return m.SaveFn(h, p)
+	return m.SaveFn(h, rd)
+}
+
+// Load loads data from the backend.
+func (m *Backend) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	if m.LoadFn == nil {
+		return nil, errors.New("not implemented")
+	}
+
+	return m.LoadFn(h, length, offset)
 }
 
 // Stat an object in the backend.
@@ -76,21 +77,21 @@ func (m *Backend) List(t restic.FileType, done <-chan struct{}) <-chan string {
 }
 
 // Remove data from the backend.
-func (m *Backend) Remove(t restic.FileType, name string) error {
+func (m *Backend) Remove(h restic.Handle) error {
 	if m.RemoveFn == nil {
 		return errors.New("not implemented")
 	}
 
-	return m.RemoveFn(t, name)
+	return m.RemoveFn(h)
 }
 
 // Test for the existence of a specific item.
-func (m *Backend) Test(t restic.FileType, name string) (bool, error) {
+func (m *Backend) Test(h restic.Handle) (bool, error) {
 	if m.TestFn == nil {
 		return false, errors.New("not implemented")
 	}
 
-	return m.TestFn(t, name)
+	return m.TestFn(h)
 }
 
 // Delete all data.

src/restic/node.go

@@ -208,7 +208,7 @@ func (node Node) createFileAt(path string, repo Repository) error {
 
 		buf = buf[:cap(buf)]
 		if uint(len(buf)) < size {
-			buf = make([]byte, size)
+			buf = NewBlobBuffer(int(size))
 		}
 
 		n, err := repo.LoadBlob(DataBlob, id, buf)

src/restic/pack/pack.go

@@ -85,15 +85,15 @@ func (p *Packer) Finalize() (uint, error) {
 		return 0, errors.Wrap(err, "Write")
 	}
 
-	hdrBytes := bytesHeader + crypto.Extension
-	if uint(n) != hdrBytes {
+	hdrBytes := restic.CiphertextLength(int(bytesHeader))
+	if n != hdrBytes {
 		return 0, errors.New("wrong number of bytes written")
 	}
 
-	bytesWritten += hdrBytes
+	bytesWritten += uint(hdrBytes)
 
 	// write length
-	err = binary.Write(p.wr, binary.LittleEndian, uint32(uint(len(p.blobs))*entrySize+crypto.Extension))
+	err = binary.Write(p.wr, binary.LittleEndian, uint32(restic.CiphertextLength(len(p.blobs)*int(entrySize))))
 	if err != nil {
 		return 0, errors.Wrap(err, "binary.Write")
 	}
@@ -233,6 +233,8 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries []restic.Blob, err
 
 	hdrRd := bytes.NewReader(buf)
 
+	entries = make([]restic.Blob, 0, uint(n)/entrySize)
+
 	pos := uint(0)
 	for {
 		e := headerEntry{}

src/restic/pack/pack_test.go

@@ -54,10 +54,9 @@ func verifyBlobs(t testing.TB, bufs []Buf, k *crypto.Key, rd io.ReaderAt, packSi
 	}
 	// header length
 	written += binary.Size(uint32(0))
-	// header
-	written += len(bufs) * (binary.Size(restic.BlobType(0)) + binary.Size(uint32(0)) + len(restic.ID{}))
-	// header crypto
-	written += crypto.Extension
+	// header + header crypto
+	headerSize := len(bufs) * (binary.Size(restic.BlobType(0)) + binary.Size(uint32(0)) + len(restic.ID{}))
+	written += restic.CiphertextLength(headerSize)
 
 	// check length
 	Equals(t, uint(written), packSize)
@@ -127,7 +126,7 @@ func TestUnpackReadSeeker(t *testing.T) {
 	id := restic.Hash(packData)
 
 	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	OK(t, b.Save(handle, packData))
+	OK(t, b.Save(handle, bytes.NewReader(packData)))
 	verifyBlobs(t, bufs, k, restic.ReaderAt(b, handle), packSize)
 }
 
@@ -140,6 +139,6 @@ func TestShortPack(t *testing.T) {
 	id := restic.Hash(packData)
 
 	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	OK(t, b.Save(handle, packData))
+	OK(t, b.Save(handle, bytes.NewReader(packData)))
 	verifyBlobs(t, bufs, k, restic.ReaderAt(b, handle), packSize)
 }

src/restic/readerat.go

@@ -2,6 +2,7 @@ package restic
 
 import (
 	"io"
+	"restic/debug"
 )
 
 type backendReaderAt struct {
@@ -9,11 +10,30 @@ type backendReaderAt struct {
 	h  Handle
 }
 
-func (brd backendReaderAt) ReadAt(p []byte, off int64) (n int, err error) {
-	return brd.be.Load(brd.h, p, off)
+func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) {
+	return ReadAt(brd.be, brd.h, offset, p)
 }
 
 // ReaderAt returns an io.ReaderAt for a file in the backend.
 func ReaderAt(be Backend, h Handle) io.ReaderAt {
 	return backendReaderAt{be: be, h: h}
 }
+
+// ReadAt reads from the backend handle h at the given position.
+func ReadAt(be Backend, h Handle, offset int64, p []byte) (n int, err error) {
+	debug.Log("ReadAt(%v) at %v, len %v", h, offset, len(p))
+	rd, err := be.Load(h, len(p), offset)
+	if err != nil {
+		return 0, err
+	}
+
+	n, err = io.ReadFull(rd, p)
+	e := rd.Close()
+	if err == nil {
+		err = e
+	}
+
+	debug.Log("ReadAt(%v) ReadFull returned %v bytes", h, n)
+
+	return n, err
+}

src/restic/repository/index.go

@@ -1,7 +1,6 @@
 package repository
 
 import (
-	"bytes"
 	"encoding/json"
 	"io"
 	"restic"
@@ -10,7 +9,6 @@ import (
 
 	"restic/errors"
 
-	"restic/crypto"
 	"restic/debug"
 )
 
@@ -177,15 +175,15 @@ func (idx *Index) Has(id restic.ID, tpe restic.BlobType) bool {
 	return false
 }
 
-// LookupSize returns the length of the cleartext content behind the
-// given id
-func (idx *Index) LookupSize(id restic.ID, tpe restic.BlobType) (cleartextLength uint, err error) {
+// LookupSize returns the length of the plaintext content of the blob with the
+// given id.
+func (idx *Index) LookupSize(id restic.ID, tpe restic.BlobType) (plaintextLength uint, err error) {
 	blobs, err := idx.Lookup(id, tpe)
 	if err != nil {
 		return 0, err
 	}
 
-	return blobs[0].Length - crypto.Extension, nil
+	return uint(restic.PlaintextLength(int(blobs[0].Length))), nil
 }
 
 // Supersedes returns the list of indexes this index supersedes, if any.
@@ -452,12 +450,11 @@ func isErrOldIndex(err error) bool {
 var ErrOldIndexFormat = errors.New("index has old format")
 
 // DecodeIndex loads and unserializes an index from rd.
-func DecodeIndex(rd io.Reader) (idx *Index, err error) {
+func DecodeIndex(buf []byte) (idx *Index, err error) {
 	debug.Log("Start decoding index")
-	idxJSON := jsonIndex{}
+	idxJSON := &jsonIndex{}
 
-	dec := json.NewDecoder(rd)
-	err = dec.Decode(&idxJSON)
+	err = json.Unmarshal(buf, idxJSON)
 	if err != nil {
 		debug.Log("Error %v", err)
 
@@ -491,12 +488,11 @@ func DecodeIndex(rd io.Reader) (idx *Index, err error) {
 }
 
 // DecodeOldIndex loads and unserializes an index in the old format from rd.
-func DecodeOldIndex(rd io.Reader) (idx *Index, err error) {
+func DecodeOldIndex(buf []byte) (idx *Index, err error) {
 	debug.Log("Start decoding old index")
 	list := []*packJSON{}
 
-	dec := json.NewDecoder(rd)
-	err = dec.Decode(&list)
+	err = json.Unmarshal(buf, &list)
 	if err != nil {
 		debug.Log("Error %#v", err)
 		return nil, errors.Wrap(err, "Decode")
@@ -523,7 +519,7 @@ func DecodeOldIndex(rd io.Reader) (idx *Index, err error) {
 }
 
 // LoadIndexWithDecoder loads the index and decodes it with fn.
-func LoadIndexWithDecoder(repo restic.Repository, id restic.ID, fn func(io.Reader) (*Index, error)) (idx *Index, err error) {
+func LoadIndexWithDecoder(repo restic.Repository, id restic.ID, fn func([]byte) (*Index, error)) (idx *Index, err error) {
 	debug.Log("Loading index %v", id.Str())
 
 	buf, err := repo.LoadAndDecrypt(restic.IndexFile, id)
@@ -531,7 +527,7 @@ func LoadIndexWithDecoder(repo restic.Repository, id restic.ID, fn func(io.Reade
 		return nil, err
 	}
 
-	idx, err = fn(bytes.NewReader(buf))
+	idx, err = fn(buf)
 	if err != nil {
 		debug.Log("error while decoding index %v: %v", id, err)
 		return nil, err

src/restic/repository/index_rebuild.go

@@ -55,7 +55,8 @@ func RebuildIndex(repo restic.Repository) error {
 	debug.Log("new index saved as %v", id.Str())
 
 	for indexID := range oldIndexes {
-		err := repo.Backend().Remove(restic.IndexFile, indexID.String())
+		h := restic.Handle{Type: restic.IndexFile, Name: indexID.String()}
+		err := repo.Backend().Remove(h)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "unable to remove index %v: %v\n", indexID.Str(), err)
 		}

src/restic/repository/index_test.go

@@ -54,7 +54,7 @@ func TestIndexSerialize(t *testing.T) {
 	err := idx.Encode(wr)
 	OK(t, err)
 
-	idx2, err := repository.DecodeIndex(wr)
+	idx2, err := repository.DecodeIndex(wr.Bytes())
 	OK(t, err)
 	Assert(t, idx2 != nil,
 		"nil returned for decoded index")
@@ -136,7 +136,7 @@ func TestIndexSerialize(t *testing.T) {
 	Assert(t, id2.Equal(id),
 		"wrong ID returned: want %v, got %v", id, id2)
 
-	idx3, err := repository.DecodeIndex(wr3)
+	idx3, err := repository.DecodeIndex(wr3.Bytes())
 	OK(t, err)
 	Assert(t, idx3 != nil,
 		"nil returned for decoded index")
@@ -288,7 +288,7 @@ var exampleLookupTest = struct {
 func TestIndexUnserialize(t *testing.T) {
 	oldIdx := restic.IDs{restic.TestParseID("ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452")}
 
-	idx, err := repository.DecodeIndex(bytes.NewReader(docExample))
+	idx, err := repository.DecodeIndex(docExample)
 	OK(t, err)
 
 	for _, test := range exampleTests {
@@ -326,8 +326,17 @@ func TestIndexUnserialize(t *testing.T) {
 	}
 }
 
+func BenchmarkDecodeIndex(b *testing.B) {
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		_, err := repository.DecodeIndex(docExample)
+		OK(b, err)
+	}
+}
+
 func TestIndexUnserializeOld(t *testing.T) {
-	idx, err := repository.DecodeOldIndex(bytes.NewReader(docOldExample))
+	idx, err := repository.DecodeOldIndex(docOldExample)
 	OK(t, err)
 
 	for _, test := range exampleTests {

src/restic/repository/key.go

@@ -1,6 +1,7 @@
 package repository
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -146,7 +147,7 @@ func SearchKey(s *Repository, password string, maxKeys int) (*Key, error) {
 // LoadKey loads a key from the backend.
 func LoadKey(s *Repository, name string) (k *Key, err error) {
 	h := restic.Handle{Type: restic.KeyFile, Name: name}
-	data, err := backend.LoadAll(s.be, h, nil)
+	data, err := backend.LoadAll(s.be, h)
 	if err != nil {
 		return nil, err
 	}
@@ -232,7 +233,7 @@ func AddKey(s *Repository, password string, template *crypto.Key) (*Key, error)
 		Name: restic.Hash(buf).String(),
 	}
 
-	err = s.be.Save(h, buf)
+	err = s.be.Save(h, bytes.NewReader(buf))
 	if err != nil {
 		return nil, err
 	}

src/restic/repository/master_index.go

@@ -30,8 +30,7 @@ func (mi *MasterIndex) Lookup(id restic.ID, tpe restic.BlobType) (blobs []restic
 	for _, idx := range mi.idx {
 		blobs, err = idx.Lookup(id, tpe)
 		if err == nil {
-			debug.Log("MasterIndex.Lookup",
-				"found id %v: %v", id.Str(), blobs)
+			debug.Log("found id %v: %v", id.Str(), blobs)
 			return
 		}
 	}
@@ -46,9 +45,8 @@ func (mi *MasterIndex) LookupSize(id restic.ID, tpe restic.BlobType) (uint, erro
 	defer mi.idxMutex.RUnlock()
 
 	for _, idx := range mi.idx {
-		length, err := idx.LookupSize(id, tpe)
-		if err == nil {
-			return length, nil
+		if idx.Has(id, tpe) {
+			return idx.LookupSize(id, tpe)
 		}
 	}
 
@@ -119,16 +117,14 @@ func (mi *MasterIndex) Remove(index *Index) {
 	}
 }
 
-// Current returns an index that is not yet finalized, so that new entries can
-// still be added. If all indexes are finalized, a new index is created and
-// returned.
-func (mi *MasterIndex) Current() *Index {
+// Store remembers the id and pack in the index.
+func (mi *MasterIndex) Store(pb restic.PackedBlob) {
 	mi.idxMutex.RLock()
-
 	for _, idx := range mi.idx {
 		if !idx.Final() {
 			mi.idxMutex.RUnlock()
-			return idx
+			idx.Store(pb)
+			return
 		}
 	}
 
@@ -137,9 +133,8 @@ func (mi *MasterIndex) Current() *Index {
 	defer mi.idxMutex.Unlock()
 
 	newIdx := NewIndex()
+	newIdx.Store(pb)
 	mi.idx = append(mi.idx, newIdx)
-
-	return newIdx
 }
 
 // NotFinalIndexes returns all indexes that have not yet been saved.

src/restic/repository/packer_manager.go

@@ -1,6 +1,7 @@
 package repository
 
 import (
+	"crypto/sha256"
 	"io"
 	"io/ioutil"
 	"os"
@@ -8,6 +9,7 @@ import (
 	"sync"
 
 	"restic/errors"
+	"restic/hashing"
 
 	"restic/crypto"
 	"restic/debug"
@@ -17,15 +19,22 @@ import (
 
 // Saver implements saving data in a backend.
 type Saver interface {
-	Save(h restic.Handle, jp []byte) error
+	Save(restic.Handle, io.Reader) error
+}
+
+// Packer holds a pack.Packer together with a hash writer.
+type Packer struct {
+	*pack.Packer
+	hw      *hashing.Writer
+	tmpfile *os.File
 }
 
 // packerManager keeps a list of open packs and creates new on demand.
 type packerManager struct {
-	be    Saver
-	key   *crypto.Key
-	pm    sync.Mutex
-	packs []*pack.Packer
+	be      Saver
+	key     *crypto.Key
+	pm      sync.Mutex
+	packers []*Packer
 
 	pool sync.Pool
 }
@@ -50,18 +59,18 @@ func newPackerManager(be Saver, key *crypto.Key) *packerManager {
 
 // findPacker returns a packer for a new blob of size bytes. Either a new one is
 // created or one is returned that already has some blobs.
-func (r *packerManager) findPacker(size uint) (packer *pack.Packer, err error) {
+func (r *packerManager) findPacker(size uint) (packer *Packer, err error) {
 	r.pm.Lock()
 	defer r.pm.Unlock()
 
 	// search for a suitable packer
-	if len(r.packs) > 0 {
+	if len(r.packers) > 0 {
 		debug.Log("searching packer for %d bytes\n", size)
-		for i, p := range r.packs {
-			if p.Size()+size < maxPackSize {
+		for i, p := range r.packers {
+			if p.Packer.Size()+size < maxPackSize {
 				debug.Log("found packer %v", p)
 				// remove from list
-				r.packs = append(r.packs[:i], r.packs[i+1:]...)
+				r.packers = append(r.packers[:i], r.packers[i+1:]...)
 				return p, nil
 			}
 		}
@@ -74,50 +83,43 @@ func (r *packerManager) findPacker(size uint) (packer *pack.Packer, err error) {
 		return nil, errors.Wrap(err, "ioutil.TempFile")
 	}
 
-	return pack.NewPacker(r.key, tmpfile), nil
+	hw := hashing.NewWriter(tmpfile, sha256.New())
+	p := pack.NewPacker(r.key, hw)
+	packer = &Packer{
+		Packer:  p,
+		hw:      hw,
+		tmpfile: tmpfile,
+	}
+
+	return packer, nil
 }
 
 // insertPacker appends p to s.packs.
-func (r *packerManager) insertPacker(p *pack.Packer) {
+func (r *packerManager) insertPacker(p *Packer) {
 	r.pm.Lock()
 	defer r.pm.Unlock()
 
-	r.packs = append(r.packs, p)
-	debug.Log("%d packers\n", len(r.packs))
+	r.packers = append(r.packers, p)
+	debug.Log("%d packers\n", len(r.packers))
 }
 
 // savePacker stores p in the backend.
-func (r *Repository) savePacker(p *pack.Packer) error {
-	debug.Log("save packer with %d blobs\n", p.Count())
-	n, err := p.Finalize()
+func (r *Repository) savePacker(p *Packer) error {
+	debug.Log("save packer with %d blobs\n", p.Packer.Count())
+	_, err := p.Packer.Finalize()
 	if err != nil {
 		return err
 	}
 
-	tmpfile := p.Writer().(*os.File)
-	f, err := fs.Open(tmpfile.Name())
+	_, err = p.tmpfile.Seek(0, 0)
 	if err != nil {
-		return errors.Wrap(err, "Open")
+		return errors.Wrap(err, "Seek")
 	}
 
-	data := make([]byte, n)
-	m, err := io.ReadFull(f, data)
-	if err != nil {
-		return errors.Wrap(err, "ReadFul")
-	}
-
-	if uint(m) != n {
-		return errors.Errorf("read wrong number of bytes from %v: want %v, got %v", tmpfile.Name(), n, m)
-	}
-
-	if err = f.Close(); err != nil {
-		return errors.Wrap(err, "Close")
-	}
-
-	id := restic.Hash(data)
+	id := restic.IDFromHash(p.hw.Sum(nil))
 	h := restic.Handle{Type: restic.DataFile, Name: id.String()}
 
-	err = r.be.Save(h, data)
+	err = r.be.Save(h, p.tmpfile)
 	if err != nil {
 		debug.Log("Save(%v) error: %v", h, err)
 		return err
@@ -125,15 +127,20 @@ func (r *Repository) savePacker(p *pack.Packer) error {
 
 	debug.Log("saved as %v", h)
 
-	err = fs.Remove(tmpfile.Name())
+	err = p.tmpfile.Close()
+	if err != nil {
+		return errors.Wrap(err, "close tempfile")
+	}
+
+	err = fs.Remove(p.tmpfile.Name())
 	if err != nil {
 		return errors.Wrap(err, "Remove")
 	}
 
 	// update blobs in the index
-	for _, b := range p.Blobs() {
+	for _, b := range p.Packer.Blobs() {
 		debug.Log("  updating blob %v to pack %v", b.ID.Str(), id.Str())
-		r.idx.Current().Store(restic.PackedBlob{
+		r.idx.Store(restic.PackedBlob{
 			Blob: restic.Blob{
 				Type:   b.Type,
 				ID:     b.ID,
@@ -152,5 +159,5 @@ func (r *packerManager) countPacker() int {
 	r.pm.Lock()
 	defer r.pm.Unlock()
 
-	return len(r.packs)
+	return len(r.packers)
 }

src/restic/repository/packer_manager_test.go

@@ -7,6 +7,7 @@ import (
 	"restic"
 	"restic/backend/mem"
 	"restic/crypto"
+	"restic/mock"
 	"testing"
 )
 
@@ -46,32 +47,19 @@ func randomID(rd io.Reader) restic.ID {
 
 const maxBlobSize = 1 << 20
 
-func saveFile(t testing.TB, be Saver, filename string, n int) {
-	f, err := os.Open(filename)
-	if err != nil {
+func saveFile(t testing.TB, be Saver, f *os.File, id restic.ID) {
+	h := restic.Handle{Type: restic.DataFile, Name: id.String()}
+	t.Logf("save file %v", h)
+
+	if err := be.Save(h, f); err != nil {
 		t.Fatal(err)
 	}
 
-	data := make([]byte, n)
-	m, err := io.ReadFull(f, data)
-
-	if m != n {
-		t.Fatalf("read wrong number of bytes from %v: want %v, got %v", filename, m, n)
-	}
-
-	if err = f.Close(); err != nil {
+	if err := f.Close(); err != nil {
 		t.Fatal(err)
 	}
 
-	h := restic.Handle{Type: restic.DataFile, Name: restic.Hash(data).String()}
-
-	err = be.Save(h, data)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = os.Remove(filename)
-	if err != nil {
+	if err := os.Remove(f.Name()); err != nil {
 		t.Fatal(err)
 	}
 }
@@ -105,13 +93,17 @@ func fillPacks(t testing.TB, rnd *randReader, be Saver, pm *packerManager, buf [
 			continue
 		}
 
-		bytesWritten, err := packer.Finalize()
+		_, err = packer.Finalize()
 		if err != nil {
 			t.Fatal(err)
 		}
 
-		tmpfile := packer.Writer().(*os.File)
-		saveFile(t, be, tmpfile.Name(), int(bytesWritten))
+		if _, err = packer.tmpfile.Seek(0, 0); err != nil {
+			t.Fatal(err)
+		}
+
+		packID := restic.IDFromHash(packer.hw.Sum(nil))
+		saveFile(t, be, packer.tmpfile, packID)
 	}
 
 	return bytes
@@ -119,27 +111,21 @@ func fillPacks(t testing.TB, rnd *randReader, be Saver, pm *packerManager, buf [
 
 func flushRemainingPacks(t testing.TB, rnd *randReader, be Saver, pm *packerManager) (bytes int) {
 	if pm.countPacker() > 0 {
-		for _, packer := range pm.packs {
+		for _, packer := range pm.packers {
 			n, err := packer.Finalize()
 			if err != nil {
 				t.Fatal(err)
 			}
 			bytes += int(n)
 
-			tmpfile := packer.Writer().(*os.File)
-			saveFile(t, be, tmpfile.Name(), bytes)
+			packID := restic.IDFromHash(packer.hw.Sum(nil))
+			saveFile(t, be, packer.tmpfile, packID)
 		}
 	}
 
 	return bytes
 }
 
-type fakeBackend struct{}
-
-func (f *fakeBackend) Save(h restic.Handle, data []byte) error {
-	return nil
-}
-
 func TestPackerManager(t *testing.T) {
 	rnd := newRandReader(rand.NewSource(23))
 
@@ -157,17 +143,18 @@ func TestPackerManager(t *testing.T) {
 func BenchmarkPackerManager(t *testing.B) {
 	rnd := newRandReader(rand.NewSource(23))
 
-	be := &fakeBackend{}
-	pm := newPackerManager(be, crypto.NewRandomKey())
+	be := &mock.Backend{
+		SaveFn: func(restic.Handle, io.Reader) error { return nil },
+	}
 	blobBuf := make([]byte, maxBlobSize)
 
 	t.ResetTimer()
 
-	bytes := 0
 	for i := 0; i < t.N; i++ {
+		bytes := 0
+		pm := newPackerManager(be, crypto.NewRandomKey())
 		bytes += fillPacks(t, rnd, be, pm, blobBuf)
+		bytes += flushRemainingPacks(t, rnd, be, pm)
+		t.Logf("saved %d bytes", bytes)
 	}
-
-	bytes += flushRemainingPacks(t, rnd, be, pm)
-	t.Logf("saved %d bytes", bytes)
 }

src/restic/repository/repack.go

@@ -1,11 +1,14 @@
 package repository
 
 import (
-	"bytes"
+	"crypto/sha256"
 	"io"
+	"io/ioutil"
+	"os"
 	"restic"
 	"restic/crypto"
 	"restic/debug"
+	"restic/hashing"
 	"restic/pack"
 
 	"restic/errors"
@@ -18,30 +21,47 @@ import (
 func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet) (err error) {
 	debug.Log("repacking %d packs while keeping %d blobs", len(packs), len(keepBlobs))
 
-	buf := make([]byte, 0, maxPackSize)
 	for packID := range packs {
-		// load the complete pack
+		// load the complete pack into a temp file
 		h := restic.Handle{Type: restic.DataFile, Name: packID.String()}
 
-		l, err := repo.Backend().Load(h, buf[:cap(buf)], 0)
-		if errors.Cause(err) == io.ErrUnexpectedEOF {
-			err = nil
-			buf = buf[:l]
+		tempfile, err := ioutil.TempFile("", "restic-temp-repack-")
+		if err != nil {
+			return errors.Wrap(err, "TempFile")
 		}
 
+		beRd, err := repo.Backend().Load(h, 0, 0)
 		if err != nil {
 			return err
 		}
 
-		debug.Log("pack %v loaded (%d bytes)", packID.Str(), len(buf))
+		defer beRd.Close()
 
-		blobs, err := pack.List(repo.Key(), bytes.NewReader(buf), int64(len(buf)))
+		hrd := hashing.NewReader(beRd, sha256.New())
+		packLength, err := io.Copy(tempfile, hrd)
+		if err != nil {
+			return errors.Wrap(err, "Copy")
+		}
+
+		hash := restic.IDFromHash(hrd.Sum(nil))
+		debug.Log("pack %v loaded (%d bytes), hash %v", packID.Str(), packLength, hash.Str())
+
+		if !packID.Equal(hash) {
+			return errors.Errorf("hash does not match id: want %v, got %v", packID, hash)
+		}
+
+		_, err = tempfile.Seek(0, 0)
+		if err != nil {
+			return errors.Wrap(err, "Seek")
+		}
+
+		blobs, err := pack.List(repo.Key(), tempfile, packLength)
 		if err != nil {
 			return err
 		}
 
 		debug.Log("processing pack %v, blobs: %v", packID.Str(), len(blobs))
-		var plaintext []byte
+		var buf []byte
 		for _, entry := range blobs {
 			h := restic.BlobHandle{ID: entry.ID, Type: entry.Type}
 			if !keepBlobs.Has(h) {
@@ -50,21 +70,36 @@ func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet
 
 			debug.Log("  process blob %v", h)
 
-			ciphertext := buf[entry.Offset : entry.Offset+entry.Length]
-			plaintext = plaintext[:len(plaintext)]
-			if len(plaintext) < len(ciphertext) {
-				plaintext = make([]byte, len(ciphertext))
+			buf = buf[:len(buf)]
+			if uint(len(buf)) < entry.Length {
+				buf = make([]byte, entry.Length)
+			}
+			buf = buf[:entry.Length]
+
+			n, err := tempfile.ReadAt(buf, int64(entry.Offset))
+			if err != nil {
+				return errors.Wrap(err, "ReadAt")
 			}
 
-			debug.Log("  ciphertext %d, plaintext %d", len(plaintext), len(ciphertext))
+			if n != len(buf) {
+				return errors.Errorf("read blob %v from %v: not enough bytes read, want %v, got %v",
+					h, tempfile.Name(), len(buf), n)
+			}
 
-			n, err := crypto.Decrypt(repo.Key(), plaintext, ciphertext)
+			n, err = crypto.Decrypt(repo.Key(), buf, buf)
 			if err != nil {
 				return err
 			}
-			plaintext = plaintext[:n]
 
-			_, err = repo.SaveBlob(entry.Type, plaintext, entry.ID)
+			buf = buf[:n]
+
+			id := restic.Hash(buf)
+			if !id.Equal(entry.ID) {
+				return errors.Errorf("read blob %v from %v: wrong data returned, hash is %v",
+					h, tempfile.Name(), id)
+			}
+
+			_, err = repo.SaveBlob(entry.Type, buf, entry.ID)
 			if err != nil {
 				return err
 			}
@@ -73,6 +108,14 @@ func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet
 
 			keepBlobs.Delete(h)
 		}
+
+		if err = tempfile.Close(); err != nil {
+			return errors.Wrap(err, "Close")
+		}
+
+		if err = os.Remove(tempfile.Name()); err != nil {
+			return errors.Wrap(err, "Remove")
+		}
 	}
 
 	if err := repo.Flush(); err != nil {
@@ -80,7 +123,8 @@ func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet
 	}
 
 	for packID := range packs {
-		err := repo.Backend().Remove(restic.DataFile, packID.String())
+		h := restic.Handle{Type: restic.DataFile, Name: packID.String()}
+		err := repo.Backend().Remove(h)
 		if err != nil {
 			debug.Log("error removing pack %v: %v", packID.Str(), err)
 			return err

src/restic/repository/repository.go

@@ -54,7 +54,7 @@ func (r *Repository) LoadAndDecrypt(t restic.FileType, id restic.ID) ([]byte, er
 	debug.Log("load %v with id %v", t, id.Str())
 
 	h := restic.Handle{Type: t, Name: id.String()}
-	buf, err := backend.LoadAll(r.be, h, nil)
+	buf, err := backend.LoadAll(r.be, h)
 	if err != nil {
 		debug.Log("error loading %v: %v", id.Str(), err)
 		return nil, err
@@ -64,33 +64,20 @@ func (r *Repository) LoadAndDecrypt(t restic.FileType, id restic.ID) ([]byte, er
 		return nil, errors.New("invalid data returned")
 	}
 
-	plain := make([]byte, len(buf))
-
 	// decrypt
-	n, err := r.decryptTo(plain, buf)
+	n, err := r.decryptTo(buf, buf)
 	if err != nil {
 		return nil, err
 	}
 
-	return plain[:n], nil
+	return buf[:n], nil
 }
 
 // loadBlob tries to load and decrypt content identified by t and id from a
 // pack from the backend, the result is stored in plaintextBuf, which must be
 // large enough to hold the complete blob.
 func (r *Repository) loadBlob(id restic.ID, t restic.BlobType, plaintextBuf []byte) (int, error) {
-	debug.Log("load %v with id %v (buf %p, len %d)", t, id.Str(), plaintextBuf, len(plaintextBuf))
-
-	// lookup plaintext size of blob
-	size, err := r.idx.LookupSize(id, t)
-	if err != nil {
-		return 0, err
-	}
-
-	// make sure the plaintext buffer is large enough, extend otherwise
-	if len(plaintextBuf) < int(size) {
-		return 0, errors.Errorf("buffer is too small: %d < %d", len(plaintextBuf), size)
-	}
+	debug.Log("load %v with id %v (buf len %v, cap %d)", t, id.Str(), len(plaintextBuf), cap(plaintextBuf))
 
 	// lookup packs
 	blobs, err := r.idx.Lookup(id, t)
@@ -109,8 +96,14 @@ func (r *Repository) loadBlob(id restic.ID, t restic.BlobType, plaintextBuf []by
 
 		// load blob from pack
 		h := restic.Handle{Type: restic.DataFile, Name: blob.PackID.String()}
-		ciphertextBuf := make([]byte, blob.Length)
-		n, err := r.be.Load(h, ciphertextBuf, int64(blob.Offset))
+
+		if uint(cap(plaintextBuf)) < blob.Length {
+			return 0, errors.Errorf("buffer is too small: %v < %v", cap(plaintextBuf), blob.Length)
+		}
+
+		plaintextBuf = plaintextBuf[:blob.Length]
+
+		n, err := restic.ReadAt(r.be, h, int64(blob.Offset), plaintextBuf)
 		if err != nil {
 			debug.Log("error loading blob %v: %v", blob, err)
 			lastError = err
@@ -125,7 +118,7 @@ func (r *Repository) loadBlob(id restic.ID, t restic.BlobType, plaintextBuf []by
 		}
 
 		// decrypt
-		n, err = r.decryptTo(plaintextBuf, ciphertextBuf)
+		n, err = r.decryptTo(plaintextBuf, plaintextBuf)
 		if err != nil {
 			lastError = errors.Errorf("decrypting blob %v failed: %v", id, err)
 			continue
@@ -224,7 +217,7 @@ func (r *Repository) SaveJSONUnpacked(t restic.FileType, item interface{}) (rest
 // SaveUnpacked encrypts data and stores it in the backend. Returned is the
 // storage hash.
 func (r *Repository) SaveUnpacked(t restic.FileType, p []byte) (id restic.ID, err error) {
-	ciphertext := make([]byte, len(p)+crypto.Extension)
+	ciphertext := restic.NewBlobBuffer(len(p))
 	ciphertext, err = r.Encrypt(ciphertext, p)
 	if err != nil {
 		return restic.ID{}, err
@@ -233,7 +226,7 @@ func (r *Repository) SaveUnpacked(t restic.FileType, p []byte) (id restic.ID, er
 	id = restic.Hash(ciphertext)
 	h := restic.Handle{Type: t, Name: id.String()}
 
-	err = r.be.Save(h, ciphertext)
+	err = r.be.Save(h, bytes.NewReader(ciphertext))
 	if err != nil {
 		debug.Log("error saving blob %v: %v", h, err)
 		return restic.ID{}, err
@@ -248,15 +241,15 @@ func (r *Repository) Flush() error {
 	r.pm.Lock()
 	defer r.pm.Unlock()
 
-	debug.Log("manually flushing %d packs", len(r.packs))
+	debug.Log("manually flushing %d packs", len(r.packerManager.packers))
 
-	for _, p := range r.packs {
+	for _, p := range r.packerManager.packers {
 		err := r.savePacker(p)
 		if err != nil {
 			return err
 		}
 	}
-	r.packs = r.packs[:0]
+	r.packerManager.packers = r.packerManager.packers[:0]
 	return nil
 }
 
@@ -387,7 +380,7 @@ func (r *Repository) SearchKey(password string, maxKeys int) error {
 // Init creates a new master key with the supplied password, initializes and
 // saves the repository config.
 func (r *Repository) Init(password string) error {
-	has, err := r.be.Test(restic.ConfigFile, "")
+	has, err := r.be.Test(restic.Handle{Type: restic.ConfigFile})
 	if err != nil {
 		return err
 	}
@@ -528,16 +521,18 @@ func (r *Repository) Close() error {
 	return r.be.Close()
 }
 
-// LoadBlob loads a blob of type t from the repository to the buffer.
+// LoadBlob loads a blob of type t from the repository to the buffer. buf must
+// be large enough to hold the encrypted blob, since it is used as scratch
+// space.
 func (r *Repository) LoadBlob(t restic.BlobType, id restic.ID, buf []byte) (int, error) {
-	debug.Log("load blob %v into buf %p", id.Str(), buf)
+	debug.Log("load blob %v into buf (len %v, cap %v)", id.Str(), len(buf), cap(buf))
 	size, err := r.idx.LookupSize(id, t)
 	if err != nil {
 		return 0, err
 	}
 
-	if len(buf) < int(size) {
-		return 0, errors.Errorf("buffer is too small for data blob (%d < %d)", len(buf), size)
+	if cap(buf) < restic.CiphertextLength(int(size)) {
+		return 0, errors.Errorf("buffer is too small for data blob (%d < %d)", cap(buf), restic.CiphertextLength(int(size)))
 	}
 
 	n, err := r.loadBlob(id, t, buf)
@@ -571,7 +566,7 @@ func (r *Repository) LoadTree(id restic.ID) (*restic.Tree, error) {
 	}
 
 	debug.Log("size is %d, create buffer", size)
-	buf := make([]byte, size)
+	buf := restic.NewBlobBuffer(int(size))
 
 	n, err := r.loadBlob(id, restic.TreeBlob, buf)
 	if err != nil {

src/restic/repository/repository_test.go

@@ -2,12 +2,12 @@ package repository_test
 
 import (
 	"bytes"
-	"crypto/rand"
 	"crypto/sha256"
 	"io"
-	mrand "math/rand"
+	"math/rand"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"restic"
 	"restic/archiver"
@@ -17,13 +17,15 @@ import (
 
 var testSizes = []int{5, 23, 2<<18 + 23, 1 << 20}
 
+var rnd = rand.New(rand.NewSource(time.Now().UnixNano()))
+
 func TestSave(t *testing.T) {
 	repo, cleanup := repository.TestRepository(t)
 	defer cleanup()
 
 	for _, size := range testSizes {
 		data := make([]byte, size)
-		_, err := io.ReadFull(rand.Reader, data)
+		_, err := io.ReadFull(rnd, data)
 		OK(t, err)
 
 		id := restic.Hash(data)
@@ -38,7 +40,7 @@ func TestSave(t *testing.T) {
 		// OK(t, repo.SaveIndex())
 
 		// read back
-		buf := make([]byte, size)
+		buf := restic.NewBlobBuffer(size)
 		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
 		OK(t, err)
 		Equals(t, len(buf), n)
@@ -59,7 +61,7 @@ func TestSaveFrom(t *testing.T) {
 
 	for _, size := range testSizes {
 		data := make([]byte, size)
-		_, err := io.ReadFull(rand.Reader, data)
+		_, err := io.ReadFull(rnd, data)
 		OK(t, err)
 
 		id := restic.Hash(data)
@@ -72,7 +74,7 @@ func TestSaveFrom(t *testing.T) {
 		OK(t, repo.Flush())
 
 		// read back
-		buf := make([]byte, size)
+		buf := restic.NewBlobBuffer(size)
 		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
 		OK(t, err)
 		Equals(t, len(buf), n)
@@ -94,7 +96,7 @@ func BenchmarkSaveAndEncrypt(t *testing.B) {
 	size := 4 << 20 // 4MiB
 
 	data := make([]byte, size)
-	_, err := io.ReadFull(rand.Reader, data)
+	_, err := io.ReadFull(rnd, data)
 	OK(t, err)
 
 	id := restic.ID(sha256.Sum256(data))
@@ -145,6 +147,113 @@ func BenchmarkLoadTree(t *testing.B) {
 	}
 }
 
+func TestLoadBlob(t *testing.T) {
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
+
+	length := 1000000
+	buf := restic.NewBlobBuffer(length)
+	_, err := io.ReadFull(rnd, buf)
+	OK(t, err)
+
+	id, err := repo.SaveBlob(restic.DataBlob, buf, restic.ID{})
+	OK(t, err)
+	OK(t, repo.Flush())
+
+	// first, test with buffers that are too small
+	for _, testlength := range []int{length - 20, length, restic.CiphertextLength(length) - 1} {
+		buf = make([]byte, 0, testlength)
+		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
+		if err == nil {
+			t.Errorf("LoadBlob() did not return an error for a buffer that is too small to hold the blob")
+			continue
+		}
+
+		if n != 0 {
+			t.Errorf("LoadBlob() returned an error and n > 0")
+			continue
+		}
+	}
+
+	// then use buffers that are large enough
+	base := restic.CiphertextLength(length)
+	for _, testlength := range []int{base, base + 7, base + 15, base + 1000} {
+		buf = make([]byte, 0, testlength)
+		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
+		if err != nil {
+			t.Errorf("LoadBlob() returned an error for buffer size %v: %v", testlength, err)
+			continue
+		}
+
+		if n != length {
+			t.Errorf("LoadBlob() returned the wrong number of bytes: want %v, got %v", length, n)
+			continue
+		}
+	}
+}
+
+func BenchmarkLoadBlob(b *testing.B) {
+	repo, cleanup := repository.TestRepository(b)
+	defer cleanup()
+
+	length := 1000000
+	buf := restic.NewBlobBuffer(length)
+	_, err := io.ReadFull(rnd, buf)
+	OK(b, err)
+
+	id, err := repo.SaveBlob(restic.DataBlob, buf, restic.ID{})
+	OK(b, err)
+	OK(b, repo.Flush())
+
+	b.ResetTimer()
+	b.SetBytes(int64(length))
+
+	for i := 0; i < b.N; i++ {
+		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
+		OK(b, err)
+		if n != length {
+			b.Errorf("wanted %d bytes, got %d", length, n)
+		}
+
+		id2 := restic.Hash(buf[:n])
+		if !id.Equal(id2) {
+			b.Errorf("wrong data returned, wanted %v, got %v", id.Str(), id2.Str())
+		}
+	}
+}
+
+func BenchmarkLoadAndDecrypt(b *testing.B) {
+	repo, cleanup := repository.TestRepository(b)
+	defer cleanup()
+
+	length := 1000000
+	buf := restic.NewBlobBuffer(length)
+	_, err := io.ReadFull(rnd, buf)
+	OK(b, err)
+
+	dataID := restic.Hash(buf)
+
+	storageID, err := repo.SaveUnpacked(restic.DataFile, buf)
+	OK(b, err)
+	// OK(b, repo.Flush())
+
+	b.ResetTimer()
+	b.SetBytes(int64(length))
+
+	for i := 0; i < b.N; i++ {
+		data, err := repo.LoadAndDecrypt(restic.DataFile, storageID)
+		OK(b, err)
+		if len(data) != length {
+			b.Errorf("wanted %d bytes, got %d", length, len(data))
+		}
+
+		id2 := restic.Hash(data)
+		if !dataID.Equal(id2) {
+			b.Errorf("wrong data returned, wanted %v, got %v", storageID.Str(), id2.Str())
+		}
+	}
+}
+
 func TestLoadJSONUnpacked(t *testing.T) {
 	repo, cleanup := repository.TestRepository(t)
 	defer cleanup()
@@ -182,25 +291,48 @@ func TestRepositoryLoadIndex(t *testing.T) {
 }
 
 func BenchmarkLoadIndex(b *testing.B) {
-	repodir, cleanup := Env(b, repoFixture)
+	repository.TestUseLowSecurityKDFParameters(b)
+
+	repo, cleanup := repository.TestRepository(b)
 	defer cleanup()
 
-	repo := repository.TestOpenLocal(b, repodir)
+	idx := repository.NewIndex()
+
+	for i := 0; i < 5000; i++ {
+		idx.Store(restic.PackedBlob{
+			Blob: restic.Blob{
+				Type:   restic.DataBlob,
+				Length: 1234,
+				ID:     restic.NewRandomID(),
+				Offset: 1235,
+			},
+			PackID: restic.NewRandomID(),
+		})
+	}
+
+	id, err := repository.SaveIndex(repo, idx)
+	OK(b, err)
+
+	b.Logf("index saved as %v (%v entries)", id.Str(), idx.Count(restic.DataBlob))
+	fi, err := repo.Backend().Stat(restic.Handle{Type: restic.IndexFile, Name: id.String()})
+	OK(b, err)
+	b.Logf("filesize is %v", fi.Size)
+
 	b.ResetTimer()
 
 	for i := 0; i < b.N; i++ {
-		repo.SetIndex(repository.NewMasterIndex())
-		OK(b, repo.LoadIndex())
+		_, err := repository.LoadIndex(repo, id)
+		OK(b, err)
 	}
 }
 
 // saveRandomDataBlobs generates random data blobs and saves them to the repository.
 func saveRandomDataBlobs(t testing.TB, repo restic.Repository, num int, sizeMax int) {
 	for i := 0; i < num; i++ {
-		size := mrand.Int() % sizeMax
+		size := rand.Int() % sizeMax
 
 		buf := make([]byte, size)
-		_, err := io.ReadFull(rand.Reader, buf)
+		_, err := io.ReadFull(rnd, buf)
 		OK(t, err)
 
 		_, err = repo.SaveBlob(restic.DataBlob, buf, restic.ID{})

src/restic/test/helpers.go

@@ -79,7 +79,7 @@ func Random(seed, count int) []byte {
 
 		for j := range data {
 			cur := i + j
-			if len(p) >= cur {
+			if cur >= len(p) {
 				break
 			}
 			p[cur] = data[j]

src/restic/testing.go

@@ -23,16 +23,26 @@ type fakeFileSystem struct {
 	repo        Repository
 	knownBlobs  IDSet
 	duplication float32
+	buf         []byte
+	chunker     *chunker.Chunker
 }
 
 // saveFile reads from rd and saves the blobs in the repository. The list of
 // IDs is returned.
-func (fs fakeFileSystem) saveFile(rd io.Reader) (blobs IDs) {
-	blobs = IDs{}
-	ch := chunker.New(rd, fs.repo.Config().ChunkerPolynomial)
+func (fs *fakeFileSystem) saveFile(rd io.Reader) (blobs IDs) {
+	if fs.buf == nil {
+		fs.buf = make([]byte, chunker.MaxSize)
+	}
 
+	if fs.chunker == nil {
+		fs.chunker = chunker.New(rd, fs.repo.Config().ChunkerPolynomial)
+	} else {
+		fs.chunker.Reset(rd, fs.repo.Config().ChunkerPolynomial)
+	}
+
+	blobs = IDs{}
 	for {
-		chunk, err := ch.Next(getBuf())
+		chunk, err := fs.chunker.Next(fs.buf)
 		if errors.Cause(err) == io.EOF {
 			break
 		}
@@ -50,7 +60,6 @@ func (fs fakeFileSystem) saveFile(rd io.Reader) (blobs IDs) {
 
 			fs.knownBlobs.Insert(id)
 		}
-		freeBuf(chunk.Data)
 
 		blobs = append(blobs, id)
 	}
@@ -64,7 +73,7 @@ const (
 	maxNodes    = 32
 )
 
-func (fs fakeFileSystem) treeIsKnown(tree *Tree) (bool, []byte, ID) {
+func (fs *fakeFileSystem) treeIsKnown(tree *Tree) (bool, []byte, ID) {
 	data, err := json.Marshal(tree)
 	if err != nil {
 		fs.t.Fatalf("json.Marshal(tree) returned error: %v", err)
@@ -76,7 +85,7 @@ func (fs fakeFileSystem) treeIsKnown(tree *Tree) (bool, []byte, ID) {
 	return fs.blobIsKnown(id, TreeBlob), data, id
 }
 
-func (fs fakeFileSystem) blobIsKnown(id ID, t BlobType) bool {
+func (fs *fakeFileSystem) blobIsKnown(id ID, t BlobType) bool {
 	if rand.Float32() < fs.duplication {
 		return false
 	}
@@ -94,7 +103,7 @@ func (fs fakeFileSystem) blobIsKnown(id ID, t BlobType) bool {
 }
 
 // saveTree saves a tree of fake files in the repo and returns the ID.
-func (fs fakeFileSystem) saveTree(seed int64, depth int) ID {
+func (fs *fakeFileSystem) saveTree(seed int64, depth int) ID {
 	rnd := rand.NewSource(seed)
 	numNodes := int(rnd.Int63() % maxNodes)
 

src/restic/testing_test.go

@@ -47,3 +47,14 @@ func TestCreateSnapshot(t *testing.T) {
 
 	checker.TestCheckRepo(t, repo)
 }
+
+func BenchmarkTestCreateSnapshot(t *testing.B) {
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
+
+	t.ResetTimer()
+
+	for i := 0; i < t.N; i++ {
+		restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth, 0)
+	}
+}

vendor/manifest

@@ -37,6 +37,12 @@
 			"revision": "17b591df37844cde689f4d5813e5cea0927d8dd2",
 			"branch": "master"
 		},
+		{
+			"importpath": "github.com/pkg/profile",
+			"repository": "https://github.com/pkg/profile",
+			"revision": "1c16f117a3ab788fdf0e334e623b8bccf5679866",
+			"branch": "HEAD"
+		},
 		{
 			"importpath": "github.com/pkg/sftp",
 			"repository": "https://github.com/pkg/sftp",

vendor/src/github.com/pkg/profile/AUTHORS

@@ -0,0 +1 @@
+Dave Cheney <dave@cheney.net>

vendor/src/github.com/pkg/profile/LICENSE

@@ -0,0 +1,24 @@
+Copyright (c) 2013 Dave Cheney. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/src/github.com/pkg/profile/README.md

@@ -0,0 +1,47 @@
+profile
+=======
+
+Simple profiling support package for Go
+
+[![Build Status](https://travis-ci.org/pkg/profile.svg?branch=master)](https://travis-ci.org/pkg/profile) [![GoDoc](http://godoc.org/github.com/pkg/profile?status.svg)](http://godoc.org/github.com/pkg/profile)
+
+
+installation
+------------
+
+    go get github.com/pkg/profile
+
+usage
+-----
+
+Enabling profiling in your application is as simple as one line at the top of your main function
+
+```go
+import "github.com/pkg/profile"
+
+func main() {
+    defer profile.Start().Stop()
+    ...
+}
+```
+
+options
+-------
+
+What to profile is controlled by config value passed to profile.Start. 
+By default CPU profiling is enabled.
+
+```go
+import "github.com/pkg/profile"
+
+func main() {
+    // p.Stop() must be called before the program exits to
+    // ensure profiling information is written to disk.
+    p := profile.Start(profile.MemProfile, profile.ProfilePath("."), profile.NoShutdownHook)
+    ...
+}
+```
+
+Several convenience package level values are provided for cpu, memory, and block (contention) profiling.
+
+For more complex options, consult the [documentation](http://godoc.org/github.com/pkg/profile).

vendor/src/github.com/pkg/profile/example_test.go

@@ -0,0 +1,56 @@
+package profile_test
+
+import (
+	"flag"
+	"os"
+
+	"github.com/pkg/profile"
+)
+
+func ExampleStart() {
+	// start a simple CPU profile and register
+	// a defer to Stop (flush) the profiling data.
+	defer profile.Start().Stop()
+}
+
+func ExampleCPUProfile() {
+	// CPU profiling is the default profiling mode, but you can specify it
+	// explicitly for completeness.
+	defer profile.Start(profile.CPUProfile).Stop()
+}
+
+func ExampleMemProfile() {
+	// use memory profiling, rather than the default cpu profiling.
+	defer profile.Start(profile.MemProfile).Stop()
+}
+
+func ExampleMemProfileRate() {
+	// use memory profiling with custom rate.
+	defer profile.Start(profile.MemProfileRate(2048)).Stop()
+}
+
+func ExampleProfilePath() {
+	// set the location that the profile will be written to
+	defer profile.Start(profile.ProfilePath(os.Getenv("HOME")))
+}
+
+func ExampleNoShutdownHook() {
+	// disable the automatic shutdown hook.
+	defer profile.Start(profile.NoShutdownHook).Stop()
+}
+
+func ExampleStart_withFlags() {
+	// use the flags package to selectively enable profiling.
+	mode := flag.String("profile.mode", "", "enable profiling mode, one of [cpu, mem, block]")
+	flag.Parse()
+	switch *mode {
+	case "cpu":
+		defer profile.Start(profile.CPUProfile).Stop()
+	case "mem":
+		defer profile.Start(profile.MemProfile).Stop()
+	case "block":
+		defer profile.Start(profile.BlockProfile).Stop()
+	default:
+		// do nothing
+	}
+}

vendor/src/github.com/pkg/profile/profile.go

@@ -0,0 +1,216 @@
+// Package profile provides a simple way to manage runtime/pprof
+// profiling of your Go application.
+package profile
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"runtime"
+	"runtime/pprof"
+	"sync/atomic"
+)
+
+const (
+	cpuMode = iota
+	memMode
+	blockMode
+	traceMode
+)
+
+type profile struct {
+	// quiet suppresses informational messages during profiling.
+	quiet bool
+
+	// noShutdownHook controls whether the profiling package should
+	// hook SIGINT to write profiles cleanly.
+	noShutdownHook bool
+
+	// mode holds the type of profiling that will be made
+	mode int
+
+	// path holds the base path where various profiling files are  written.
+	// If blank, the base path will be generated by ioutil.TempDir.
+	path string
+
+	// memProfileRate holds the rate for the memory profile.
+	memProfileRate int
+
+	// closer holds a cleanup function that run after each profile
+	closer func()
+
+	// stopped records if a call to profile.Stop has been made
+	stopped uint32
+}
+
+// NoShutdownHook controls whether the profiling package should
+// hook SIGINT to write profiles cleanly.
+// Programs with more sophisticated signal handling should set
+// this to true and ensure the Stop() function returned from Start()
+// is called during shutdown.
+func NoShutdownHook(p *profile) { p.noShutdownHook = true }
+
+// Quiet suppresses informational messages during profiling.
+func Quiet(p *profile) { p.quiet = true }
+
+// CPUProfile enables cpu profiling.
+// It disables any previous profiling settings.
+func CPUProfile(p *profile) { p.mode = cpuMode }
+
+// DefaultMemProfileRate is the default memory profiling rate.
+// See also http://golang.org/pkg/runtime/#pkg-variables
+const DefaultMemProfileRate = 4096
+
+// MemProfile enables memory profiling.
+// It disables any previous profiling settings.
+func MemProfile(p *profile) {
+	p.memProfileRate = DefaultMemProfileRate
+	p.mode = memMode
+}
+
+// MemProfileRate enables memory profiling at the preferred rate.
+// It disables any previous profiling settings.
+func MemProfileRate(rate int) func(*profile) {
+	return func(p *profile) {
+		p.memProfileRate = rate
+		p.mode = memMode
+	}
+}
+
+// BlockProfile enables block (contention) profiling.
+// It disables any previous profiling settings.
+func BlockProfile(p *profile) { p.mode = blockMode }
+
+// ProfilePath controls the base path where various profiling
+// files are written. If blank, the base path will be generated
+// by ioutil.TempDir.
+func ProfilePath(path string) func(*profile) {
+	return func(p *profile) {
+		p.path = path
+	}
+}
+
+// Stop stops the profile and flushes any unwritten data.
+func (p *profile) Stop() {
+	if !atomic.CompareAndSwapUint32(&p.stopped, 0, 1) {
+		// someone has already called close
+		return
+	}
+	p.closer()
+	atomic.StoreUint32(&started, 0)
+}
+
+// started is non zero if a profile is running.
+var started uint32
+
+// Start starts a new profiling session.
+// The caller should call the Stop method on the value returned
+// to cleanly stop profiling.
+func Start(options ...func(*profile)) interface {
+	Stop()
+} {
+	if !atomic.CompareAndSwapUint32(&started, 0, 1) {
+		log.Fatal("profile: Start() already called")
+	}
+
+	var prof profile
+	for _, option := range options {
+		option(&prof)
+	}
+
+	path, err := func() (string, error) {
+		if p := prof.path; p != "" {
+			return p, os.MkdirAll(p, 0777)
+		}
+		return ioutil.TempDir("", "profile")
+	}()
+
+	if err != nil {
+		log.Fatalf("profile: could not create initial output directory: %v", err)
+	}
+
+	logf := func(format string, args ...interface{}) {
+		if !prof.quiet {
+			log.Printf(format, args...)
+		}
+	}
+
+	switch prof.mode {
+	case cpuMode:
+		fn := filepath.Join(path, "cpu.pprof")
+		f, err := os.Create(fn)
+		if err != nil {
+			log.Fatalf("profile: could not create cpu profile %q: %v", fn, err)
+		}
+		logf("profile: cpu profiling enabled, %s", fn)
+		pprof.StartCPUProfile(f)
+		prof.closer = func() {
+			pprof.StopCPUProfile()
+			f.Close()
+			logf("profile: cpu profiling disabled, %s", fn)
+		}
+
+	case memMode:
+		fn := filepath.Join(path, "mem.pprof")
+		f, err := os.Create(fn)
+		if err != nil {
+			log.Fatalf("profile: could not create memory profile %q: %v", fn, err)
+		}
+		old := runtime.MemProfileRate
+		runtime.MemProfileRate = prof.memProfileRate
+		logf("profile: memory profiling enabled (rate %d), %s", runtime.MemProfileRate, fn)
+		prof.closer = func() {
+			pprof.Lookup("heap").WriteTo(f, 0)
+			f.Close()
+			runtime.MemProfileRate = old
+			logf("profile: memory profiling disabled, %s", fn)
+		}
+
+	case blockMode:
+		fn := filepath.Join(path, "block.pprof")
+		f, err := os.Create(fn)
+		if err != nil {
+			log.Fatalf("profile: could not create block profile %q: %v", fn, err)
+		}
+		runtime.SetBlockProfileRate(1)
+		logf("profile: block profiling enabled, %s", fn)
+		prof.closer = func() {
+			pprof.Lookup("block").WriteTo(f, 0)
+			f.Close()
+			runtime.SetBlockProfileRate(0)
+			logf("profile: block profiling disabled, %s", fn)
+		}
+
+	case traceMode:
+		fn := filepath.Join(path, "trace.out")
+		f, err := os.Create(fn)
+		if err != nil {
+			log.Fatalf("profile: could not create trace output file %q: %v", fn, err)
+		}
+		if err := startTrace(f); err != nil {
+			log.Fatalf("profile: could not start trace: %v", err)
+		}
+		logf("profile: trace enabled, %s", fn)
+		prof.closer = func() {
+			stopTrace()
+			logf("profile: trace disabled, %s", fn)
+		}
+	}
+
+	if !prof.noShutdownHook {
+		go func() {
+			c := make(chan os.Signal, 1)
+			signal.Notify(c, os.Interrupt)
+			<-c
+
+			log.Println("profile: caught interrupt, stopping profiles")
+			prof.Stop()
+
+			os.Exit(0)
+		}()
+	}
+
+	return &prof
+}

vendor/src/github.com/pkg/profile/profile_test.go

@@ -0,0 +1,304 @@
+package profile
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+type checkFn func(t *testing.T, stdout, stderr []byte, err error)
+
+var profileTests = []struct {
+	name   string
+	code   string
+	checks []checkFn
+}{{
+	name: "default profile (cpu)",
+	code: `
+package main
+
+import "github.com/pkg/profile"
+
+func main() {
+	defer profile.Start().Stop()
+}	
+`,
+	checks: []checkFn{
+		NoStdout,
+		Stderr("profile: cpu profiling enabled"),
+		NoErr,
+	},
+}, {
+	name: "memory profile",
+	code: `
+package main
+
+import "github.com/pkg/profile"
+
+func main() {
+	defer profile.Start(profile.MemProfile).Stop()
+}	
+`,
+	checks: []checkFn{
+		NoStdout,
+		Stderr("profile: memory profiling enabled"),
+		NoErr,
+	},
+}, {
+	name: "memory profile (rate 2048)",
+	code: `
+package main
+
+import "github.com/pkg/profile"
+
+func main() {
+	defer profile.Start(profile.MemProfileRate(2048)).Stop()
+}	
+`,
+	checks: []checkFn{
+		NoStdout,
+		Stderr("profile: memory profiling enabled (rate 2048)"),
+		NoErr,
+	},
+}, {
+	name: "double start",
+	code: `
+package main
+
+import "github.com/pkg/profile"
+
+func main() {
+	profile.Start()
+	profile.Start()
+}	
+`,
+	checks: []checkFn{
+		NoStdout,
+		Stderr("cpu profiling enabled", "profile: Start() already called"),
+		Err,
+	},
+}, {
+	name: "block profile",
+	code: `
+package main
+
+import "github.com/pkg/profile"
+
+func main() {
+	defer profile.Start(profile.BlockProfile).Stop()
+}	
+`,
+	checks: []checkFn{
+		NoStdout,
+		Stderr("profile: block profiling enabled"),
+		NoErr,
+	},
+}, {
+	name: "profile path",
+	code: `
+package main
+
+import "github.com/pkg/profile"
+
+func main() {
+	defer profile.Start(profile.ProfilePath(".")).Stop()
+}	
+`,
+	checks: []checkFn{
+		NoStdout,
+		Stderr("profile: cpu profiling enabled, cpu.pprof"),
+		NoErr,
+	},
+}, {
+	name: "profile path error",
+	code: `
+package main
+
+import "github.com/pkg/profile"
+
+func main() {
+		defer profile.Start(profile.ProfilePath("README.md")).Stop()
+}	
+`,
+	checks: []checkFn{
+		NoStdout,
+		Stderr("could not create initial output"),
+		Err,
+	},
+}, {
+	name: "multiple profile sessions",
+	code: `
+package main
+
+import "github.com/pkg/profile"
+
+func main() {
+	profile.Start(profile.CPUProfile).Stop()
+	profile.Start(profile.MemProfile).Stop()
+	profile.Start(profile.BlockProfile).Stop()
+	profile.Start(profile.CPUProfile).Stop()
+}
+`,
+	checks: []checkFn{
+		NoStdout,
+		Stderr("profile: cpu profiling enabled",
+			"profile: cpu profiling disabled",
+			"profile: memory profiling enabled",
+			"profile: memory profiling disabled",
+			"profile: block profiling enabled",
+			"profile: block profiling disabled"),
+		NoErr,
+	},
+}, {
+	name: "profile quiet",
+	code: `
+package main
+
+import "github.com/pkg/profile"
+
+func main() {
+        defer profile.Start(profile.Quiet).Stop()
+}       
+`,
+	checks: []checkFn{NoStdout, NoStderr, NoErr},
+}}
+
+func TestProfile(t *testing.T) {
+	for _, tt := range profileTests {
+		t.Log(tt.name)
+		stdout, stderr, err := runTest(t, tt.code)
+		for _, f := range tt.checks {
+			f(t, stdout, stderr, err)
+		}
+	}
+}
+
+// NoStdout checks that stdout was blank.
+func NoStdout(t *testing.T, stdout, _ []byte, _ error) {
+	if len := len(stdout); len > 0 {
+		t.Errorf("stdout: wanted 0 bytes, got %d", len)
+	}
+}
+
+// Stderr verifies that the given lines match the output from stderr
+func Stderr(lines ...string) checkFn {
+	return func(t *testing.T, _, stderr []byte, _ error) {
+		r := bytes.NewReader(stderr)
+		if !validateOutput(r, lines) {
+			t.Errorf("stderr: wanted '%s', got '%s'", lines, stderr)
+		}
+	}
+}
+
+// NoStderr checks that stderr was blank.
+func NoStderr(t *testing.T, _, stderr []byte, _ error) {
+	if len := len(stderr); len > 0 {
+		t.Errorf("stderr: wanted 0 bytes, got %d", len)
+	}
+}
+
+// Err checks that there was an error returned
+func Err(t *testing.T, _, _ []byte, err error) {
+	if err == nil {
+		t.Errorf("expected error")
+	}
+}
+
+// NoErr checks that err was nil
+func NoErr(t *testing.T, _, _ []byte, err error) {
+	if err != nil {
+		t.Errorf("error: expected nil, got %v", err)
+	}
+}
+
+// validatedOutput validates the given slice of lines against data from the given reader.
+func validateOutput(r io.Reader, want []string) bool {
+	s := bufio.NewScanner(r)
+	for _, line := range want {
+		if !s.Scan() || !strings.Contains(s.Text(), line) {
+			return false
+		}
+	}
+	return true
+}
+
+var validateOutputTests = []struct {
+	input string
+	lines []string
+	want  bool
+}{{
+	input: "",
+	want:  true,
+}, {
+	input: `profile: yes
+`,
+	want: true,
+}, {
+	input: `profile: yes
+`,
+	lines: []string{"profile: yes"},
+	want:  true,
+}, {
+	input: `profile: yes
+profile: no
+`,
+	lines: []string{"profile: yes"},
+	want:  true,
+}, {
+	input: `profile: yes
+profile: no
+`,
+	lines: []string{"profile: yes", "profile: no"},
+	want:  true,
+}, {
+	input: `profile: yes
+profile: no
+`,
+	lines: []string{"profile: no"},
+	want:  false,
+}}
+
+func TestValidateOutput(t *testing.T) {
+	for _, tt := range validateOutputTests {
+		r := strings.NewReader(tt.input)
+		got := validateOutput(r, tt.lines)
+		if tt.want != got {
+			t.Errorf("validateOutput(%q, %q), want %v, got %v", tt.input, tt.lines, tt.want, got)
+		}
+	}
+}
+
+// runTest executes the go program supplied and returns the contents of stdout,
+// stderr, and an error which may contain status information about the result
+// of the program.
+func runTest(t *testing.T, code string) ([]byte, []byte, error) {
+	chk := func(err error) {
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	gopath, err := ioutil.TempDir("", "profile-gopath")
+	chk(err)
+	defer os.RemoveAll(gopath)
+
+	srcdir := filepath.Join(gopath, "src")
+	err = os.Mkdir(srcdir, 0755)
+	chk(err)
+	src := filepath.Join(srcdir, "main.go")
+	err = ioutil.WriteFile(src, []byte(code), 0644)
+	chk(err)
+
+	cmd := exec.Command("go", "run", src)
+
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	err = cmd.Run()
+	return stdout.Bytes(), stderr.Bytes(), err
+}

vendor/src/github.com/pkg/profile/trace.go

@@ -0,0 +1,11 @@
+// +build go1.7
+
+package profile
+
+import "runtime/trace"
+
+// Trace profile controls if execution tracing will be enabled. It disables any previous profiling settings.
+func TraceProfile(p *profile) { p.mode = traceMode }
+
+var startTrace = trace.Start
+var stopTrace = trace.Stop

vendor/src/github.com/pkg/profile/trace16.go

@@ -0,0 +1,10 @@
+// +build !go1.7
+
+package profile
+
+import "io"
+
+// mock trace support for Go 1.6 and earlier.
+
+func startTrace(w io.Writer) error { return nil }
+func stopTrace()                   {}

vendor/src/github.com/pkg/profile/trace_test.go

@@ -0,0 +1,10 @@
+// +build go1.7
+
+package profile_test
+
+import "github.com/pkg/profile"
+
+func ExampleTraceProfile() {
+	// use execution tracing, rather than the default cpu profiling.
+	defer profile.Start(profile.TraceProfile).Stop()
+}