Moves files

internal/backend/b2/b2.go

@@ -0,0 +1,377 @@
+package b2
+
+import (
+	"context"
+	"io"
+	"path"
+	"restic"
+	"strings"
+
+	"restic/backend"
+	"restic/debug"
+	"restic/errors"
+
+	"github.com/kurin/blazer/b2"
+)
+
+// b2Backend is a backend which stores its data on Backblaze B2.
+type b2Backend struct {
+	client *b2.Client
+	bucket *b2.Bucket
+	cfg    Config
+	backend.Layout
+	sem *backend.Semaphore
+}
+
+// ensure statically that *b2Backend implements restic.Backend.
+var _ restic.Backend = &b2Backend{}
+
+func newClient(ctx context.Context, cfg Config) (*b2.Client, error) {
+	opts := []b2.ClientOption{b2.Transport(backend.Transport())}
+
+	c, err := b2.NewClient(ctx, cfg.AccountID, cfg.Key, opts...)
+	if err != nil {
+		return nil, errors.Wrap(err, "b2.NewClient")
+	}
+	return c, nil
+}
+
+// Open opens a connection to the B2 service.
+func Open(cfg Config) (restic.Backend, error) {
+	debug.Log("cfg %#v", cfg)
+
+	ctx, cancel := context.WithCancel(context.TODO())
+	defer cancel()
+
+	client, err := newClient(ctx, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	bucket, err := client.Bucket(ctx, cfg.Bucket)
+	if err != nil {
+		return nil, errors.Wrap(err, "Bucket")
+	}
+
+	sem, err := backend.NewSemaphore(cfg.Connections)
+	if err != nil {
+		return nil, err
+	}
+
+	be := &b2Backend{
+		client: client,
+		bucket: bucket,
+		cfg:    cfg,
+		Layout: &backend.DefaultLayout{
+			Join: path.Join,
+			Path: cfg.Prefix,
+		},
+		sem: sem,
+	}
+
+	return be, nil
+}
+
+// Create opens a connection to the B2 service. If the bucket does not exist yet,
+// it is created.
+func Create(cfg Config) (restic.Backend, error) {
+	debug.Log("cfg %#v", cfg)
+
+	ctx, cancel := context.WithCancel(context.TODO())
+	defer cancel()
+
+	client, err := newClient(ctx, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	attr := b2.BucketAttrs{
+		Type: b2.Private,
+	}
+	bucket, err := client.NewBucket(ctx, cfg.Bucket, &attr)
+	if err != nil {
+		return nil, errors.Wrap(err, "NewBucket")
+	}
+
+	sem, err := backend.NewSemaphore(cfg.Connections)
+	if err != nil {
+		return nil, err
+	}
+
+	be := &b2Backend{
+		client: client,
+		bucket: bucket,
+		cfg:    cfg,
+		Layout: &backend.DefaultLayout{
+			Join: path.Join,
+			Path: cfg.Prefix,
+		},
+		sem: sem,
+	}
+
+	present, err := be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile})
+	if err != nil {
+		return nil, err
+	}
+
+	if present {
+		return nil, errors.New("config already exists")
+	}
+
+	return be, nil
+}
+
+// Location returns the location for the backend.
+func (be *b2Backend) Location() string {
+	return be.cfg.Bucket
+}
+
+// wrapReader wraps an io.ReadCloser to run an additional function on Close.
+type wrapReader struct {
+	io.ReadCloser
+	eofSeen bool
+	f       func()
+}
+
+func (wr *wrapReader) Read(p []byte) (int, error) {
+	if wr.eofSeen {
+		return 0, io.EOF
+	}
+
+	n, err := wr.ReadCloser.Read(p)
+	if err == io.EOF {
+		wr.eofSeen = true
+	}
+	return n, err
+}
+
+func (wr *wrapReader) Close() error {
+	err := wr.ReadCloser.Close()
+	wr.f()
+	return err
+}
+
+// IsNotExist returns true if the error is caused by a non-existing file.
+func (be *b2Backend) IsNotExist(err error) bool {
+	return b2.IsNotExist(errors.Cause(err))
+}
+
+// Load returns the data stored in the backend for h at the given offset
+// and saves it in p. Load has the same semantics as io.ReaderAt.
+func (be *b2Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h))
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if length < 0 {
+		return nil, errors.Errorf("invalid length %d", length)
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+
+	be.sem.GetToken()
+
+	name := be.Layout.Filename(h)
+	obj := be.bucket.Object(name)
+
+	if offset == 0 && length == 0 {
+		rd := obj.NewReader(ctx)
+		wrapper := &wrapReader{
+			ReadCloser: rd,
+			f: func() {
+				cancel()
+				be.sem.ReleaseToken()
+			},
+		}
+		return wrapper, nil
+	}
+
+	// pass a negative length to NewRangeReader so that the remainder of the
+	// file is read.
+	if length == 0 {
+		length = -1
+	}
+
+	rd := obj.NewRangeReader(ctx, offset, int64(length))
+	wrapper := &wrapReader{
+		ReadCloser: rd,
+		f: func() {
+			cancel()
+			be.sem.ReleaseToken()
+		},
+	}
+	return wrapper, nil
+}
+
+// Save stores data in the backend at the handle.
+func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	if err := h.Valid(); err != nil {
+		return err
+	}
+
+	be.sem.GetToken()
+	defer be.sem.ReleaseToken()
+
+	name := be.Filename(h)
+	debug.Log("Save %v, name %v", h, name)
+	obj := be.bucket.Object(name)
+
+	_, err = obj.Attrs(ctx)
+	if err == nil {
+		debug.Log("  %v already exists", h)
+		return errors.New("key already exists")
+	}
+
+	w := obj.NewWriter(ctx)
+	n, err := io.Copy(w, rd)
+	debug.Log("  saved %d bytes, err %v", n, err)
+
+	if err != nil {
+		_ = w.Close()
+		return errors.Wrap(err, "Copy")
+	}
+
+	return errors.Wrap(w.Close(), "Close")
+}
+
+// Stat returns information about a blob.
+func (be *b2Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
+	debug.Log("Stat %v", h)
+
+	be.sem.GetToken()
+	defer be.sem.ReleaseToken()
+
+	name := be.Filename(h)
+	obj := be.bucket.Object(name)
+	info, err := obj.Attrs(ctx)
+	if err != nil {
+		debug.Log("Attrs() err %v", err)
+		return restic.FileInfo{}, errors.Wrap(err, "Stat")
+	}
+	return restic.FileInfo{Size: info.Size}, nil
+}
+
+// Test returns true if a blob of the given type and name exists in the backend.
+func (be *b2Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
+	debug.Log("Test %v", h)
+
+	be.sem.GetToken()
+	defer be.sem.ReleaseToken()
+
+	found := false
+	name := be.Filename(h)
+	obj := be.bucket.Object(name)
+	info, err := obj.Attrs(ctx)
+	if err == nil && info != nil && info.Status == b2.Uploaded {
+		found = true
+	}
+	return found, nil
+}
+
+// Remove removes the blob with the given name and type.
+func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error {
+	debug.Log("Remove %v", h)
+
+	be.sem.GetToken()
+	defer be.sem.ReleaseToken()
+
+	obj := be.bucket.Object(be.Filename(h))
+	return errors.Wrap(obj.Delete(ctx), "Delete")
+}
+
+// List returns a channel that yields all names of blobs of type t. A
+// goroutine is started for this. If the channel done is closed, sending
+// stops.
+func (be *b2Backend) List(ctx context.Context, t restic.FileType) <-chan string {
+	debug.Log("List %v", t)
+	ch := make(chan string)
+
+	ctx, cancel := context.WithCancel(ctx)
+
+	be.sem.GetToken()
+
+	go func() {
+		defer close(ch)
+		defer cancel()
+		defer be.sem.ReleaseToken()
+
+		prefix := be.Dirname(restic.Handle{Type: t})
+		cur := &b2.Cursor{Prefix: prefix}
+
+		for {
+			objs, c, err := be.bucket.ListCurrentObjects(ctx, 1000, cur)
+			if err != nil && err != io.EOF {
+				return
+			}
+			for _, obj := range objs {
+				// Skip objects returned that do not have the specified prefix.
+				if !strings.HasPrefix(obj.Name(), prefix) {
+					continue
+				}
+
+				m := path.Base(obj.Name())
+				if m == "" {
+					continue
+				}
+
+				select {
+				case ch <- m:
+				case <-ctx.Done():
+					return
+				}
+			}
+			if err == io.EOF {
+				return
+			}
+			cur = c
+		}
+	}()
+
+	return ch
+}
+
+// Remove keys for a specified backend type.
+func (be *b2Backend) removeKeys(ctx context.Context, t restic.FileType) error {
+	debug.Log("removeKeys %v", t)
+	for key := range be.List(ctx, t) {
+		err := be.Remove(ctx, restic.Handle{Type: t, Name: key})
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
+func (be *b2Backend) Delete(ctx context.Context) error {
+	alltypes := []restic.FileType{
+		restic.DataFile,
+		restic.KeyFile,
+		restic.LockFile,
+		restic.SnapshotFile,
+		restic.IndexFile}
+
+	for _, t := range alltypes {
+		err := be.removeKeys(ctx, t)
+		if err != nil {
+			return nil
+		}
+	}
+	err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile})
+	if err != nil && b2.IsNotExist(errors.Cause(err)) {
+		err = nil
+	}
+
+	return err
+}
+
+// Close does nothing
+func (be *b2Backend) Close() error { return nil }

internal/backend/b2/b2_test.go

@@ -0,0 +1,97 @@
+package b2_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"restic"
+	"restic/backend/b2"
+	"restic/backend/test"
+
+	. "restic/test"
+)
+
+func newB2TestSuite(t testing.TB) *test.Suite {
+	return &test.Suite{
+		// do not use excessive data
+		MinimalData: true,
+
+		// wait for at most 10 seconds for removed files to disappear
+		WaitForDelayedRemoval: 10 * time.Second,
+
+		// NewConfig returns a config for a new temporary backend that will be used in tests.
+		NewConfig: func() (interface{}, error) {
+			b2cfg, err := b2.ParseConfig(os.Getenv("RESTIC_TEST_B2_REPOSITORY"))
+			if err != nil {
+				return nil, err
+			}
+
+			cfg := b2cfg.(b2.Config)
+			cfg.AccountID = os.Getenv("RESTIC_TEST_B2_ACCOUNT_ID")
+			cfg.Key = os.Getenv("RESTIC_TEST_B2_ACCOUNT_KEY")
+			cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
+			return cfg, nil
+		},
+
+		// CreateFn is a function that creates a temporary repository for the tests.
+		Create: func(config interface{}) (restic.Backend, error) {
+			cfg := config.(b2.Config)
+			return b2.Create(cfg)
+		},
+
+		// OpenFn is a function that opens a previously created temporary repository.
+		Open: func(config interface{}) (restic.Backend, error) {
+			cfg := config.(b2.Config)
+			return b2.Open(cfg)
+		},
+
+		// CleanupFn removes data created during the tests.
+		Cleanup: func(config interface{}) error {
+			cfg := config.(b2.Config)
+			be, err := b2.Open(cfg)
+			if err != nil {
+				return err
+			}
+
+			if err := be.(restic.Deleter).Delete(context.TODO()); err != nil {
+				return err
+			}
+
+			return nil
+		},
+	}
+}
+
+func testVars(t testing.TB) {
+	vars := []string{
+		"RESTIC_TEST_B2_ACCOUNT_ID",
+		"RESTIC_TEST_B2_ACCOUNT_KEY",
+		"RESTIC_TEST_B2_REPOSITORY",
+	}
+
+	for _, v := range vars {
+		if os.Getenv(v) == "" {
+			t.Skipf("environment variable %v not set", v)
+			return
+		}
+	}
+}
+
+func TestBackendB2(t *testing.T) {
+	defer func() {
+		if t.Skipped() {
+			SkipDisallowed(t, "restic/backend/b2.TestBackendB2")
+		}
+	}()
+
+	testVars(t)
+	newB2TestSuite(t).RunTests(t)
+}
+
+func BenchmarkBackendb2(t *testing.B) {
+	testVars(t)
+	newB2TestSuite(t).RunBenchmarks(t)
+}

internal/backend/b2/config.go

@@ -0,0 +1,93 @@
+package b2
+
+import (
+	"path"
+	"regexp"
+	"strings"
+
+	"restic/errors"
+	"restic/options"
+)
+
+// Config contains all configuration necessary to connect to an b2 compatible
+// server.
+type Config struct {
+	AccountID string
+	Key       string
+	Bucket    string
+	Prefix    string
+
+	Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
+}
+
+// NewConfig returns a new config with default options applied.
+func NewConfig() Config {
+	return Config{
+		Connections: 5,
+	}
+}
+
+func init() {
+	options.Register("b2", Config{})
+}
+
+var bucketName = regexp.MustCompile("^[a-zA-Z0-9-]+$")
+
+// checkBucketName tests the bucket name against the rules at
+// https://help.backblaze.com/hc/en-us/articles/217666908-What-you-need-to-know-about-B2-Bucket-names
+func checkBucketName(name string) error {
+	if name == "" {
+		return errors.New("bucket name is empty")
+	}
+
+	if len(name) < 6 {
+		return errors.New("bucket name is too short")
+	}
+
+	if len(name) > 50 {
+		return errors.New("bucket name is too long")
+	}
+
+	if !bucketName.MatchString(name) {
+		return errors.New("bucket name contains invalid characters, allowed are: a-z, 0-9, dash (-)")
+	}
+
+	return nil
+}
+
+// ParseConfig parses the string s and extracts the b2 config. The supported
+// configuration format is b2:bucketname/prefix. If no prefix is given the
+// prefix "restic" will be used.
+func ParseConfig(s string) (interface{}, error) {
+	if !strings.HasPrefix(s, "b2:") {
+		return nil, errors.New("invalid format, want: b2:bucket-name[:path]")
+	}
+
+	s = s[3:]
+	data := strings.SplitN(s, ":", 2)
+	if len(data) == 0 || len(data[0]) == 0 {
+		return nil, errors.New("bucket name not found")
+	}
+
+	cfg := NewConfig()
+	cfg.Bucket = data[0]
+
+	if err := checkBucketName(cfg.Bucket); err != nil {
+		return nil, err
+	}
+
+	if len(data) == 2 {
+		p := data[1]
+		if len(p) > 0 {
+			p = path.Clean(p)
+		}
+
+		if len(p) > 0 && path.IsAbs(p) {
+			p = p[1:]
+		}
+
+		cfg.Prefix = p
+	}
+
+	return cfg, nil
+}

internal/backend/b2/config_test.go

@@ -0,0 +1,92 @@
+package b2
+
+import "testing"
+
+var configTests = []struct {
+	s   string
+	cfg Config
+}{
+	{"b2:bucketname", Config{
+		Bucket:      "bucketname",
+		Prefix:      "",
+		Connections: 5,
+	}},
+	{"b2:bucketname:", Config{
+		Bucket:      "bucketname",
+		Prefix:      "",
+		Connections: 5,
+	}},
+	{"b2:bucketname:/prefix/directory", Config{
+		Bucket:      "bucketname",
+		Prefix:      "prefix/directory",
+		Connections: 5,
+	}},
+	{"b2:foobar", Config{
+		Bucket:      "foobar",
+		Prefix:      "",
+		Connections: 5,
+	}},
+	{"b2:foobar:", Config{
+		Bucket:      "foobar",
+		Prefix:      "",
+		Connections: 5,
+	}},
+	{"b2:foobar:/", Config{
+		Bucket:      "foobar",
+		Prefix:      "",
+		Connections: 5,
+	}},
+}
+
+func TestParseConfig(t *testing.T) {
+	for _, test := range configTests {
+		t.Run("", func(t *testing.T) {
+			cfg, err := ParseConfig(test.s)
+			if err != nil {
+				t.Fatalf("%s failed: %v", test.s, err)
+			}
+
+			if cfg != test.cfg {
+				t.Fatalf("input: %s\n wrong config, want:\n  %#v\ngot:\n  %#v",
+					test.s, test.cfg, cfg)
+			}
+		})
+	}
+}
+
+var invalidConfigTests = []struct {
+	s   string
+	err string
+}{
+	{
+		"b2",
+		"invalid format, want: b2:bucket-name[:path]",
+	},
+	{
+		"b2:",
+		"bucket name not found",
+	},
+	{
+		"b2:bucket_name",
+		"bucket name contains invalid characters, allowed are: a-z, 0-9, dash (-)",
+	},
+	{
+		"b2:bucketname/prefix/directory/",
+		"bucket name contains invalid characters, allowed are: a-z, 0-9, dash (-)",
+	},
+}
+
+func TestInvalidConfig(t *testing.T) {
+	for _, test := range invalidConfigTests {
+		t.Run("", func(t *testing.T) {
+			cfg, err := ParseConfig(test.s)
+			if err == nil {
+				t.Fatalf("expected error not found for invalid config: %v, cfg is:\n%#v", test.s, cfg)
+			}
+
+			if err.Error() != test.err {
+				t.Fatalf("unexpected error found, want:\n  %v\ngot:\n  %v", test.err, err.Error())
+			}
+		})
+	}
+}

internal/backend/doc.go

@@ -0,0 +1,4 @@
+// Package backend provides local and remote storage for restic repositories.
+// All backends need to implement the Backend interface. There is a MemBackend,
+// which stores all data in a map internally and can be used for testing.
+package backend

internal/backend/http_transport.go

@@ -0,0 +1,29 @@
+package backend
+
+import (
+	"net"
+	"net/http"
+	"restic/debug"
+	"time"
+)
+
+// Transport returns a new http.RoundTripper with default settings applied.
+func Transport() http.RoundTripper {
+	// copied from net/http
+	tr := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: true,
+		}).DialContext,
+		MaxIdleConns:          100,
+		MaxIdleConnsPerHost:   100,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+	}
+
+	// wrap in the debug round tripper
+	return debug.RoundTripper(tr)
+}

internal/backend/layout.go

@@ -0,0 +1,167 @@
+package backend
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"restic"
+	"restic/debug"
+	"restic/errors"
+	"restic/fs"
+)
+
+// Layout computes paths for file name storage.
+type Layout interface {
+	Filename(restic.Handle) string
+	Dirname(restic.Handle) string
+	Basedir(restic.FileType) string
+	Paths() []string
+	Name() string
+}
+
+// Filesystem is the abstraction of a file system used for a backend.
+type Filesystem interface {
+	Join(...string) string
+	ReadDir(string) ([]os.FileInfo, error)
+	IsNotExist(error) bool
+}
+
+// ensure statically that *LocalFilesystem implements Filesystem.
+var _ Filesystem = &LocalFilesystem{}
+
+// LocalFilesystem implements Filesystem in a local path.
+type LocalFilesystem struct {
+}
+
+// ReadDir returns all entries of a directory.
+func (l *LocalFilesystem) ReadDir(dir string) ([]os.FileInfo, error) {
+	f, err := fs.Open(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	entries, err := f.Readdir(-1)
+	if err != nil {
+		return nil, errors.Wrap(err, "Readdir")
+	}
+
+	err = f.Close()
+	if err != nil {
+		return nil, errors.Wrap(err, "Close")
+	}
+
+	return entries, nil
+}
+
+// Join combines several path components to one.
+func (l *LocalFilesystem) Join(paths ...string) string {
+	return filepath.Join(paths...)
+}
+
+// IsNotExist returns true for errors that are caused by not existing files.
+func (l *LocalFilesystem) IsNotExist(err error) bool {
+	return os.IsNotExist(err)
+}
+
+var backendFilenameLength = len(restic.ID{}) * 2
+var backendFilename = regexp.MustCompile(fmt.Sprintf("^[a-fA-F0-9]{%d}$", backendFilenameLength))
+
+func hasBackendFile(fs Filesystem, dir string) (bool, error) {
+	entries, err := fs.ReadDir(dir)
+	if err != nil && fs.IsNotExist(errors.Cause(err)) {
+		return false, nil
+	}
+
+	if err != nil {
+		return false, errors.Wrap(err, "ReadDir")
+	}
+
+	for _, e := range entries {
+		if backendFilename.MatchString(e.Name()) {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+// ErrLayoutDetectionFailed is returned by DetectLayout() when the layout
+// cannot be detected automatically.
+var ErrLayoutDetectionFailed = errors.New("auto-detecting the filesystem layout failed")
+
+// DetectLayout tries to find out which layout is used in a local (or sftp)
+// filesystem at the given path. If repo is nil, an instance of LocalFilesystem
+// is used.
+func DetectLayout(repo Filesystem, dir string) (Layout, error) {
+	debug.Log("detect layout at %v", dir)
+	if repo == nil {
+		repo = &LocalFilesystem{}
+	}
+
+	// key file in the "keys" dir (DefaultLayout)
+	foundKeysFile, err := hasBackendFile(repo, repo.Join(dir, defaultLayoutPaths[restic.KeyFile]))
+	if err != nil {
+		return nil, err
+	}
+
+	// key file in the "key" dir (S3LegacyLayout)
+	foundKeyFile, err := hasBackendFile(repo, repo.Join(dir, s3LayoutPaths[restic.KeyFile]))
+	if err != nil {
+		return nil, err
+	}
+
+	if foundKeysFile && !foundKeyFile {
+		debug.Log("found default layout at %v", dir)
+		return &DefaultLayout{
+			Path: dir,
+			Join: repo.Join,
+		}, nil
+	}
+
+	if foundKeyFile && !foundKeysFile {
+		debug.Log("found s3 layout at %v", dir)
+		return &S3LegacyLayout{
+			Path: dir,
+			Join: repo.Join,
+		}, nil
+	}
+
+	debug.Log("layout detection failed")
+	return nil, ErrLayoutDetectionFailed
+}
+
+// ParseLayout parses the config string and returns a Layout. When layout is
+// the empty string, DetectLayout is used. If that fails, defaultLayout is used.
+func ParseLayout(repo Filesystem, layout, defaultLayout, path string) (l Layout, err error) {
+	debug.Log("parse layout string %q for backend at %v", layout, path)
+	switch layout {
+	case "default":
+		l = &DefaultLayout{
+			Path: path,
+			Join: repo.Join,
+		}
+	case "s3legacy":
+		l = &S3LegacyLayout{
+			Path: path,
+			Join: repo.Join,
+		}
+	case "":
+		l, err = DetectLayout(repo, path)
+
+		// use the default layout if auto detection failed
+		if errors.Cause(err) == ErrLayoutDetectionFailed && defaultLayout != "" {
+			debug.Log("error: %v, use default layout %v", err, defaultLayout)
+			return ParseLayout(repo, defaultLayout, "", path)
+		}
+
+		if err != nil {
+			return nil, err
+		}
+		debug.Log("layout detected: %v", l)
+	default:
+		return nil, errors.Errorf("unknown backend layout string %q, may be one of: default, s3legacy", layout)
+	}
+
+	return l, nil
+}

internal/backend/layout_default.go

@@ -0,0 +1,73 @@
+package backend
+
+import (
+	"encoding/hex"
+	"restic"
+)
+
+// DefaultLayout implements the default layout for local and sftp backends, as
+// described in the Design document. The `data` directory has one level of
+// subdirs, two characters each (taken from the first two characters of the
+// file name).
+type DefaultLayout struct {
+	Path string
+	Join func(...string) string
+}
+
+var defaultLayoutPaths = map[restic.FileType]string{
+	restic.DataFile:     "data",
+	restic.SnapshotFile: "snapshots",
+	restic.IndexFile:    "index",
+	restic.LockFile:     "locks",
+	restic.KeyFile:      "keys",
+}
+
+func (l *DefaultLayout) String() string {
+	return "<DefaultLayout>"
+}
+
+// Name returns the name for this layout.
+func (l *DefaultLayout) Name() string {
+	return "default"
+}
+
+// Dirname returns the directory path for a given file type and name.
+func (l *DefaultLayout) Dirname(h restic.Handle) string {
+	p := defaultLayoutPaths[h.Type]
+
+	if h.Type == restic.DataFile && len(h.Name) > 2 {
+		p = l.Join(p, h.Name[:2]) + "/"
+	}
+
+	return l.Join(l.Path, p) + "/"
+}
+
+// Filename returns a path to a file, including its name.
+func (l *DefaultLayout) Filename(h restic.Handle) string {
+	name := h.Name
+	if h.Type == restic.ConfigFile {
+		return l.Join(l.Path, "config")
+	}
+
+	return l.Join(l.Dirname(h), name)
+}
+
+// Paths returns all directory names needed for a repo.
+func (l *DefaultLayout) Paths() (dirs []string) {
+	for _, p := range defaultLayoutPaths {
+		dirs = append(dirs, l.Join(l.Path, p))
+	}
+
+	// also add subdirs
+	for i := 0; i < 256; i++ {
+		subdir := hex.EncodeToString([]byte{byte(i)})
+		dirs = append(dirs, l.Join(l.Path, defaultLayoutPaths[restic.DataFile], subdir))
+	}
+
+	return dirs
+}
+
+// Basedir returns the base dir name for type t.
+func (l *DefaultLayout) Basedir(t restic.FileType) string {
+	return l.Join(l.Path, defaultLayoutPaths[t])
+}

internal/backend/layout_rest.go

@@ -0,0 +1,54 @@
+package backend
+
+import "restic"
+
+// RESTLayout implements the default layout for the REST protocol.
+type RESTLayout struct {
+	URL  string
+	Path string
+	Join func(...string) string
+}
+
+var restLayoutPaths = defaultLayoutPaths
+
+func (l *RESTLayout) String() string {
+	return "<RESTLayout>"
+}
+
+// Name returns the name for this layout.
+func (l *RESTLayout) Name() string {
+	return "rest"
+}
+
+// Dirname returns the directory path for a given file type and name.
+func (l *RESTLayout) Dirname(h restic.Handle) string {
+	if h.Type == restic.ConfigFile {
+		return l.URL + l.Join(l.Path, "/")
+	}
+
+	return l.URL + l.Join(l.Path, "/", restLayoutPaths[h.Type]) + "/"
+}
+
+// Filename returns a path to a file, including its name.
+func (l *RESTLayout) Filename(h restic.Handle) string {
+	name := h.Name
+
+	if h.Type == restic.ConfigFile {
+		name = "config"
+	}
+
+	return l.URL + l.Join(l.Path, "/", restLayoutPaths[h.Type], name)
+}
+
+// Paths returns all directory names
+func (l *RESTLayout) Paths() (dirs []string) {
+	for _, p := range restLayoutPaths {
+		dirs = append(dirs, l.URL+l.Join(l.Path, p))
+	}
+	return dirs
+}
+
+// Basedir returns the base dir name for files of type t.
+func (l *RESTLayout) Basedir(t restic.FileType) string {
+	return l.URL + l.Join(l.Path, restLayoutPaths[t])
+}

internal/backend/layout_s3legacy.go

@@ -0,0 +1,77 @@
+package backend
+
+import "restic"
+
+// S3LegacyLayout implements the old layout used for s3 cloud storage backends, as
+// described in the Design document.
+type S3LegacyLayout struct {
+	URL  string
+	Path string
+	Join func(...string) string
+}
+
+var s3LayoutPaths = map[restic.FileType]string{
+	restic.DataFile:     "data",
+	restic.SnapshotFile: "snapshot",
+	restic.IndexFile:    "index",
+	restic.LockFile:     "lock",
+	restic.KeyFile:      "key",
+}
+
+func (l *S3LegacyLayout) String() string {
+	return "<S3LegacyLayout>"
+}
+
+// Name returns the name for this layout.
+func (l *S3LegacyLayout) Name() string {
+	return "s3legacy"
+}
+
+// join calls Join with the first empty elements removed.
+func (l *S3LegacyLayout) join(url string, items ...string) string {
+	for len(items) > 0 && items[0] == "" {
+		items = items[1:]
+	}
+
+	path := l.Join(items...)
+	if path == "" || path[0] != '/' {
+		if url != "" && url[len(url)-1] != '/' {
+			url += "/"
+		}
+	}
+
+	return url + path
+}
+
+// Dirname returns the directory path for a given file type and name.
+func (l *S3LegacyLayout) Dirname(h restic.Handle) string {
+	if h.Type == restic.ConfigFile {
+		return l.URL + l.Join(l.Path, "/")
+	}
+
+	return l.join(l.URL, l.Path, s3LayoutPaths[h.Type]) + "/"
+}
+
+// Filename returns a path to a file, including its name.
+func (l *S3LegacyLayout) Filename(h restic.Handle) string {
+	name := h.Name
+
+	if h.Type == restic.ConfigFile {
+		name = "config"
+	}
+
+	return l.join(l.URL, l.Path, s3LayoutPaths[h.Type], name)
+}
+
+// Paths returns all directory names
+func (l *S3LegacyLayout) Paths() (dirs []string) {
+	for _, p := range s3LayoutPaths {
+		dirs = append(dirs, l.Join(l.Path, p))
+	}
+	return dirs
+}
+
+// Basedir returns the base dir name for type t.
+func (l *S3LegacyLayout) Basedir(t restic.FileType) string {
+	return l.Join(l.Path, s3LayoutPaths[t])
+}

internal/backend/layout_test.go

@@ -0,0 +1,449 @@
+package backend
+
+import (
+	"fmt"
+	"path"
+	"path/filepath"
+	"reflect"
+	"restic"
+	. "restic/test"
+	"sort"
+	"testing"
+)
+
+func TestDefaultLayout(t *testing.T) {
+	tempdir, cleanup := TempDir(t)
+	defer cleanup()
+
+	var tests = []struct {
+		path string
+		join func(...string) string
+		restic.Handle
+		filename string
+	}{
+		{
+			tempdir,
+			filepath.Join,
+			restic.Handle{Type: restic.DataFile, Name: "0123456"},
+			filepath.Join(tempdir, "data", "01", "0123456"),
+		},
+		{
+			tempdir,
+			filepath.Join,
+			restic.Handle{Type: restic.ConfigFile, Name: "CFG"},
+			filepath.Join(tempdir, "config"),
+		},
+		{
+			tempdir,
+			filepath.Join,
+			restic.Handle{Type: restic.SnapshotFile, Name: "123456"},
+			filepath.Join(tempdir, "snapshots", "123456"),
+		},
+		{
+			tempdir,
+			filepath.Join,
+			restic.Handle{Type: restic.IndexFile, Name: "123456"},
+			filepath.Join(tempdir, "index", "123456"),
+		},
+		{
+			tempdir,
+			filepath.Join,
+			restic.Handle{Type: restic.LockFile, Name: "123456"},
+			filepath.Join(tempdir, "locks", "123456"),
+		},
+		{
+			tempdir,
+			filepath.Join,
+			restic.Handle{Type: restic.KeyFile, Name: "123456"},
+			filepath.Join(tempdir, "keys", "123456"),
+		},
+		{
+			"",
+			path.Join,
+			restic.Handle{Type: restic.DataFile, Name: "0123456"},
+			"data/01/0123456",
+		},
+		{
+			"",
+			path.Join,
+			restic.Handle{Type: restic.ConfigFile, Name: "CFG"},
+			"config",
+		},
+		{
+			"",
+			path.Join,
+			restic.Handle{Type: restic.SnapshotFile, Name: "123456"},
+			"snapshots/123456",
+		},
+		{
+			"",
+			path.Join,
+			restic.Handle{Type: restic.IndexFile, Name: "123456"},
+			"index/123456",
+		},
+		{
+			"",
+			path.Join,
+			restic.Handle{Type: restic.LockFile, Name: "123456"},
+			"locks/123456",
+		},
+		{
+			"",
+			path.Join,
+			restic.Handle{Type: restic.KeyFile, Name: "123456"},
+			"keys/123456",
+		},
+	}
+
+	t.Run("Paths", func(t *testing.T) {
+		l := &DefaultLayout{
+			Path: tempdir,
+			Join: filepath.Join,
+		}
+
+		dirs := l.Paths()
+
+		want := []string{
+			filepath.Join(tempdir, "data"),
+			filepath.Join(tempdir, "snapshots"),
+			filepath.Join(tempdir, "index"),
+			filepath.Join(tempdir, "locks"),
+			filepath.Join(tempdir, "keys"),
+		}
+
+		for i := 0; i < 256; i++ {
+			want = append(want, filepath.Join(tempdir, "data", fmt.Sprintf("%02x", i)))
+		}
+
+		sort.Sort(sort.StringSlice(want))
+		sort.Sort(sort.StringSlice(dirs))
+
+		if !reflect.DeepEqual(dirs, want) {
+			t.Fatalf("wrong paths returned, want:\n  %v\ngot:\n  %v", want, dirs)
+		}
+	})
+
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("%v/%v", test.Type, test.Handle.Name), func(t *testing.T) {
+			l := &DefaultLayout{
+				Path: test.path,
+				Join: test.join,
+			}
+
+			filename := l.Filename(test.Handle)
+			if filename != test.filename {
+				t.Fatalf("wrong filename, want %v, got %v", test.filename, filename)
+			}
+		})
+	}
+}
+
+func TestRESTLayout(t *testing.T) {
+	path, cleanup := TempDir(t)
+	defer cleanup()
+
+	var tests = []struct {
+		restic.Handle
+		filename string
+	}{
+		{
+			restic.Handle{Type: restic.DataFile, Name: "0123456"},
+			filepath.Join(path, "data", "0123456"),
+		},
+		{
+			restic.Handle{Type: restic.ConfigFile, Name: "CFG"},
+			filepath.Join(path, "config"),
+		},
+		{
+			restic.Handle{Type: restic.SnapshotFile, Name: "123456"},
+			filepath.Join(path, "snapshots", "123456"),
+		},
+		{
+			restic.Handle{Type: restic.IndexFile, Name: "123456"},
+			filepath.Join(path, "index", "123456"),
+		},
+		{
+			restic.Handle{Type: restic.LockFile, Name: "123456"},
+			filepath.Join(path, "locks", "123456"),
+		},
+		{
+			restic.Handle{Type: restic.KeyFile, Name: "123456"},
+			filepath.Join(path, "keys", "123456"),
+		},
+	}
+
+	l := &RESTLayout{
+		Path: path,
+		Join: filepath.Join,
+	}
+
+	t.Run("Paths", func(t *testing.T) {
+		dirs := l.Paths()
+
+		want := []string{
+			filepath.Join(path, "data"),
+			filepath.Join(path, "snapshots"),
+			filepath.Join(path, "index"),
+			filepath.Join(path, "locks"),
+			filepath.Join(path, "keys"),
+		}
+
+		sort.Sort(sort.StringSlice(want))
+		sort.Sort(sort.StringSlice(dirs))
+
+		if !reflect.DeepEqual(dirs, want) {
+			t.Fatalf("wrong paths returned, want:\n  %v\ngot:\n  %v", want, dirs)
+		}
+	})
+
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("%v/%v", test.Type, test.Handle.Name), func(t *testing.T) {
+			filename := l.Filename(test.Handle)
+			if filename != test.filename {
+				t.Fatalf("wrong filename, want %v, got %v", test.filename, filename)
+			}
+		})
+	}
+}
+
+func TestRESTLayoutURLs(t *testing.T) {
+	var tests = []struct {
+		l   Layout
+		h   restic.Handle
+		fn  string
+		dir string
+	}{
+		{
+			&RESTLayout{URL: "https://hostname.foo", Path: "", Join: path.Join},
+			restic.Handle{Type: restic.DataFile, Name: "foobar"},
+			"https://hostname.foo/data/foobar",
+			"https://hostname.foo/data/",
+		},
+		{
+			&RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
+			restic.Handle{Type: restic.LockFile, Name: "foobar"},
+			"https://hostname.foo:1234/prefix/repo/locks/foobar",
+			"https://hostname.foo:1234/prefix/repo/locks/",
+		},
+		{
+			&RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
+			restic.Handle{Type: restic.ConfigFile, Name: "foobar"},
+			"https://hostname.foo:1234/prefix/repo/config",
+			"https://hostname.foo:1234/prefix/repo/",
+		},
+		{
+			&S3LegacyLayout{URL: "https://hostname.foo", Path: "/", Join: path.Join},
+			restic.Handle{Type: restic.DataFile, Name: "foobar"},
+			"https://hostname.foo/data/foobar",
+			"https://hostname.foo/data/",
+		},
+		{
+			&S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "", Join: path.Join},
+			restic.Handle{Type: restic.LockFile, Name: "foobar"},
+			"https://hostname.foo:1234/prefix/repo/lock/foobar",
+			"https://hostname.foo:1234/prefix/repo/lock/",
+		},
+		{
+			&S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
+			restic.Handle{Type: restic.ConfigFile, Name: "foobar"},
+			"https://hostname.foo:1234/prefix/repo/config",
+			"https://hostname.foo:1234/prefix/repo/",
+		},
+		{
+			&S3LegacyLayout{URL: "", Path: "", Join: path.Join},
+			restic.Handle{Type: restic.DataFile, Name: "foobar"},
+			"data/foobar",
+			"data/",
+		},
+		{
+			&S3LegacyLayout{URL: "", Path: "", Join: path.Join},
+			restic.Handle{Type: restic.LockFile, Name: "foobar"},
+			"lock/foobar",
+			"lock/",
+		},
+		{
+			&S3LegacyLayout{URL: "", Path: "/", Join: path.Join},
+			restic.Handle{Type: restic.ConfigFile, Name: "foobar"},
+			"/config",
+			"/",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("%T", test.l), func(t *testing.T) {
+			fn := test.l.Filename(test.h)
+			if fn != test.fn {
+				t.Fatalf("wrong filename, want %v, got %v", test.fn, fn)
+			}
+
+			dir := test.l.Dirname(test.h)
+			if dir != test.dir {
+				t.Fatalf("wrong dirname, want %v, got %v", test.dir, dir)
+			}
+		})
+	}
+}
+
+func TestS3LegacyLayout(t *testing.T) {
+	path, cleanup := TempDir(t)
+	defer cleanup()
+
+	var tests = []struct {
+		restic.Handle
+		filename string
+	}{
+		{
+			restic.Handle{Type: restic.DataFile, Name: "0123456"},
+			filepath.Join(path, "data", "0123456"),
+		},
+		{
+			restic.Handle{Type: restic.ConfigFile, Name: "CFG"},
+			filepath.Join(path, "config"),
+		},
+		{
+			restic.Handle{Type: restic.SnapshotFile, Name: "123456"},
+			filepath.Join(path, "snapshot", "123456"),
+		},
+		{
+			restic.Handle{Type: restic.IndexFile, Name: "123456"},
+			filepath.Join(path, "index", "123456"),
+		},
+		{
+			restic.Handle{Type: restic.LockFile, Name: "123456"},
+			filepath.Join(path, "lock", "123456"),
+		},
+		{
+			restic.Handle{Type: restic.KeyFile, Name: "123456"},
+			filepath.Join(path, "key", "123456"),
+		},
+	}
+
+	l := &S3LegacyLayout{
+		Path: path,
+		Join: filepath.Join,
+	}
+
+	t.Run("Paths", func(t *testing.T) {
+		dirs := l.Paths()
+
+		want := []string{
+			filepath.Join(path, "data"),
+			filepath.Join(path, "snapshot"),
+			filepath.Join(path, "index"),
+			filepath.Join(path, "lock"),
+			filepath.Join(path, "key"),
+		}
+
+		sort.Sort(sort.StringSlice(want))
+		sort.Sort(sort.StringSlice(dirs))
+
+		if !reflect.DeepEqual(dirs, want) {
+			t.Fatalf("wrong paths returned, want:\n  %v\ngot:\n  %v", want, dirs)
+		}
+	})
+
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("%v/%v", test.Type, test.Handle.Name), func(t *testing.T) {
+			filename := l.Filename(test.Handle)
+			if filename != test.filename {
+				t.Fatalf("wrong filename, want %v, got %v", test.filename, filename)
+			}
+		})
+	}
+}
+
+func TestDetectLayout(t *testing.T) {
+	path, cleanup := TempDir(t)
+	defer cleanup()
+
+	var tests = []struct {
+		filename string
+		want     string
+	}{
+		{"repo-layout-default.tar.gz", "*backend.DefaultLayout"},
+		{"repo-layout-s3legacy.tar.gz", "*backend.S3LegacyLayout"},
+	}
+
+	var fs = &LocalFilesystem{}
+	for _, test := range tests {
+		for _, fs := range []Filesystem{fs, nil} {
+			t.Run(fmt.Sprintf("%v/fs-%T", test.filename, fs), func(t *testing.T) {
+				SetupTarTestFixture(t, path, filepath.Join("testdata", test.filename))
+
+				layout, err := DetectLayout(fs, filepath.Join(path, "repo"))
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				if layout == nil {
+					t.Fatal("wanted some layout, but detect returned nil")
+				}
+
+				layoutName := fmt.Sprintf("%T", layout)
+				if layoutName != test.want {
+					t.Fatalf("want layout %v, got %v", test.want, layoutName)
+				}
+
+				RemoveAll(t, filepath.Join(path, "repo"))
+			})
+		}
+	}
+}
+
+func TestParseLayout(t *testing.T) {
+	path, cleanup := TempDir(t)
+	defer cleanup()
+
+	var tests = []struct {
+		layoutName        string
+		defaultLayoutName string
+		want              string
+	}{
+		{"default", "", "*backend.DefaultLayout"},
+		{"s3legacy", "", "*backend.S3LegacyLayout"},
+		{"", "", "*backend.DefaultLayout"},
+	}
+
+	SetupTarTestFixture(t, path, filepath.Join("testdata", "repo-layout-default.tar.gz"))
+
+	for _, test := range tests {
+		t.Run(test.layoutName, func(t *testing.T) {
+			layout, err := ParseLayout(&LocalFilesystem{}, test.layoutName, test.defaultLayoutName, filepath.Join(path, "repo"))
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if layout == nil {
+				t.Fatal("wanted some layout, but detect returned nil")
+			}
+
+			// test that the functions work (and don't panic)
+			_ = layout.Dirname(restic.Handle{Type: restic.DataFile})
+			_ = layout.Filename(restic.Handle{Type: restic.DataFile, Name: "1234"})
+			_ = layout.Paths()
+
+			layoutName := fmt.Sprintf("%T", layout)
+			if layoutName != test.want {
+				t.Fatalf("want layout %v, got %v", test.want, layoutName)
+			}
+		})
+	}
+}
+
+func TestParseLayoutInvalid(t *testing.T) {
+	path, cleanup := TempDir(t)
+	defer cleanup()
+
+	var invalidNames = []string{
+		"foo", "bar", "local",
+	}
+
+	for _, name := range invalidNames {
+		t.Run(name, func(t *testing.T) {
+			layout, err := ParseLayout(nil, name, "", path)
+			if err == nil {
+				t.Fatalf("expected error not found for layout name %v, layout is %v", name, layout)
+			}
+		})
+	}
+}

internal/backend/local/config.go

@@ -0,0 +1,27 @@
+package local
+
+import (
+	"strings"
+
+	"restic/errors"
+	"restic/options"
+)
+
+// Config holds all information needed to open a local repository.
+type Config struct {
+	Path   string
+	Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect)"`
+}
+
+func init() {
+	options.Register("local", Config{})
+}
+
+// ParseConfig parses a local backend config.
+func ParseConfig(cfg string) (interface{}, error) {
+	if !strings.HasPrefix(cfg, "local:") {
+		return nil, errors.New(`invalid format, prefix "local" not found`)
+	}
+
+	return Config{Path: cfg[6:]}, nil
+}

internal/backend/local/doc.go

@@ -0,0 +1,2 @@
+// Package local implements repository storage in a local directory.
+package local

internal/backend/local/layout_test.go

@@ -0,0 +1,80 @@
+package local
+
+import (
+	"context"
+	"path/filepath"
+	"restic"
+	. "restic/test"
+	"testing"
+)
+
+func TestLayout(t *testing.T) {
+	path, cleanup := TempDir(t)
+	defer cleanup()
+
+	var tests = []struct {
+		filename        string
+		layout          string
+		failureExpected bool
+		datafiles       map[string]bool
+	}{
+		{"repo-layout-default.tar.gz", "", false, map[string]bool{
+			"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
+			"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
+			"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
+		}},
+		{"repo-layout-s3legacy.tar.gz", "", false, map[string]bool{
+			"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
+			"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
+			"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
+		}},
+	}
+
+	for _, test := range tests {
+		t.Run(test.filename, func(t *testing.T) {
+			SetupTarTestFixture(t, path, filepath.Join("..", "testdata", test.filename))
+
+			repo := filepath.Join(path, "repo")
+			be, err := Open(Config{
+				Path:   repo,
+				Layout: test.layout,
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if be == nil {
+				t.Fatalf("Open() returned nil but no error")
+			}
+
+			datafiles := make(map[string]bool)
+			for id := range be.List(context.TODO(), restic.DataFile) {
+				datafiles[id] = false
+			}
+
+			if len(datafiles) == 0 {
+				t.Errorf("List() returned zero data files")
+			}
+
+			for id := range test.datafiles {
+				if _, ok := datafiles[id]; !ok {
+					t.Errorf("datafile with id %v not found", id)
+				}
+
+				datafiles[id] = true
+			}
+
+			for id, v := range datafiles {
+				if !v {
+					t.Errorf("unexpected id %v found", id)
+				}
+			}
+
+			if err = be.Close(); err != nil {
+				t.Errorf("Close() returned error %v", err)
+			}
+
+			RemoveAll(t, filepath.Join(path, "repo"))
+		})
+	}
+}

internal/backend/local/local.go

@@ -0,0 +1,256 @@
+package local
+
+import (
+	"context"
+	"io"
+	"os"
+	"path/filepath"
+	"restic"
+
+	"restic/errors"
+
+	"restic/backend"
+	"restic/debug"
+	"restic/fs"
+)
+
+// Local is a backend in a local directory.
+type Local struct {
+	Config
+	backend.Layout
+}
+
+// ensure statically that *Local implements restic.Backend.
+var _ restic.Backend = &Local{}
+
+const defaultLayout = "default"
+
+// Open opens the local backend as specified by config.
+func Open(cfg Config) (*Local, error) {
+	debug.Log("open local backend at %v (layout %q)", cfg.Path, cfg.Layout)
+	l, err := backend.ParseLayout(&backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path)
+	if err != nil {
+		return nil, err
+	}
+
+	be := &Local{Config: cfg, Layout: l}
+
+	// create paths for data and refs. MkdirAll does nothing if the directory already exists.
+	for _, d := range be.Paths() {
+		err := fs.MkdirAll(d, backend.Modes.Dir)
+		if err != nil {
+			return nil, errors.Wrap(err, "MkdirAll")
+		}
+	}
+
+	return be, nil
+}
+
+// Create creates all the necessary files and directories for a new local
+// backend at dir. Afterwards a new config blob should be created.
+func Create(cfg Config) (*Local, error) {
+	debug.Log("create local backend at %v (layout %q)", cfg.Path, cfg.Layout)
+
+	l, err := backend.ParseLayout(&backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path)
+	if err != nil {
+		return nil, err
+	}
+
+	be := &Local{
+		Config: cfg,
+		Layout: l,
+	}
+
+	// test if config file already exists
+	_, err = fs.Lstat(be.Filename(restic.Handle{Type: restic.ConfigFile}))
+	if err == nil {
+		return nil, errors.New("config file already exists")
+	}
+
+	// create paths for data and refs
+	for _, d := range be.Paths() {
+		err := fs.MkdirAll(d, backend.Modes.Dir)
+		if err != nil {
+			return nil, errors.Wrap(err, "MkdirAll")
+		}
+	}
+
+	return be, nil
+}
+
+// Location returns this backend's location (the directory name).
+func (b *Local) Location() string {
+	return b.Path
+}
+
+// IsNotExist returns true if the error is caused by a non existing file.
+func (b *Local) IsNotExist(err error) bool {
+	return os.IsNotExist(errors.Cause(err))
+}
+
+// Save stores data in the backend at the handle.
+func (b *Local) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) {
+	debug.Log("Save %v", h)
+	if err := h.Valid(); err != nil {
+		return err
+	}
+
+	filename := b.Filename(h)
+
+	// create new file
+	f, err := fs.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, backend.Modes.File)
+	if err != nil {
+		return errors.Wrap(err, "OpenFile")
+	}
+
+	// save data, then sync
+	_, err = io.Copy(f, rd)
+	if err != nil {
+		_ = f.Close()
+		return errors.Wrap(err, "Write")
+	}
+
+	if err = f.Sync(); err != nil {
+		_ = f.Close()
+		return errors.Wrap(err, "Sync")
+	}
+
+	err = f.Close()
+	if err != nil {
+		return errors.Wrap(err, "Close")
+	}
+
+	// set mode to read-only
+	fi, err := fs.Stat(filename)
+	if err != nil {
+		return errors.Wrap(err, "Stat")
+	}
+
+	return setNewFileMode(filename, fi)
+}
+
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (b *Local) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	f, err := fs.Open(b.Filename(h))
+	if err != nil {
+		return nil, err
+	}
+
+	if offset > 0 {
+		_, err = f.Seek(offset, 0)
+		if err != nil {
+			_ = f.Close()
+			return nil, err
+		}
+	}
+
+	if length > 0 {
+		return backend.LimitReadCloser(f, int64(length)), nil
+	}
+
+	return f, nil
+}
+
+// Stat returns information about a blob.
+func (b *Local) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
+	debug.Log("Stat %v", h)
+	if err := h.Valid(); err != nil {
+		return restic.FileInfo{}, err
+	}
+
+	fi, err := fs.Stat(b.Filename(h))
+	if err != nil {
+		return restic.FileInfo{}, errors.Wrap(err, "Stat")
+	}
+
+	return restic.FileInfo{Size: fi.Size()}, nil
+}
+
+// Test returns true if a blob of the given type and name exists in the backend.
+func (b *Local) Test(ctx context.Context, h restic.Handle) (bool, error) {
+	debug.Log("Test %v", h)
+	_, err := fs.Stat(b.Filename(h))
+	if err != nil {
+		if os.IsNotExist(errors.Cause(err)) {
+			return false, nil
+		}
+		return false, errors.Wrap(err, "Stat")
+	}
+
+	return true, nil
+}
+
+// Remove removes the blob with the given name and type.
+func (b *Local) Remove(ctx context.Context, h restic.Handle) error {
+	debug.Log("Remove %v", h)
+	fn := b.Filename(h)
+
+	// reset read-only flag
+	err := fs.Chmod(fn, 0666)
+	if err != nil {
+		return errors.Wrap(err, "Chmod")
+	}
+
+	return fs.Remove(fn)
+}
+
+func isFile(fi os.FileInfo) bool {
+	return fi.Mode()&(os.ModeType|os.ModeCharDevice) == 0
+}
+
+// List returns a channel that yields all names of blobs of type t. A
+// goroutine is started for this.
+func (b *Local) List(ctx context.Context, t restic.FileType) <-chan string {
+	debug.Log("List %v", t)
+
+	ch := make(chan string)
+
+	go func() {
+		defer close(ch)
+
+		fs.Walk(b.Basedir(t), func(path string, fi os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+
+			if !isFile(fi) {
+				return err
+			}
+
+			select {
+			case ch <- filepath.Base(path):
+			case <-ctx.Done():
+				return err
+			}
+
+			return err
+		})
+	}()
+
+	return ch
+}
+
+// Delete removes the repository and all files.
+func (b *Local) Delete() error {
+	debug.Log("Delete()")
+	return fs.RemoveAll(b.Path)
+}
+
+// Close closes all open files.
+func (b *Local) Close() error {
+	debug.Log("Close()")
+	// this does not need to do anything, all open files are closed within the
+	// same function.
+	return nil
+}

internal/backend/local/local_test.go

@@ -0,0 +1,61 @@
+package local_test
+
+import (
+	"io/ioutil"
+	"restic"
+	"testing"
+
+	"restic/backend/local"
+	"restic/backend/test"
+	. "restic/test"
+)
+
+func newTestSuite(t testing.TB) *test.Suite {
+	return &test.Suite{
+		// NewConfig returns a config for a new temporary backend that will be used in tests.
+		NewConfig: func() (interface{}, error) {
+			dir, err := ioutil.TempDir(TestTempDir, "restic-test-local-")
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			t.Logf("create new backend at %v", dir)
+
+			cfg := local.Config{
+				Path: dir,
+			}
+			return cfg, nil
+		},
+
+		// CreateFn is a function that creates a temporary repository for the tests.
+		Create: func(config interface{}) (restic.Backend, error) {
+			cfg := config.(local.Config)
+			return local.Create(cfg)
+		},
+
+		// OpenFn is a function that opens a previously created temporary repository.
+		Open: func(config interface{}) (restic.Backend, error) {
+			cfg := config.(local.Config)
+			return local.Open(cfg)
+		},
+
+		// CleanupFn removes data created during the tests.
+		Cleanup: func(config interface{}) error {
+			cfg := config.(local.Config)
+			if !TestCleanupTempDirs {
+				t.Logf("leaving test backend dir at %v", cfg.Path)
+			}
+
+			RemoveAll(t, cfg.Path)
+			return nil
+		},
+	}
+}
+
+func TestBackend(t *testing.T) {
+	newTestSuite(t).RunTests(t)
+}
+
+func BenchmarkBackend(t *testing.B) {
+	newTestSuite(t).RunBenchmarks(t)
+}

internal/backend/local/local_unix.go

@@ -0,0 +1,13 @@
+// +build !windows
+
+package local
+
+import (
+	"os"
+	"restic/fs"
+)
+
+// set file to readonly
+func setNewFileMode(f string, fi os.FileInfo) error {
+	return fs.Chmod(f, fi.Mode()&os.FileMode(^uint32(0222)))
+}

internal/backend/local/local_windows.go

@@ -0,0 +1,12 @@
+package local
+
+import (
+	"os"
+)
+
+// We don't modify read-only on windows,
+// since it will make us unable to delete the file,
+// and this isn't common practice on this platform.
+func setNewFileMode(f string, fi os.FileInfo) error {
+	return nil
+}

internal/backend/location/location.go

@@ -0,0 +1,107 @@
+// Package location implements parsing the restic repository location from a string.
+package location
+
+import (
+	"strings"
+
+	"restic/backend/b2"
+	"restic/backend/local"
+	"restic/backend/rest"
+	"restic/backend/s3"
+	"restic/backend/sftp"
+	"restic/backend/swift"
+	"restic/errors"
+)
+
+// Location specifies the location of a repository, including the method of
+// access and (possibly) credentials needed for access.
+type Location struct {
+	Scheme string
+	Config interface{}
+}
+
+type parser struct {
+	scheme string
+	parse  func(string) (interface{}, error)
+}
+
+// parsers is a list of valid config parsers for the backends. The first parser
+// is the fallback and should always be set to the local backend.
+var parsers = []parser{
+	{"b2", b2.ParseConfig},
+	{"local", local.ParseConfig},
+	{"sftp", sftp.ParseConfig},
+	{"s3", s3.ParseConfig},
+	{"swift", swift.ParseConfig},
+	{"rest", rest.ParseConfig},
+}
+
+func isPath(s string) bool {
+	if strings.HasPrefix(s, "../") || strings.HasPrefix(s, `..\`) {
+		return true
+	}
+
+	if strings.HasPrefix(s, "/") || strings.HasPrefix(s, `\`) {
+		return true
+	}
+
+	if len(s) < 3 {
+		return false
+	}
+
+	// check for drive paths
+	drive := s[0]
+	if !(drive >= 'a' && drive <= 'z') && !(drive >= 'A' && drive <= 'Z') {
+		return false
+	}
+
+	if s[1] != ':' {
+		return false
+	}
+
+	if s[2] != '\\' && s[2] != '/' {
+		return false
+	}
+
+	return true
+}
+
+// Parse extracts repository location information from the string s. If s
+// starts with a backend name followed by a colon, that backend's Parse()
+// function is called. Otherwise, the local backend is used which interprets s
+// as the name of a directory.
+func Parse(s string) (u Location, err error) {
+	scheme := extractScheme(s)
+	u.Scheme = scheme
+
+	for _, parser := range parsers {
+		if parser.scheme != scheme {
+			continue
+		}
+
+		u.Config, err = parser.parse(s)
+		if err != nil {
+			return Location{}, err
+		}
+
+		return u, nil
+	}
+
+	// if s is not a path or contains ":", it's ambiguous
+	if !isPath(s) && strings.ContainsRune(s, ':') {
+		return Location{}, errors.New("invalid backend\nIf the repo is in a local directory, you need to add a `local:` prefix")
+	}
+
+	u.Scheme = "local"
+	u.Config, err = local.ParseConfig("local:" + s)
+	if err != nil {
+		return Location{}, err
+	}
+
+	return u, nil
+}
+
+func extractScheme(s string) string {
+	data := strings.SplitN(s, ":", 2)
+	return data[0]
+}

internal/backend/location/location_test.go

@@ -0,0 +1,339 @@
+package location
+
+import (
+	"net/url"
+	"reflect"
+	"testing"
+
+	"restic/backend/b2"
+	"restic/backend/local"
+	"restic/backend/rest"
+	"restic/backend/s3"
+	"restic/backend/sftp"
+	"restic/backend/swift"
+)
+
+func parseURL(s string) *url.URL {
+	u, err := url.Parse(s)
+	if err != nil {
+		panic(err)
+	}
+
+	return u
+}
+
+var parseTests = []struct {
+	s string
+	u Location
+}{
+	{
+		"local:/srv/repo",
+		Location{Scheme: "local",
+			Config: local.Config{
+				Path: "/srv/repo",
+			},
+		},
+	},
+	{
+		"local:dir1/dir2",
+		Location{Scheme: "local",
+			Config: local.Config{
+				Path: "dir1/dir2",
+			},
+		},
+	},
+	{
+		"local:dir1/dir2",
+		Location{Scheme: "local",
+			Config: local.Config{
+				Path: "dir1/dir2",
+			},
+		},
+	},
+	{
+		"dir1/dir2",
+		Location{Scheme: "local",
+			Config: local.Config{
+				Path: "dir1/dir2",
+			},
+		},
+	},
+	{
+		"/dir1/dir2",
+		Location{Scheme: "local",
+			Config: local.Config{
+				Path: "/dir1/dir2",
+			},
+		},
+	},
+	{
+		"local:../dir1/dir2",
+		Location{Scheme: "local",
+			Config: local.Config{
+				Path: "../dir1/dir2",
+			},
+		},
+	},
+	{
+		"/dir1/dir2",
+		Location{Scheme: "local",
+			Config: local.Config{
+				Path: "/dir1/dir2",
+			},
+		},
+	},
+	{
+		"/dir1:foobar/dir2",
+		Location{Scheme: "local",
+			Config: local.Config{
+				Path: "/dir1:foobar/dir2",
+			},
+		},
+	},
+	{
+		`\dir1\foobar\dir2`,
+		Location{Scheme: "local",
+			Config: local.Config{
+				Path: `\dir1\foobar\dir2`,
+			},
+		},
+	},
+	{
+		`c:\dir1\foobar\dir2`,
+		Location{Scheme: "local",
+			Config: local.Config{
+				Path: `c:\dir1\foobar\dir2`,
+			},
+		},
+	},
+	{
+		`C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`,
+		Location{Scheme: "local",
+			Config: local.Config{
+				Path: `C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`,
+			},
+		},
+	},
+	{
+		`c:/dir1/foobar/dir2`,
+		Location{Scheme: "local",
+			Config: local.Config{
+				Path: `c:/dir1/foobar/dir2`,
+			},
+		},
+	},
+	{
+		"sftp:user@host:/srv/repo",
+		Location{Scheme: "sftp",
+			Config: sftp.Config{
+				User: "user",
+				Host: "host",
+				Path: "/srv/repo",
+			},
+		},
+	},
+	{
+		"sftp:host:/srv/repo",
+		Location{Scheme: "sftp",
+			Config: sftp.Config{
+				User: "",
+				Host: "host",
+				Path: "/srv/repo",
+			},
+		},
+	},
+	{
+		"sftp://user@host/srv/repo",
+		Location{Scheme: "sftp",
+			Config: sftp.Config{
+				User: "user",
+				Host: "host",
+				Path: "srv/repo",
+			},
+		},
+	},
+	{
+		"sftp://user@host//srv/repo",
+		Location{Scheme: "sftp",
+			Config: sftp.Config{
+				User: "user",
+				Host: "host",
+				Path: "/srv/repo",
+			},
+		},
+	},
+
+	{
+		"s3://eu-central-1/bucketname",
+		Location{Scheme: "s3",
+			Config: s3.Config{
+				Endpoint:    "eu-central-1",
+				Bucket:      "bucketname",
+				Prefix:      "restic",
+				Connections: 5,
+			},
+		},
+	},
+	{
+		"s3://hostname.foo/bucketname",
+		Location{Scheme: "s3",
+			Config: s3.Config{
+				Endpoint:    "hostname.foo",
+				Bucket:      "bucketname",
+				Prefix:      "restic",
+				Connections: 5,
+			},
+		},
+	},
+	{
+		"s3://hostname.foo/bucketname/prefix/directory",
+		Location{Scheme: "s3",
+			Config: s3.Config{
+				Endpoint:    "hostname.foo",
+				Bucket:      "bucketname",
+				Prefix:      "prefix/directory",
+				Connections: 5,
+			},
+		},
+	},
+	{
+		"s3:eu-central-1/repo",
+		Location{Scheme: "s3",
+			Config: s3.Config{
+				Endpoint:    "eu-central-1",
+				Bucket:      "repo",
+				Prefix:      "restic",
+				Connections: 5,
+			},
+		},
+	},
+	{
+		"s3:eu-central-1/repo/prefix/directory",
+		Location{Scheme: "s3",
+			Config: s3.Config{
+				Endpoint:    "eu-central-1",
+				Bucket:      "repo",
+				Prefix:      "prefix/directory",
+				Connections: 5,
+			},
+		},
+	},
+	{
+		"s3:https://hostname.foo/repo",
+		Location{Scheme: "s3",
+			Config: s3.Config{
+				Endpoint:    "hostname.foo",
+				Bucket:      "repo",
+				Prefix:      "restic",
+				Connections: 5,
+			},
+		},
+	},
+	{
+		"s3:https://hostname.foo/repo/prefix/directory",
+		Location{Scheme: "s3",
+			Config: s3.Config{
+				Endpoint:    "hostname.foo",
+				Bucket:      "repo",
+				Prefix:      "prefix/directory",
+				Connections: 5,
+			},
+		},
+	},
+	{
+		"s3:http://hostname.foo/repo",
+		Location{Scheme: "s3",
+			Config: s3.Config{
+				Endpoint:    "hostname.foo",
+				Bucket:      "repo",
+				Prefix:      "restic",
+				UseHTTP:     true,
+				Connections: 5,
+			},
+		},
+	},
+	{
+		"swift:container17:/",
+		Location{Scheme: "swift",
+			Config: swift.Config{
+				Container:   "container17",
+				Prefix:      "",
+				Connections: 5,
+			},
+		},
+	},
+	{
+		"swift:container17:/prefix97",
+		Location{Scheme: "swift",
+			Config: swift.Config{
+				Container:   "container17",
+				Prefix:      "prefix97",
+				Connections: 5,
+			},
+		},
+	},
+	{
+		"rest:http://hostname.foo:1234/",
+		Location{Scheme: "rest",
+			Config: rest.Config{
+				URL:         parseURL("http://hostname.foo:1234/"),
+				Connections: 5,
+			},
+		},
+	},
+	{
+		"b2:bucketname:/prefix", Location{Scheme: "b2",
+			Config: b2.Config{
+				Bucket:      "bucketname",
+				Prefix:      "prefix",
+				Connections: 5,
+			},
+		},
+	},
+	{
+		"b2:bucketname", Location{Scheme: "b2",
+			Config: b2.Config{
+				Bucket:      "bucketname",
+				Prefix:      "",
+				Connections: 5,
+			},
+		},
+	},
+}
+
+func TestParse(t *testing.T) {
+	for i, test := range parseTests {
+		t.Run(test.s, func(t *testing.T) {
+			u, err := Parse(test.s)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			if test.u.Scheme != u.Scheme {
+				t.Errorf("test %d: scheme does not match, want %q, got %q",
+					i, test.u.Scheme, u.Scheme)
+			}
+
+			if !reflect.DeepEqual(test.u.Config, u.Config) {
+				t.Errorf("test %d: cfg map does not match, want:\n  %#v\ngot: \n  %#v",
+					i, test.u.Config, u.Config)
+			}
+		})
+	}
+}
+
+func TestInvalidScheme(t *testing.T) {
+	var invalidSchemes = []string{
+		"foobar:xxx",
+		"foobar:/dir/dir2",
+	}
+
+	for _, s := range invalidSchemes {
+		t.Run(s, func(t *testing.T) {
+			_, err := Parse(s)
+			if err == nil {
+				t.Fatalf("error for invalid location %q not found", s)
+			}
+		})
+	}
+}

internal/backend/mem/mem_backend.go

@@ -0,0 +1,213 @@
+package mem
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"io/ioutil"
+	"restic"
+	"sync"
+
+	"restic/errors"
+
+	"restic/debug"
+)
+
+type memMap map[restic.Handle][]byte
+
+// make sure that MemoryBackend implements backend.Backend
+var _ restic.Backend = &MemoryBackend{}
+
+var errNotFound = errors.New("not found")
+
+// MemoryBackend is a mock backend that uses a map for storing all data in
+// memory. This should only be used for tests.
+type MemoryBackend struct {
+	data memMap
+	m    sync.Mutex
+}
+
+// New returns a new backend that saves all data in a map in memory.
+func New() *MemoryBackend {
+	be := &MemoryBackend{
+		data: make(memMap),
+	}
+
+	debug.Log("created new memory backend")
+
+	return be
+}
+
+// Test returns whether a file exists.
+func (be *MemoryBackend) Test(ctx context.Context, h restic.Handle) (bool, error) {
+	be.m.Lock()
+	defer be.m.Unlock()
+
+	debug.Log("Test %v", h)
+
+	if _, ok := be.data[h]; ok {
+		return true, nil
+	}
+
+	return false, nil
+}
+
+// IsNotExist returns true if the file does not exist.
+func (be *MemoryBackend) IsNotExist(err error) bool {
+	return errors.Cause(err) == errNotFound
+}
+
+// Save adds new Data to the backend.
+func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd io.Reader) error {
+	if err := h.Valid(); err != nil {
+		return err
+	}
+
+	be.m.Lock()
+	defer be.m.Unlock()
+
+	if h.Type == restic.ConfigFile {
+		h.Name = ""
+	}
+
+	if _, ok := be.data[h]; ok {
+		return errors.New("file already exists")
+	}
+
+	buf, err := ioutil.ReadAll(rd)
+	if err != nil {
+		return err
+	}
+
+	be.data[h] = buf
+	debug.Log("saved %v bytes at %v", len(buf), h)
+
+	return nil
+}
+
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (be *MemoryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	be.m.Lock()
+	defer be.m.Unlock()
+
+	if h.Type == restic.ConfigFile {
+		h.Name = ""
+	}
+
+	debug.Log("Load %v offset %v len %v", h, offset, length)
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if _, ok := be.data[h]; !ok {
+		return nil, errNotFound
+	}
+
+	buf := be.data[h]
+	if offset > int64(len(buf)) {
+		return nil, errors.New("offset beyond end of file")
+	}
+
+	buf = buf[offset:]
+	if length > 0 && len(buf) > length {
+		buf = buf[:length]
+	}
+
+	return ioutil.NopCloser(bytes.NewReader(buf)), nil
+}
+
+// Stat returns information about a file in the backend.
+func (be *MemoryBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
+	be.m.Lock()
+	defer be.m.Unlock()
+
+	if err := h.Valid(); err != nil {
+		return restic.FileInfo{}, err
+	}
+
+	if h.Type == restic.ConfigFile {
+		h.Name = ""
+	}
+
+	debug.Log("stat %v", h)
+
+	e, ok := be.data[h]
+	if !ok {
+		return restic.FileInfo{}, errNotFound
+	}
+
+	return restic.FileInfo{Size: int64(len(e))}, nil
+}
+
+// Remove deletes a file from the backend.
+func (be *MemoryBackend) Remove(ctx context.Context, h restic.Handle) error {
+	be.m.Lock()
+	defer be.m.Unlock()
+
+	debug.Log("Remove %v", h)
+
+	if _, ok := be.data[h]; !ok {
+		return errNotFound
+	}
+
+	delete(be.data, h)
+
+	return nil
+}
+
+// List returns a channel which yields entries from the backend.
+func (be *MemoryBackend) List(ctx context.Context, t restic.FileType) <-chan string {
+	be.m.Lock()
+	defer be.m.Unlock()
+
+	ch := make(chan string)
+
+	var ids []string
+	for entry := range be.data {
+		if entry.Type != t {
+			continue
+		}
+		ids = append(ids, entry.Name)
+	}
+
+	debug.Log("list %v: %v", t, ids)
+
+	go func() {
+		defer close(ch)
+		for _, id := range ids {
+			select {
+			case ch <- id:
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	return ch
+}
+
+// Location returns the location of the backend (RAM).
+func (be *MemoryBackend) Location() string {
+	return "RAM"
+}
+
+// Delete removes all data in the backend.
+func (be *MemoryBackend) Delete(ctx context.Context) error {
+	be.m.Lock()
+	defer be.m.Unlock()
+
+	be.data = make(memMap)
+	return nil
+}
+
+// Close closes the backend.
+func (be *MemoryBackend) Close() error {
+	return nil
+}

internal/backend/mem/mem_backend_test.go

@@ -0,0 +1,66 @@
+package mem_test
+
+import (
+	"context"
+	"restic"
+	"testing"
+
+	"restic/errors"
+
+	"restic/backend/mem"
+	"restic/backend/test"
+)
+
+type memConfig struct {
+	be restic.Backend
+}
+
+func newTestSuite() *test.Suite {
+	return &test.Suite{
+		// NewConfig returns a config for a new temporary backend that will be used in tests.
+		NewConfig: func() (interface{}, error) {
+			return &memConfig{}, nil
+		},
+
+		// CreateFn is a function that creates a temporary repository for the tests.
+		Create: func(cfg interface{}) (restic.Backend, error) {
+			c := cfg.(*memConfig)
+			if c.be != nil {
+				ok, err := c.be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile})
+				if err != nil {
+					return nil, err
+				}
+
+				if ok {
+					return nil, errors.New("config already exists")
+				}
+			}
+
+			c.be = mem.New()
+			return c.be, nil
+		},
+
+		// OpenFn is a function that opens a previously created temporary repository.
+		Open: func(cfg interface{}) (restic.Backend, error) {
+			c := cfg.(*memConfig)
+			if c.be == nil {
+				c.be = mem.New()
+			}
+			return c.be, nil
+		},
+
+		// CleanupFn removes data created during the tests.
+		Cleanup: func(cfg interface{}) error {
+			// no cleanup needed
+			return nil
+		},
+	}
+}
+
+func TestSuiteBackendMem(t *testing.T) {
+	newTestSuite().RunTests(t)
+}
+
+func BenchmarkSuiteBackendMem(t *testing.B) {
+	newTestSuite().RunBenchmarks(t)
+}

internal/backend/paths.go

@@ -0,0 +1,26 @@
+package backend
+
+import "os"
+
+// Paths contains the default paths for file-based backends (e.g. local).
+var Paths = struct {
+	Data      string
+	Snapshots string
+	Index     string
+	Locks     string
+	Keys      string
+	Temp      string
+	Config    string
+}{
+	"data",
+	"snapshots",
+	"index",
+	"locks",
+	"keys",
+	"tmp",
+	"config",
+}
+
+// Modes holds the default modes for directories and files for file-based
+// backends.
+var Modes = struct{ Dir, File os.FileMode }{0700, 0600}

internal/backend/rest/config.go

@@ -0,0 +1,44 @@
+package rest
+
+import (
+	"net/url"
+	"strings"
+
+	"restic/errors"
+	"restic/options"
+)
+
+// Config contains all configuration necessary to connect to a REST server.
+type Config struct {
+	URL         *url.URL
+	Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
+}
+
+func init() {
+	options.Register("rest", Config{})
+}
+
+// NewConfig returns a new Config with the default values filled in.
+func NewConfig() Config {
+	return Config{
+		Connections: 5,
+	}
+}
+
+// ParseConfig parses the string s and extracts the REST server URL.
+func ParseConfig(s string) (interface{}, error) {
+	if !strings.HasPrefix(s, "rest:") {
+		return nil, errors.New("invalid REST backend specification")
+	}
+
+	s = s[5:]
+	u, err := url.Parse(s)
+
+	if err != nil {
+		return nil, errors.Wrap(err, "url.Parse")
+	}
+
+	cfg := NewConfig()
+	cfg.URL = u
+	return cfg, nil
+}

internal/backend/rest/config_test.go

@@ -0,0 +1,42 @@
+package rest
+
+import (
+	"net/url"
+	"reflect"
+	"testing"
+)
+
+func parseURL(s string) *url.URL {
+	u, err := url.Parse(s)
+	if err != nil {
+		panic(err)
+	}
+
+	return u
+}
+
+var configTests = []struct {
+	s   string
+	cfg Config
+}{
+	{"rest:http://localhost:1234", Config{
+		URL:         parseURL("http://localhost:1234"),
+		Connections: 5,
+	}},
+}
+
+func TestParseConfig(t *testing.T) {
+	for i, test := range configTests {
+		cfg, err := ParseConfig(test.s)
+		if err != nil {
+			t.Errorf("test %d:%s failed: %v", i, test.s, err)
+			continue
+		}
+
+		if !reflect.DeepEqual(cfg, test.cfg) {
+			t.Errorf("test %d:\ninput:\n  %s\n wrong config, want:\n  %v\ngot:\n  %v",
+				i, test.s, test.cfg, cfg)
+			continue
+		}
+	}
+}

internal/backend/rest/rest.go

@@ -0,0 +1,351 @@
+package rest
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"path"
+	"restic"
+	"strings"
+
+	"golang.org/x/net/context/ctxhttp"
+
+	"restic/debug"
+	"restic/errors"
+
+	"restic/backend"
+)
+
+// make sure the rest backend implements restic.Backend
+var _ restic.Backend = &restBackend{}
+
+type restBackend struct {
+	url    *url.URL
+	sem    *backend.Semaphore
+	client *http.Client
+	backend.Layout
+}
+
+// Open opens the REST backend with the given config.
+func Open(cfg Config) (restic.Backend, error) {
+	client := &http.Client{Transport: backend.Transport()}
+
+	sem, err := backend.NewSemaphore(cfg.Connections)
+	if err != nil {
+		return nil, err
+	}
+
+	// use url without trailing slash for layout
+	url := cfg.URL.String()
+	if url[len(url)-1] == '/' {
+		url = url[:len(url)-1]
+	}
+
+	be := &restBackend{
+		url:    cfg.URL,
+		client: client,
+		Layout: &backend.RESTLayout{URL: url, Join: path.Join},
+		sem:    sem,
+	}
+
+	return be, nil
+}
+
+// Create creates a new REST on server configured in config.
+func Create(cfg Config) (restic.Backend, error) {
+	be, err := Open(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
+	if err == nil {
+		return nil, errors.Fatal("config file already exists")
+	}
+
+	url := *cfg.URL
+	values := url.Query()
+	values.Set("create", "true")
+	url.RawQuery = values.Encode()
+
+	resp, err := http.Post(url.String(), "binary/octet-stream", strings.NewReader(""))
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.Fatalf("server response unexpected: %v (%v)", resp.Status, resp.StatusCode)
+	}
+
+	_, err = io.Copy(ioutil.Discard, resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	err = resp.Body.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	return be, nil
+}
+
+// Location returns this backend's location (the server's URL).
+func (b *restBackend) Location() string {
+	return b.url.String()
+}
+
+// Save stores data in the backend at the handle.
+func (b *restBackend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) {
+	if err := h.Valid(); err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	// make sure that client.Post() cannot close the reader by wrapping it
+	rd = ioutil.NopCloser(rd)
+
+	b.sem.GetToken()
+	resp, err := ctxhttp.Post(ctx, b.client, b.Filename(h), "binary/octet-stream", rd)
+	b.sem.ReleaseToken()
+
+	if resp != nil {
+		defer func() {
+			_, _ = io.Copy(ioutil.Discard, resp.Body)
+			e := resp.Body.Close()
+
+			if err == nil {
+				err = errors.Wrap(e, "Close")
+			}
+		}()
+	}
+
+	if err != nil {
+		return errors.Wrap(err, "client.Post")
+	}
+
+	if resp.StatusCode != 200 {
+		return errors.Errorf("server response unexpected: %v (%v)", resp.Status, resp.StatusCode)
+	}
+
+	return nil
+}
+
+// ErrIsNotExist is returned whenever the requested file does not exist on the
+// server.
+type ErrIsNotExist struct {
+	restic.Handle
+}
+
+func (e ErrIsNotExist) Error() string {
+	return fmt.Sprintf("%v does not exist", e.Handle)
+}
+
+// IsNotExist returns true if the error was caused by a non-existing file.
+func (b *restBackend) IsNotExist(err error) bool {
+	err = errors.Cause(err)
+	_, ok := err.(ErrIsNotExist)
+	return ok
+}
+
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (b *restBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if length < 0 {
+		return nil, errors.Errorf("invalid length %d", length)
+	}
+
+	req, err := http.NewRequest("GET", b.Filename(h), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "http.NewRequest")
+	}
+
+	byteRange := fmt.Sprintf("bytes=%d-", offset)
+	if length > 0 {
+		byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1)
+	}
+	req.Header.Add("Range", byteRange)
+	debug.Log("Load(%v) send range %v", h, byteRange)
+
+	b.sem.GetToken()
+	resp, err := ctxhttp.Do(ctx, b.client, req)
+	b.sem.ReleaseToken()
+
+	if err != nil {
+		if resp != nil {
+			_, _ = io.Copy(ioutil.Discard, resp.Body)
+			_ = resp.Body.Close()
+		}
+		return nil, errors.Wrap(err, "client.Do")
+	}
+
+	if resp.StatusCode == http.StatusNotFound {
+		_ = resp.Body.Close()
+		return nil, ErrIsNotExist{h}
+	}
+
+	if resp.StatusCode != 200 && resp.StatusCode != 206 {
+		_ = resp.Body.Close()
+		return nil, errors.Errorf("unexpected HTTP response (%v): %v", resp.StatusCode, resp.Status)
+	}
+
+	return resp.Body, nil
+}
+
+// Stat returns information about a blob.
+func (b *restBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
+	if err := h.Valid(); err != nil {
+		return restic.FileInfo{}, err
+	}
+
+	b.sem.GetToken()
+	resp, err := ctxhttp.Head(ctx, b.client, b.Filename(h))
+	b.sem.ReleaseToken()
+	if err != nil {
+		return restic.FileInfo{}, errors.Wrap(err, "client.Head")
+	}
+
+	_, _ = io.Copy(ioutil.Discard, resp.Body)
+	if err = resp.Body.Close(); err != nil {
+		return restic.FileInfo{}, errors.Wrap(err, "Close")
+	}
+
+	if resp.StatusCode == http.StatusNotFound {
+		_ = resp.Body.Close()
+		return restic.FileInfo{}, ErrIsNotExist{h}
+	}
+
+	if resp.StatusCode != 200 {
+		return restic.FileInfo{}, errors.Errorf("unexpected HTTP response (%v): %v", resp.StatusCode, resp.Status)
+	}
+
+	if resp.ContentLength < 0 {
+		return restic.FileInfo{}, errors.New("negative content length")
+	}
+
+	bi := restic.FileInfo{
+		Size: resp.ContentLength,
+	}
+
+	return bi, nil
+}
+
+// Test returns true if a blob of the given type and name exists in the backend.
+func (b *restBackend) Test(ctx context.Context, h restic.Handle) (bool, error) {
+	_, err := b.Stat(ctx, h)
+	if err != nil {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+// Remove removes the blob with the given name and type.
+func (b *restBackend) Remove(ctx context.Context, h restic.Handle) error {
+	if err := h.Valid(); err != nil {
+		return err
+	}
+
+	req, err := http.NewRequest("DELETE", b.Filename(h), nil)
+	if err != nil {
+		return errors.Wrap(err, "http.NewRequest")
+	}
+	b.sem.GetToken()
+	resp, err := ctxhttp.Do(ctx, b.client, req)
+	b.sem.ReleaseToken()
+
+	if err != nil {
+		return errors.Wrap(err, "client.Do")
+	}
+
+	if resp.StatusCode == http.StatusNotFound {
+		_ = resp.Body.Close()
+		return ErrIsNotExist{h}
+	}
+
+	if resp.StatusCode != 200 {
+		return errors.Errorf("blob not removed, server response: %v (%v)", resp.Status, resp.StatusCode)
+	}
+
+	_, err = io.Copy(ioutil.Discard, resp.Body)
+	if err != nil {
+		return errors.Wrap(err, "Copy")
+	}
+
+	return errors.Wrap(resp.Body.Close(), "Close")
+}
+
+// List returns a channel that yields all names of blobs of type t. A
+// goroutine is started for this. If the channel done is closed, sending
+// stops.
+func (b *restBackend) List(ctx context.Context, t restic.FileType) <-chan string {
+	ch := make(chan string)
+
+	url := b.Dirname(restic.Handle{Type: t})
+	if !strings.HasSuffix(url, "/") {
+		url += "/"
+	}
+
+	b.sem.GetToken()
+	resp, err := ctxhttp.Get(ctx, b.client, url)
+	b.sem.ReleaseToken()
+
+	if resp != nil {
+		defer func() {
+			_, _ = io.Copy(ioutil.Discard, resp.Body)
+			e := resp.Body.Close()
+
+			if err == nil {
+				err = errors.Wrap(e, "Close")
+			}
+		}()
+	}
+
+	if err != nil {
+		close(ch)
+		return ch
+	}
+
+	dec := json.NewDecoder(resp.Body)
+	var list []string
+	if err = dec.Decode(&list); err != nil {
+		close(ch)
+		return ch
+	}
+
+	go func() {
+		defer close(ch)
+		for _, m := range list {
+			select {
+			case ch <- m:
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	return ch
+}
+
+// Close closes all open files.
+func (b *restBackend) Close() error {
+	// this does not need to do anything, all open files are closed within the
+	// same function.
+	return nil
+}

internal/backend/rest/rest_test.go

@@ -0,0 +1,133 @@
+package rest_test
+
+import (
+	"context"
+	"io/ioutil"
+	"net"
+	"net/url"
+	"os"
+	"os/exec"
+	"restic"
+	"testing"
+	"time"
+
+	"restic/backend/rest"
+	"restic/backend/test"
+	. "restic/test"
+)
+
+func runRESTServer(ctx context.Context, t testing.TB, dir string) func() {
+	srv, err := exec.LookPath("rest-server")
+	if err != nil {
+		t.Skip(err)
+	}
+
+	cmd := exec.CommandContext(ctx, srv, "--path", dir)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stdout
+	if err := cmd.Start(); err != nil {
+		t.Fatal(err)
+	}
+
+	// wait until the TCP port is reachable
+	var success bool
+	for i := 0; i < 10; i++ {
+		time.Sleep(200 * time.Millisecond)
+
+		c, err := net.Dial("tcp", "localhost:8000")
+		if err != nil {
+			continue
+		}
+
+		success = true
+		if err := c.Close(); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if !success {
+		t.Fatal("unable to connect to rest server")
+		return nil
+	}
+
+	return func() {
+		if err := cmd.Process.Kill(); err != nil {
+			t.Fatal(err)
+		}
+
+		// ignore errors, we've killed the process
+		_ = cmd.Wait()
+	}
+}
+
+func newTestSuite(ctx context.Context, t testing.TB) *test.Suite {
+	return &test.Suite{
+		// NewConfig returns a config for a new temporary backend that will be used in tests.
+		NewConfig: func() (interface{}, error) {
+			dir, err := ioutil.TempDir(TestTempDir, "restic-test-rest-")
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			t.Logf("create new backend at %v", dir)
+
+			url, err := url.Parse("http://localhost:8000/restic-test")
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			cfg := rest.NewConfig()
+			cfg.URL = url
+			return cfg, nil
+		},
+
+		// CreateFn is a function that creates a temporary repository for the tests.
+		Create: func(config interface{}) (restic.Backend, error) {
+			cfg := config.(rest.Config)
+			return rest.Create(cfg)
+		},
+
+		// OpenFn is a function that opens a previously created temporary repository.
+		Open: func(config interface{}) (restic.Backend, error) {
+			cfg := config.(rest.Config)
+			return rest.Open(cfg)
+		},
+
+		// CleanupFn removes data created during the tests.
+		Cleanup: func(config interface{}) error {
+			return nil
+		},
+	}
+}
+
+func TestBackendREST(t *testing.T) {
+	defer func() {
+		if t.Skipped() {
+			SkipDisallowed(t, "restic/backend/rest.TestBackendREST")
+		}
+	}()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	dir, cleanup := TempDir(t)
+	defer cleanup()
+
+	cleanup = runRESTServer(ctx, t, dir)
+	defer cleanup()
+
+	newTestSuite(ctx, t).RunTests(t)
+}
+
+func BenchmarkBackendREST(t *testing.B) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	dir, cleanup := TempDir(t)
+	defer cleanup()
+
+	cleanup = runRESTServer(ctx, t, dir)
+	defer cleanup()
+
+	newTestSuite(ctx, t).RunBenchmarks(t)
+}

internal/backend/s3/config.go

@@ -0,0 +1,89 @@
+package s3
+
+import (
+	"net/url"
+	"path"
+	"strings"
+
+	"restic/errors"
+	"restic/options"
+)
+
+// Config contains all configuration necessary to connect to an s3 compatible
+// server.
+type Config struct {
+	Endpoint      string
+	UseHTTP       bool
+	KeyID, Secret string
+	Bucket        string
+	Prefix        string
+	Layout        string `option:"layout" help:"use this backend layout (default: auto-detect)"`
+
+	Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
+	MaxRetries  uint `option:"retries" help:"set the number of retries attempted"`
+}
+
+// NewConfig returns a new Config with the default values filled in.
+func NewConfig() Config {
+	return Config{
+		Connections: 5,
+	}
+}
+
+func init() {
+	options.Register("s3", Config{})
+}
+
+const defaultPrefix = "restic"
+
+// ParseConfig parses the string s and extracts the s3 config. The two
+// supported configuration formats are s3://host/bucketname/prefix and
+// s3:host:bucketname/prefix. The host can also be a valid s3 region
+// name. If no prefix is given the prefix "restic" will be used.
+func ParseConfig(s string) (interface{}, error) {
+	switch {
+	case strings.HasPrefix(s, "s3:http"):
+		// assume that a URL has been specified, parse it and
+		// use the host as the endpoint and the path as the
+		// bucket name and prefix
+		url, err := url.Parse(s[3:])
+		if err != nil {
+			return nil, errors.Wrap(err, "url.Parse")
+		}
+
+		if url.Path == "" {
+			return nil, errors.New("s3: bucket name not found")
+		}
+
+		path := strings.SplitN(url.Path[1:], "/", 2)
+		return createConfig(url.Host, path, url.Scheme == "http")
+	case strings.HasPrefix(s, "s3://"):
+		s = s[5:]
+	case strings.HasPrefix(s, "s3:"):
+		s = s[3:]
+	default:
+		return nil, errors.New("s3: invalid format")
+	}
+	// use the first entry of the path as the endpoint and the
+	// remainder as bucket name and prefix
+	path := strings.SplitN(s, "/", 3)
+	return createConfig(path[0], path[1:], false)
+}
+
+func createConfig(endpoint string, p []string, useHTTP bool) (interface{}, error) {
+	var prefix string
+	switch {
+	case len(p) < 1:
+		return nil, errors.New("s3: invalid format, host/region or bucket name not found")
+	case len(p) == 1 || p[1] == "":
+		prefix = defaultPrefix
+	default:
+		prefix = path.Clean(p[1])
+	}
+	cfg := NewConfig()
+	cfg.Endpoint = endpoint
+	cfg.UseHTTP = useHTTP
+	cfg.Bucket = p[0]
+	cfg.Prefix = prefix
+	return cfg, nil
+}

internal/backend/s3/config_test.go

@@ -0,0 +1,113 @@
+package s3
+
+import "testing"
+
+var configTests = []struct {
+	s   string
+	cfg Config
+}{
+	{"s3://eu-central-1/bucketname", Config{
+		Endpoint:    "eu-central-1",
+		Bucket:      "bucketname",
+		Prefix:      "restic",
+		Connections: 5,
+	}},
+	{"s3://eu-central-1/bucketname/", Config{
+		Endpoint:    "eu-central-1",
+		Bucket:      "bucketname",
+		Prefix:      "restic",
+		Connections: 5,
+	}},
+	{"s3://eu-central-1/bucketname/prefix/directory", Config{
+		Endpoint:    "eu-central-1",
+		Bucket:      "bucketname",
+		Prefix:      "prefix/directory",
+		Connections: 5,
+	}},
+	{"s3://eu-central-1/bucketname/prefix/directory/", Config{
+		Endpoint:    "eu-central-1",
+		Bucket:      "bucketname",
+		Prefix:      "prefix/directory",
+		Connections: 5,
+	}},
+	{"s3:eu-central-1/foobar", Config{
+		Endpoint:    "eu-central-1",
+		Bucket:      "foobar",
+		Prefix:      "restic",
+		Connections: 5,
+	}},
+	{"s3:eu-central-1/foobar/", Config{
+		Endpoint:    "eu-central-1",
+		Bucket:      "foobar",
+		Prefix:      "restic",
+		Connections: 5,
+	}},
+	{"s3:eu-central-1/foobar/prefix/directory", Config{
+		Endpoint:    "eu-central-1",
+		Bucket:      "foobar",
+		Prefix:      "prefix/directory",
+		Connections: 5,
+	}},
+	{"s3:eu-central-1/foobar/prefix/directory/", Config{
+		Endpoint:    "eu-central-1",
+		Bucket:      "foobar",
+		Prefix:      "prefix/directory",
+		Connections: 5,
+	}},
+	{"s3:https://hostname:9999/foobar", Config{
+		Endpoint:    "hostname:9999",
+		Bucket:      "foobar",
+		Prefix:      "restic",
+		Connections: 5,
+	}},
+	{"s3:https://hostname:9999/foobar/", Config{
+		Endpoint:    "hostname:9999",
+		Bucket:      "foobar",
+		Prefix:      "restic",
+		Connections: 5,
+	}},
+	{"s3:http://hostname:9999/foobar", Config{
+		Endpoint:    "hostname:9999",
+		Bucket:      "foobar",
+		Prefix:      "restic",
+		UseHTTP:     true,
+		Connections: 5,
+	}},
+	{"s3:http://hostname:9999/foobar/", Config{
+		Endpoint:    "hostname:9999",
+		Bucket:      "foobar",
+		Prefix:      "restic",
+		UseHTTP:     true,
+		Connections: 5,
+	}},
+	{"s3:http://hostname:9999/bucket/prefix/directory", Config{
+		Endpoint:    "hostname:9999",
+		Bucket:      "bucket",
+		Prefix:      "prefix/directory",
+		UseHTTP:     true,
+		Connections: 5,
+	}},
+	{"s3:http://hostname:9999/bucket/prefix/directory/", Config{
+		Endpoint:    "hostname:9999",
+		Bucket:      "bucket",
+		Prefix:      "prefix/directory",
+		UseHTTP:     true,
+		Connections: 5,
+	}},
+}
+
+func TestParseConfig(t *testing.T) {
+	for i, test := range configTests {
+		cfg, err := ParseConfig(test.s)
+		if err != nil {
+			t.Errorf("test %d:%s failed: %v", i, test.s, err)
+			continue
+		}
+
+		if cfg != test.cfg {
+			t.Errorf("test %d:\ninput:\n  %s\n wrong config, want:\n  %v\ngot:\n  %v",
+				i, test.s, test.cfg, cfg)
+			continue
+		}
+	}
+}

internal/backend/s3/s3.go

@@ -0,0 +1,447 @@
+package s3
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"restic"
+	"strings"
+	"time"
+
+	"restic/backend"
+	"restic/errors"
+
+	"github.com/minio/minio-go"
+	"github.com/minio/minio-go/pkg/credentials"
+
+	"restic/debug"
+)
+
+// Backend stores data on an S3 endpoint.
+type Backend struct {
+	client *minio.Client
+	sem    *backend.Semaphore
+	cfg    Config
+	backend.Layout
+}
+
+// make sure that *Backend implements backend.Backend
+var _ restic.Backend = &Backend{}
+
+const defaultLayout = "default"
+
+func open(cfg Config) (*Backend, error) {
+	debug.Log("open, config %#v", cfg)
+
+	if cfg.MaxRetries > 0 {
+		minio.MaxRetry = int(cfg.MaxRetries)
+	}
+
+	var client *minio.Client
+	var err error
+
+	if cfg.KeyID == "" || cfg.Secret == "" {
+		debug.Log("key/secret not found, trying to get them from IAM")
+		creds := credentials.NewIAM("")
+		client, err = minio.NewWithCredentials(cfg.Endpoint, creds, !cfg.UseHTTP, "")
+
+		if err != nil {
+			return nil, errors.Wrap(err, "minio.NewWithCredentials")
+		}
+	} else {
+		debug.Log("key/secret found")
+		client, err = minio.New(cfg.Endpoint, cfg.KeyID, cfg.Secret, !cfg.UseHTTP)
+
+		if err != nil {
+			return nil, errors.Wrap(err, "minio.New")
+		}
+	}
+
+	sem, err := backend.NewSemaphore(cfg.Connections)
+	if err != nil {
+		return nil, err
+	}
+
+	be := &Backend{
+		client: client,
+		sem:    sem,
+		cfg:    cfg,
+	}
+
+	client.SetCustomTransport(backend.Transport())
+
+	l, err := backend.ParseLayout(be, cfg.Layout, defaultLayout, cfg.Prefix)
+	if err != nil {
+		return nil, err
+	}
+
+	be.Layout = l
+
+	return be, nil
+}
+
+// Open opens the S3 backend at bucket and region. The bucket is created if it
+// does not exist yet.
+func Open(cfg Config) (restic.Backend, error) {
+	return open(cfg)
+}
+
+// Create opens the S3 backend at bucket and region and creates the bucket if
+// it does not exist yet.
+func Create(cfg Config) (restic.Backend, error) {
+	be, err := open(cfg)
+	if err != nil {
+		return nil, errors.Wrap(err, "open")
+	}
+	found, err := be.client.BucketExists(cfg.Bucket)
+	if err != nil {
+		debug.Log("BucketExists(%v) returned err %v", cfg.Bucket, err)
+		return nil, errors.Wrap(err, "client.BucketExists")
+	}
+
+	if !found {
+		// create new bucket with default ACL in default region
+		err = be.client.MakeBucket(cfg.Bucket, "")
+		if err != nil {
+			return nil, errors.Wrap(err, "client.MakeBucket")
+		}
+	}
+
+	return be, nil
+}
+
+// IsNotExist returns true if the error is caused by a not existing file.
+func (be *Backend) IsNotExist(err error) bool {
+	debug.Log("IsNotExist(%T, %#v)", err, err)
+	if os.IsNotExist(errors.Cause(err)) {
+		return true
+	}
+
+	if e, ok := errors.Cause(err).(minio.ErrorResponse); ok && e.Code == "NoSuchKey" {
+		return true
+	}
+
+	return false
+}
+
+// Join combines path components with slashes.
+func (be *Backend) Join(p ...string) string {
+	return path.Join(p...)
+}
+
+type fileInfo struct {
+	name    string
+	size    int64
+	mode    os.FileMode
+	modTime time.Time
+	isDir   bool
+}
+
+func (fi fileInfo) Name() string       { return fi.name }    // base name of the file
+func (fi fileInfo) Size() int64        { return fi.size }    // length in bytes for regular files; system-dependent for others
+func (fi fileInfo) Mode() os.FileMode  { return fi.mode }    // file mode bits
+func (fi fileInfo) ModTime() time.Time { return fi.modTime } // modification time
+func (fi fileInfo) IsDir() bool        { return fi.isDir }   // abbreviation for Mode().IsDir()
+func (fi fileInfo) Sys() interface{}   { return nil }        // underlying data source (can return nil)
+
+// ReadDir returns the entries for a directory.
+func (be *Backend) ReadDir(dir string) (list []os.FileInfo, err error) {
+	debug.Log("ReadDir(%v)", dir)
+
+	// make sure dir ends with a slash
+	if dir[len(dir)-1] != '/' {
+		dir += "/"
+	}
+
+	done := make(chan struct{})
+	defer close(done)
+
+	for obj := range be.client.ListObjects(be.cfg.Bucket, dir, false, done) {
+		if obj.Key == "" {
+			continue
+		}
+
+		name := strings.TrimPrefix(obj.Key, dir)
+		if name == "" {
+			return nil, errors.Errorf("invalid key name %v, removing prefix %v yielded empty string", obj.Key, dir)
+		}
+		entry := fileInfo{
+			name:    name,
+			size:    obj.Size,
+			modTime: obj.LastModified,
+		}
+
+		if name[len(name)-1] == '/' {
+			entry.isDir = true
+			entry.mode = os.ModeDir | 0755
+			entry.name = name[:len(name)-1]
+		} else {
+			entry.mode = 0644
+		}
+
+		list = append(list, entry)
+	}
+
+	return list, nil
+}
+
+// Location returns this backend's location (the bucket name).
+func (be *Backend) Location() string {
+	return be.Join(be.cfg.Bucket, be.cfg.Prefix)
+}
+
+// Path returns the path in the bucket that is used for this backend.
+func (be *Backend) Path() string {
+	return be.cfg.Prefix
+}
+
+// Save stores data in the backend at the handle.
+func (be *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) {
+	debug.Log("Save %v", h)
+
+	if err := h.Valid(); err != nil {
+		return err
+	}
+
+	objName := be.Filename(h)
+
+	// Check key does not already exist
+	_, err = be.client.StatObject(be.cfg.Bucket, objName)
+	if err == nil {
+		debug.Log("%v already exists", h)
+		return errors.New("key already exists")
+	}
+
+	// prevent the HTTP client from closing a file
+	rd = ioutil.NopCloser(rd)
+
+	be.sem.GetToken()
+	debug.Log("PutObject(%v, %v)", be.cfg.Bucket, objName)
+	n, err := be.client.PutObject(be.cfg.Bucket, objName, rd, "application/octet-stream")
+	be.sem.ReleaseToken()
+
+	debug.Log("%v -> %v bytes, err %#v: %v", objName, n, err, err)
+
+	return errors.Wrap(err, "client.PutObject")
+}
+
+// wrapReader wraps an io.ReadCloser to run an additional function on Close.
+type wrapReader struct {
+	io.ReadCloser
+	f func()
+}
+
+func (wr wrapReader) Close() error {
+	err := wr.ReadCloser.Close()
+	wr.f()
+	return err
+}
+
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h))
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if length < 0 {
+		return nil, errors.Errorf("invalid length %d", length)
+	}
+
+	objName := be.Filename(h)
+
+	byteRange := fmt.Sprintf("bytes=%d-", offset)
+	if length > 0 {
+		byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1)
+	}
+	headers := minio.NewGetReqHeaders()
+	headers.Add("Range", byteRange)
+
+	be.sem.GetToken()
+	debug.Log("Load(%v) send range %v", h, byteRange)
+
+	coreClient := minio.Core{Client: be.client}
+	rd, _, err := coreClient.GetObject(be.cfg.Bucket, objName, headers)
+	if err != nil {
+		be.sem.ReleaseToken()
+		return nil, err
+	}
+
+	closeRd := wrapReader{
+		ReadCloser: rd,
+		f: func() {
+			debug.Log("Close()")
+			be.sem.ReleaseToken()
+		},
+	}
+
+	return closeRd, err
+}
+
+// Stat returns information about a blob.
+func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
+	debug.Log("%v", h)
+
+	objName := be.Filename(h)
+	var obj *minio.Object
+
+	obj, err = be.client.GetObject(be.cfg.Bucket, objName)
+	if err != nil {
+		debug.Log("GetObject() err %v", err)
+		return restic.FileInfo{}, errors.Wrap(err, "client.GetObject")
+	}
+
+	// make sure that the object is closed properly.
+	defer func() {
+		e := obj.Close()
+		if err == nil {
+			err = errors.Wrap(e, "Close")
+		}
+	}()
+
+	fi, err := obj.Stat()
+	if err != nil {
+		debug.Log("Stat() err %v", err)
+		return restic.FileInfo{}, errors.Wrap(err, "Stat")
+	}
+
+	return restic.FileInfo{Size: fi.Size}, nil
+}
+
+// Test returns true if a blob of the given type and name exists in the backend.
+func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
+	found := false
+	objName := be.Filename(h)
+	_, err := be.client.StatObject(be.cfg.Bucket, objName)
+	if err == nil {
+		found = true
+	}
+
+	// If error, then not found
+	return found, nil
+}
+
+// Remove removes the blob with the given name and type.
+func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
+	objName := be.Filename(h)
+	err := be.client.RemoveObject(be.cfg.Bucket, objName)
+	debug.Log("Remove(%v) at %v -> err %v", h, objName, err)
+
+	if be.IsNotExist(err) {
+		err = nil
+	}
+
+	return errors.Wrap(err, "client.RemoveObject")
+}
+
+// List returns a channel that yields all names of blobs of type t. A
+// goroutine is started for this. If the channel done is closed, sending
+// stops.
+func (be *Backend) List(ctx context.Context, t restic.FileType) <-chan string {
+	debug.Log("listing %v", t)
+	ch := make(chan string)
+
+	prefix := be.Dirname(restic.Handle{Type: t})
+
+	// make sure prefix ends with a slash
+	if prefix[len(prefix)-1] != '/' {
+		prefix += "/"
+	}
+
+	listresp := be.client.ListObjects(be.cfg.Bucket, prefix, true, ctx.Done())
+
+	go func() {
+		defer close(ch)
+		for obj := range listresp {
+			m := strings.TrimPrefix(obj.Key, prefix)
+			if m == "" {
+				continue
+			}
+
+			select {
+			case ch <- path.Base(m):
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	return ch
+}
+
+// Remove keys for a specified backend type.
+func (be *Backend) removeKeys(ctx context.Context, t restic.FileType) error {
+	for key := range be.List(ctx, restic.DataFile) {
+		err := be.Remove(ctx, restic.Handle{Type: restic.DataFile, Name: key})
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
+func (be *Backend) Delete(ctx context.Context) error {
+	alltypes := []restic.FileType{
+		restic.DataFile,
+		restic.KeyFile,
+		restic.LockFile,
+		restic.SnapshotFile,
+		restic.IndexFile}
+
+	for _, t := range alltypes {
+		err := be.removeKeys(ctx, t)
+		if err != nil {
+			return nil
+		}
+	}
+
+	return be.Remove(ctx, restic.Handle{Type: restic.ConfigFile})
+}
+
+// Close does nothing
+func (be *Backend) Close() error { return nil }
+
+// Rename moves a file based on the new layout l.
+func (be *Backend) Rename(h restic.Handle, l backend.Layout) error {
+	debug.Log("Rename %v to %v", h, l)
+	oldname := be.Filename(h)
+	newname := l.Filename(h)
+
+	if oldname == newname {
+		debug.Log("  %v is already renamed", newname)
+		return nil
+	}
+
+	debug.Log("  %v -> %v", oldname, newname)
+
+	src := minio.NewSourceInfo(be.cfg.Bucket, oldname, nil)
+
+	dst, err := minio.NewDestinationInfo(be.cfg.Bucket, newname, nil, nil)
+	if err != nil {
+		return errors.Wrap(err, "NewDestinationInfo")
+	}
+
+	err = be.client.CopyObject(dst, src)
+	if err != nil && be.IsNotExist(err) {
+		debug.Log("copy failed: %v, seems to already have been renamed", err)
+		return nil
+	}
+
+	if err != nil {
+		debug.Log("copy failed: %v", err)
+		return err
+	}
+
+	return be.client.RemoveObject(be.cfg.Bucket, oldname)
+}

internal/backend/s3/s3_test.go

@@ -0,0 +1,320 @@
+package s3_test
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"restic"
+	"testing"
+	"time"
+
+	"restic/backend/s3"
+	"restic/backend/test"
+	. "restic/test"
+)
+
+func mkdir(t testing.TB, dir string) {
+	err := os.MkdirAll(dir, 0700)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func runMinio(ctx context.Context, t testing.TB, dir, key, secret string) func() {
+	mkdir(t, filepath.Join(dir, "config"))
+	mkdir(t, filepath.Join(dir, "root"))
+
+	cmd := exec.CommandContext(ctx, "minio",
+		"server",
+		"--address", "127.0.0.1:9000",
+		"--config-dir", filepath.Join(dir, "config"),
+		filepath.Join(dir, "root"))
+	cmd.Env = append(os.Environ(),
+		"MINIO_ACCESS_KEY="+key,
+		"MINIO_SECRET_KEY="+secret,
+	)
+	cmd.Stderr = os.Stderr
+
+	err := cmd.Start()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// wait until the TCP port is reachable
+	var success bool
+	for i := 0; i < 100; i++ {
+		time.Sleep(200 * time.Millisecond)
+
+		c, err := net.Dial("tcp", "localhost:9000")
+		if err == nil {
+			success = true
+			if err := c.Close(); err != nil {
+				t.Fatal(err)
+			}
+			break
+		}
+	}
+
+	if !success {
+		t.Fatal("unable to connect to minio server")
+		return nil
+	}
+
+	return func() {
+		err = cmd.Process.Kill()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// ignore errors, we've killed the process
+		_ = cmd.Wait()
+	}
+}
+
+func newRandomCredentials(t testing.TB) (key, secret string) {
+	buf := make([]byte, 10)
+	_, err := io.ReadFull(rand.Reader, buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	key = hex.EncodeToString(buf)
+
+	_, err = io.ReadFull(rand.Reader, buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	secret = hex.EncodeToString(buf)
+
+	return key, secret
+}
+
+type MinioTestConfig struct {
+	s3.Config
+
+	tempdir       string
+	removeTempdir func()
+	stopServer    func()
+}
+
+func createS3(t testing.TB, cfg MinioTestConfig) (be restic.Backend, err error) {
+	for i := 0; i < 10; i++ {
+		be, err = s3.Create(cfg.Config)
+		if err != nil {
+			t.Logf("s3 open: try %d: error %v", i, err)
+			time.Sleep(500 * time.Millisecond)
+			continue
+		}
+
+		break
+	}
+
+	return be, err
+}
+
+func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite {
+	return &test.Suite{
+		// NewConfig returns a config for a new temporary backend that will be used in tests.
+		NewConfig: func() (interface{}, error) {
+			cfg := MinioTestConfig{}
+
+			cfg.tempdir, cfg.removeTempdir = TempDir(t)
+			key, secret := newRandomCredentials(t)
+			cfg.stopServer = runMinio(ctx, t, cfg.tempdir, key, secret)
+
+			cfg.Config = s3.NewConfig()
+			cfg.Config.Endpoint = "localhost:9000"
+			cfg.Config.Bucket = "restictestbucket"
+			cfg.Config.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
+			cfg.Config.UseHTTP = true
+			cfg.Config.KeyID = key
+			cfg.Config.Secret = secret
+			return cfg, nil
+		},
+
+		// CreateFn is a function that creates a temporary repository for the tests.
+		Create: func(config interface{}) (restic.Backend, error) {
+			cfg := config.(MinioTestConfig)
+
+			be, err := createS3(t, cfg)
+			if err != nil {
+				return nil, err
+			}
+
+			exists, err := be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile})
+			if err != nil {
+				return nil, err
+			}
+
+			if exists {
+				return nil, errors.New("config already exists")
+			}
+
+			return be, nil
+		},
+
+		// OpenFn is a function that opens a previously created temporary repository.
+		Open: func(config interface{}) (restic.Backend, error) {
+			cfg := config.(MinioTestConfig)
+			return s3.Open(cfg.Config)
+		},
+
+		// CleanupFn removes data created during the tests.
+		Cleanup: func(config interface{}) error {
+			cfg := config.(MinioTestConfig)
+			if cfg.stopServer != nil {
+				cfg.stopServer()
+			}
+			if cfg.removeTempdir != nil {
+				cfg.removeTempdir()
+			}
+			return nil
+		},
+	}
+}
+
+func TestBackendMinio(t *testing.T) {
+	defer func() {
+		if t.Skipped() {
+			SkipDisallowed(t, "restic/backend/s3.TestBackendMinio")
+		}
+	}()
+
+	// try to find a minio binary
+	_, err := exec.LookPath("minio")
+	if err != nil {
+		t.Skip(err)
+		return
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	newMinioTestSuite(ctx, t).RunTests(t)
+}
+
+func BenchmarkBackendMinio(t *testing.B) {
+	// try to find a minio binary
+	_, err := exec.LookPath("minio")
+	if err != nil {
+		t.Skip(err)
+		return
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	newMinioTestSuite(ctx, t).RunBenchmarks(t)
+}
+
+func newS3TestSuite(t testing.TB) *test.Suite {
+	return &test.Suite{
+		// do not use excessive data
+		MinimalData: true,
+
+		// NewConfig returns a config for a new temporary backend that will be used in tests.
+		NewConfig: func() (interface{}, error) {
+			s3cfg, err := s3.ParseConfig(os.Getenv("RESTIC_TEST_S3_REPOSITORY"))
+			if err != nil {
+				return nil, err
+			}
+
+			cfg := s3cfg.(s3.Config)
+			cfg.KeyID = os.Getenv("RESTIC_TEST_S3_KEY")
+			cfg.Secret = os.Getenv("RESTIC_TEST_S3_SECRET")
+			cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
+			return cfg, nil
+		},
+
+		// CreateFn is a function that creates a temporary repository for the tests.
+		Create: func(config interface{}) (restic.Backend, error) {
+			cfg := config.(s3.Config)
+
+			be, err := s3.Create(cfg)
+			if err != nil {
+				return nil, err
+			}
+
+			exists, err := be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile})
+			if err != nil {
+				return nil, err
+			}
+
+			if exists {
+				return nil, errors.New("config already exists")
+			}
+
+			return be, nil
+		},
+
+		// OpenFn is a function that opens a previously created temporary repository.
+		Open: func(config interface{}) (restic.Backend, error) {
+			cfg := config.(s3.Config)
+			return s3.Open(cfg)
+		},
+
+		// CleanupFn removes data created during the tests.
+		Cleanup: func(config interface{}) error {
+			cfg := config.(s3.Config)
+
+			be, err := s3.Open(cfg)
+			if err != nil {
+				return err
+			}
+
+			if err := be.(restic.Deleter).Delete(context.TODO()); err != nil {
+				return err
+			}
+
+			return nil
+		},
+	}
+}
+
+func TestBackendS3(t *testing.T) {
+	defer func() {
+		if t.Skipped() {
+			SkipDisallowed(t, "restic/backend/s3.TestBackendS3")
+		}
+	}()
+
+	vars := []string{
+		"RESTIC_TEST_S3_KEY",
+		"RESTIC_TEST_S3_SECRET",
+		"RESTIC_TEST_S3_REPOSITORY",
+	}
+
+	for _, v := range vars {
+		if os.Getenv(v) == "" {
+			t.Skipf("environment variable %v not set", v)
+			return
+		}
+	}
+
+	t.Logf("run tests")
+	newS3TestSuite(t).RunTests(t)
+}
+
+func BenchmarkBackendS3(t *testing.B) {
+	vars := []string{
+		"RESTIC_TEST_S3_KEY",
+		"RESTIC_TEST_S3_SECRET",
+		"RESTIC_TEST_S3_REPOSITORY",
+	}
+
+	for _, v := range vars {
+		if os.Getenv(v) == "" {
+			t.Skipf("environment variable %v not set", v)
+			return
+		}
+	}
+
+	t.Logf("run tests")
+	newS3TestSuite(t).RunBenchmarks(t)
+}

internal/backend/semaphore.go

@@ -0,0 +1,28 @@
+package backend
+
+import "restic/errors"
+
+// Semaphore limits access to a restricted resource.
+type Semaphore struct {
+	ch chan struct{}
+}
+
+// NewSemaphore returns a new semaphore with capacity n.
+func NewSemaphore(n uint) (*Semaphore, error) {
+	if n <= 0 {
+		return nil, errors.New("must be a positive number")
+	}
+	return &Semaphore{
+		ch: make(chan struct{}, n),
+	}, nil
+}
+
+// GetToken blocks until a Token is available.
+func (s *Semaphore) GetToken() {
+	s.ch <- struct{}{}
+}
+
+// ReleaseToken returns a token.
+func (s *Semaphore) ReleaseToken() {
+	<-s.ch
+}

internal/backend/sftp/config.go

@@ -0,0 +1,72 @@
+package sftp
+
+import (
+	"net/url"
+	"path"
+	"strings"
+
+	"restic/errors"
+	"restic/options"
+)
+
+// Config collects all information required to connect to an sftp server.
+type Config struct {
+	User, Host, Path string
+	Layout           string `option:"layout" help:"use this backend directory layout (default: auto-detect)"`
+	Command          string `option:"command" help:"specify command to create sftp connection"`
+}
+
+func init() {
+	options.Register("sftp", Config{})
+}
+
+// ParseConfig parses the string s and extracts the sftp config. The
+// supported configuration formats are sftp://user@host/directory
+//  and sftp:user@host:directory.  The directory will be path Cleaned and can
+//  be an absolute path if it starts with a '/' (e.g.
+//  sftp://user@host//absolute and sftp:user@host:/absolute).
+func ParseConfig(s string) (interface{}, error) {
+	var user, host, dir string
+	switch {
+	case strings.HasPrefix(s, "sftp://"):
+		// parse the "sftp://user@host/path" url format
+		url, err := url.Parse(s)
+		if err != nil {
+			return nil, errors.Wrap(err, "url.Parse")
+		}
+		if url.User != nil {
+			user = url.User.Username()
+		}
+		host = url.Host
+		dir = url.Path
+		if dir == "" {
+			return nil, errors.Errorf("invalid backend %q, no directory specified", s)
+		}
+
+		dir = dir[1:]
+	case strings.HasPrefix(s, "sftp:"):
+		// parse the sftp:user@host:path format, which means we'll get
+		// "user@host:path" in s
+		s = s[5:]
+		// split user@host and path at the colon
+		data := strings.SplitN(s, ":", 2)
+		if len(data) < 2 {
+			return nil, errors.New("sftp: invalid format, hostname or path not found")
+		}
+		host = data[0]
+		dir = data[1]
+		// split user and host at the "@"
+		data = strings.SplitN(host, "@", 2)
+		if len(data) == 2 {
+			user = data[0]
+			host = data[1]
+		}
+	default:
+		return nil, errors.New(`invalid format, does not start with "sftp:"`)
+	}
+	return Config{
+		User: user,
+		Host: host,
+		Path: path.Clean(dir),
+	}, nil
+}

internal/backend/sftp/config_test.go

@@ -0,0 +1,90 @@
+package sftp
+
+import "testing"
+
+var configTests = []struct {
+	in  string
+	cfg Config
+}{
+	// first form, user specified sftp://user@host/dir
+	{
+		"sftp://user@host/dir/subdir",
+		Config{User: "user", Host: "host", Path: "dir/subdir"},
+	},
+	{
+		"sftp://host/dir/subdir",
+		Config{Host: "host", Path: "dir/subdir"},
+	},
+	{
+		"sftp://host//dir/subdir",
+		Config{Host: "host", Path: "/dir/subdir"},
+	},
+	{
+		"sftp://host:10022//dir/subdir",
+		Config{Host: "host:10022", Path: "/dir/subdir"},
+	},
+	{
+		"sftp://user@host:10022//dir/subdir",
+		Config{User: "user", Host: "host:10022", Path: "/dir/subdir"},
+	},
+	{
+		"sftp://user@host/dir/subdir/../other",
+		Config{User: "user", Host: "host", Path: "dir/other"},
+	},
+	{
+		"sftp://user@host/dir///subdir",
+		Config{User: "user", Host: "host", Path: "dir/subdir"},
+	},
+
+	// second form, user specified sftp:user@host:/dir
+	{
+		"sftp:user@host:/dir/subdir",
+		Config{User: "user", Host: "host", Path: "/dir/subdir"},
+	},
+	{
+		"sftp:host:../dir/subdir",
+		Config{Host: "host", Path: "../dir/subdir"},
+	},
+	{
+		"sftp:user@host:dir/subdir:suffix",
+		Config{User: "user", Host: "host", Path: "dir/subdir:suffix"},
+	},
+	{
+		"sftp:user@host:dir/subdir/../other",
+		Config{User: "user", Host: "host", Path: "dir/other"},
+	},
+	{
+		"sftp:user@host:dir///subdir",
+		Config{User: "user", Host: "host", Path: "dir/subdir"},
+	},
+}
+
+func TestParseConfig(t *testing.T) {
+	for i, test := range configTests {
+		cfg, err := ParseConfig(test.in)
+		if err != nil {
+			t.Errorf("test %d:%s failed: %v", i, test.in, err)
+			continue
+		}
+
+		if cfg != test.cfg {
+			t.Errorf("test %d:\ninput:\n  %s\n wrong config, want:\n  %v\ngot:\n  %v",
+				i, test.in, test.cfg, cfg)
+			continue
+		}
+	}
+}
+
+var configTestsInvalid = []string{
+	"sftp://host:dir",
+}
+
+func TestParseConfigInvalid(t *testing.T) {
+	for i, test := range configTestsInvalid {
+		_, err := ParseConfig(test)
+		if err == nil {
+			t.Errorf("test %d: invalid config %s did not return an error", i, test)
+			continue
+		}
+	}
+}

internal/backend/sftp/doc.go

@@ -0,0 +1,3 @@
+// Package sftp implements repository storage in a directory on a remote server
+// via the sftp protocol.
+package sftp

internal/backend/sftp/layout_test.go

@@ -0,0 +1,87 @@
+package sftp_test
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"restic"
+	"restic/backend/sftp"
+	. "restic/test"
+	"testing"
+)
+
+func TestLayout(t *testing.T) {
+	if sftpServer == "" {
+		t.Skip("sftp server binary not available")
+	}
+
+	path, cleanup := TempDir(t)
+	defer cleanup()
+
+	var tests = []struct {
+		filename        string
+		layout          string
+		failureExpected bool
+		datafiles       map[string]bool
+	}{
+		{"repo-layout-default.tar.gz", "", false, map[string]bool{
+			"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
+			"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
+			"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
+		}},
+		{"repo-layout-s3legacy.tar.gz", "", false, map[string]bool{
+			"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
+			"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
+			"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
+		}},
+	}
+
+	for _, test := range tests {
+		t.Run(test.filename, func(t *testing.T) {
+			SetupTarTestFixture(t, path, filepath.Join("..", "testdata", test.filename))
+
+			repo := filepath.Join(path, "repo")
+			be, err := sftp.Open(sftp.Config{
+				Command: fmt.Sprintf("%q -e", sftpServer),
+				Path:    repo,
+				Layout:  test.layout,
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if be == nil {
+				t.Fatalf("Open() returned nil but no error")
+			}
+
+			datafiles := make(map[string]bool)
+			for id := range be.List(context.TODO(), restic.DataFile) {
+				datafiles[id] = false
+			}
+
+			if len(datafiles) == 0 {
+				t.Errorf("List() returned zero data files")
+			}
+
+			for id := range test.datafiles {
+				if _, ok := datafiles[id]; !ok {
+					t.Errorf("datafile with id %v not found", id)
+				}
+
+				datafiles[id] = true
+			}
+
+			for id, v := range datafiles {
+				if !v {
+					t.Errorf("unexpected id %v found", id)
+				}
+			}
+
+			if err = be.Close(); err != nil {
+				t.Errorf("Close() returned error %v", err)
+			}
+
+			RemoveAll(t, filepath.Join(path, "repo"))
+		})
+	}
+}

internal/backend/sftp/sftp.go

@@ -0,0 +1,497 @@
+package sftp
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path"
+	"restic"
+	"strings"
+	"time"
+
+	"restic/errors"
+
+	"restic/backend"
+	"restic/debug"
+
+	"github.com/pkg/sftp"
+)
+
+// SFTP is a backend in a directory accessed via SFTP.
+type SFTP struct {
+	c *sftp.Client
+	p string
+
+	cmd    *exec.Cmd
+	result <-chan error
+
+	backend.Layout
+	Config
+}
+
+var _ restic.Backend = &SFTP{}
+
+const defaultLayout = "default"
+
+func startClient(program string, args ...string) (*SFTP, error) {
+	debug.Log("start client %v %v", program, args)
+	// Connect to a remote host and request the sftp subsystem via the 'ssh'
+	// command.  This assumes that passwordless login is correctly configured.
+	cmd := exec.Command(program, args...)
+
+	// prefix the errors with the program name
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		return nil, errors.Wrap(err, "cmd.StderrPipe")
+	}
+
+	go func() {
+		sc := bufio.NewScanner(stderr)
+		for sc.Scan() {
+			fmt.Fprintf(os.Stderr, "subprocess %v: %v\n", program, sc.Text())
+		}
+	}()
+
+	// ignore signals sent to the parent (e.g. SIGINT)
+	cmd.SysProcAttr = ignoreSigIntProcAttr()
+
+	// get stdin and stdout
+	wr, err := cmd.StdinPipe()
+	if err != nil {
+		return nil, errors.Wrap(err, "cmd.StdinPipe")
+	}
+	rd, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, errors.Wrap(err, "cmd.StdoutPipe")
+	}
+
+	// start the process
+	if err := cmd.Start(); err != nil {
+		return nil, errors.Wrap(err, "cmd.Start")
+	}
+
+	// wait in a different goroutine
+	ch := make(chan error, 1)
+	go func() {
+		err := cmd.Wait()
+		debug.Log("ssh command exited, err %v", err)
+		ch <- errors.Wrap(err, "cmd.Wait")
+	}()
+
+	// open the SFTP session
+	client, err := sftp.NewClientPipe(rd, wr)
+	if err != nil {
+		return nil, errors.Errorf("unable to start the sftp session, error: %v", err)
+	}
+
+	return &SFTP{c: client, cmd: cmd, result: ch}, nil
+}
+
+// clientError returns an error if the client has exited. Otherwise, nil is
+// returned immediately.
+func (r *SFTP) clientError() error {
+	select {
+	case err := <-r.result:
+		debug.Log("client has exited with err %v", err)
+		return err
+	default:
+	}
+
+	return nil
+}
+
+// Open opens an sftp backend as described by the config by running
+// "ssh" with the appropriate arguments (or cfg.Command, if set).
+func Open(cfg Config) (*SFTP, error) {
+	debug.Log("open backend with config %#v", cfg)
+
+	cmd, args, err := buildSSHCommand(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	sftp, err := startClient(cmd, args...)
+	if err != nil {
+		debug.Log("unable to start program: %v", err)
+		return nil, err
+	}
+
+	sftp.Layout, err = backend.ParseLayout(sftp, cfg.Layout, defaultLayout, cfg.Path)
+	if err != nil {
+		return nil, err
+	}
+
+	debug.Log("layout: %v\n", sftp.Layout)
+
+	if err := sftp.checkDataSubdirs(); err != nil {
+		debug.Log("checkDataSubdirs returned %v", err)
+		return nil, err
+	}
+
+	sftp.Config = cfg
+	sftp.p = cfg.Path
+	return sftp, nil
+}
+
+func (r *SFTP) checkDataSubdirs() error {
+	datadir := r.Dirname(restic.Handle{Type: restic.DataFile})
+
+	// check if all paths for data/ exist
+	entries, err := r.c.ReadDir(datadir)
+	if err != nil {
+		return err
+	}
+
+	subdirs := make(map[string]struct{}, len(entries))
+	for _, entry := range entries {
+		subdirs[entry.Name()] = struct{}{}
+	}
+
+	for i := 0; i < 256; i++ {
+		subdir := fmt.Sprintf("%02x", i)
+		if _, ok := subdirs[subdir]; !ok {
+			debug.Log("subdir %v is missing, creating", subdir)
+			err := r.mkdirAll(path.Join(datadir, subdir), backend.Modes.Dir)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (r *SFTP) mkdirAllDataSubdirs() error {
+	for _, d := range r.Paths() {
+		err := r.mkdirAll(d, backend.Modes.Dir)
+		debug.Log("mkdirAll %v -> %v", d, err)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Join combines path components with slashes (according to the sftp spec).
+func (r *SFTP) Join(p ...string) string {
+	return path.Join(p...)
+}
+
+// ReadDir returns the entries for a directory.
+func (r *SFTP) ReadDir(dir string) ([]os.FileInfo, error) {
+	return r.c.ReadDir(dir)
+}
+
+// IsNotExist returns true if the error is caused by a not existing file.
+func (r *SFTP) IsNotExist(err error) bool {
+	if os.IsNotExist(err) {
+		return true
+	}
+
+	statusError, ok := err.(*sftp.StatusError)
+	if !ok {
+		return false
+	}
+
+	return statusError.Error() == `sftp: "No such file" (SSH_FX_NO_SUCH_FILE)`
+}
+
+func buildSSHCommand(cfg Config) (cmd string, args []string, err error) {
+	if cfg.Command != "" {
+		return SplitShellArgs(cfg.Command)
+	}
+
+	cmd = "ssh"
+
+	hostport := strings.Split(cfg.Host, ":")
+	args = []string{hostport[0]}
+	if len(hostport) > 1 {
+		args = append(args, "-p", hostport[1])
+	}
+	if cfg.User != "" {
+		args = append(args, "-l")
+		args = append(args, cfg.User)
+	}
+	args = append(args, "-s")
+	args = append(args, "sftp")
+	return cmd, args, nil
+}
+
+// Create creates an sftp backend as described by the config by running
+// "ssh" with the appropriate arguments (or cfg.Command, if set).
+func Create(cfg Config) (*SFTP, error) {
+	cmd, args, err := buildSSHCommand(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	sftp, err := startClient(cmd, args...)
+	if err != nil {
+		debug.Log("unable to start program: %v", err)
+		return nil, err
+	}
+
+	sftp.Layout, err = backend.ParseLayout(sftp, cfg.Layout, defaultLayout, cfg.Path)
+	if err != nil {
+		return nil, err
+	}
+
+	// test if config file already exists
+	_, err = sftp.c.Lstat(Join(cfg.Path, backend.Paths.Config))
+	if err == nil {
+		return nil, errors.New("config file already exists")
+	}
+
+	// create paths for data and refs
+	if err = sftp.mkdirAllDataSubdirs(); err != nil {
+		return nil, err
+	}
+
+	err = sftp.Close()
+	if err != nil {
+		return nil, errors.Wrap(err, "Close")
+	}
+
+	// open backend
+	return Open(cfg)
+}
+
+// Location returns this backend's location (the directory name).
+func (r *SFTP) Location() string {
+	return r.p
+}
+
+func (r *SFTP) mkdirAll(dir string, mode os.FileMode) error {
+	// check if directory already exists
+	fi, err := r.c.Lstat(dir)
+	if err == nil {
+		if fi.IsDir() {
+			return nil
+		}
+
+		return errors.Errorf("mkdirAll(%s): entry exists but is not a directory", dir)
+	}
+
+	// create parent directories
+	errMkdirAll := r.mkdirAll(path.Dir(dir), backend.Modes.Dir)
+
+	// create directory
+	errMkdir := r.c.Mkdir(dir)
+
+	// test if directory was created successfully
+	fi, err = r.c.Lstat(dir)
+	if err != nil {
+		// return previous errors
+		return errors.Errorf("mkdirAll(%s): unable to create directories: %v, %v", dir, errMkdirAll, errMkdir)
+	}
+
+	if !fi.IsDir() {
+		return errors.Errorf("mkdirAll(%s): entry exists but is not a directory", dir)
+	}
+
+	// set mode
+	return r.c.Chmod(dir, mode)
+}
+
+// Join joins the given paths and cleans them afterwards. This always uses
+// forward slashes, which is required by sftp.
+func Join(parts ...string) string {
+	return path.Clean(path.Join(parts...))
+}
+
+// Save stores data in the backend at the handle.
+func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) {
+	debug.Log("Save %v", h)
+	if err := r.clientError(); err != nil {
+		return err
+	}
+
+	if err := h.Valid(); err != nil {
+		return err
+	}
+
+	filename := r.Filename(h)
+
+	// create new file
+	f, err := r.c.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY)
+	if r.IsNotExist(errors.Cause(err)) {
+		// create the locks dir, then try again
+		err = r.mkdirAll(r.Dirname(h), backend.Modes.Dir)
+		if err != nil {
+			return errors.Wrap(err, "MkdirAll")
+		}
+
+		return r.Save(ctx, h, rd)
+	}
+
+	if err != nil {
+		return errors.Wrap(err, "OpenFile")
+	}
+
+	// save data
+	_, err = io.Copy(f, rd)
+	if err != nil {
+		_ = f.Close()
+		return errors.Wrap(err, "Write")
+	}
+
+	err = f.Close()
+	if err != nil {
+		return errors.Wrap(err, "Close")
+	}
+
+	// set mode to read-only
+	fi, err := r.c.Lstat(filename)
+	if err != nil {
+		return errors.Wrap(err, "Lstat")
+	}
+
+	err = r.c.Chmod(filename, fi.Mode()&os.FileMode(^uint32(0222)))
+	return errors.Wrap(err, "Chmod")
+}
+
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (r *SFTP) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	f, err := r.c.Open(r.Filename(h))
+	if err != nil {
+		return nil, err
+	}
+
+	if offset > 0 {
+		_, err = f.Seek(offset, 0)
+		if err != nil {
+			_ = f.Close()
+			return nil, err
+		}
+	}
+
+	if length > 0 {
+		return backend.LimitReadCloser(f, int64(length)), nil
+	}
+
+	return f, nil
+}
+
+// Stat returns information about a blob.
+func (r *SFTP) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
+	debug.Log("Stat(%v)", h)
+	if err := r.clientError(); err != nil {
+		return restic.FileInfo{}, err
+	}
+
+	if err := h.Valid(); err != nil {
+		return restic.FileInfo{}, err
+	}
+
+	fi, err := r.c.Lstat(r.Filename(h))
+	if err != nil {
+		return restic.FileInfo{}, errors.Wrap(err, "Lstat")
+	}
+
+	return restic.FileInfo{Size: fi.Size()}, nil
+}
+
+// Test returns true if a blob of the given type and name exists in the backend.
+func (r *SFTP) Test(ctx context.Context, h restic.Handle) (bool, error) {
+	debug.Log("Test(%v)", h)
+	if err := r.clientError(); err != nil {
+		return false, err
+	}
+
+	_, err := r.c.Lstat(r.Filename(h))
+	if os.IsNotExist(errors.Cause(err)) {
+		return false, nil
+	}
+
+	if err != nil {
+		return false, errors.Wrap(err, "Lstat")
+	}
+
+	return true, nil
+}
+
+// Remove removes the content stored at name.
+func (r *SFTP) Remove(ctx context.Context, h restic.Handle) error {
+	debug.Log("Remove(%v)", h)
+	if err := r.clientError(); err != nil {
+		return err
+	}
+
+	return r.c.Remove(r.Filename(h))
+}
+
+// List returns a channel that yields all names of blobs of type t. A
+// goroutine is started for this. If the channel done is closed, sending
+// stops.
+func (r *SFTP) List(ctx context.Context, t restic.FileType) <-chan string {
+	debug.Log("List %v", t)
+
+	ch := make(chan string)
+
+	go func() {
+		defer close(ch)
+
+		walker := r.c.Walk(r.Basedir(t))
+		for walker.Step() {
+			if walker.Err() != nil {
+				continue
+			}
+
+			if !walker.Stat().Mode().IsRegular() {
+				continue
+			}
+
+			select {
+			case ch <- path.Base(walker.Path()):
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	return ch
+
+}
+
+var closeTimeout = 2 * time.Second
+
+// Close closes the sftp connection and terminates the underlying command.
+func (r *SFTP) Close() error {
+	debug.Log("")
+	if r == nil {
+		return nil
+	}
+
+	err := r.c.Close()
+	debug.Log("Close returned error %v", err)
+
+	// wait for closeTimeout before killing the process
+	select {
+	case err := <-r.result:
+		return err
+	case <-time.After(closeTimeout):
+	}
+
+	if err := r.cmd.Process.Kill(); err != nil {
+		return err
+	}
+
+	// get the error, but ignore it
+	<-r.result
+	return nil
+}

internal/backend/sftp/sftp_test.go

@@ -0,0 +1,95 @@
+package sftp_test
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"restic"
+	"restic/backend/sftp"
+	"restic/backend/test"
+	"restic/errors"
+	"strings"
+	"testing"
+
+	. "restic/test"
+)
+
+func findSFTPServerBinary() string {
+	for _, dir := range strings.Split(TestSFTPPath, ":") {
+		testpath := filepath.Join(dir, "sftp-server")
+		_, err := os.Stat(testpath)
+		if !os.IsNotExist(errors.Cause(err)) {
+			return testpath
+		}
+	}
+
+	return ""
+}
+
+var sftpServer = findSFTPServerBinary()
+
+func newTestSuite(t testing.TB) *test.Suite {
+	return &test.Suite{
+		// NewConfig returns a config for a new temporary backend that will be used in tests.
+		NewConfig: func() (interface{}, error) {
+			dir, err := ioutil.TempDir(TestTempDir, "restic-test-sftp-")
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			t.Logf("create new backend at %v", dir)
+
+			cfg := sftp.Config{
+				Path:    dir,
+				Command: fmt.Sprintf("%q -e", sftpServer),
+			}
+			return cfg, nil
+		},
+
+		// CreateFn is a function that creates a temporary repository for the tests.
+		Create: func(config interface{}) (restic.Backend, error) {
+			cfg := config.(sftp.Config)
+			return sftp.Create(cfg)
+		},
+
+		// OpenFn is a function that opens a previously created temporary repository.
+		Open: func(config interface{}) (restic.Backend, error) {
+			cfg := config.(sftp.Config)
+			return sftp.Open(cfg)
+		},
+
+		// CleanupFn removes data created during the tests.
+		Cleanup: func(config interface{}) error {
+			cfg := config.(sftp.Config)
+			if !TestCleanupTempDirs {
+				t.Logf("leaving test backend dir at %v", cfg.Path)
+			}
+
+			RemoveAll(t, cfg.Path)
+			return nil
+		},
+	}
+}
+
+func TestBackendSFTP(t *testing.T) {
+	defer func() {
+		if t.Skipped() {
+			SkipDisallowed(t, "restic/backend/sftp.TestBackendSFTP")
+		}
+	}()
+
+	if sftpServer == "" {
+		t.Skip("sftp server binary not found")
+	}
+
+	newTestSuite(t).RunTests(t)
+}
+
+func BenchmarkBackendSFTP(t *testing.B) {
+	if sftpServer == "" {
+		t.Skip("sftp server binary not found")
+	}
+
+	newTestSuite(t).RunBenchmarks(t)
+}

internal/backend/sftp/sftp_unix.go

@@ -0,0 +1,13 @@
+// +build !windows
+
+package sftp
+
+import (
+	"syscall"
+)
+
+// ignoreSigIntProcAttr returns a syscall.SysProcAttr that
+// disables SIGINT on parent.
+func ignoreSigIntProcAttr() *syscall.SysProcAttr {
+	return &syscall.SysProcAttr{Setsid: true}
+}

internal/backend/sftp/sftp_windows.go

@@ -0,0 +1,11 @@
+package sftp
+
+import (
+	"syscall"
+)
+
+// ignoreSigIntProcAttr returns a default syscall.SysProcAttr
+// on Windows.
+func ignoreSigIntProcAttr() *syscall.SysProcAttr {
+	return &syscall.SysProcAttr{}
+}

internal/backend/sftp/split.go

@@ -0,0 +1,77 @@
+package sftp
+
+import (
+	"restic/errors"
+	"unicode"
+)
+
+// shellSplitter splits a command string into separater arguments. It supports
+// single and double quoted strings.
+type shellSplitter struct {
+	quote    rune
+	lastChar rune
+}
+
+func (s *shellSplitter) isSplitChar(c rune) bool {
+	// only test for quotes if the last char was not a backslash
+	if s.lastChar != '\\' {
+
+		// quote ended
+		if s.quote != 0 && c == s.quote {
+			s.quote = 0
+			return true
+		}
+
+		// quote starts
+		if s.quote == 0 && (c == '"' || c == '\'') {
+			s.quote = c
+			return true
+		}
+	}
+
+	s.lastChar = c
+
+	// within quote
+	if s.quote != 0 {
+		return false
+	}
+
+	// outside quote
+	return c == '\\' || unicode.IsSpace(c)
+}
+
+// SplitShellArgs returns the list of arguments from a shell command string.
+func SplitShellArgs(data string) (cmd string, args []string, err error) {
+	s := &shellSplitter{}
+
+	// derived from strings.SplitFunc
+	fieldStart := -1 // Set to -1 when looking for start of field.
+	for i, rune := range data {
+		if s.isSplitChar(rune) {
+			if fieldStart >= 0 {
+				args = append(args, data[fieldStart:i])
+				fieldStart = -1
+			}
+		} else if fieldStart == -1 {
+			fieldStart = i
+		}
+	}
+	if fieldStart >= 0 { // Last field might end at EOF.
+		args = append(args, data[fieldStart:])
+	}
+
+	switch s.quote {
+	case '\'':
+		return "", nil, errors.New("single-quoted string not terminated")
+	case '"':
+		return "", nil, errors.New("double-quoted string not terminated")
+	}
+
+	if len(args) == 0 {
+		return "", nil, errors.New("command string is empty")
+	}
+
+	cmd, args = args[0], args[1:]
+
+	return cmd, args, nil
+}

internal/backend/sftp/split_test.go

@@ -0,0 +1,115 @@
+package sftp
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestShellSplitter(t *testing.T) {
+	var tests = []struct {
+		data string
+		cmd  string
+		args []string
+	}{
+		{
+			`foo`,
+			"foo", []string{},
+		},
+		{
+			`'foo'`,
+			"foo", []string{},
+		},
+		{
+			`foo bar baz`,
+			"foo", []string{"bar", "baz"},
+		},
+		{
+			`foo 'bar' baz`,
+			"foo", []string{"bar", "baz"},
+		},
+		{
+			`'bar box' baz`,
+			"bar box", []string{"baz"},
+		},
+		{
+			`"bar 'box'" baz`,
+			"bar 'box'", []string{"baz"},
+		},
+		{
+			`'bar "box"' baz`,
+			`bar "box"`, []string{"baz"},
+		},
+		{
+			`\"bar box baz`,
+			`"bar`, []string{"box", "baz"},
+		},
+		{
+			`"bar/foo/x" "box baz"`,
+			"bar/foo/x", []string{"box baz"},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run("", func(t *testing.T) {
+			cmd, args, err := SplitShellArgs(test.data)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if cmd != test.cmd {
+				t.Fatalf("wrong cmd returned, want:\n  %#v\ngot:\n  %#v",
+					test.cmd, cmd)
+			}
+
+			if !reflect.DeepEqual(args, test.args) {
+				t.Fatalf("wrong args returned, want:\n  %#v\ngot:\n  %#v",
+					test.args, args)
+			}
+		})
+	}
+}
+
+func TestShellSplitterInvalid(t *testing.T) {
+	var tests = []struct {
+		data string
+		err  string
+	}{
+		{
+			"foo'",
+			"single-quoted string not terminated",
+		},
+		{
+			`foo"`,
+			"double-quoted string not terminated",
+		},
+		{
+			"foo 'bar",
+			"single-quoted string not terminated",
+		},
+		{
+			`foo "bar`,
+			"double-quoted string not terminated",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run("", func(t *testing.T) {
+			cmd, args, err := SplitShellArgs(test.data)
+			if err == nil {
+				t.Fatalf("expected error not found: %v", test.err)
+			}
+
+			if err.Error() != test.err {
+				t.Fatalf("expected error not found, want:\n  %q\ngot:\n  %q", test.err, err.Error())
+			}
+
+			if cmd != "" {
+				t.Fatalf("splitter returned cmd from invalid data: %v", cmd)
+			}
+
+			if len(args) > 0 {
+				t.Fatalf("splitter returned fields from invalid data: %v", args)
+			}
+		})
+	}
+}

internal/backend/sftp/sshcmd_test.go

@@ -0,0 +1,52 @@
+package sftp
+
+import (
+	"reflect"
+	"testing"
+)
+
+var sshcmdTests = []struct {
+	cfg  Config
+	cmd  string
+	args []string
+}{
+	{
+		Config{User: "user", Host: "host", Path: "dir/subdir"},
+		"ssh",
+		[]string{"host", "-l", "user", "-s", "sftp"},
+	},
+	{
+		Config{Host: "host", Path: "dir/subdir"},
+		"ssh",
+		[]string{"host", "-s", "sftp"},
+	},
+	{
+		Config{Host: "host:10022", Path: "/dir/subdir"},
+		"ssh",
+		[]string{"host", "-p", "10022", "-s", "sftp"},
+	},
+	{
+		Config{User: "user", Host: "host:10022", Path: "/dir/subdir"},
+		"ssh",
+		[]string{"host", "-p", "10022", "-l", "user", "-s", "sftp"},
+	},
+}
+
+func TestBuildSSHCommand(t *testing.T) {
+	for _, test := range sshcmdTests {
+		t.Run("", func(t *testing.T) {
+			cmd, args, err := buildSSHCommand(test.cfg)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if cmd != test.cmd {
+				t.Fatalf("cmd: want %v, got %v", test.cmd, cmd)
+			}
+
+			if !reflect.DeepEqual(test.args, args) {
+				t.Fatalf("wrong args, want:\n  %v\ngot:\n  %v", test.args, args)
+			}
+		})
+	}
+}

internal/backend/swift/config.go

@@ -0,0 +1,109 @@
+package swift
+
+import (
+	"os"
+	"restic/errors"
+	"restic/options"
+	"strings"
+)
+
+// Config contains basic configuration needed to specify swift location for a swift server
+type Config struct {
+	UserName     string
+	Domain       string
+	APIKey       string
+	AuthURL      string
+	Region       string
+	Tenant       string
+	TenantID     string
+	TenantDomain string
+	TrustID      string
+
+	StorageURL string
+	AuthToken  string
+
+	Container              string
+	Prefix                 string
+	DefaultContainerPolicy string
+
+	Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
+}
+
+func init() {
+	options.Register("swift", Config{})
+}
+
+// NewConfig returns a new config with the default values filled in.
+func NewConfig() Config {
+	return Config{
+		Connections: 5,
+	}
+}
+
+// ParseConfig parses the string s and extract swift's container name and prefix.
+func ParseConfig(s string) (interface{}, error) {
+	data := strings.SplitN(s, ":", 3)
+	if len(data) != 3 {
+		return nil, errors.New("invalid URL, expected: swift:container-name:/[prefix]")
+	}
+
+	scheme, container, prefix := data[0], data[1], data[2]
+	if scheme != "swift" {
+		return nil, errors.Errorf("unexpected prefix: %s", data[0])
+	}
+
+	if len(prefix) == 0 {
+		return nil, errors.Errorf("prefix is empty")
+	}
+
+	if prefix[0] != '/' {
+		return nil, errors.Errorf("prefix does not start with slash (/)")
+	}
+	prefix = prefix[1:]
+
+	cfg := NewConfig()
+	cfg.Container = container
+	cfg.Prefix = prefix
+
+	return cfg, nil
+}
+
+// ApplyEnvironment saves values from the environment to the config.
+func ApplyEnvironment(prefix string, cfg interface{}) error {
+	c := cfg.(*Config)
+	for _, val := range []struct {
+		s   *string
+		env string
+	}{
+		// v2/v3 specific
+		{&c.UserName, prefix + "OS_USERNAME"},
+		{&c.APIKey, prefix + "OS_PASSWORD"},
+		{&c.Region, prefix + "OS_REGION_NAME"},
+		{&c.AuthURL, prefix + "OS_AUTH_URL"},
+
+		// v3 specific
+		{&c.Domain, prefix + "OS_USER_DOMAIN_NAME"},
+		{&c.Tenant, prefix + "OS_PROJECT_NAME"},
+		{&c.TenantDomain, prefix + "OS_PROJECT_DOMAIN_NAME"},
+
+		// v2 specific
+		{&c.TenantID, prefix + "OS_TENANT_ID"},
+		{&c.Tenant, prefix + "OS_TENANT_NAME"},
+
+		// v1 specific
+		{&c.AuthURL, prefix + "ST_AUTH"},
+		{&c.UserName, prefix + "ST_USER"},
+		{&c.APIKey, prefix + "ST_KEY"},
+
+		// Manual authentication
+		{&c.StorageURL, prefix + "OS_STORAGE_URL"},
+		{&c.AuthToken, prefix + "OS_AUTH_TOKEN"},
+
+		{&c.DefaultContainerPolicy, prefix + "SWIFT_DEFAULT_CONTAINER_POLICY"},
+	} {
+		if *val.s == "" {
+			*val.s = os.Getenv(val.env)
+		}
+	}
+	return nil
+}

internal/backend/swift/config_test.go

@@ -0,0 +1,72 @@
+package swift
+
+import "testing"
+
+var configTests = []struct {
+	s   string
+	cfg Config
+}{
+	{
+		"swift:cnt1:/",
+		Config{
+			Container:   "cnt1",
+			Prefix:      "",
+			Connections: 5,
+		},
+	},
+	{
+		"swift:cnt2:/prefix",
+		Config{Container: "cnt2",
+			Prefix:      "prefix",
+			Connections: 5,
+		},
+	},
+	{
+		"swift:cnt3:/prefix/longer",
+		Config{Container: "cnt3",
+			Prefix:      "prefix/longer",
+			Connections: 5,
+		},
+	},
+}
+
+func TestParseConfig(t *testing.T) {
+	for _, test := range configTests {
+		t.Run("", func(t *testing.T) {
+			v, err := ParseConfig(test.s)
+			if err != nil {
+				t.Fatalf("parsing %q failed: %v", test.s, err)
+			}
+
+			cfg, ok := v.(Config)
+			if !ok {
+				t.Fatalf("wrong type returned, want Config, got %T", cfg)
+			}
+
+			if cfg != test.cfg {
+				t.Fatalf("wrong output for %q, want:\n  %#v\ngot:\n  %#v",
+					test.s, test.cfg, cfg)
+			}
+		})
+	}
+}
+
+var configTestsInvalid = []string{
+	"swift://hostname/container",
+	"swift:////",
+	"swift://",
+	"swift:////prefix",
+	"swift:container",
+	"swift:container:",
+	"swift:container/prefix",
+}
+
+func TestParseConfigInvalid(t *testing.T) {
+	for i, test := range configTestsInvalid {
+		_, err := ParseConfig(test)
+		if err == nil {
+			t.Errorf("test %d: invalid config %s did not return an error", i, test)
+			continue
+		}
+	}
+}

internal/backend/swift/swift.go

@@ -0,0 +1,321 @@
+package swift
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"path"
+	"path/filepath"
+	"restic"
+	"restic/backend"
+	"restic/debug"
+	"restic/errors"
+	"strings"
+	"time"
+
+	"github.com/ncw/swift"
+)
+
+const connLimit = 10
+
+// beSwift is a backend which stores the data on a swift endpoint.
+type beSwift struct {
+	conn      *swift.Connection
+	sem       *backend.Semaphore
+	container string // Container name
+	prefix    string // Prefix of object names in the container
+	backend.Layout
+}
+
+// ensure statically that *beSwift implements restic.Backend.
+var _ restic.Backend = &beSwift{}
+
+// Open opens the swift backend at a container in region. The container is
+// created if it does not exist yet.
+func Open(cfg Config) (restic.Backend, error) {
+	debug.Log("config %#v", cfg)
+
+	sem, err := backend.NewSemaphore(cfg.Connections)
+	if err != nil {
+		return nil, err
+	}
+
+	be := &beSwift{
+		conn: &swift.Connection{
+			UserName:       cfg.UserName,
+			Domain:         cfg.Domain,
+			ApiKey:         cfg.APIKey,
+			AuthUrl:        cfg.AuthURL,
+			Region:         cfg.Region,
+			Tenant:         cfg.Tenant,
+			TenantId:       cfg.TenantID,
+			TenantDomain:   cfg.TenantDomain,
+			TrustId:        cfg.TrustID,
+			StorageUrl:     cfg.StorageURL,
+			AuthToken:      cfg.AuthToken,
+			ConnectTimeout: time.Minute,
+			Timeout:        time.Minute,
+
+			Transport: backend.Transport(),
+		},
+		sem:       sem,
+		container: cfg.Container,
+		prefix:    cfg.Prefix,
+		Layout: &backend.DefaultLayout{
+			Path: cfg.Prefix,
+			Join: path.Join,
+		},
+	}
+
+	// Authenticate if needed
+	if !be.conn.Authenticated() {
+		if err := be.conn.Authenticate(); err != nil {
+			return nil, errors.Wrap(err, "conn.Authenticate")
+		}
+	}
+
+	// Ensure container exists
+	switch _, _, err := be.conn.Container(be.container); err {
+	case nil:
+		// Container exists
+
+	case swift.ContainerNotFound:
+		err = be.createContainer(cfg.DefaultContainerPolicy)
+		if err != nil {
+			return nil, errors.Wrap(err, "beSwift.createContainer")
+		}
+
+	default:
+		return nil, errors.Wrap(err, "conn.Container")
+	}
+
+	return be, nil
+}
+
+func (be *beSwift) createContainer(policy string) error {
+	var h swift.Headers
+	if policy != "" {
+		h = swift.Headers{
+			"X-Storage-Policy": policy,
+		}
+	}
+
+	return be.conn.ContainerCreate(be.container, h)
+}
+
+// Location returns this backend's location (the container name).
+func (be *beSwift) Location() string {
+	return be.container
+}
+
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (be *beSwift) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if length < 0 {
+		return nil, errors.Errorf("invalid length %d", length)
+	}
+
+	objName := be.Filename(h)
+
+	be.sem.GetToken()
+	defer func() {
+		be.sem.ReleaseToken()
+	}()
+
+	headers := swift.Headers{}
+	if offset > 0 {
+		headers["Range"] = fmt.Sprintf("bytes=%d-", offset)
+	}
+
+	if length > 0 {
+		headers["Range"] = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1)
+	}
+
+	if _, ok := headers["Range"]; ok {
+		debug.Log("Load(%v) send range %v", h, headers["Range"])
+	}
+
+	obj, _, err := be.conn.ObjectOpen(be.container, objName, false, headers)
+	if err != nil {
+		debug.Log("  err %v", err)
+		return nil, errors.Wrap(err, "conn.ObjectOpen")
+	}
+
+	return obj, nil
+}
+
+// Save stores data in the backend at the handle.
+func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) {
+	if err = h.Valid(); err != nil {
+		return err
+	}
+
+	objName := be.Filename(h)
+
+	debug.Log("Save %v at %v", h, objName)
+
+	// Check key does not already exist
+	switch _, _, err = be.conn.Object(be.container, objName); err {
+	case nil:
+		debug.Log("%v already exists", h)
+		return errors.New("key already exists")
+
+	case swift.ObjectNotFound:
+		// Ok, that's what we want
+
+	default:
+		return errors.Wrap(err, "conn.Object")
+	}
+
+	be.sem.GetToken()
+	defer func() {
+		be.sem.ReleaseToken()
+	}()
+
+	encoding := "binary/octet-stream"
+
+	debug.Log("PutObject(%v, %v, %v)", be.container, objName, encoding)
+	_, err = be.conn.ObjectPut(be.container, objName, rd, true, "", encoding, nil)
+	debug.Log("%v, err %#v", objName, err)
+
+	return errors.Wrap(err, "client.PutObject")
+}
+
+// Stat returns information about a blob.
+func (be *beSwift) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
+	debug.Log("%v", h)
+
+	objName := be.Filename(h)
+
+	obj, _, err := be.conn.Object(be.container, objName)
+	if err != nil {
+		debug.Log("Object() err %v", err)
+		return restic.FileInfo{}, errors.Wrap(err, "conn.Object")
+	}
+
+	return restic.FileInfo{Size: obj.Bytes}, nil
+}
+
+// Test returns true if a blob of the given type and name exists in the backend.
+func (be *beSwift) Test(ctx context.Context, h restic.Handle) (bool, error) {
+	objName := be.Filename(h)
+	switch _, _, err := be.conn.Object(be.container, objName); err {
+	case nil:
+		return true, nil
+
+	case swift.ObjectNotFound:
+		return false, nil
+
+	default:
+		return false, errors.Wrap(err, "conn.Object")
+	}
+}
+
+// Remove removes the blob with the given name and type.
+func (be *beSwift) Remove(ctx context.Context, h restic.Handle) error {
+	objName := be.Filename(h)
+	err := be.conn.ObjectDelete(be.container, objName)
+	debug.Log("Remove(%v) -> err %v", h, err)
+	return errors.Wrap(err, "conn.ObjectDelete")
+}
+
+// List returns a channel that yields all names of blobs of type t. A
+// goroutine is started for this. If the channel done is closed, sending
+// stops.
+func (be *beSwift) List(ctx context.Context, t restic.FileType) <-chan string {
+	debug.Log("listing %v", t)
+	ch := make(chan string)
+
+	prefix := be.Filename(restic.Handle{Type: t}) + "/"
+
+	go func() {
+		defer close(ch)
+
+		err := be.conn.ObjectsWalk(be.container, &swift.ObjectsOpts{Prefix: prefix},
+			func(opts *swift.ObjectsOpts) (interface{}, error) {
+				newObjects, err := be.conn.ObjectNames(be.container, opts)
+				if err != nil {
+					return nil, errors.Wrap(err, "conn.ObjectNames")
+				}
+				for _, obj := range newObjects {
+					m := filepath.Base(strings.TrimPrefix(obj, prefix))
+					if m == "" {
+						continue
+					}
+
+					select {
+					case ch <- m:
+					case <-ctx.Done():
+						return nil, io.EOF
+					}
+				}
+				return newObjects, nil
+			})
+
+		if err != nil {
+			debug.Log("ObjectsWalk returned error: %v", err)
+		}
+	}()
+
+	return ch
+}
+
+// Remove keys for a specified backend type.
+func (be *beSwift) removeKeys(ctx context.Context, t restic.FileType) error {
+	for key := range be.List(ctx, t) {
+		err := be.Remove(ctx, restic.Handle{Type: t, Name: key})
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// IsNotExist returns true if the error is caused by a not existing file.
+func (be *beSwift) IsNotExist(err error) bool {
+	if e, ok := errors.Cause(err).(*swift.Error); ok {
+		return e.StatusCode == http.StatusNotFound
+	}
+
+	return false
+}
+
+// Delete removes all restic objects in the container.
+// It will not remove the container itself.
+func (be *beSwift) Delete(ctx context.Context) error {
+	alltypes := []restic.FileType{
+		restic.DataFile,
+		restic.KeyFile,
+		restic.LockFile,
+		restic.SnapshotFile,
+		restic.IndexFile}
+
+	for _, t := range alltypes {
+		err := be.removeKeys(ctx, t)
+		if err != nil {
+			return nil
+		}
+	}
+
+	err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile})
+	if err != nil && !be.IsNotExist(err) {
+		return err
+	}
+
+	return nil
+}
+
+// Close does nothing
+func (be *beSwift) Close() error { return nil }

internal/backend/swift/swift_test.go

@@ -0,0 +1,111 @@
+package swift_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"restic"
+	"testing"
+	"time"
+
+	"restic/errors"
+	. "restic/test"
+
+	"restic/backend/swift"
+	"restic/backend/test"
+)
+
+func newSwiftTestSuite(t testing.TB) *test.Suite {
+	return &test.Suite{
+		// do not use excessive data
+		MinimalData: true,
+
+		// wait for removals for at least 60s
+		WaitForDelayedRemoval: 60 * time.Second,
+
+		// NewConfig returns a config for a new temporary backend that will be used in tests.
+		NewConfig: func() (interface{}, error) {
+			swiftcfg, err := swift.ParseConfig(os.Getenv("RESTIC_TEST_SWIFT"))
+			if err != nil {
+				return nil, err
+			}
+
+			cfg := swiftcfg.(swift.Config)
+			if err = swift.ApplyEnvironment("RESTIC_TEST_", &cfg); err != nil {
+				return nil, err
+			}
+			cfg.Prefix += fmt.Sprintf("/test-%d", time.Now().UnixNano())
+			t.Logf("using prefix %v", cfg.Prefix)
+			return cfg, nil
+		},
+
+		// CreateFn is a function that creates a temporary repository for the tests.
+		Create: func(config interface{}) (restic.Backend, error) {
+			cfg := config.(swift.Config)
+
+			be, err := swift.Open(cfg)
+			if err != nil {
+				return nil, err
+			}
+
+			exists, err := be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile})
+			if err != nil {
+				return nil, err
+			}
+
+			if exists {
+				return nil, errors.New("config already exists")
+			}
+
+			return be, nil
+		},
+
+		// OpenFn is a function that opens a previously created temporary repository.
+		Open: func(config interface{}) (restic.Backend, error) {
+			cfg := config.(swift.Config)
+			return swift.Open(cfg)
+		},
+
+		// CleanupFn removes data created during the tests.
+		Cleanup: func(config interface{}) error {
+			cfg := config.(swift.Config)
+
+			be, err := swift.Open(cfg)
+			if err != nil {
+				return err
+			}
+
+			if err := be.(restic.Deleter).Delete(context.TODO()); err != nil {
+				return err
+			}
+
+			return nil
+		},
+	}
+}
+
+func TestBackendSwift(t *testing.T) {
+	defer func() {
+		if t.Skipped() {
+			SkipDisallowed(t, "restic/backend/swift.TestBackendSwift")
+		}
+	}()
+
+	if os.Getenv("RESTIC_TEST_SWIFT") == "" {
+		t.Skip("RESTIC_TEST_SWIFT unset, skipping test")
+		return
+	}
+
+	t.Logf("run tests")
+	newSwiftTestSuite(t).RunTests(t)
+}
+
+func BenchmarkBackendSwift(t *testing.B) {
+	if os.Getenv("RESTIC_TEST_SWIFT") == "" {
+		t.Skip("RESTIC_TEST_SWIFT unset, skipping test")
+		return
+	}
+
+	t.Logf("run tests")
+	newSwiftTestSuite(t).RunBenchmarks(t)
+}

internal/backend/test/benchmarks.go

@@ -0,0 +1,183 @@
+package test
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"restic"
+	"restic/test"
+	"testing"
+)
+
+func saveRandomFile(t testing.TB, be restic.Backend, length int) ([]byte, restic.Handle) {
+	data := test.Random(23, length)
+	id := restic.Hash(data)
+	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
+	if err := be.Save(context.TODO(), handle, bytes.NewReader(data)); err != nil {
+		t.Fatalf("Save() error: %+v", err)
+	}
+	return data, handle
+}
+
+func remove(t testing.TB, be restic.Backend, h restic.Handle) {
+	if err := be.Remove(context.TODO(), h); err != nil {
+		t.Fatalf("Remove() returned error: %v", err)
+	}
+}
+
+// BenchmarkLoadFile benchmarks the Load() method of a backend by
+// loading a complete file.
+func (s *Suite) BenchmarkLoadFile(t *testing.B) {
+	be := s.open(t)
+	defer s.close(t, be)
+
+	length := 1<<24 + 2123
+	data, handle := saveRandomFile(t, be, length)
+	defer remove(t, be, handle)
+
+	buf := make([]byte, length)
+
+	t.SetBytes(int64(length))
+	t.ResetTimer()
+
+	for i := 0; i < t.N; i++ {
+		rd, err := be.Load(context.TODO(), handle, 0, 0)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		n, err := io.ReadFull(rd, buf)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if err = rd.Close(); err != nil {
+			t.Fatalf("Close() returned error: %v", err)
+		}
+
+		if n != length {
+			t.Fatalf("wrong number of bytes read: want %v, got %v", length, n)
+		}
+
+		if !bytes.Equal(data, buf) {
+			t.Fatalf("wrong bytes returned")
+		}
+	}
+}
+
+// BenchmarkLoadPartialFile benchmarks the Load() method of a backend by
+// loading the remainder of a file starting at a given offset.
+func (s *Suite) BenchmarkLoadPartialFile(t *testing.B) {
+	be := s.open(t)
+	defer s.close(t, be)
+
+	datalength := 1<<24 + 2123
+	data, handle := saveRandomFile(t, be, datalength)
+	defer remove(t, be, handle)
+
+	testLength := datalength/4 + 555
+
+	buf := make([]byte, testLength)
+
+	t.SetBytes(int64(testLength))
+	t.ResetTimer()
+
+	for i := 0; i < t.N; i++ {
+		rd, err := be.Load(context.TODO(), handle, testLength, 0)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		n, err := io.ReadFull(rd, buf)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if err = rd.Close(); err != nil {
+			t.Fatalf("Close() returned error: %v", err)
+		}
+
+		if n != testLength {
+			t.Fatalf("wrong number of bytes read: want %v, got %v", testLength, n)
+		}
+
+		if !bytes.Equal(data[:testLength], buf) {
+			t.Fatalf("wrong bytes returned")
+		}
+
+	}
+}
+
+// BenchmarkLoadPartialFileOffset benchmarks the Load() method of a
+// backend by loading a number of bytes of a file starting at a given offset.
+func (s *Suite) BenchmarkLoadPartialFileOffset(t *testing.B) {
+	be := s.open(t)
+	defer s.close(t, be)
+
+	datalength := 1<<24 + 2123
+	data, handle := saveRandomFile(t, be, datalength)
+	defer remove(t, be, handle)
+
+	testLength := datalength/4 + 555
+	testOffset := 8273
+
+	buf := make([]byte, testLength)
+
+	t.SetBytes(int64(testLength))
+	t.ResetTimer()
+
+	for i := 0; i < t.N; i++ {
+		rd, err := be.Load(context.TODO(), handle, testLength, int64(testOffset))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		n, err := io.ReadFull(rd, buf)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if err = rd.Close(); err != nil {
+			t.Fatalf("Close() returned error: %v", err)
+		}
+
+		if n != testLength {
+			t.Fatalf("wrong number of bytes read: want %v, got %v", testLength, n)
+		}
+
+		if !bytes.Equal(data[testOffset:testOffset+testLength], buf) {
+			t.Fatalf("wrong bytes returned")
+		}
+
+	}
+}
+
+// BenchmarkSave benchmarks the Save() method of a backend.
+func (s *Suite) BenchmarkSave(t *testing.B) {
+	be := s.open(t)
+	defer s.close(t, be)
+
+	length := 1<<24 + 2123
+	data := test.Random(23, length)
+	id := restic.Hash(data)
+	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
+
+	rd := bytes.NewReader(data)
+
+	t.SetBytes(int64(length))
+	t.ResetTimer()
+
+	for i := 0; i < t.N; i++ {
+		if _, err := rd.Seek(0, 0); err != nil {
+			t.Fatal(err)
+		}
+
+		if err := be.Save(context.TODO(), handle, rd); err != nil {
+			t.Fatal(err)
+		}
+
+		if err := be.Remove(context.TODO(), handle); err != nil {
+			t.Fatal(err)
+		}
+	}
+}

internal/backend/test/doc.go

@@ -0,0 +1,42 @@
+// Package test contains a test suite with benchmarks for restic backends.
+//
+// Overview
+//
+// For the test suite to work a few functions need to be implemented to create
+// new config, create a backend, open it and run cleanup tasks afterwards. The
+// Suite struct has fields for each function.
+//
+// So for a new backend, a Suite needs to be built with callback functions,
+// then the methods RunTests() and RunBenchmarks() can be used to run the
+// individual tests and benchmarks as subtests/subbenchmarks.
+//
+// Example
+//
+// Assuming a *Suite is returned by newTestSuite(), the tests and benchmarks
+// can be run like this:
+//   func newTestSuite(t testing.TB) *test.Suite {
+//   	return &test.Suite{
+//   		Create: func(cfg interface{}) (restic.Backend, error) {
+//   			[...]
+//   		},
+//   		[...]
+//   	}
+//   }
+//
+//   func TestSuiteBackendMem(t *testing.T) {
+//   	newTestSuite(t).RunTests(t)
+//   }
+//
+//   func BenchmarkSuiteBackendMem(b *testing.B) {
+//   	newTestSuite(b).RunBenchmarks(b)
+//   }
+//
+// The functions are run in alphabetical order.
+//
+// Add new tests
+//
+// A new test or benchmark can be added by implementing a method on *Suite
+// with the name starting with "Test" and a single *testing.T parameter for
+// test. For benchmarks, the name must start with "Benchmark" and the parameter
+// is a *testing.B
+package test

internal/backend/test/suite.go

@@ -0,0 +1,181 @@
+package test
+
+import (
+	"reflect"
+	"restic"
+	"restic/test"
+	"strings"
+	"testing"
+	"time"
+)
+
+// Suite implements a test suite for restic backends.
+type Suite struct {
+	// Config should be used to configure the backend.
+	Config interface{}
+
+	// NewConfig returns a config for a new temporary backend that will be used in tests.
+	NewConfig func() (interface{}, error)
+
+	// CreateFn is a function that creates a temporary repository for the tests.
+	Create func(cfg interface{}) (restic.Backend, error)
+
+	// OpenFn is a function that opens a previously created temporary repository.
+	Open func(cfg interface{}) (restic.Backend, error)
+
+	// CleanupFn removes data created during the tests.
+	Cleanup func(cfg interface{}) error
+
+	// MinimalData instructs the tests to not use excessive data.
+	MinimalData bool
+
+	// WaitForDelayedRemoval is set to a non-zero value to instruct the test
+	// suite to wait for this amount of time until a file that was removed
+	// really disappeared.
+	WaitForDelayedRemoval time.Duration
+}
+
+// RunTests executes all defined tests as subtests of t.
+func (s *Suite) RunTests(t *testing.T) {
+	var err error
+	s.Config, err = s.NewConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// test create/open functions first
+	be := s.create(t)
+	s.close(t, be)
+
+	for _, test := range s.testFuncs(t) {
+		t.Run(test.Name, test.Fn)
+	}
+
+	if !test.TestCleanupTempDirs {
+		t.Logf("not cleaning up backend")
+		return
+	}
+
+	if err = s.Cleanup(s.Config); err != nil {
+		t.Fatal(err)
+	}
+}
+
+type testFunction struct {
+	Name string
+	Fn   func(*testing.T)
+}
+
+func (s *Suite) testFuncs(t testing.TB) (funcs []testFunction) {
+	tpe := reflect.TypeOf(s)
+	v := reflect.ValueOf(s)
+
+	for i := 0; i < tpe.NumMethod(); i++ {
+		methodType := tpe.Method(i)
+		name := methodType.Name
+
+		// discard functions which do not have the right name
+		if !strings.HasPrefix(name, "Test") {
+			continue
+		}
+
+		iface := v.Method(i).Interface()
+		f, ok := iface.(func(*testing.T))
+		if !ok {
+			t.Logf("warning: function %v of *Suite has the wrong signature for a test function\nwant: func(*testing.T),\nhave: %T",
+				name, iface)
+			continue
+		}
+
+		funcs = append(funcs, testFunction{
+			Name: name,
+			Fn:   f,
+		})
+	}
+
+	return funcs
+}
+
+type benchmarkFunction struct {
+	Name string
+	Fn   func(*testing.B)
+}
+
+func (s *Suite) benchmarkFuncs(t testing.TB) (funcs []benchmarkFunction) {
+	tpe := reflect.TypeOf(s)
+	v := reflect.ValueOf(s)
+
+	for i := 0; i < tpe.NumMethod(); i++ {
+		methodType := tpe.Method(i)
+		name := methodType.Name
+
+		// discard functions which do not have the right name
+		if !strings.HasPrefix(name, "Benchmark") {
+			continue
+		}
+
+		iface := v.Method(i).Interface()
+		f, ok := iface.(func(*testing.B))
+		if !ok {
+			t.Logf("warning: function %v of *Suite has the wrong signature for a test function\nwant: func(*testing.T),\nhave: %T",
+				name, iface)
+			continue
+		}
+
+		funcs = append(funcs, benchmarkFunction{
+			Name: name,
+			Fn:   f,
+		})
+	}
+
+	return funcs
+}
+
+// RunBenchmarks executes all defined benchmarks as subtests of b.
+func (s *Suite) RunBenchmarks(b *testing.B) {
+	var err error
+	s.Config, err = s.NewConfig()
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// test create/open functions first
+	be := s.create(b)
+	s.close(b, be)
+
+	for _, test := range s.benchmarkFuncs(b) {
+		b.Run(test.Name, test.Fn)
+	}
+
+	if !test.TestCleanupTempDirs {
+		b.Logf("not cleaning up backend")
+		return
+	}
+
+	if err = s.Cleanup(s.Config); err != nil {
+		b.Fatal(err)
+	}
+}
+
+func (s *Suite) create(t testing.TB) restic.Backend {
+	be, err := s.Create(s.Config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return be
+}
+
+func (s *Suite) open(t testing.TB) restic.Backend {
+	be, err := s.Open(s.Config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return be
+}
+
+func (s *Suite) close(t testing.TB, be restic.Backend) {
+	err := be.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}

internal/backend/test/tests.go

@@ -0,0 +1,638 @@
+package test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"reflect"
+	"restic"
+	"restic/errors"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"restic/test"
+
+	"restic/backend"
+)
+
+func seedRand(t testing.TB) {
+	seed := time.Now().UnixNano()
+	rand.Seed(seed)
+	t.Logf("rand initialized with seed %d", seed)
+}
+
+// TestCreateWithConfig tests that creating a backend in a location which already
+// has a config file fails.
+func (s *Suite) TestCreateWithConfig(t *testing.T) {
+	b := s.open(t)
+	defer s.close(t, b)
+
+	// remove a config if present
+	cfgHandle := restic.Handle{Type: restic.ConfigFile}
+	cfgPresent, err := b.Test(context.TODO(), cfgHandle)
+	if err != nil {
+		t.Fatalf("unable to test for config: %+v", err)
+	}
+
+	if cfgPresent {
+		remove(t, b, cfgHandle)
+	}
+
+	// save a config
+	store(t, b, restic.ConfigFile, []byte("test config"))
+
+	// now create the backend again, this must fail
+	_, err = s.Create(s.Config)
+	if err == nil {
+		t.Fatalf("expected error not found for creating a backend with an existing config file")
+	}
+
+	// remove config
+	err = b.Remove(context.TODO(), restic.Handle{Type: restic.ConfigFile, Name: ""})
+	if err != nil {
+		t.Fatalf("unexpected error removing config: %+v", err)
+	}
+}
+
+// TestLocation tests that a location string is returned.
+func (s *Suite) TestLocation(t *testing.T) {
+	b := s.open(t)
+	defer s.close(t, b)
+
+	l := b.Location()
+	if l == "" {
+		t.Fatalf("invalid location string %q", l)
+	}
+}
+
+// TestConfig saves and loads a config from the backend.
+func (s *Suite) TestConfig(t *testing.T) {
+	b := s.open(t)
+	defer s.close(t, b)
+
+	var testString = "Config"
+
+	// create config and read it back
+	_, err := backend.LoadAll(context.TODO(), b, restic.Handle{Type: restic.ConfigFile})
+	if err == nil {
+		t.Fatalf("did not get expected error for non-existing config")
+	}
+
+	err = b.Save(context.TODO(), restic.Handle{Type: restic.ConfigFile}, strings.NewReader(testString))
+	if err != nil {
+		t.Fatalf("Save() error: %+v", err)
+	}
+
+	// try accessing the config with different names, should all return the
+	// same config
+	for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} {
+		h := restic.Handle{Type: restic.ConfigFile, Name: name}
+		buf, err := backend.LoadAll(context.TODO(), b, h)
+		if err != nil {
+			t.Fatalf("unable to read config with name %q: %+v", name, err)
+		}
+
+		if string(buf) != testString {
+			t.Fatalf("wrong data returned, want %q, got %q", testString, string(buf))
+		}
+	}
+
+	// remove the config
+	remove(t, b, restic.Handle{Type: restic.ConfigFile})
+}
+
+// TestLoad tests the backend's Load function.
+func (s *Suite) TestLoad(t *testing.T) {
+	seedRand(t)
+
+	b := s.open(t)
+	defer s.close(t, b)
+
+	rd, err := b.Load(context.TODO(), restic.Handle{}, 0, 0)
+	if err == nil {
+		t.Fatalf("Load() did not return an error for invalid handle")
+	}
+	if rd != nil {
+		_ = rd.Close()
+	}
+
+	err = testLoad(b, restic.Handle{Type: restic.DataFile, Name: "foobar"}, 0, 0)
+	if err == nil {
+		t.Fatalf("Load() did not return an error for non-existing blob")
+	}
+
+	length := rand.Intn(1<<24) + 2000
+
+	data := test.Random(23, length)
+	id := restic.Hash(data)
+
+	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
+	err = b.Save(context.TODO(), handle, bytes.NewReader(data))
+	if err != nil {
+		t.Fatalf("Save() error: %+v", err)
+	}
+
+	t.Logf("saved %d bytes as %v", length, handle)
+
+	rd, err = b.Load(context.TODO(), handle, 100, -1)
+	if err == nil {
+		t.Fatalf("Load() returned no error for negative offset!")
+	}
+
+	if rd != nil {
+		t.Fatalf("Load() returned a non-nil reader for negative offset!")
+	}
+
+	loadTests := 50
+	if s.MinimalData {
+		loadTests = 10
+	}
+
+	for i := 0; i < loadTests; i++ {
+		l := rand.Intn(length + 2000)
+		o := rand.Intn(length + 2000)
+
+		d := data
+		if o < len(d) {
+			d = d[o:]
+		} else {
+			t.Logf("offset == length, skipping test")
+			continue
+		}
+
+		getlen := l
+		if l >= len(d) && rand.Float32() >= 0.5 {
+			getlen = 0
+		}
+
+		if l > 0 && l < len(d) {
+			d = d[:l]
+		}
+
+		rd, err := b.Load(context.TODO(), handle, getlen, int64(o))
+		if err != nil {
+			t.Logf("Load, l %v, o %v, len(d) %v, getlen %v", l, o, len(d), getlen)
+			t.Errorf("Load(%d, %d) returned unexpected error: %+v", l, o, err)
+			continue
+		}
+
+		buf, err := ioutil.ReadAll(rd)
+		if err != nil {
+			t.Logf("Load, l %v, o %v, len(d) %v, getlen %v", l, o, len(d), getlen)
+			t.Errorf("Load(%d, %d) ReadAll() returned unexpected error: %+v", l, o, err)
+			if err = rd.Close(); err != nil {
+				t.Errorf("Load(%d, %d) rd.Close() returned error: %+v", l, o, err)
+			}
+			continue
+		}
+
+		if l == 0 && len(buf) != len(d) {
+			t.Logf("Load, l %v, o %v, len(d) %v, getlen %v", l, o, len(d), getlen)
+			t.Errorf("Load(%d, %d) wrong number of bytes read: want %d, got %d", l, o, len(d), len(buf))
+			if err = rd.Close(); err != nil {
+				t.Errorf("Load(%d, %d) rd.Close() returned error: %+v", l, o, err)
+			}
+			continue
+		}
+
+		if l > 0 && l <= len(d) && len(buf) != l {
+			t.Logf("Load, l %v, o %v, len(d) %v, getlen %v", l, o, len(d), getlen)
+			t.Errorf("Load(%d, %d) wrong number of bytes read: want %d, got %d", l, o, l, len(buf))
+			if err = rd.Close(); err != nil {
+				t.Errorf("Load(%d, %d) rd.Close() returned error: %+v", l, o, err)
+			}
+			continue
+		}
+
+		if l > len(d) && len(buf) != len(d) {
+			t.Logf("Load, l %v, o %v, len(d) %v, getlen %v", l, o, len(d), getlen)
+			t.Errorf("Load(%d, %d) wrong number of bytes read for overlong read: want %d, got %d", l, o, l, len(buf))
+			if err = rd.Close(); err != nil {
+				t.Errorf("Load(%d, %d) rd.Close() returned error: %+v", l, o, err)
+			}
+			continue
+		}
+
+		if !bytes.Equal(buf, d) {
+			t.Logf("Load, l %v, o %v, len(d) %v, getlen %v", l, o, len(d), getlen)
+			t.Errorf("Load(%d, %d) returned wrong bytes", l, o)
+			if err = rd.Close(); err != nil {
+				t.Errorf("Load(%d, %d) rd.Close() returned error: %+v", l, o, err)
+			}
+			continue
+		}
+
+		err = rd.Close()
+		if err != nil {
+			t.Logf("Load, l %v, o %v, len(d) %v, getlen %v", l, o, len(d), getlen)
+			t.Errorf("Load(%d, %d) rd.Close() returned unexpected error: %+v", l, o, err)
+			continue
+		}
+	}
+
+	test.OK(t, b.Remove(context.TODO(), handle))
+}
+
+type errorCloser struct {
+	io.Reader
+	l int
+	t testing.TB
+}
+
+func (ec errorCloser) Close() error {
+	ec.t.Error("forbidden method close was called")
+	return errors.New("forbidden method close was called")
+}
+
+func (ec errorCloser) Len() int {
+	return ec.l
+}
+
+// TestSave tests saving data in the backend.
+func (s *Suite) TestSave(t *testing.T) {
+	seedRand(t)
+
+	b := s.open(t)
+	defer s.close(t, b)
+	var id restic.ID
+
+	saveTests := 10
+	if s.MinimalData {
+		saveTests = 2
+	}
+
+	for i := 0; i < saveTests; i++ {
+		length := rand.Intn(1<<23) + 200000
+		data := test.Random(23, length)
+		// use the first 32 byte as the ID
+		copy(id[:], data)
+
+		h := restic.Handle{
+			Type: restic.DataFile,
+			Name: fmt.Sprintf("%s-%d", id, i),
+		}
+		err := b.Save(context.TODO(), h, bytes.NewReader(data))
+		test.OK(t, err)
+
+		buf, err := backend.LoadAll(context.TODO(), b, h)
+		test.OK(t, err)
+		if len(buf) != len(data) {
+			t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf))
+		}
+
+		if !bytes.Equal(buf, data) {
+			t.Fatalf("data not equal")
+		}
+
+		fi, err := b.Stat(context.TODO(), h)
+		test.OK(t, err)
+
+		if fi.Size != int64(len(data)) {
+			t.Fatalf("Stat() returned different size, want %q, got %d", len(data), fi.Size)
+		}
+
+		err = b.Remove(context.TODO(), h)
+		if err != nil {
+			t.Fatalf("error removing item: %+v", err)
+		}
+	}
+
+	// test saving from a tempfile
+	tmpfile, err := ioutil.TempFile("", "restic-backend-save-test-")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	length := rand.Intn(1<<23) + 200000
+	data := test.Random(23, length)
+	copy(id[:], data)
+
+	if _, err = tmpfile.Write(data); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err = tmpfile.Seek(0, io.SeekStart); err != nil {
+		t.Fatal(err)
+	}
+
+	h := restic.Handle{Type: restic.DataFile, Name: id.String()}
+
+	// wrap the tempfile in an errorCloser, so we can detect if the backend
+	// closes the reader
+	err = b.Save(context.TODO(), h, errorCloser{t: t, l: length, Reader: tmpfile})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = delayedRemove(t, b, s.WaitForDelayedRemoval, h)
+	if err != nil {
+		t.Fatalf("error removing item: %+v", err)
+	}
+
+	// try again directly with the temp file
+	if _, err = tmpfile.Seek(588, io.SeekStart); err != nil {
+		t.Fatal(err)
+	}
+
+	err = b.Save(context.TODO(), h, tmpfile)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err = tmpfile.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	err = b.Remove(context.TODO(), h)
+	if err != nil {
+		t.Fatalf("error removing item: %+v", err)
+	}
+
+	if err = os.Remove(tmpfile.Name()); err != nil {
+		t.Fatal(err)
+	}
+}
+
+var filenameTests = []struct {
+	name string
+	data string
+}{
+	{"1dfc6bc0f06cb255889e9ea7860a5753e8eb9665c9a96627971171b444e3113e", "x"},
+	{"f00b4r", "foobar"},
+	{
+		"1dfc6bc0f06cb255889e9ea7860a5753e8eb9665c9a96627971171b444e3113e4bf8f2d9144cc5420a80f04a4880ad6155fc58903a4fb6457c476c43541dcaa6-5",
+		"foobar content of data blob",
+	},
+}
+
+// TestSaveFilenames tests saving data with various file names in the backend.
+func (s *Suite) TestSaveFilenames(t *testing.T) {
+	b := s.open(t)
+	defer s.close(t, b)
+
+	for i, test := range filenameTests {
+		h := restic.Handle{Name: test.name, Type: restic.DataFile}
+		err := b.Save(context.TODO(), h, strings.NewReader(test.data))
+		if err != nil {
+			t.Errorf("test %d failed: Save() returned %+v", i, err)
+			continue
+		}
+
+		buf, err := backend.LoadAll(context.TODO(), b, h)
+		if err != nil {
+			t.Errorf("test %d failed: Load() returned %+v", i, err)
+			continue
+		}
+
+		if !bytes.Equal(buf, []byte(test.data)) {
+			t.Errorf("test %d: returned wrong bytes", i)
+		}
+
+		err = b.Remove(context.TODO(), h)
+		if err != nil {
+			t.Errorf("test %d failed: Remove() returned %+v", i, err)
+			continue
+		}
+	}
+}
+
+var testStrings = []struct {
+	id   string
+	data string
+}{
+	{"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", "foobar"},
+	{"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"},
+	{"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", "foo/bar"},
+	{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"},
+}
+
+func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) restic.Handle {
+	id := restic.Hash(data)
+	h := restic.Handle{Name: id.String(), Type: tpe}
+	err := b.Save(context.TODO(), h, bytes.NewReader(data))
+	test.OK(t, err)
+	return h
+}
+
+// testLoad loads a blob (but discards its contents).
+func testLoad(b restic.Backend, h restic.Handle, length int, offset int64) error {
+	rd, err := b.Load(context.TODO(), h, 0, 0)
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(ioutil.Discard, rd)
+	cerr := rd.Close()
+	if err == nil {
+		err = cerr
+	}
+	return err
+}
+
+func delayedRemove(t testing.TB, be restic.Backend, maxwait time.Duration, handles ...restic.Handle) error {
+	// Some backend (swift, I'm looking at you) may implement delayed
+	// removal of data. Let's wait a bit if this happens.
+
+	for _, h := range handles {
+		err := be.Remove(context.TODO(), h)
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, h := range handles {
+		start := time.Now()
+		attempt := 0
+		var found bool
+		var err error
+		for time.Since(start) <= maxwait {
+			found, err = be.Test(context.TODO(), h)
+			if err != nil {
+				return err
+			}
+
+			if !found {
+				break
+			}
+
+			time.Sleep(2 * time.Second)
+			attempt++
+		}
+
+		if found {
+			t.Fatalf("removed blob %v still present after %v (%d attempts)", h, time.Since(start), attempt)
+		}
+	}
+
+	return nil
+}
+
+func delayedList(t testing.TB, b restic.Backend, tpe restic.FileType, max int, maxwait time.Duration) restic.IDs {
+	list := restic.NewIDSet()
+	start := time.Now()
+	for i := 0; i < max; i++ {
+		for s := range b.List(context.TODO(), tpe) {
+			id := restic.TestParseID(s)
+			list.Insert(id)
+		}
+		if len(list) < max && time.Since(start) < maxwait {
+			time.Sleep(500 * time.Millisecond)
+		}
+	}
+
+	return list.List()
+}
+
+// TestBackend tests all functions of the backend.
+func (s *Suite) TestBackend(t *testing.T) {
+	b := s.open(t)
+	defer s.close(t, b)
+
+	for _, tpe := range []restic.FileType{
+		restic.DataFile, restic.KeyFile, restic.LockFile,
+		restic.SnapshotFile, restic.IndexFile,
+	} {
+		// detect non-existing files
+		for _, ts := range testStrings {
+			id, err := restic.ParseID(ts.id)
+			test.OK(t, err)
+
+			// test if blob is already in repository
+			h := restic.Handle{Type: tpe, Name: id.String()}
+			ret, err := b.Test(context.TODO(), h)
+			test.OK(t, err)
+			test.Assert(t, !ret, "blob was found to exist before creating")
+
+			// try to stat a not existing blob
+			_, err = b.Stat(context.TODO(), h)
+			test.Assert(t, err != nil, "blob data could be extracted before creation")
+
+			// try to read not existing blob
+			err = testLoad(b, h, 0, 0)
+			test.Assert(t, err != nil, "blob could be read before creation")
+
+			// try to get string out, should fail
+			ret, err = b.Test(context.TODO(), h)
+			test.OK(t, err)
+			test.Assert(t, !ret, "id %q was found (but should not have)", ts.id)
+		}
+
+		// add files
+		for _, ts := range testStrings {
+			store(t, b, tpe, []byte(ts.data))
+
+			// test Load()
+			h := restic.Handle{Type: tpe, Name: ts.id}
+			buf, err := backend.LoadAll(context.TODO(), b, h)
+			test.OK(t, err)
+			test.Equals(t, ts.data, string(buf))
+
+			// try to read it out with an offset and a length
+			start := 1
+			end := len(ts.data) - 2
+			length := end - start
+
+			buf2 := make([]byte, length)
+			rd, err := b.Load(context.TODO(), h, len(buf2), int64(start))
+			test.OK(t, err)
+			n, err := io.ReadFull(rd, buf2)
+			test.OK(t, err)
+			test.Equals(t, len(buf2), n)
+
+			remaining, err := io.Copy(ioutil.Discard, rd)
+			test.OK(t, err)
+			test.Equals(t, int64(0), remaining)
+
+			test.OK(t, rd.Close())
+
+			test.Equals(t, ts.data[start:end], string(buf2))
+		}
+
+		// test adding the first file again
+		ts := testStrings[0]
+
+		// create blob
+		h := restic.Handle{Type: tpe, Name: ts.id}
+		err := b.Save(context.TODO(), h, strings.NewReader(ts.data))
+		test.Assert(t, err != nil, "expected error for %v, got %v", h, err)
+
+		// remove and recreate
+		err = delayedRemove(t, b, s.WaitForDelayedRemoval, h)
+		test.OK(t, err)
+
+		// test that the blob is gone
+		ok, err := b.Test(context.TODO(), h)
+		test.OK(t, err)
+		test.Assert(t, !ok, "removed blob still present")
+
+		// create blob
+		err = b.Save(context.TODO(), h, strings.NewReader(ts.data))
+		test.OK(t, err)
+
+		// list items
+		IDs := restic.IDs{}
+
+		for _, ts := range testStrings {
+			id, err := restic.ParseID(ts.id)
+			test.OK(t, err)
+			IDs = append(IDs, id)
+		}
+
+		list := delayedList(t, b, tpe, len(IDs), s.WaitForDelayedRemoval)
+		if len(IDs) != len(list) {
+			t.Fatalf("wrong number of IDs returned: want %d, got %d", len(IDs), len(list))
+		}
+
+		sort.Sort(IDs)
+		sort.Sort(list)
+
+		if !reflect.DeepEqual(IDs, list) {
+			t.Fatalf("lists aren't equal, want:\n  %v\n  got:\n%v\n", IDs, list)
+		}
+
+		// remove content if requested
+		if test.TestCleanupTempDirs {
+			var handles []restic.Handle
+			for _, ts := range testStrings {
+				id, err := restic.ParseID(ts.id)
+				test.OK(t, err)
+
+				h := restic.Handle{Type: tpe, Name: id.String()}
+
+				found, err := b.Test(context.TODO(), h)
+				test.OK(t, err)
+				test.Assert(t, found, fmt.Sprintf("id %q not found", id))
+
+				handles = append(handles, h)
+			}
+
+			test.OK(t, delayedRemove(t, b, s.WaitForDelayedRemoval, handles...))
+		}
+	}
+}
+
+// TestDelete tests the Delete function.
+func (s *Suite) TestDelete(t *testing.T) {
+	if !test.TestCleanupTempDirs {
+		t.Skipf("not removing backend, TestCleanupTempDirs is false")
+	}
+
+	b := s.open(t)
+	defer s.close(t, b)
+
+	be, ok := b.(restic.Deleter)
+	if !ok {
+		return
+	}
+
+	err := be.Delete(context.TODO())
+	if err != nil {
+		t.Fatalf("error deleting backend: %+v", err)
+	}
+}

internal/backend/test/tests_test.go

@@ -0,0 +1,67 @@
+package test_test
+
+import (
+	"context"
+	"restic"
+	"restic/errors"
+	"testing"
+
+	"restic/backend/mem"
+	"restic/backend/test"
+)
+
+//go:generate go run generate_test_list.go
+
+type memConfig struct {
+	be restic.Backend
+}
+
+func newTestSuite(t testing.TB) *test.Suite {
+	return &test.Suite{
+		// NewConfig returns a config for a new temporary backend that will be used in tests.
+		NewConfig: func() (interface{}, error) {
+			return &memConfig{}, nil
+		},
+
+		// CreateFn is a function that creates a temporary repository for the tests.
+		Create: func(cfg interface{}) (restic.Backend, error) {
+			c := cfg.(*memConfig)
+			if c.be != nil {
+				ok, err := c.be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile})
+				if err != nil {
+					return nil, err
+				}
+
+				if ok {
+					return nil, errors.New("config already exists")
+				}
+			}
+
+			c.be = mem.New()
+			return c.be, nil
+		},
+
+		// OpenFn is a function that opens a previously created temporary repository.
+		Open: func(cfg interface{}) (restic.Backend, error) {
+			c := cfg.(*memConfig)
+			if c.be == nil {
+				c.be = mem.New()
+			}
+			return c.be, nil
+		},
+
+		// CleanupFn removes data created during the tests.
+		Cleanup: func(cfg interface{}) error {
+			// no cleanup needed
+			return nil
+		},
+	}
+}
+
+func TestSuiteBackendMem(t *testing.T) {
+	newTestSuite(t).RunTests(t)
+}
+
+func BenchmarkSuiteBackendMem(b *testing.B) {
+	newTestSuite(b).RunBenchmarks(b)
+}

internal/backend/utils.go

@@ -0,0 +1,47 @@
+package backend
+
+import (
+	"context"
+	"io"
+	"io/ioutil"
+	"restic"
+)
+
+// LoadAll reads all data stored in the backend for the handle.
+func LoadAll(ctx context.Context, be restic.Backend, h restic.Handle) (buf []byte, err error) {
+	rd, err := be.Load(ctx, h, 0, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		_, e := io.Copy(ioutil.Discard, rd)
+		if err == nil {
+			err = e
+		}
+
+		e = rd.Close()
+		if err == nil {
+			err = e
+		}
+	}()
+
+	return ioutil.ReadAll(rd)
+}
+
+// LimitedReadCloser wraps io.LimitedReader and exposes the Close() method.
+type LimitedReadCloser struct {
+	io.ReadCloser
+	io.Reader
+}
+
+// Read reads data from the limited reader.
+func (l *LimitedReadCloser) Read(p []byte) (int, error) {
+	return l.Reader.Read(p)
+}
+
+// LimitReadCloser returns a new reader wraps r in an io.LimitReader, but also
+// exposes the Close() method.
+func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser {
+	return &LimitedReadCloser{ReadCloser: r, Reader: io.LimitReader(r, n)}
+}

internal/backend/utils_test.go

@@ -0,0 +1,91 @@
+package backend_test
+
+import (
+	"bytes"
+	"context"
+	"math/rand"
+	"restic"
+	"testing"
+
+	"restic/backend"
+	"restic/backend/mem"
+	. "restic/test"
+)
+
+const KiB = 1 << 10
+const MiB = 1 << 20
+
+func TestLoadAll(t *testing.T) {
+	b := mem.New()
+
+	for i := 0; i < 20; i++ {
+		data := Random(23+i, rand.Intn(MiB)+500*KiB)
+
+		id := restic.Hash(data)
+		err := b.Save(context.TODO(), restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
+		OK(t, err)
+
+		buf, err := backend.LoadAll(context.TODO(), b, restic.Handle{Type: restic.DataFile, Name: id.String()})
+		OK(t, err)
+
+		if len(buf) != len(data) {
+			t.Errorf("length of returned buffer does not match, want %d, got %d", len(data), len(buf))
+			continue
+		}
+
+		if !bytes.Equal(buf, data) {
+			t.Errorf("wrong data returned")
+			continue
+		}
+	}
+}
+
+func TestLoadSmallBuffer(t *testing.T) {
+	b := mem.New()
+
+	for i := 0; i < 20; i++ {
+		data := Random(23+i, rand.Intn(MiB)+500*KiB)
+
+		id := restic.Hash(data)
+		err := b.Save(context.TODO(), restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
+		OK(t, err)
+
+		buf, err := backend.LoadAll(context.TODO(), b, restic.Handle{Type: restic.DataFile, Name: id.String()})
+		OK(t, err)
+
+		if len(buf) != len(data) {
+			t.Errorf("length of returned buffer does not match, want %d, got %d", len(data), len(buf))
+			continue
+		}
+
+		if !bytes.Equal(buf, data) {
+			t.Errorf("wrong data returned")
+			continue
+		}
+	}
+}
+
+func TestLoadLargeBuffer(t *testing.T) {
+	b := mem.New()
+
+	for i := 0; i < 20; i++ {
+		data := Random(23+i, rand.Intn(MiB)+500*KiB)
+
+		id := restic.Hash(data)
+		err := b.Save(context.TODO(), restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
+		OK(t, err)
+
+		buf, err := backend.LoadAll(context.TODO(), b, restic.Handle{Type: restic.DataFile, Name: id.String()})
+		OK(t, err)
+
+		if len(buf) != len(data) {
+			t.Errorf("length of returned buffer does not match, want %d, got %d", len(data), len(buf))
+			continue
+		}
+
+		if !bytes.Equal(buf, data) {
+			t.Errorf("wrong data returned")
+			continue
+		}
+	}
+}