Moves files

internal/backend/swift/config.go

@@ -0,0 +1,109 @@
+package swift
+
+import (
+	"os"
+	"restic/errors"
+	"restic/options"
+	"strings"
+)
+
+// Config contains basic configuration needed to specify swift location for a swift server
+type Config struct {
+	UserName     string
+	Domain       string
+	APIKey       string
+	AuthURL      string
+	Region       string
+	Tenant       string
+	TenantID     string
+	TenantDomain string
+	TrustID      string
+
+	StorageURL string
+	AuthToken  string
+
+	Container              string
+	Prefix                 string
+	DefaultContainerPolicy string
+
+	Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
+}
+
+func init() {
+	options.Register("swift", Config{})
+}
+
+// NewConfig returns a new config with the default values filled in.
+func NewConfig() Config {
+	return Config{
+		Connections: 5,
+	}
+}
+
+// ParseConfig parses the string s and extract swift's container name and prefix.
+func ParseConfig(s string) (interface{}, error) {
+	data := strings.SplitN(s, ":", 3)
+	if len(data) != 3 {
+		return nil, errors.New("invalid URL, expected: swift:container-name:/[prefix]")
+	}
+
+	scheme, container, prefix := data[0], data[1], data[2]
+	if scheme != "swift" {
+		return nil, errors.Errorf("unexpected prefix: %s", data[0])
+	}
+
+	if len(prefix) == 0 {
+		return nil, errors.Errorf("prefix is empty")
+	}
+
+	if prefix[0] != '/' {
+		return nil, errors.Errorf("prefix does not start with slash (/)")
+	}
+	prefix = prefix[1:]
+
+	cfg := NewConfig()
+	cfg.Container = container
+	cfg.Prefix = prefix
+
+	return cfg, nil
+}
+
+// ApplyEnvironment saves values from the environment to the config.
+func ApplyEnvironment(prefix string, cfg interface{}) error {
+	c := cfg.(*Config)
+	for _, val := range []struct {
+		s   *string
+		env string
+	}{
+		// v2/v3 specific
+		{&c.UserName, prefix + "OS_USERNAME"},
+		{&c.APIKey, prefix + "OS_PASSWORD"},
+		{&c.Region, prefix + "OS_REGION_NAME"},
+		{&c.AuthURL, prefix + "OS_AUTH_URL"},
+
+		// v3 specific
+		{&c.Domain, prefix + "OS_USER_DOMAIN_NAME"},
+		{&c.Tenant, prefix + "OS_PROJECT_NAME"},
+		{&c.TenantDomain, prefix + "OS_PROJECT_DOMAIN_NAME"},
+
+		// v2 specific
+		{&c.TenantID, prefix + "OS_TENANT_ID"},
+		{&c.Tenant, prefix + "OS_TENANT_NAME"},
+
+		// v1 specific
+		{&c.AuthURL, prefix + "ST_AUTH"},
+		{&c.UserName, prefix + "ST_USER"},
+		{&c.APIKey, prefix + "ST_KEY"},
+
+		// Manual authentication
+		{&c.StorageURL, prefix + "OS_STORAGE_URL"},
+		{&c.AuthToken, prefix + "OS_AUTH_TOKEN"},
+
+		{&c.DefaultContainerPolicy, prefix + "SWIFT_DEFAULT_CONTAINER_POLICY"},
+	} {
+		if *val.s == "" {
+			*val.s = os.Getenv(val.env)
+		}
+	}
+	return nil
+}

internal/backend/swift/config_test.go

@@ -0,0 +1,72 @@
+package swift
+
+import "testing"
+
+var configTests = []struct {
+	s   string
+	cfg Config
+}{
+	{
+		"swift:cnt1:/",
+		Config{
+			Container:   "cnt1",
+			Prefix:      "",
+			Connections: 5,
+		},
+	},
+	{
+		"swift:cnt2:/prefix",
+		Config{Container: "cnt2",
+			Prefix:      "prefix",
+			Connections: 5,
+		},
+	},
+	{
+		"swift:cnt3:/prefix/longer",
+		Config{Container: "cnt3",
+			Prefix:      "prefix/longer",
+			Connections: 5,
+		},
+	},
+}
+
+func TestParseConfig(t *testing.T) {
+	for _, test := range configTests {
+		t.Run("", func(t *testing.T) {
+			v, err := ParseConfig(test.s)
+			if err != nil {
+				t.Fatalf("parsing %q failed: %v", test.s, err)
+			}
+
+			cfg, ok := v.(Config)
+			if !ok {
+				t.Fatalf("wrong type returned, want Config, got %T", cfg)
+			}
+
+			if cfg != test.cfg {
+				t.Fatalf("wrong output for %q, want:\n  %#v\ngot:\n  %#v",
+					test.s, test.cfg, cfg)
+			}
+		})
+	}
+}
+
+var configTestsInvalid = []string{
+	"swift://hostname/container",
+	"swift:////",
+	"swift://",
+	"swift:////prefix",
+	"swift:container",
+	"swift:container:",
+	"swift:container/prefix",
+}
+
+func TestParseConfigInvalid(t *testing.T) {
+	for i, test := range configTestsInvalid {
+		_, err := ParseConfig(test)
+		if err == nil {
+			t.Errorf("test %d: invalid config %s did not return an error", i, test)
+			continue
+		}
+	}
+}

internal/backend/swift/swift.go

@@ -0,0 +1,321 @@
+package swift
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"path"
+	"path/filepath"
+	"restic"
+	"restic/backend"
+	"restic/debug"
+	"restic/errors"
+	"strings"
+	"time"
+
+	"github.com/ncw/swift"
+)
+
+const connLimit = 10
+
+// beSwift is a backend which stores the data on a swift endpoint.
+type beSwift struct {
+	conn      *swift.Connection
+	sem       *backend.Semaphore
+	container string // Container name
+	prefix    string // Prefix of object names in the container
+	backend.Layout
+}
+
+// ensure statically that *beSwift implements restic.Backend.
+var _ restic.Backend = &beSwift{}
+
+// Open opens the swift backend at a container in region. The container is
+// created if it does not exist yet.
+func Open(cfg Config) (restic.Backend, error) {
+	debug.Log("config %#v", cfg)
+
+	sem, err := backend.NewSemaphore(cfg.Connections)
+	if err != nil {
+		return nil, err
+	}
+
+	be := &beSwift{
+		conn: &swift.Connection{
+			UserName:       cfg.UserName,
+			Domain:         cfg.Domain,
+			ApiKey:         cfg.APIKey,
+			AuthUrl:        cfg.AuthURL,
+			Region:         cfg.Region,
+			Tenant:         cfg.Tenant,
+			TenantId:       cfg.TenantID,
+			TenantDomain:   cfg.TenantDomain,
+			TrustId:        cfg.TrustID,
+			StorageUrl:     cfg.StorageURL,
+			AuthToken:      cfg.AuthToken,
+			ConnectTimeout: time.Minute,
+			Timeout:        time.Minute,
+
+			Transport: backend.Transport(),
+		},
+		sem:       sem,
+		container: cfg.Container,
+		prefix:    cfg.Prefix,
+		Layout: &backend.DefaultLayout{
+			Path: cfg.Prefix,
+			Join: path.Join,
+		},
+	}
+
+	// Authenticate if needed
+	if !be.conn.Authenticated() {
+		if err := be.conn.Authenticate(); err != nil {
+			return nil, errors.Wrap(err, "conn.Authenticate")
+		}
+	}
+
+	// Ensure container exists
+	switch _, _, err := be.conn.Container(be.container); err {
+	case nil:
+		// Container exists
+
+	case swift.ContainerNotFound:
+		err = be.createContainer(cfg.DefaultContainerPolicy)
+		if err != nil {
+			return nil, errors.Wrap(err, "beSwift.createContainer")
+		}
+
+	default:
+		return nil, errors.Wrap(err, "conn.Container")
+	}
+
+	return be, nil
+}
+
+func (be *beSwift) createContainer(policy string) error {
+	var h swift.Headers
+	if policy != "" {
+		h = swift.Headers{
+			"X-Storage-Policy": policy,
+		}
+	}
+
+	return be.conn.ContainerCreate(be.container, h)
+}
+
+// Location returns this backend's location (the container name).
+func (be *beSwift) Location() string {
+	return be.container
+}
+
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (be *beSwift) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if length < 0 {
+		return nil, errors.Errorf("invalid length %d", length)
+	}
+
+	objName := be.Filename(h)
+
+	be.sem.GetToken()
+	defer func() {
+		be.sem.ReleaseToken()
+	}()
+
+	headers := swift.Headers{}
+	if offset > 0 {
+		headers["Range"] = fmt.Sprintf("bytes=%d-", offset)
+	}
+
+	if length > 0 {
+		headers["Range"] = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1)
+	}
+
+	if _, ok := headers["Range"]; ok {
+		debug.Log("Load(%v) send range %v", h, headers["Range"])
+	}
+
+	obj, _, err := be.conn.ObjectOpen(be.container, objName, false, headers)
+	if err != nil {
+		debug.Log("  err %v", err)
+		return nil, errors.Wrap(err, "conn.ObjectOpen")
+	}
+
+	return obj, nil
+}
+
+// Save stores data in the backend at the handle.
+func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) {
+	if err = h.Valid(); err != nil {
+		return err
+	}
+
+	objName := be.Filename(h)
+
+	debug.Log("Save %v at %v", h, objName)
+
+	// Check key does not already exist
+	switch _, _, err = be.conn.Object(be.container, objName); err {
+	case nil:
+		debug.Log("%v already exists", h)
+		return errors.New("key already exists")
+
+	case swift.ObjectNotFound:
+		// Ok, that's what we want
+
+	default:
+		return errors.Wrap(err, "conn.Object")
+	}
+
+	be.sem.GetToken()
+	defer func() {
+		be.sem.ReleaseToken()
+	}()
+
+	encoding := "binary/octet-stream"
+
+	debug.Log("PutObject(%v, %v, %v)", be.container, objName, encoding)
+	_, err = be.conn.ObjectPut(be.container, objName, rd, true, "", encoding, nil)
+	debug.Log("%v, err %#v", objName, err)
+
+	return errors.Wrap(err, "client.PutObject")
+}
+
+// Stat returns information about a blob.
+func (be *beSwift) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
+	debug.Log("%v", h)
+
+	objName := be.Filename(h)
+
+	obj, _, err := be.conn.Object(be.container, objName)
+	if err != nil {
+		debug.Log("Object() err %v", err)
+		return restic.FileInfo{}, errors.Wrap(err, "conn.Object")
+	}
+
+	return restic.FileInfo{Size: obj.Bytes}, nil
+}
+
+// Test returns true if a blob of the given type and name exists in the backend.
+func (be *beSwift) Test(ctx context.Context, h restic.Handle) (bool, error) {
+	objName := be.Filename(h)
+	switch _, _, err := be.conn.Object(be.container, objName); err {
+	case nil:
+		return true, nil
+
+	case swift.ObjectNotFound:
+		return false, nil
+
+	default:
+		return false, errors.Wrap(err, "conn.Object")
+	}
+}
+
+// Remove removes the blob with the given name and type.
+func (be *beSwift) Remove(ctx context.Context, h restic.Handle) error {
+	objName := be.Filename(h)
+	err := be.conn.ObjectDelete(be.container, objName)
+	debug.Log("Remove(%v) -> err %v", h, err)
+	return errors.Wrap(err, "conn.ObjectDelete")
+}
+
+// List returns a channel that yields all names of blobs of type t. A
+// goroutine is started for this. If the channel done is closed, sending
+// stops.
+func (be *beSwift) List(ctx context.Context, t restic.FileType) <-chan string {
+	debug.Log("listing %v", t)
+	ch := make(chan string)
+
+	prefix := be.Filename(restic.Handle{Type: t}) + "/"
+
+	go func() {
+		defer close(ch)
+
+		err := be.conn.ObjectsWalk(be.container, &swift.ObjectsOpts{Prefix: prefix},
+			func(opts *swift.ObjectsOpts) (interface{}, error) {
+				newObjects, err := be.conn.ObjectNames(be.container, opts)
+				if err != nil {
+					return nil, errors.Wrap(err, "conn.ObjectNames")
+				}
+				for _, obj := range newObjects {
+					m := filepath.Base(strings.TrimPrefix(obj, prefix))
+					if m == "" {
+						continue
+					}
+
+					select {
+					case ch <- m:
+					case <-ctx.Done():
+						return nil, io.EOF
+					}
+				}
+				return newObjects, nil
+			})
+
+		if err != nil {
+			debug.Log("ObjectsWalk returned error: %v", err)
+		}
+	}()
+
+	return ch
+}
+
+// Remove keys for a specified backend type.
+func (be *beSwift) removeKeys(ctx context.Context, t restic.FileType) error {
+	for key := range be.List(ctx, t) {
+		err := be.Remove(ctx, restic.Handle{Type: t, Name: key})
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// IsNotExist returns true if the error is caused by a not existing file.
+func (be *beSwift) IsNotExist(err error) bool {
+	if e, ok := errors.Cause(err).(*swift.Error); ok {
+		return e.StatusCode == http.StatusNotFound
+	}
+
+	return false
+}
+
+// Delete removes all restic objects in the container.
+// It will not remove the container itself.
+func (be *beSwift) Delete(ctx context.Context) error {
+	alltypes := []restic.FileType{
+		restic.DataFile,
+		restic.KeyFile,
+		restic.LockFile,
+		restic.SnapshotFile,
+		restic.IndexFile}
+
+	for _, t := range alltypes {
+		err := be.removeKeys(ctx, t)
+		if err != nil {
+			return nil
+		}
+	}
+
+	err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile})
+	if err != nil && !be.IsNotExist(err) {
+		return err
+	}
+
+	return nil
+}
+
+// Close does nothing
+func (be *beSwift) Close() error { return nil }

internal/backend/swift/swift_test.go

@@ -0,0 +1,111 @@
+package swift_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"restic"
+	"testing"
+	"time"
+
+	"restic/errors"
+	. "restic/test"
+
+	"restic/backend/swift"
+	"restic/backend/test"
+)
+
+func newSwiftTestSuite(t testing.TB) *test.Suite {
+	return &test.Suite{
+		// do not use excessive data
+		MinimalData: true,
+
+		// wait for removals for at least 60s
+		WaitForDelayedRemoval: 60 * time.Second,
+
+		// NewConfig returns a config for a new temporary backend that will be used in tests.
+		NewConfig: func() (interface{}, error) {
+			swiftcfg, err := swift.ParseConfig(os.Getenv("RESTIC_TEST_SWIFT"))
+			if err != nil {
+				return nil, err
+			}
+
+			cfg := swiftcfg.(swift.Config)
+			if err = swift.ApplyEnvironment("RESTIC_TEST_", &cfg); err != nil {
+				return nil, err
+			}
+			cfg.Prefix += fmt.Sprintf("/test-%d", time.Now().UnixNano())
+			t.Logf("using prefix %v", cfg.Prefix)
+			return cfg, nil
+		},
+
+		// CreateFn is a function that creates a temporary repository for the tests.
+		Create: func(config interface{}) (restic.Backend, error) {
+			cfg := config.(swift.Config)
+
+			be, err := swift.Open(cfg)
+			if err != nil {
+				return nil, err
+			}
+
+			exists, err := be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile})
+			if err != nil {
+				return nil, err
+			}
+
+			if exists {
+				return nil, errors.New("config already exists")
+			}
+
+			return be, nil
+		},
+
+		// OpenFn is a function that opens a previously created temporary repository.
+		Open: func(config interface{}) (restic.Backend, error) {
+			cfg := config.(swift.Config)
+			return swift.Open(cfg)
+		},
+
+		// CleanupFn removes data created during the tests.
+		Cleanup: func(config interface{}) error {
+			cfg := config.(swift.Config)
+
+			be, err := swift.Open(cfg)
+			if err != nil {
+				return err
+			}
+
+			if err := be.(restic.Deleter).Delete(context.TODO()); err != nil {
+				return err
+			}
+
+			return nil
+		},
+	}
+}
+
+func TestBackendSwift(t *testing.T) {
+	defer func() {
+		if t.Skipped() {
+			SkipDisallowed(t, "restic/backend/swift.TestBackendSwift")
+		}
+	}()
+
+	if os.Getenv("RESTIC_TEST_SWIFT") == "" {
+		t.Skip("RESTIC_TEST_SWIFT unset, skipping test")
+		return
+	}
+
+	t.Logf("run tests")
+	newSwiftTestSuite(t).RunTests(t)
+}
+
+func BenchmarkBackendSwift(t *testing.B) {
+	if os.Getenv("RESTIC_TEST_SWIFT") == "" {
+		t.Skip("RESTIC_TEST_SWIFT unset, skipping test")
+		return
+	}
+
+	t.Logf("run tests")
+	newSwiftTestSuite(t).RunBenchmarks(t)
+}