diff --git a/backend/s3/config.go b/backend/s3/config.go index b0224925a..5cd74ce79 100644 --- a/backend/s3/config.go +++ b/backend/s3/config.go @@ -3,6 +3,7 @@ package s3 import ( "errors" "net/url" + "path" "strings" ) @@ -13,54 +14,22 @@ type Config struct { UseHTTP bool KeyID, Secret string Bucket string + Prefix string } +const defaultPrefix = "restic" + // ParseConfig parses the string s and extracts the s3 config. The two -// supported configuration formats are s3://host/bucketname and -// s3:host:bucketname. The host can also be a valid s3 region name. +// supported configuration formats are s3://host/bucketname/prefix and +// s3:host:bucketname/prefix. The host can also be a valid s3 region +// name. If no prefix is given the prefix "restic" will be used. func ParseConfig(s string) (interface{}, error) { - if strings.HasPrefix(s, "s3://") { - s = s[5:] - - data := strings.SplitN(s, "/", 2) - if len(data) != 2 { - return nil, errors.New("s3: invalid format, host/region or bucket name not found") - } - - cfg := Config{ - Endpoint: data[0], - Bucket: data[1], - } - - return cfg, nil - } - - data := strings.SplitN(s, ":", 2) - if len(data) != 2 { - return nil, errors.New("s3: invalid format") - } - - if data[0] != "s3" { - return nil, errors.New(`s3: config does not start with "s3"`) - } - - s = data[1] - - cfg := Config{} - rest := strings.Split(s, "/") - if len(rest) < 2 { - return nil, errors.New("s3: region or bucket not found") - } - - if len(rest) == 2 { - // assume that just a region name and a bucket has been specified, in - // the format region/bucket - cfg.Endpoint = rest[0] - cfg.Bucket = rest[1] - } else { - // assume that a URL has been specified, parse it and use the path as - // the bucket name. - url, err := url.Parse(s) + switch { + case strings.HasPrefix(s, "s3:http"): + // assume that a URL has been specified, parse it and + // use the host as the endpoint and the path as the + // bucket name and prefix + url, err := url.Parse(s[3:]) if err != nil { return nil, err } @@ -69,13 +38,35 @@ func ParseConfig(s string) (interface{}, error) { return nil, errors.New("s3: bucket name not found") } - cfg.Endpoint = url.Host - if url.Scheme == "http" { - cfg.UseHTTP = true - } - - cfg.Bucket = url.Path[1:] + path := strings.SplitN(url.Path[1:], "/", 2) + return createConfig(url.Host, path, url.Scheme == "http") + case strings.HasPrefix(s, "s3://"): + s = s[5:] + case strings.HasPrefix(s, "s3:"): + s = s[3:] + default: + return nil, errors.New("s3: invalid format") } - - return cfg, nil + // use the first entry of the path as the endpoint and the + // remainder as bucket name and prefix + path := strings.SplitN(s, "/", 3) + return createConfig(path[0], path[1:], false) +} + +func createConfig(endpoint string, p []string, useHTTP bool) (interface{}, error) { + var prefix string + switch { + case len(p) < 1: + return nil, errors.New("s3: invalid format, host/region or bucket name not found") + case len(p) == 1 || p[1] == "": + prefix = defaultPrefix + default: + prefix = path.Clean(p[1]) + } + return Config{ + Endpoint: endpoint, + UseHTTP: useHTTP, + Bucket: p[0], + Prefix: prefix, + }, nil } diff --git a/backend/s3/config_test.go b/backend/s3/config_test.go index 54fc4718a..3a04d59a2 100644 --- a/backend/s3/config_test.go +++ b/backend/s3/config_test.go @@ -9,18 +9,75 @@ var configTests = []struct { {"s3://eu-central-1/bucketname", Config{ Endpoint: "eu-central-1", Bucket: "bucketname", + Prefix: "restic", + }}, + {"s3://eu-central-1/bucketname/", Config{ + Endpoint: "eu-central-1", + Bucket: "bucketname", + Prefix: "restic", + }}, + {"s3://eu-central-1/bucketname/prefix/directory", Config{ + Endpoint: "eu-central-1", + Bucket: "bucketname", + Prefix: "prefix/directory", + }}, + {"s3://eu-central-1/bucketname/prefix/directory/", Config{ + Endpoint: "eu-central-1", + Bucket: "bucketname", + Prefix: "prefix/directory", }}, {"s3:eu-central-1/foobar", Config{ Endpoint: "eu-central-1", Bucket: "foobar", + Prefix: "restic", + }}, + {"s3:eu-central-1/foobar/", Config{ + Endpoint: "eu-central-1", + Bucket: "foobar", + Prefix: "restic", + }}, + {"s3:eu-central-1/foobar/prefix/directory", Config{ + Endpoint: "eu-central-1", + Bucket: "foobar", + Prefix: "prefix/directory", + }}, + {"s3:eu-central-1/foobar/prefix/directory/", Config{ + Endpoint: "eu-central-1", + Bucket: "foobar", + Prefix: "prefix/directory", }}, {"s3:https://hostname:9999/foobar", Config{ Endpoint: "hostname:9999", Bucket: "foobar", + Prefix: "restic", + }}, + {"s3:https://hostname:9999/foobar/", Config{ + Endpoint: "hostname:9999", + Bucket: "foobar", + Prefix: "restic", }}, {"s3:http://hostname:9999/foobar", Config{ Endpoint: "hostname:9999", Bucket: "foobar", + Prefix: "restic", + UseHTTP: true, + }}, + {"s3:http://hostname:9999/foobar/", Config{ + Endpoint: "hostname:9999", + Bucket: "foobar", + Prefix: "restic", + UseHTTP: true, + }}, + {"s3:http://hostname:9999/bucket/prefix/directory", Config{ + Endpoint: "hostname:9999", + Bucket: "bucket", + Prefix: "prefix/directory", + UseHTTP: true, + }}, + {"s3:http://hostname:9999/bucket/prefix/directory/", Config{ + Endpoint: "hostname:9999", + Bucket: "bucket", + Prefix: "prefix/directory", UseHTTP: true, }}, } @@ -29,13 +86,13 @@ func TestParseConfig(t *testing.T) { for i, test := range configTests { cfg, err := ParseConfig(test.s) if err != nil { - t.Errorf("test %d failed: %v", i, err) + t.Errorf("test %d:%s failed: %v", i, test.s, err) continue } if cfg != test.cfg { - t.Errorf("test %d: wrong config, want:\n %v\ngot:\n %v", - i, test.cfg, cfg) + t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v", + i, test.s, test.cfg, cfg) continue } } diff --git a/backend/s3/s3.go b/backend/s3/s3.go index ccb6cd42c..a6657fd2b 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -13,20 +13,13 @@ import ( ) const connLimit = 10 -const backendPrefix = "restic" - -func s3path(t backend.Type, name string) string { - if t == backend.Config { - return backendPrefix + "/" + string(t) - } - return backendPrefix + "/" + string(t) + "/" + name -} // s3 is a backend which stores the data on an S3 endpoint. type s3 struct { client minio.CloudStorageClient connChan chan struct{} bucketname string + prefix string } // Open opens the S3 backend at bucket and region. The bucket is created if it @@ -39,7 +32,7 @@ func Open(cfg Config) (backend.Backend, error) { return nil, err } - be := &s3{client: client, bucketname: cfg.Bucket} + be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix} be.createConnections() if err := client.BucketExists(cfg.Bucket); err != nil { @@ -56,6 +49,20 @@ func Open(cfg Config) (backend.Backend, error) { return be, nil } +func (be *s3) s3path(t backend.Type, name string) string { + var path string + + if be.prefix != "" { + path = be.prefix + "/" + } + path += string(t) + + if t == backend.Config { + return path + } + return path + "/" + name +} + func (be *s3) createConnections() { be.connChan = make(chan struct{}, connLimit) for i := 0; i < connLimit; i++ { @@ -72,7 +79,7 @@ func (be *s3) Location() string { // and saves it in p. Load has the same semantics as io.ReaderAt. func (be s3) Load(h backend.Handle, p []byte, off int64) (int, error) { debug.Log("s3.Load", "%v, offset %v, len %v", h, off, len(p)) - path := s3path(h.Type, h.Name) + path := be.s3path(h.Type, h.Name) obj, err := be.client.GetObject(be.bucketname, path) if err != nil { debug.Log("s3.GetReader", " err %v", err) @@ -101,7 +108,7 @@ func (be s3) Save(h backend.Handle, p []byte) (err error) { debug.Log("s3.Save", "%v bytes at %d", len(p), h) - path := s3path(h.Type, h.Name) + path := be.s3path(h.Type, h.Name) // Check key does not already exist _, err = be.client.StatObject(be.bucketname, path) @@ -126,7 +133,7 @@ func (be s3) Save(h backend.Handle, p []byte) (err error) { // Stat returns information about a blob. func (be s3) Stat(h backend.Handle) (backend.BlobInfo, error) { debug.Log("s3.Stat", "%v") - path := s3path(h.Type, h.Name) + path := be.s3path(h.Type, h.Name) obj, err := be.client.GetObject(be.bucketname, path) if err != nil { debug.Log("s3.Stat", "GetObject() err %v", err) @@ -145,7 +152,7 @@ func (be s3) Stat(h backend.Handle) (backend.BlobInfo, error) { // Test returns true if a blob of the given type and name exists in the backend. func (be *s3) Test(t backend.Type, name string) (bool, error) { found := false - path := s3path(t, name) + path := be.s3path(t, name) _, err := be.client.StatObject(be.bucketname, path) if err == nil { found = true @@ -157,7 +164,7 @@ func (be *s3) Test(t backend.Type, name string) (bool, error) { // Remove removes the blob with the given name and type. func (be *s3) Remove(t backend.Type, name string) error { - path := s3path(t, name) + path := be.s3path(t, name) err := be.client.RemoveObject(be.bucketname, path) debug.Log("s3.Remove", "%v %v -> err %v", t, name, err) return err @@ -170,7 +177,7 @@ func (be *s3) List(t backend.Type, done <-chan struct{}) <-chan string { debug.Log("s3.List", "listing %v", t) ch := make(chan string) - prefix := s3path(t, "") + prefix := be.s3path(t, "") listresp := be.client.ListObjects(be.bucketname, prefix, true, done) diff --git a/location/location_test.go b/location/location_test.go index b0303fad1..ef827dcdd 100644 --- a/location/location_test.go +++ b/location/location_test.go @@ -48,30 +48,56 @@ var parseTests = []struct { Config: s3.Config{ Endpoint: "eu-central-1", Bucket: "bucketname", + Prefix: "restic", }}, }, {"s3://hostname.foo/bucketname", Location{Scheme: "s3", Config: s3.Config{ Endpoint: "hostname.foo", Bucket: "bucketname", + Prefix: "restic", + }}, + }, + {"s3://hostname.foo/bucketname/prefix/directory", Location{Scheme: "s3", + Config: s3.Config{ + Endpoint: "hostname.foo", + Bucket: "bucketname", + Prefix: "prefix/directory", }}, }, {"s3:eu-central-1/repo", Location{Scheme: "s3", Config: s3.Config{ Endpoint: "eu-central-1", Bucket: "repo", + Prefix: "restic", + }}, + }, + {"s3:eu-central-1/repo/prefix/directory", Location{Scheme: "s3", + Config: s3.Config{ + Endpoint: "eu-central-1", + Bucket: "repo", + Prefix: "prefix/directory", }}, }, {"s3:https://hostname.foo/repo", Location{Scheme: "s3", Config: s3.Config{ Endpoint: "hostname.foo", Bucket: "repo", + Prefix: "restic", + }}, + }, + {"s3:https://hostname.foo/repo/prefix/directory", Location{Scheme: "s3", + Config: s3.Config{ + Endpoint: "hostname.foo", + Bucket: "repo", + Prefix: "prefix/directory", }}, }, {"s3:http://hostname.foo/repo", Location{Scheme: "s3", Config: s3.Config{ Endpoint: "hostname.foo", Bucket: "repo", + Prefix: "restic", UseHTTP: true, }}, },