From 86b5d8ffaac04301a28e23348460fe8b13616368 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood <nick@craig-wood.com>
Date: Fri, 3 Jan 2020 19:37:43 +0000
Subject: [PATCH] s3: add bucket-lookup parameter to select path or dns style
bucket lookup
This is to enable restic working with Alibaba cloud
Fixes #2528
---
changelog/unreleased/issue-2528 | 20 ++++++++++++++++
doc/030_preparing_a_new_repo.rst | 40 ++++++++++++++++++++++++++++++++
internal/backend/s3/config.go | 7 +++---
internal/backend/s3/s3.go | 19 +++++++++++++--
4 files changed, 81 insertions(+), 5 deletions(-)
create mode 100644 changelog/unreleased/issue-2528
diff --git a/changelog/unreleased/issue-2528 b/changelog/unreleased/issue-2528
new file mode 100644
index 000000000..680e03840
--- /dev/null
+++ b/changelog/unreleased/issue-2528
@@ -0,0 +1,20 @@
+Enhancement: support Alibaba/Aliyun OSS with S3 backend
+
+We've added a new flag to the s3 backend `s3.bucket-lookup` which can
+be set to these 3 values:
+
+- `auto` - existing behaviour
+- `dns` - use DNS style bucket access
+- `path` - use path style bucket access
+
+To make the s3 backend work with Alibaba/Aliyun OSS you must set
+`s3.bucket-lookup` to `dns` and set the `s3.region` parameter. For
+example:
+
+ restic -o s3.bucket-lookup=dns -o s3.region=oss-eu-west-1 -r s3:https://oss-eu-west-1.aliyuncs.com/bucketname init
+
+Note that s3.region must be set otherwise the minio SDK tries to look
+it up and it seems that Alibaba doesn't support that properly.
+
+https://github.com/restic/restic/issues/2528
+https://github.com/restic/restic/pull/2535
diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst
index d7907645a..f470cf383 100644
--- a/doc/030_preparing_a_new_repo.rst
+++ b/doc/030_preparing_a_new_repo.rst
@@ -299,6 +299,46 @@ this command.
Please note that knowledge of your password is required to access
the repository. Losing your password means that your data is irrecoverably lost.
+Alibaba Cloud (Aliyun) Object Storage System (OSS)
+**************************************************
+
+`Alibaba OSS <https://www.alibabacloud.com/product/oss/>`__ is an
+encrypted, secure, cost-effective, and easy-to-use object storage
+service that enables you to store, back up, and archive large amounts
+of data in the cloud.
+
+Alibaba OSS is S3 compatible so it can be used as a storage provider
+for a restic repository with a couple of extra parameters.
+
+- Determine the correct `Alibaba OSS region endpoint <https://www.alibabacloud.com/help/doc-detail/31837.htm>`__ - this will be something like ``oss-eu-west-1.aliyuncs.com``
+- You'll need the region name too - this will be something like ``oss-eu-west-1``
+
+You must first setup the following environment variables with the
+credentials of your Alibaba OSS account.
+
+.. code-block:: console
+
+ $ export AWS_ACCESS_KEY_ID=<YOUR-OSS-ACCESS-KEY-ID>
+ $ export AWS_SECRET_ACCESS_KEY=<YOUR-OSS-SECRET-ACCESS-KEY>
+
+Now you can easily initialize restic to use Alibaba OSS as a backend with
+this command.
+
+.. code-block:: console
+
+ $ ./restic -o s3.bucket-lookup=dns -o s3.region=<OSS-REGION> -r s3:https://<OSS-ENDPOINT>/<OSS-BUCKET-NAME> init
+ enter password for new backend:
+ enter password again:
+ created restic backend xxxxxxxxxx at s3:https://<OSS-ENDPOINT>/<OSS-BUCKET-NAME>
+ Please note that knowledge of your password is required to access
+ the repository. Losing your password means that your data is irrecoverably lost.
+
+For example with an actual endpoint:
+
+.. code-block:: console
+
+ $ restic -o s3.bucket-lookup=dns -o s3.region=oss-eu-west-1 -r s3:https://oss-eu-west-1.aliyuncs.com/bucketname init
+
OpenStack Swift
***************
diff --git a/internal/backend/s3/config.go b/internal/backend/s3/config.go
index 499e05094..93de42152 100644
--- a/internal/backend/s3/config.go
+++ b/internal/backend/s3/config.go
@@ -20,9 +20,10 @@ type Config struct {
Layout string `option:"layout" help:"use this backend layout (default: auto-detect)"`
StorageClass string `option:"storage-class" help:"set S3 storage class (STANDARD, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING or REDUCED_REDUNDANCY)"`
- Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
- MaxRetries uint `option:"retries" help:"set the number of retries attempted"`
- Region string `option:"region" help:"set region"`
+ Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
+ MaxRetries uint `option:"retries" help:"set the number of retries attempted"`
+ Region string `option:"region" help:"set region"`
+ BucketLookup string `option:"bucket-lookup" help:"bucket lookup style: 'auto', 'dns', or 'path'."`
}
// NewConfig returns a new Config with the default values filled in.
diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go
index d1243bfd5..3c40f7c49 100644
--- a/internal/backend/s3/s3.go
+++ b/internal/backend/s3/s3.go
@@ -2,6 +2,7 @@ package s3
import (
"context"
+ "fmt"
"io"
"io/ioutil"
"net/http"
@@ -66,12 +67,26 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro
},
},
})
- client, err := minio.New(cfg.Endpoint, &minio.Options{
+
+ options := &minio.Options{
Creds: creds,
Secure: !cfg.UseHTTP,
Region: cfg.Region,
Transport: rt,
- })
+ }
+
+ switch strings.ToLower(cfg.BucketLookup) {
+ case "", "auto":
+ options.BucketLookup = minio.BucketLookupAuto
+ case "dns":
+ options.BucketLookup = minio.BucketLookupDNS
+ case "path":
+ options.BucketLookup = minio.BucketLookupPath
+ default:
+ return nil, fmt.Errorf(`bad bucket-lookup style %q must be "auto", "path" or "dns"`, cfg.BucketLookup)
+ }
+
+ client, err := minio.New(cfg.Endpoint, options)
if err != nil {
return nil, errors.Wrap(err, "minio.New")
}