mirror of
https://github.com/restic/restic.git
synced 2025-08-18 16:17:47 +00:00
Compare commits
31 Commits
Author | SHA1 | Date | |
---|---|---|---|
![]() |
81473f4538 | ||
![]() |
e1a847e4d1 | ||
![]() |
0f426c3795 | ||
![]() |
6df3d169b8 | ||
![]() |
5479daa6d4 | ||
![]() |
397fec0152 | ||
![]() |
d7e644272f | ||
![]() |
e91749bbb0 | ||
![]() |
bcd1e45ba7 | ||
![]() |
4c6b626db6 | ||
![]() |
835ba16c27 | ||
![]() |
3b6a580b32 | ||
![]() |
01c486d486 | ||
![]() |
6342a08a16 | ||
![]() |
94c8ee11f8 | ||
![]() |
9b38980ed9 | ||
![]() |
649c536250 | ||
![]() |
dd49e2b12d | ||
![]() |
f61dab1774 | ||
![]() |
40edf00182 | ||
![]() |
c35518a865 | ||
![]() |
7a0b4428e3 | ||
![]() |
c784a15aaa | ||
![]() |
ce180de9b8 | ||
![]() |
fca9a523e9 | ||
![]() |
8a3889be11 | ||
![]() |
2a1633621b | ||
![]() |
e2deeceb1b | ||
![]() |
d4e994de7b | ||
![]() |
a60e751217 | ||
![]() |
81c5d8a968 |
10
.travis.yml
10
.travis.yml
@@ -2,8 +2,8 @@ language: go
|
||||
sudo: false
|
||||
|
||||
go:
|
||||
- 1.8.3
|
||||
- 1.9
|
||||
- 1.8.x
|
||||
- 1.9.x
|
||||
|
||||
os:
|
||||
- linux
|
||||
@@ -16,12 +16,12 @@ env:
|
||||
matrix:
|
||||
exclude:
|
||||
- os: osx
|
||||
go: 1.8.3
|
||||
go: 1.8.x
|
||||
- os: linux
|
||||
go: 1.9
|
||||
go: 1.9.x
|
||||
include:
|
||||
- os: linux
|
||||
go: 1.9
|
||||
go: 1.9.x
|
||||
sudo: true
|
||||
env:
|
||||
RESTIC_TEST_FUSE=1
|
||||
|
11
CHANGELOG.md
11
CHANGELOG.md
@@ -1,6 +1,17 @@
|
||||
This file describes changes relevant to all users that are made in each
|
||||
released version of restic from the perspective of the user.
|
||||
|
||||
Important Changes in 0.7.3
|
||||
==========================
|
||||
|
||||
* For large backups stored in Google Cloud Storage, the `prune` command fails
|
||||
because listing only returns the first 1000 files. This has been corrected,
|
||||
no data is lost in the process. In addition, a plausibility check was added
|
||||
to `prune`.
|
||||
https://github.com/restic/restic/issues/1246
|
||||
https://github.com/restic/restic/pull/1247
|
||||
|
||||
|
||||
Important Changes in 0.7.2
|
||||
==========================
|
||||
|
||||
|
@@ -4,8 +4,6 @@ import (
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var autocompleteTarget string
|
||||
|
||||
var cmdAutocomplete = &cobra.Command{
|
||||
Use: "autocomplete",
|
||||
Short: "Generate shell autocompletion script",
|
||||
@@ -28,10 +26,12 @@ $ sudo restic autocomplete`,
|
||||
},
|
||||
}
|
||||
|
||||
var autocompleteTarget string
|
||||
|
||||
func init() {
|
||||
cmdRoot.AddCommand(cmdAutocomplete)
|
||||
|
||||
cmdAutocomplete.Flags().StringVarP(&autocompleteTarget, "completionfile", "", "/etc/bash_completion.d/restic.sh", "autocompletion file")
|
||||
cmdAutocomplete.Flags().StringVarP(&autocompleteTarget, "completionfile", "", "/usr/share/bash-completion/completions/restic", "autocompletion file")
|
||||
// For bash-completion
|
||||
cmdAutocomplete.Flags().SetAnnotation("completionfile", cobra.BashCompFilenameExt, []string{})
|
||||
}
|
||||
|
@@ -179,6 +179,12 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
|
||||
}
|
||||
bar.Done()
|
||||
|
||||
if len(usedBlobs) > stats.blobs {
|
||||
return errors.Fatalf("number of used blobs is larger than number of available blobs!\n" +
|
||||
"Please report this error (along with the output of the 'prune' run) at\n" +
|
||||
"https://github.com/restic/restic/issues/new")
|
||||
}
|
||||
|
||||
Verbosef("found %d of %d data blobs still in use, removing %d blobs\n",
|
||||
len(usedBlobs), stats.blobs, stats.blobs-len(usedBlobs))
|
||||
|
||||
|
@@ -71,7 +71,7 @@ func sameModTime(fi1, fi2 os.FileInfo) bool {
|
||||
}
|
||||
}
|
||||
|
||||
return fi1.ModTime() == fi2.ModTime()
|
||||
return fi1.ModTime().Equal(fi2.ModTime())
|
||||
}
|
||||
|
||||
// directoriesEqualContents checks if both directories contain exactly the same
|
||||
|
@@ -19,7 +19,7 @@ import os
|
||||
# Add any Sphinx extension module names here, as strings. They can be
|
||||
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
|
||||
# ones.
|
||||
extensions = []
|
||||
extensions = ['sphinx.ext.extlinks']
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
templates_path = ['_templates']
|
||||
@@ -104,3 +104,7 @@ html_static_path = ['_static']
|
||||
|
||||
# Output file base name for HTML help builder.
|
||||
htmlhelp_basename = 'resticdoc'
|
||||
|
||||
extlinks = {
|
||||
'issue': ('https://github.com/restic/restic/issues/%s', '#'),
|
||||
}
|
||||
|
23
doc/faq.rst
23
doc/faq.rst
@@ -26,3 +26,26 @@ The message means that there is more data stored in the repo than
|
||||
strictly necessary. With high probability this is duplicate data. In
|
||||
order to clean it up, the command ``restic prune`` can be used. The
|
||||
cause of this bug is not yet known.
|
||||
|
||||
How can I specify encryption passwords automatically?
|
||||
-----------------------------------------------------
|
||||
|
||||
When you run ``restic backup``, you need to enter the passphrase on
|
||||
the console. This is not very convenient for automated backups, so you
|
||||
can also provide the password through the ``--password-file`` option
|
||||
or ``RESTIC_PASSWORD`` environment. A discussion is in progress over
|
||||
implementing unattended backups happens in :issue:`533`.
|
||||
|
||||
.. important:: Be careful how you set the environment; using the env
|
||||
command, a `system()` call or using inline shell
|
||||
scripts (e.g. `RESTIC_PASSWORD=password borg ...`)
|
||||
might expose the credentials in the process list
|
||||
directly and they will be readable to all users on a
|
||||
system. Using export in a shell script file should be
|
||||
safe, however, as the environment of a process is
|
||||
`accessible only to that user`_. Please make sure that
|
||||
the permissions on the files where the password is
|
||||
eventually stored are safe (e.g. `0600` and owned by
|
||||
root).
|
||||
|
||||
.. _accessible only to that user: https://security.stackexchange.com/questions/14000/environment-variable-accessibility-in-linux/14009#14009
|
||||
|
@@ -31,7 +31,7 @@ $ sudo restic autocomplete
|
||||
|
||||
.SH OPTIONS
|
||||
.PP
|
||||
\fB\-\-completionfile\fP="/etc/bash\_completion.d/restic.sh"
|
||||
\fB\-\-completionfile\fP="/usr/share/bash\-completion/completions/restic"
|
||||
autocompletion file
|
||||
|
||||
.PP
|
||||
|
@@ -1075,8 +1075,11 @@ statements originating in functions that match the pattern ``*unlock*``
|
||||
|
||||
$ DEBUG_FUNCS=*unlock* restic check
|
||||
|
||||
Under the hood: Browse repository objects
|
||||
-----------------------------------------
|
||||
Under the hood
|
||||
--------------
|
||||
|
||||
Browse repository objects
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Internally, a repository stores data of several different types
|
||||
described in the `design
|
||||
@@ -1121,6 +1124,33 @@ objects or its raw content.
|
||||
"gid": 20
|
||||
}
|
||||
|
||||
Metadata handling
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
Restic saves and restores most default attributes, including extended attributes like ACLs.
|
||||
Sparse files are not handled in a special way yet, and aren't restored.
|
||||
|
||||
The following metadata is handled by restic:
|
||||
|
||||
- Name
|
||||
- Type
|
||||
- Mode
|
||||
- ModTime
|
||||
- AccessTime
|
||||
- ChangeTime
|
||||
- UID
|
||||
- GID
|
||||
- User
|
||||
- Group
|
||||
- Inode
|
||||
- Size
|
||||
- Links
|
||||
- LinkTarget
|
||||
- Device
|
||||
- Content
|
||||
- Subtree
|
||||
- ExtendedAttributes
|
||||
|
||||
Scripting
|
||||
---------
|
||||
|
||||
|
@@ -134,7 +134,7 @@ func (arch *Archiver) reloadFileIfChanged(node *restic.Node, file fs.File) (*res
|
||||
return nil, errors.Wrap(err, "restic.Stat")
|
||||
}
|
||||
|
||||
if fi.ModTime() == node.ModTime {
|
||||
if fi.ModTime().Equal(node.ModTime) {
|
||||
return node, nil
|
||||
}
|
||||
|
||||
@@ -162,6 +162,7 @@ func (arch *Archiver) saveChunk(ctx context.Context, chunk chunker.Chunk, p *res
|
||||
// TODO handle error
|
||||
if err != nil {
|
||||
debug.Log("Save(%v) failed: %v", id.Str(), err)
|
||||
fmt.Printf("\nerror while saving data to the repo: %+v\n", err)
|
||||
panic(err)
|
||||
}
|
||||
|
||||
|
@@ -7,7 +7,6 @@ import (
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/storage"
|
||||
"github.com/restic/restic/internal/backend"
|
||||
@@ -18,13 +17,16 @@ import (
|
||||
|
||||
// Backend stores data on an azure endpoint.
|
||||
type Backend struct {
|
||||
accountName string
|
||||
container *storage.Container
|
||||
sem *backend.Semaphore
|
||||
prefix string
|
||||
accountName string
|
||||
container *storage.Container
|
||||
sem *backend.Semaphore
|
||||
prefix string
|
||||
listMaxItems int
|
||||
backend.Layout
|
||||
}
|
||||
|
||||
const defaultListMaxItems = 5000
|
||||
|
||||
// make sure that *Backend implements backend.Backend
|
||||
var _ restic.Backend = &Backend{}
|
||||
|
||||
@@ -54,6 +56,7 @@ func open(cfg Config) (*Backend, error) {
|
||||
Path: cfg.Prefix,
|
||||
Join: path.Join,
|
||||
},
|
||||
listMaxItems: defaultListMaxItems,
|
||||
}
|
||||
|
||||
return be, nil
|
||||
@@ -85,6 +88,11 @@ func Create(cfg Config) (restic.Backend, error) {
|
||||
return be, nil
|
||||
}
|
||||
|
||||
// SetListMaxItems sets the number of list items to load per request.
|
||||
func (be *Backend) SetListMaxItems(i int) {
|
||||
be.listMaxItems = i
|
||||
}
|
||||
|
||||
// IsNotExist returns true if the error is caused by a not existing file.
|
||||
func (be *Backend) IsNotExist(err error) bool {
|
||||
debug.Log("IsNotExist(%T, %#v)", err, err)
|
||||
@@ -96,62 +104,6 @@ func (be *Backend) Join(p ...string) string {
|
||||
return path.Join(p...)
|
||||
}
|
||||
|
||||
type fileInfo struct {
|
||||
name string
|
||||
size int64
|
||||
mode os.FileMode
|
||||
modTime time.Time
|
||||
isDir bool
|
||||
}
|
||||
|
||||
func (fi fileInfo) Name() string { return fi.name } // base name of the file
|
||||
func (fi fileInfo) Size() int64 { return fi.size } // length in bytes for regular files; system-dependent for others
|
||||
func (fi fileInfo) Mode() os.FileMode { return fi.mode } // file mode bits
|
||||
func (fi fileInfo) ModTime() time.Time { return fi.modTime } // modification time
|
||||
func (fi fileInfo) IsDir() bool { return fi.isDir } // abbreviation for Mode().IsDir()
|
||||
func (fi fileInfo) Sys() interface{} { return nil } // underlying data source (can return nil)
|
||||
|
||||
// ReadDir returns the entries for a directory.
|
||||
func (be *Backend) ReadDir(dir string) (list []os.FileInfo, err error) {
|
||||
debug.Log("ReadDir(%v)", dir)
|
||||
|
||||
// make sure dir ends with a slash
|
||||
if dir[len(dir)-1] != '/' {
|
||||
dir += "/"
|
||||
}
|
||||
|
||||
obj, err := be.container.ListBlobs(storage.ListBlobsParameters{Prefix: dir, Delimiter: "/"})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, item := range obj.BlobPrefixes {
|
||||
entry := fileInfo{
|
||||
name: strings.TrimPrefix(item, dir),
|
||||
isDir: true,
|
||||
mode: os.ModeDir | 0755,
|
||||
}
|
||||
if entry.name != "" {
|
||||
list = append(list, entry)
|
||||
}
|
||||
}
|
||||
|
||||
for _, item := range obj.Blobs {
|
||||
entry := fileInfo{
|
||||
name: strings.TrimPrefix(item.Name, dir),
|
||||
isDir: false,
|
||||
mode: 0644,
|
||||
size: item.Properties.ContentLength,
|
||||
modTime: time.Time(item.Properties.LastModified),
|
||||
}
|
||||
if entry.name != "" {
|
||||
list = append(list, entry)
|
||||
}
|
||||
}
|
||||
|
||||
return list, nil
|
||||
}
|
||||
|
||||
// Location returns this backend's location (the container name).
|
||||
func (be *Backend) Location() string {
|
||||
return be.Join(be.container.Name, be.prefix)
|
||||
@@ -321,25 +273,39 @@ func (be *Backend) List(ctx context.Context, t restic.FileType) <-chan string {
|
||||
prefix += "/"
|
||||
}
|
||||
|
||||
params := storage.ListBlobsParameters{
|
||||
MaxResults: uint(be.listMaxItems),
|
||||
Prefix: prefix,
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer close(ch)
|
||||
|
||||
obj, err := be.container.ListBlobs(storage.ListBlobsParameters{Prefix: prefix})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, item := range obj.Blobs {
|
||||
m := strings.TrimPrefix(item.Name, prefix)
|
||||
if m == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case ch <- path.Base(m):
|
||||
case <-ctx.Done():
|
||||
for {
|
||||
obj, err := be.container.ListBlobs(params)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
debug.Log("got %v objects", len(obj.Blobs))
|
||||
|
||||
for _, item := range obj.Blobs {
|
||||
m := strings.TrimPrefix(item.Name, prefix)
|
||||
if m == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case ch <- path.Base(m):
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if obj.NextMarker == "" {
|
||||
break
|
||||
}
|
||||
params.Marker = obj.NextMarker
|
||||
}
|
||||
}()
|
||||
|
||||
|
@@ -16,13 +16,16 @@ import (
|
||||
|
||||
// b2Backend is a backend which stores its data on Backblaze B2.
|
||||
type b2Backend struct {
|
||||
client *b2.Client
|
||||
bucket *b2.Bucket
|
||||
cfg Config
|
||||
client *b2.Client
|
||||
bucket *b2.Bucket
|
||||
cfg Config
|
||||
listMaxItems int
|
||||
backend.Layout
|
||||
sem *backend.Semaphore
|
||||
}
|
||||
|
||||
const defaultListMaxItems = 1000
|
||||
|
||||
// ensure statically that *b2Backend implements restic.Backend.
|
||||
var _ restic.Backend = &b2Backend{}
|
||||
|
||||
@@ -121,6 +124,11 @@ func Create(cfg Config) (restic.Backend, error) {
|
||||
return be, nil
|
||||
}
|
||||
|
||||
// SetListMaxItems sets the number of list items to load per request.
|
||||
func (be *b2Backend) SetListMaxItems(i int) {
|
||||
be.listMaxItems = i
|
||||
}
|
||||
|
||||
// Location returns the location for the backend.
|
||||
func (be *b2Backend) Location() string {
|
||||
return be.cfg.Bucket
|
||||
@@ -307,10 +315,11 @@ func (be *b2Backend) List(ctx context.Context, t restic.FileType) <-chan string
|
||||
cur := &b2.Cursor{Prefix: prefix}
|
||||
|
||||
for {
|
||||
objs, c, err := be.bucket.ListCurrentObjects(ctx, 1000, cur)
|
||||
objs, c, err := be.bucket.ListCurrentObjects(ctx, be.listMaxItems, cur)
|
||||
if err != nil && err != io.EOF {
|
||||
return
|
||||
}
|
||||
debug.Log("returned %v items", len(objs))
|
||||
for _, obj := range objs {
|
||||
// Skip objects returned that do not have the specified prefix.
|
||||
if !strings.HasPrefix(obj.Name(), prefix) {
|
||||
|
@@ -7,7 +7,6 @@ import (
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/restic/restic/internal/backend"
|
||||
@@ -23,11 +22,12 @@ import (
|
||||
|
||||
// Backend stores data on an gs endpoint.
|
||||
type Backend struct {
|
||||
service *storage.Service
|
||||
projectID string
|
||||
sem *backend.Semaphore
|
||||
bucketName string
|
||||
prefix string
|
||||
service *storage.Service
|
||||
projectID string
|
||||
sem *backend.Semaphore
|
||||
bucketName string
|
||||
prefix string
|
||||
listMaxItems int
|
||||
backend.Layout
|
||||
}
|
||||
|
||||
@@ -56,6 +56,8 @@ func getStorageService(jsonKeyPath string) (*storage.Service, error) {
|
||||
return service, nil
|
||||
}
|
||||
|
||||
const defaultListMaxItems = 1000
|
||||
|
||||
func open(cfg Config) (*Backend, error) {
|
||||
debug.Log("open, config %#v", cfg)
|
||||
|
||||
@@ -79,6 +81,7 @@ func open(cfg Config) (*Backend, error) {
|
||||
Path: cfg.Prefix,
|
||||
Join: path.Join,
|
||||
},
|
||||
listMaxItems: defaultListMaxItems,
|
||||
}
|
||||
|
||||
return be, nil
|
||||
@@ -112,6 +115,11 @@ func Create(cfg Config) (restic.Backend, error) {
|
||||
return be, nil
|
||||
}
|
||||
|
||||
// SetListMaxItems sets the number of list items to load per request.
|
||||
func (be *Backend) SetListMaxItems(i int) {
|
||||
be.listMaxItems = i
|
||||
}
|
||||
|
||||
// IsNotExist returns true if the error is caused by a not existing file.
|
||||
func (be *Backend) IsNotExist(err error) bool {
|
||||
debug.Log("IsNotExist(%T, %#v)", err, err)
|
||||
@@ -134,59 +142,6 @@ func (be *Backend) Join(p ...string) string {
|
||||
return path.Join(p...)
|
||||
}
|
||||
|
||||
type fileInfo struct {
|
||||
name string
|
||||
size int64
|
||||
mode os.FileMode
|
||||
modTime time.Time
|
||||
isDir bool
|
||||
}
|
||||
|
||||
func (fi fileInfo) Name() string { return fi.name } // base name of the file
|
||||
func (fi fileInfo) Size() int64 { return fi.size } // length in bytes for regular files; system-dependent for others
|
||||
func (fi fileInfo) Mode() os.FileMode { return fi.mode } // file mode bits
|
||||
func (fi fileInfo) ModTime() time.Time { return fi.modTime } // modification time
|
||||
func (fi fileInfo) IsDir() bool { return fi.isDir } // abbreviation for Mode().IsDir()
|
||||
func (fi fileInfo) Sys() interface{} { return nil } // underlying data source (can return nil)
|
||||
|
||||
// ReadDir returns the entries for a directory.
|
||||
func (be *Backend) ReadDir(dir string) (list []os.FileInfo, err error) {
|
||||
debug.Log("ReadDir(%v)", dir)
|
||||
|
||||
// make sure dir ends with a slash
|
||||
if dir[len(dir)-1] != '/' {
|
||||
dir += "/"
|
||||
}
|
||||
|
||||
obj, err := be.service.Objects.List(be.bucketName).Prefix(dir).Delimiter("/").Do()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, item := range obj.Prefixes {
|
||||
entry := fileInfo{
|
||||
name: strings.TrimPrefix(item, dir),
|
||||
isDir: true,
|
||||
mode: os.ModeDir | 0755,
|
||||
}
|
||||
list = append(list, entry)
|
||||
}
|
||||
for _, item := range obj.Items {
|
||||
entry := fileInfo{
|
||||
name: strings.TrimPrefix(item.Name, dir),
|
||||
isDir: false,
|
||||
mode: 0644,
|
||||
size: int64(item.Size),
|
||||
//modTime: item.Updated,
|
||||
}
|
||||
if entry.name != "" {
|
||||
list = append(list, entry)
|
||||
}
|
||||
}
|
||||
|
||||
return list, nil
|
||||
}
|
||||
|
||||
// Location returns this backend's location (the bucket name).
|
||||
func (be *Backend) Location() string {
|
||||
return be.Join(be.bucketName, be.prefix)
|
||||
@@ -352,22 +307,33 @@ func (be *Backend) List(ctx context.Context, t restic.FileType) <-chan string {
|
||||
go func() {
|
||||
defer close(ch)
|
||||
|
||||
obj, err := be.service.Objects.List(be.bucketName).Prefix(prefix).Do()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, item := range obj.Items {
|
||||
m := strings.TrimPrefix(item.Name, prefix)
|
||||
if m == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case ch <- path.Base(m):
|
||||
case <-ctx.Done():
|
||||
listReq := be.service.Objects.List(be.bucketName).Prefix(prefix).MaxResults(int64(be.listMaxItems))
|
||||
for {
|
||||
obj, err := listReq.Do()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error listing %v: %v\n", prefix, err)
|
||||
return
|
||||
}
|
||||
|
||||
debug.Log("returned %v items", len(obj.Items))
|
||||
|
||||
for _, item := range obj.Items {
|
||||
m := strings.TrimPrefix(item.Name, prefix)
|
||||
if m == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case ch <- path.Base(m):
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if obj.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
listReq.PageToken(obj.NextPageToken)
|
||||
}
|
||||
}()
|
||||
|
||||
|
@@ -20,8 +20,21 @@ func newSwiftTestSuite(t testing.TB) *test.Suite {
|
||||
// do not use excessive data
|
||||
MinimalData: true,
|
||||
|
||||
// wait for removals for at least 60s
|
||||
WaitForDelayedRemoval: 60 * time.Second,
|
||||
// wait for removals for at least 5m
|
||||
WaitForDelayedRemoval: 5 * time.Minute,
|
||||
|
||||
ErrorHandler: func(t testing.TB, be restic.Backend, err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if be.IsNotExist(err) {
|
||||
t.Logf("swift: ignoring error %v", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
return err
|
||||
},
|
||||
|
||||
// NewConfig returns a config for a new temporary backend that will be used in tests.
|
||||
NewConfig: func() (interface{}, error) {
|
||||
|
@@ -34,6 +34,9 @@ type Suite struct {
|
||||
// suite to wait for this amount of time until a file that was removed
|
||||
// really disappeared.
|
||||
WaitForDelayedRemoval time.Duration
|
||||
|
||||
// ErrorHandler allows ignoring certain errors.
|
||||
ErrorHandler func(testing.TB, restic.Backend, error) error
|
||||
}
|
||||
|
||||
// RunTests executes all defined tests as subtests of t.
|
||||
|
@@ -240,6 +240,78 @@ func (s *Suite) TestLoad(t *testing.T) {
|
||||
test.OK(t, b.Remove(context.TODO(), handle))
|
||||
}
|
||||
|
||||
// TestList makes sure that the backend implements List() pagination correctly.
|
||||
func (s *Suite) TestList(t *testing.T) {
|
||||
seedRand(t)
|
||||
|
||||
numTestFiles := rand.Intn(20) + 20
|
||||
|
||||
b := s.open(t)
|
||||
defer s.close(t, b)
|
||||
|
||||
list1 := restic.NewIDSet()
|
||||
|
||||
for i := 0; i < numTestFiles; i++ {
|
||||
data := []byte(fmt.Sprintf("random test blob %v", i))
|
||||
id := restic.Hash(data)
|
||||
h := restic.Handle{Type: restic.DataFile, Name: id.String()}
|
||||
err := b.Save(context.TODO(), h, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
list1.Insert(id)
|
||||
}
|
||||
|
||||
t.Logf("wrote %v files", len(list1))
|
||||
|
||||
var tests = []struct {
|
||||
maxItems int
|
||||
}{
|
||||
{11}, {23}, {numTestFiles}, {numTestFiles + 10}, {numTestFiles + 1123},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(fmt.Sprintf("max-%v", test.maxItems), func(t *testing.T) {
|
||||
list2 := restic.NewIDSet()
|
||||
|
||||
type setter interface {
|
||||
SetListMaxItems(int)
|
||||
}
|
||||
|
||||
if s, ok := b.(setter); ok {
|
||||
t.Logf("setting max list items to %d", test.maxItems)
|
||||
s.SetListMaxItems(test.maxItems)
|
||||
}
|
||||
|
||||
for name := range b.List(context.TODO(), restic.DataFile) {
|
||||
id, err := restic.ParseID(name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
list2.Insert(id)
|
||||
}
|
||||
|
||||
t.Logf("loaded %v IDs from backend", len(list2))
|
||||
|
||||
if !list1.Equals(list2) {
|
||||
t.Errorf("lists are not equal, list1 %d entries, list2 %d entries",
|
||||
len(list1), len(list2))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Logf("remove %d files", numTestFiles)
|
||||
handles := make([]restic.Handle, 0, len(list1))
|
||||
for id := range list1 {
|
||||
handles = append(handles, restic.Handle{Type: restic.DataFile, Name: id.String()})
|
||||
}
|
||||
|
||||
err := s.delayedRemove(t, b, handles...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
type errorCloser struct {
|
||||
io.Reader
|
||||
l int
|
||||
@@ -331,7 +403,7 @@ func (s *Suite) TestSave(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = delayedRemove(t, b, s.WaitForDelayedRemoval, h)
|
||||
err = s.delayedRemove(t, b, h)
|
||||
if err != nil {
|
||||
t.Fatalf("error removing item: %+v", err)
|
||||
}
|
||||
@@ -436,12 +508,15 @@ func testLoad(b restic.Backend, h restic.Handle, length int, offset int64) error
|
||||
return err
|
||||
}
|
||||
|
||||
func delayedRemove(t testing.TB, be restic.Backend, maxwait time.Duration, handles ...restic.Handle) error {
|
||||
func (s *Suite) delayedRemove(t testing.TB, be restic.Backend, handles ...restic.Handle) error {
|
||||
// Some backend (swift, I'm looking at you) may implement delayed
|
||||
// removal of data. Let's wait a bit if this happens.
|
||||
|
||||
for _, h := range handles {
|
||||
err := be.Remove(context.TODO(), h)
|
||||
if s.ErrorHandler != nil {
|
||||
err = s.ErrorHandler(t, be, err)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -452,8 +527,11 @@ func delayedRemove(t testing.TB, be restic.Backend, maxwait time.Duration, handl
|
||||
attempt := 0
|
||||
var found bool
|
||||
var err error
|
||||
for time.Since(start) <= maxwait {
|
||||
for time.Since(start) <= s.WaitForDelayedRemoval {
|
||||
found, err = be.Test(context.TODO(), h)
|
||||
if s.ErrorHandler != nil {
|
||||
err = s.ErrorHandler(t, be, err)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -564,7 +642,7 @@ func (s *Suite) TestBackend(t *testing.T) {
|
||||
test.Assert(t, err != nil, "expected error for %v, got %v", h, err)
|
||||
|
||||
// remove and recreate
|
||||
err = delayedRemove(t, b, s.WaitForDelayedRemoval, h)
|
||||
err = s.delayedRemove(t, b, h)
|
||||
test.OK(t, err)
|
||||
|
||||
// test that the blob is gone
|
||||
@@ -613,7 +691,7 @@ func (s *Suite) TestBackend(t *testing.T) {
|
||||
handles = append(handles, h)
|
||||
}
|
||||
|
||||
test.OK(t, delayedRemove(t, b, s.WaitForDelayedRemoval, handles...))
|
||||
test.OK(t, s.delayedRemove(t, b, handles...))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@@ -10,7 +10,7 @@ import (
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
// Statically ensure that *file implements the given interface
|
||||
// Statically ensure that *link implements the given interface
|
||||
var _ = fs.NodeReadlinker(&link{})
|
||||
|
||||
type link struct {
|
||||
|
@@ -25,7 +25,7 @@ type SnapshotsDir struct {
|
||||
names map[string]*restic.Snapshot
|
||||
}
|
||||
|
||||
// ensure that *DirSnapshots implements these interfaces
|
||||
// ensure that *SnapshotsDir implements these interfaces
|
||||
var _ = fs.HandleReadDirAller(&SnapshotsDir{})
|
||||
var _ = fs.NodeStringLookuper(&SnapshotsDir{})
|
||||
|
||||
|
@@ -503,7 +503,7 @@ func (node *Node) IsNewer(path string, fi os.FileInfo) bool {
|
||||
|
||||
extendedStat, ok := toStatT(fi.Sys())
|
||||
if !ok {
|
||||
if node.ModTime != fi.ModTime() ||
|
||||
if !node.ModTime.Equal(fi.ModTime()) ||
|
||||
node.Size != size {
|
||||
debug.Log("node %v is newer: timestamp or size changed", path)
|
||||
return true
|
||||
@@ -513,8 +513,8 @@ func (node *Node) IsNewer(path string, fi os.FileInfo) bool {
|
||||
|
||||
inode := extendedStat.ino()
|
||||
|
||||
if node.ModTime != fi.ModTime() ||
|
||||
node.ChangeTime != changeTime(extendedStat) ||
|
||||
if !node.ModTime.Equal(fi.ModTime()) ||
|
||||
!node.ChangeTime.Equal(changeTime(extendedStat)) ||
|
||||
node.Inode != uint64(inode) ||
|
||||
node.Size != size {
|
||||
debug.Log("node %v is newer: timestamp, size or inode changed", path)
|
||||
|
Reference in New Issue
Block a user