mirror of
https://github.com/restic/restic.git
synced 2025-12-04 00:21:46 +00:00
Refactor backends
This commit is contained in:
Alexander Neumann
119
backend/backend_test.go
Normal file
119
backend/backend_test.go
Normal file
@@ -0,0 +1,119 @@
|
||||
package backend_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/backend"
|
||||
)
|
||||
|
||||
func testBackend(b backend.Backend, t *testing.T) {
|
||||
for _, tpe := range []backend.Type{backend.Data, backend.Key, backend.Lock, backend.Snapshot, backend.Tree} {
|
||||
// detect non-existing files
|
||||
for _, test := range TestStrings {
|
||||
id, err := backend.ParseID(test.id)
|
||||
ok(t, err)
|
||||
|
||||
// test if blob is already in repository
|
||||
ret, err := b.Test(tpe, id.String())
|
||||
ok(t, err)
|
||||
assert(t, !ret, "blob was found to exist before creating")
|
||||
|
||||
// try to open not existing blob
|
||||
_, err = b.Get(tpe, id.String())
|
||||
assert(t, err != nil, "blob data could be extracted before creation")
|
||||
|
||||
// try to get string out, should fail
|
||||
ret, err = b.Test(tpe, id.String())
|
||||
ok(t, err)
|
||||
assert(t, !ret, "id %q was found (but should not have)", test.id)
|
||||
}
|
||||
|
||||
// add files
|
||||
for _, test := range TestStrings {
|
||||
// store string in backend
|
||||
blob, err := b.Create()
|
||||
ok(t, err)
|
||||
|
||||
_, err = blob.Write([]byte(test.data))
|
||||
ok(t, err)
|
||||
ok(t, blob.Finalize(tpe, test.id))
|
||||
|
||||
// try to get it out again
|
||||
rd, err := b.Get(tpe, test.id)
|
||||
ok(t, err)
|
||||
assert(t, rd != nil, "Get() returned nil")
|
||||
|
||||
buf, err := ioutil.ReadAll(rd)
|
||||
ok(t, err)
|
||||
equals(t, test.data, string(buf))
|
||||
|
||||
// compare content
|
||||
equals(t, test.data, string(buf))
|
||||
}
|
||||
|
||||
// test adding the first file again
|
||||
test := TestStrings[0]
|
||||
|
||||
// create blob
|
||||
blob, err := b.Create()
|
||||
ok(t, err)
|
||||
|
||||
_, err = blob.Write([]byte(test.data))
|
||||
ok(t, err)
|
||||
err = blob.Finalize(tpe, test.id)
|
||||
assert(t, err != nil, "expected error, got %v", err)
|
||||
|
||||
// remove and recreate
|
||||
err = b.Remove(tpe, test.id)
|
||||
ok(t, err)
|
||||
|
||||
// create blob
|
||||
blob, err = b.Create()
|
||||
ok(t, err)
|
||||
|
||||
_, err = io.Copy(blob, bytes.NewReader([]byte(test.data)))
|
||||
ok(t, err)
|
||||
ok(t, blob.Finalize(tpe, test.id))
|
||||
|
||||
// list items
|
||||
IDs := backend.IDs{}
|
||||
|
||||
for _, test := range TestStrings {
|
||||
id, err := backend.ParseID(test.id)
|
||||
ok(t, err)
|
||||
IDs = append(IDs, id)
|
||||
}
|
||||
|
||||
sort.Sort(IDs)
|
||||
|
||||
i := 0
|
||||
for s := range b.List(tpe, nil) {
|
||||
equals(t, IDs[i].String(), s)
|
||||
i++
|
||||
}
|
||||
|
||||
// remove content if requested
|
||||
if *testCleanup {
|
||||
for _, test := range TestStrings {
|
||||
id, err := backend.ParseID(test.id)
|
||||
ok(t, err)
|
||||
|
||||
found, err := b.Test(tpe, id.String())
|
||||
ok(t, err)
|
||||
assert(t, found, fmt.Sprintf("id %q was not found before removal", id))
|
||||
|
||||
ok(t, b.Remove(tpe, id.String()))
|
||||
|
||||
found, err = b.Test(tpe, id.String())
|
||||
ok(t, err)
|
||||
assert(t, !found, fmt.Sprintf("id %q not found after removal", id))
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,15 +1,12 @@
|
||||
package backend
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"sort"
|
||||
)
|
||||
|
||||
const (
|
||||
MinPrefixLength = 4
|
||||
MinPrefixLength = 8
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -24,21 +21,6 @@ var (
|
||||
|
||||
const hashSize = sha256.Size
|
||||
|
||||
// Each lists all entries of type t in the backend and calls function f() with
|
||||
// the id.
|
||||
func EachID(be Lister, t Type, f func(ID)) error {
|
||||
ids, err := be.List(t)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, id := range ids {
|
||||
f(id)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Hash returns the ID for data.
|
||||
func Hash(data []byte) ID {
|
||||
h := hashData(data)
|
||||
@@ -47,78 +29,67 @@ func Hash(data []byte) ID {
|
||||
return id
|
||||
}
|
||||
|
||||
// Find loads the list of all blobs of type t and searches for IDs which start
|
||||
// with prefix. If none is found, nil and ErrNoIDPrefixFound is returned. If
|
||||
// more than one is found, nil and ErrMultipleIDMatches is returned.
|
||||
func Find(be Lister, t Type, prefix string) (ID, error) {
|
||||
p, err := hex.DecodeString(prefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Find loads the list of all blobs of type t and searches for names which
|
||||
// start with prefix. If none is found, nil and ErrNoIDPrefixFound is returned.
|
||||
// If more than one is found, nil and ErrMultipleIDMatches is returned.
|
||||
func Find(be Lister, t Type, prefix string) (string, error) {
|
||||
done := make(chan struct{})
|
||||
defer close(done)
|
||||
|
||||
list, err := be.List(t)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
match := ID(nil)
|
||||
match := ""
|
||||
|
||||
// TODO: optimize by sorting list etc.
|
||||
for _, id := range list {
|
||||
if bytes.Equal(p, id[:len(p)]) {
|
||||
if match == nil {
|
||||
match = id
|
||||
for name := range be.List(t, done) {
|
||||
if prefix == name[:len(prefix)] {
|
||||
if match == "" {
|
||||
match = name
|
||||
} else {
|
||||
return nil, ErrMultipleIDMatches
|
||||
return "", ErrMultipleIDMatches
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if match != nil {
|
||||
if match != "" {
|
||||
return match, nil
|
||||
}
|
||||
|
||||
return nil, ErrNoIDPrefixFound
|
||||
return "", ErrNoIDPrefixFound
|
||||
}
|
||||
|
||||
// FindSnapshot takes a string and tries to find a snapshot whose ID matches
|
||||
// the string as closely as possible.
|
||||
func FindSnapshot(be Lister, s string) (ID, error) {
|
||||
// parse ID directly
|
||||
if id, err := ParseID(s); err == nil {
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func FindSnapshot(be Lister, s string) (string, error) {
|
||||
// find snapshot id with prefix
|
||||
id, err := Find(be, Snapshot, s)
|
||||
name, err := Find(be, Snapshot, s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return "", err
|
||||
}
|
||||
|
||||
return id, nil
|
||||
return name, nil
|
||||
}
|
||||
|
||||
// PrefixLength returns the number of bytes required so that all prefixes of
|
||||
// all IDs of type t are unique.
|
||||
// all names of type t are unique.
|
||||
func PrefixLength(be Lister, t Type) (int, error) {
|
||||
// load all IDs of the given type
|
||||
list, err := be.List(t)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
done := make(chan struct{})
|
||||
defer close(done)
|
||||
|
||||
sort.Sort(list)
|
||||
// load all IDs of the given type
|
||||
list := make([]string, 0, 100)
|
||||
for name := range be.List(t, done) {
|
||||
list = append(list, name)
|
||||
}
|
||||
|
||||
// select prefixes of length l, test if the last one is the same as the current one
|
||||
outer:
|
||||
for l := MinPrefixLength; l < IDSize; l++ {
|
||||
var last ID
|
||||
var last string
|
||||
|
||||
for _, id := range list {
|
||||
if bytes.Equal(last, id[:l]) {
|
||||
for _, name := range list {
|
||||
if last == name[:l] {
|
||||
continue outer
|
||||
}
|
||||
last = id[:l]
|
||||
last = name[:l]
|
||||
}
|
||||
|
||||
return l, nil
|
||||
|
||||
@@ -48,7 +48,7 @@ func str2id(s string) backend.ID {
|
||||
}
|
||||
|
||||
type mockBackend struct {
|
||||
list func(backend.Type) (backend.IDs, error)
|
||||
list func(backend.Type, <-chan struct{}) <-chan string
|
||||
get func(backend.Type, backend.ID) ([]byte, error)
|
||||
getReader func(backend.Type, backend.ID) (io.ReadCloser, error)
|
||||
create func(backend.Type, []byte) (backend.ID, error)
|
||||
@@ -58,8 +58,8 @@ type mockBackend struct {
|
||||
close func() error
|
||||
}
|
||||
|
||||
func (m mockBackend) List(t backend.Type) (backend.IDs, error) {
|
||||
return m.list(t)
|
||||
func (m mockBackend) List(t backend.Type, done <-chan struct{}) <-chan string {
|
||||
return m.list(t, done)
|
||||
}
|
||||
|
||||
func (m mockBackend) Get(t backend.Type, id backend.ID) ([]byte, error) {
|
||||
@@ -105,21 +105,32 @@ func TestPrefixLength(t *testing.T) {
|
||||
list := samples
|
||||
|
||||
m := mockBackend{}
|
||||
m.list = func(t backend.Type) (backend.IDs, error) {
|
||||
return list, nil
|
||||
m.list = func(t backend.Type, done <-chan struct{}) <-chan string {
|
||||
ch := make(chan string)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
for _, id := range list {
|
||||
select {
|
||||
case ch <- id.String():
|
||||
case <-done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return ch
|
||||
}
|
||||
|
||||
l, err := backend.PrefixLength(m, backend.Snapshot)
|
||||
ok(t, err)
|
||||
equals(t, 10, l)
|
||||
equals(t, 19, l)
|
||||
|
||||
list = samples[:3]
|
||||
l, err = backend.PrefixLength(m, backend.Snapshot)
|
||||
ok(t, err)
|
||||
equals(t, 10, l)
|
||||
equals(t, 19, l)
|
||||
|
||||
list = samples[3:]
|
||||
l, err = backend.PrefixLength(m, backend.Snapshot)
|
||||
ok(t, err)
|
||||
equals(t, 4, l)
|
||||
equals(t, 8, l)
|
||||
}
|
||||
|
||||
@@ -6,6 +6,16 @@ import (
|
||||
"github.com/restic/restic/backend"
|
||||
)
|
||||
|
||||
var TestStrings = []struct {
|
||||
id string
|
||||
data string
|
||||
}{
|
||||
{"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", "foobar"},
|
||||
{"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"},
|
||||
{"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", "foo/bar"},
|
||||
{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"},
|
||||
}
|
||||
|
||||
func TestID(t *testing.T) {
|
||||
for _, test := range TestStrings {
|
||||
id, err := backend.ParseID(test.id)
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
package backend
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
)
|
||||
import "io"
|
||||
|
||||
// Type is the type of a Blob.
|
||||
type Type string
|
||||
|
||||
const (
|
||||
@@ -16,62 +14,60 @@ const (
|
||||
)
|
||||
|
||||
const (
|
||||
BackendVersion = 1
|
||||
Version = 1
|
||||
)
|
||||
|
||||
var (
|
||||
ErrAlreadyPresent = errors.New("blob is already present in backend")
|
||||
)
|
||||
// A Backend manages blobs of data.
|
||||
type Backend interface {
|
||||
// Location returns a string that specifies the location of the repository,
|
||||
// like a URL.
|
||||
Location() string
|
||||
|
||||
type Blob interface {
|
||||
io.WriteCloser
|
||||
ID() (ID, error)
|
||||
Size() uint
|
||||
// Create creates a new Blob. The data is available only after Finalize()
|
||||
// has been called on the returned Blob.
|
||||
Create() (Blob, error)
|
||||
|
||||
// Get returns an io.ReadCloser for the Blob with the given name of type t.
|
||||
Get(t Type, name string) (io.ReadCloser, error)
|
||||
|
||||
// Test a boolean value whether a Blob with the name and type exists.
|
||||
Test(t Type, name string) (bool, error)
|
||||
|
||||
// Remove removes a Blob with type t and name.
|
||||
Remove(t Type, name string) error
|
||||
|
||||
// Close the backend
|
||||
Close() error
|
||||
|
||||
Identifier
|
||||
Lister
|
||||
}
|
||||
|
||||
type Identifier interface {
|
||||
// ID returns a unique ID for a specific repository. This means restic can
|
||||
// recognize repositories accessed via different methods (e.g. local file
|
||||
// access and sftp).
|
||||
ID() string
|
||||
}
|
||||
|
||||
type Lister interface {
|
||||
List(Type) (IDs, error)
|
||||
}
|
||||
|
||||
type Getter interface {
|
||||
Get(Type, ID) ([]byte, error)
|
||||
GetReader(Type, ID) (io.ReadCloser, error)
|
||||
}
|
||||
|
||||
type Creater interface {
|
||||
Create(Type) (Blob, error)
|
||||
}
|
||||
|
||||
type Tester interface {
|
||||
Test(Type, ID) (bool, error)
|
||||
}
|
||||
|
||||
type Remover interface {
|
||||
Remove(Type, ID) error
|
||||
}
|
||||
|
||||
type Closer interface {
|
||||
Close() error
|
||||
// List returns a channel that yields all names of blobs of type t in
|
||||
// lexicographic order. A goroutine is started for this. If the channel
|
||||
// done is closed, sending stops.
|
||||
List(t Type, done <-chan struct{}) <-chan string
|
||||
}
|
||||
|
||||
type Deleter interface {
|
||||
// Delete the complete repository.
|
||||
Delete() error
|
||||
}
|
||||
|
||||
type Locationer interface {
|
||||
Location() string
|
||||
}
|
||||
type Blob interface {
|
||||
io.Writer
|
||||
|
||||
type IDer interface {
|
||||
ID() ID
|
||||
}
|
||||
// Finalize moves the data blob to the final location for type and name.
|
||||
Finalize(t Type, name string) error
|
||||
|
||||
type Backend interface {
|
||||
Lister
|
||||
Getter
|
||||
Creater
|
||||
Tester
|
||||
Remover
|
||||
Closer
|
||||
IDer
|
||||
// Size returns the number of bytes written to the backend so far.
|
||||
Size() uint
|
||||
}
|
||||
|
||||
456
backend/local.go
456
backend/local.go
@@ -1,456 +0,0 @@
|
||||
package backend
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
dirMode = 0700
|
||||
dataPath = "data"
|
||||
snapshotPath = "snapshots"
|
||||
treePath = "trees"
|
||||
lockPath = "locks"
|
||||
keyPath = "keys"
|
||||
tempPath = "tmp"
|
||||
versionFileName = "version"
|
||||
idFileName = "id"
|
||||
)
|
||||
|
||||
var ErrWrongData = errors.New("wrong data returned by backend, checksum does not match")
|
||||
|
||||
type Local struct {
|
||||
p string
|
||||
ver uint
|
||||
id ID
|
||||
}
|
||||
|
||||
// OpenLocal opens the local backend at dir.
|
||||
func OpenLocal(dir string) (*Local, error) {
|
||||
items := []string{
|
||||
dir,
|
||||
filepath.Join(dir, dataPath),
|
||||
filepath.Join(dir, snapshotPath),
|
||||
filepath.Join(dir, treePath),
|
||||
filepath.Join(dir, lockPath),
|
||||
filepath.Join(dir, keyPath),
|
||||
filepath.Join(dir, tempPath),
|
||||
}
|
||||
|
||||
// test if all necessary dirs and files are there
|
||||
for _, d := range items {
|
||||
if _, err := os.Stat(d); err != nil {
|
||||
return nil, fmt.Errorf("%s does not exist", d)
|
||||
}
|
||||
}
|
||||
|
||||
// read version file
|
||||
f, err := os.Open(filepath.Join(dir, versionFileName))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to read version file: %v\n", err)
|
||||
}
|
||||
|
||||
var version uint
|
||||
n, err := fmt.Fscanf(f, "%d", &version)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if n != 1 {
|
||||
return nil, errors.New("could not read version from file")
|
||||
}
|
||||
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// check version
|
||||
if version != BackendVersion {
|
||||
return nil, fmt.Errorf("wrong version %d", version)
|
||||
}
|
||||
|
||||
// read ID
|
||||
f, err = os.Open(filepath.Join(dir, idFileName))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
buf, err := ioutil.ReadAll(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
id, err := ParseID(strings.TrimSpace(string(buf)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Local{p: dir, ver: version, id: id}, nil
|
||||
}
|
||||
|
||||
// CreateLocal creates all the necessary files and directories for a new local
|
||||
// backend at dir.
|
||||
func CreateLocal(dir string) (*Local, error) {
|
||||
versionFile := filepath.Join(dir, versionFileName)
|
||||
idFile := filepath.Join(dir, idFileName)
|
||||
dirs := []string{
|
||||
dir,
|
||||
filepath.Join(dir, dataPath),
|
||||
filepath.Join(dir, snapshotPath),
|
||||
filepath.Join(dir, treePath),
|
||||
filepath.Join(dir, lockPath),
|
||||
filepath.Join(dir, keyPath),
|
||||
filepath.Join(dir, tempPath),
|
||||
}
|
||||
|
||||
// test if files already exist
|
||||
_, err := os.Lstat(versionFile)
|
||||
if err == nil {
|
||||
return nil, errors.New("version file already exists")
|
||||
}
|
||||
|
||||
_, err = os.Lstat(idFile)
|
||||
if err == nil {
|
||||
return nil, errors.New("id file already exists")
|
||||
}
|
||||
|
||||
// test if directories already exist
|
||||
for _, d := range dirs[1:] {
|
||||
if _, err := os.Stat(d); err == nil {
|
||||
return nil, fmt.Errorf("dir %s already exists", d)
|
||||
}
|
||||
}
|
||||
|
||||
// create paths for data, refs and temp
|
||||
for _, d := range dirs {
|
||||
err := os.MkdirAll(d, dirMode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// create version file
|
||||
f, err := os.Create(versionFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, err = fmt.Fprintf(f, "%d\n", BackendVersion)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// create ID file
|
||||
id := make([]byte, sha256.Size)
|
||||
_, err = rand.Read(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
f, err = os.Create(idFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, err = fmt.Fprintf(f, "%s\n", ID(id).String())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// open backend
|
||||
return OpenLocal(dir)
|
||||
}
|
||||
|
||||
// Location returns this backend's location (the directory name).
|
||||
func (b *Local) Location() string {
|
||||
return b.p
|
||||
}
|
||||
|
||||
// Return temp directory in correct directory for this backend.
|
||||
func (b *Local) tempFile() (*os.File, error) {
|
||||
return ioutil.TempFile(filepath.Join(b.p, tempPath), "temp-")
|
||||
}
|
||||
|
||||
// Rename temp file to final name according to type and ID.
|
||||
func (b *Local) renameFile(file *os.File, t Type, id ID) error {
|
||||
filename := b.filename(t, id)
|
||||
oldname := file.Name()
|
||||
|
||||
if t == Data || t == Tree {
|
||||
// create directories if necessary, ignore errors
|
||||
os.MkdirAll(filepath.Dir(filename), dirMode)
|
||||
}
|
||||
|
||||
err := os.Rename(oldname, filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// set mode to read-only
|
||||
fi, err := os.Stat(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.Chmod(filename, fi.Mode()&os.FileMode(^uint32(0222)))
|
||||
}
|
||||
|
||||
// Construct directory for given Type.
|
||||
func (b *Local) dirname(t Type, id ID) string {
|
||||
var n string
|
||||
switch t {
|
||||
case Data:
|
||||
n = dataPath
|
||||
if id != nil {
|
||||
n = filepath.Join(dataPath, fmt.Sprintf("%02x", id[0]))
|
||||
}
|
||||
case Snapshot:
|
||||
n = snapshotPath
|
||||
case Tree:
|
||||
n = treePath
|
||||
if id != nil {
|
||||
n = filepath.Join(treePath, fmt.Sprintf("%02x", id[0]))
|
||||
}
|
||||
case Lock:
|
||||
n = lockPath
|
||||
case Key:
|
||||
n = keyPath
|
||||
}
|
||||
return filepath.Join(b.p, n)
|
||||
}
|
||||
|
||||
type localBlob struct {
|
||||
f *os.File
|
||||
hw *HashingWriter
|
||||
backend *Local
|
||||
tpe Type
|
||||
id ID
|
||||
size uint
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (lb *localBlob) Close() error {
|
||||
if lb.closed {
|
||||
return errors.New("Close() called on closed file")
|
||||
|
||||
}
|
||||
lb.closed = true
|
||||
|
||||
err := lb.f.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("local: file.Close: %v", err)
|
||||
}
|
||||
|
||||
// get ID
|
||||
lb.id = ID(lb.hw.Sum(nil))
|
||||
|
||||
// check for duplicate ID
|
||||
res, err := lb.backend.Test(lb.tpe, lb.id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("testing presence of ID %v failed: %v", lb.id, err)
|
||||
}
|
||||
|
||||
if res {
|
||||
return ErrAlreadyPresent
|
||||
}
|
||||
|
||||
// rename file
|
||||
err = lb.backend.renameFile(lb.f, lb.tpe, lb.id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (lb *localBlob) Write(p []byte) (int, error) {
|
||||
n, err := lb.hw.Write(p)
|
||||
lb.size += uint(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (lb *localBlob) ID() (ID, error) {
|
||||
if lb.id == nil {
|
||||
return nil, errors.New("blob is not closed, ID unavailable")
|
||||
}
|
||||
|
||||
return lb.id, nil
|
||||
}
|
||||
|
||||
func (lb *localBlob) Size() uint {
|
||||
return lb.size
|
||||
}
|
||||
|
||||
// Create creates a new blob of type t. Blob implements io.WriteCloser. Once
|
||||
// Close() has been called, ID() can be used to retrieve the ID. If the blob is
|
||||
// already present, Close() returns ErrAlreadyPresent.
|
||||
func (b *Local) Create(t Type) (Blob, error) {
|
||||
// TODO: make sure that tempfile is removed upon error
|
||||
|
||||
// create tempfile in backend
|
||||
file, err := b.tempFile()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
hw := NewHashingWriter(file, newHash())
|
||||
blob := localBlob{
|
||||
hw: hw,
|
||||
f: file,
|
||||
backend: b,
|
||||
tpe: t,
|
||||
}
|
||||
|
||||
return &blob, nil
|
||||
}
|
||||
|
||||
// Construct path for given Type and ID.
|
||||
func (b *Local) filename(t Type, id ID) string {
|
||||
return filepath.Join(b.dirname(t, id), id.String())
|
||||
}
|
||||
|
||||
// Get returns the content stored under the given ID. If the data doesn't match
|
||||
// the requested ID, ErrWrongData is returned.
|
||||
func (b *Local) Get(t Type, id ID) ([]byte, error) {
|
||||
if id == nil {
|
||||
return nil, errors.New("unable to load nil ID")
|
||||
}
|
||||
|
||||
// try to open file
|
||||
file, err := os.Open(b.filename(t, id))
|
||||
defer file.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// read all
|
||||
buf, err := ioutil.ReadAll(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// check id
|
||||
if !Hash(buf).Equal(id) {
|
||||
return nil, ErrWrongData
|
||||
}
|
||||
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
// GetReader returns a reader that yields the content stored under the given
|
||||
// ID. The content is not verified. The reader should be closed after draining
|
||||
// it.
|
||||
func (b *Local) GetReader(t Type, id ID) (io.ReadCloser, error) {
|
||||
if id == nil {
|
||||
return nil, errors.New("unable to load nil ID")
|
||||
}
|
||||
|
||||
// try to open file
|
||||
file, err := os.Open(b.filename(t, id))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return file, nil
|
||||
}
|
||||
|
||||
// Test returns true if a blob of the given type and ID exists in the backend.
|
||||
func (b *Local) Test(t Type, id ID) (bool, error) {
|
||||
// try to open file
|
||||
file, err := os.Open(b.filename(t, id))
|
||||
defer func() {
|
||||
file.Close()
|
||||
}()
|
||||
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Remove removes the content stored at ID.
|
||||
func (b *Local) Remove(t Type, id ID) error {
|
||||
return os.Remove(b.filename(t, id))
|
||||
}
|
||||
|
||||
// List lists all objects of a given type.
|
||||
func (b *Local) List(t Type) (IDs, error) {
|
||||
// TODO: use os.Open() and d.Readdirnames() instead of Glob()
|
||||
var pattern string
|
||||
if t == Data || t == Tree {
|
||||
pattern = filepath.Join(b.dirname(t, nil), "*", "*")
|
||||
} else {
|
||||
pattern = filepath.Join(b.dirname(t, nil), "*")
|
||||
}
|
||||
|
||||
matches, err := filepath.Glob(pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ids := make(IDs, 0, len(matches))
|
||||
|
||||
for _, m := range matches {
|
||||
base := filepath.Base(m)
|
||||
|
||||
if base == "" {
|
||||
continue
|
||||
}
|
||||
id, err := ParseID(base)
|
||||
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
ids = append(ids, id)
|
||||
}
|
||||
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
// Version returns the version of this local backend.
|
||||
func (b *Local) Version() uint {
|
||||
return b.ver
|
||||
}
|
||||
|
||||
// ID returns the ID of this local backend.
|
||||
func (b *Local) ID() ID {
|
||||
return b.id
|
||||
}
|
||||
|
||||
// Close closes the backend
|
||||
func (b *Local) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete removes the repository and all files.
|
||||
func (b *Local) Delete() error {
|
||||
return os.RemoveAll(b.p)
|
||||
}
|
||||
36
backend/local/generic_test.go
Normal file
36
backend/local/generic_test.go
Normal file
@@ -0,0 +1,36 @@
|
||||
package local_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// assert fails the test if the condition is false.
|
||||
func assert(tb testing.TB, condition bool, msg string, v ...interface{}) {
|
||||
if !condition {
|
||||
_, file, line, _ := runtime.Caller(1)
|
||||
fmt.Printf("\033[31m%s:%d: "+msg+"\033[39m\n\n", append([]interface{}{filepath.Base(file), line}, v...)...)
|
||||
tb.FailNow()
|
||||
}
|
||||
}
|
||||
|
||||
// ok fails the test if an err is not nil.
|
||||
func ok(tb testing.TB, err error) {
|
||||
if err != nil {
|
||||
_, file, line, _ := runtime.Caller(1)
|
||||
fmt.Printf("\033[31m%s:%d: unexpected error: %s\033[39m\n\n", filepath.Base(file), line, err.Error())
|
||||
tb.FailNow()
|
||||
}
|
||||
}
|
||||
|
||||
// equals fails the test if exp is not equal to act.
|
||||
func equals(tb testing.TB, exp, act interface{}) {
|
||||
if !reflect.DeepEqual(exp, act) {
|
||||
_, file, line, _ := runtime.Caller(1)
|
||||
fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act)
|
||||
tb.FailNow()
|
||||
}
|
||||
}
|
||||
377
backend/local/local.go
Normal file
377
backend/local/local.go
Normal file
@@ -0,0 +1,377 @@
|
||||
package local
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/restic/restic/backend"
|
||||
)
|
||||
|
||||
var ErrWrongData = errors.New("wrong data returned by backend, checksum does not match")
|
||||
|
||||
type Local struct {
|
||||
p string
|
||||
ver uint
|
||||
name string
|
||||
id string
|
||||
}
|
||||
|
||||
// Open opens the local backend at dir.
|
||||
func Open(dir string) (*Local, error) {
|
||||
items := []string{
|
||||
dir,
|
||||
filepath.Join(dir, backend.Paths.Data),
|
||||
filepath.Join(dir, backend.Paths.Snapshots),
|
||||
filepath.Join(dir, backend.Paths.Trees),
|
||||
filepath.Join(dir, backend.Paths.Locks),
|
||||
filepath.Join(dir, backend.Paths.Keys),
|
||||
filepath.Join(dir, backend.Paths.Temp),
|
||||
}
|
||||
|
||||
// test if all necessary dirs and files are there
|
||||
for _, d := range items {
|
||||
if _, err := os.Stat(d); err != nil {
|
||||
return nil, fmt.Errorf("%s does not exist", d)
|
||||
}
|
||||
}
|
||||
|
||||
// read version file
|
||||
f, err := os.Open(filepath.Join(dir, backend.Paths.Version))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to read version file: %v\n", err)
|
||||
}
|
||||
|
||||
var version uint
|
||||
n, err := fmt.Fscanf(f, "%d", &version)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if n != 1 {
|
||||
return nil, errors.New("could not read version from file")
|
||||
}
|
||||
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// check version
|
||||
if version != backend.Version {
|
||||
return nil, fmt.Errorf("wrong version %d", version)
|
||||
}
|
||||
|
||||
// read ID
|
||||
f, err = os.Open(filepath.Join(dir, backend.Paths.ID))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
buf, err := ioutil.ReadAll(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
id := strings.TrimSpace(string(buf))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Local{p: dir, ver: version, id: id}, nil
|
||||
}
|
||||
|
||||
// Create creates all the necessary files and directories for a new local
|
||||
// backend at dir.
|
||||
func Create(dir string) (*Local, error) {
|
||||
versionFile := filepath.Join(dir, backend.Paths.Version)
|
||||
idFile := filepath.Join(dir, backend.Paths.ID)
|
||||
dirs := []string{
|
||||
dir,
|
||||
filepath.Join(dir, backend.Paths.Data),
|
||||
filepath.Join(dir, backend.Paths.Snapshots),
|
||||
filepath.Join(dir, backend.Paths.Trees),
|
||||
filepath.Join(dir, backend.Paths.Locks),
|
||||
filepath.Join(dir, backend.Paths.Keys),
|
||||
filepath.Join(dir, backend.Paths.Temp),
|
||||
}
|
||||
|
||||
// test if files already exist
|
||||
_, err := os.Lstat(versionFile)
|
||||
if err == nil {
|
||||
return nil, errors.New("version file already exists")
|
||||
}
|
||||
|
||||
_, err = os.Lstat(idFile)
|
||||
if err == nil {
|
||||
return nil, errors.New("id file already exists")
|
||||
}
|
||||
|
||||
// test if directories already exist
|
||||
for _, d := range dirs[1:] {
|
||||
if _, err := os.Stat(d); err == nil {
|
||||
return nil, fmt.Errorf("dir %s already exists", d)
|
||||
}
|
||||
}
|
||||
|
||||
// create paths for data, refs and temp
|
||||
for _, d := range dirs {
|
||||
err := os.MkdirAll(d, backend.Modes.Dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// create version file
|
||||
f, err := os.Create(versionFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, err = fmt.Fprintf(f, "%d\n", backend.Version)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// create ID file
|
||||
id := make([]byte, sha256.Size)
|
||||
_, err = rand.Read(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
f, err = os.Create(idFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, err = fmt.Fprintln(f, hex.EncodeToString(id))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// open backend
|
||||
return Open(dir)
|
||||
}
|
||||
|
||||
// Location returns this backend's location (the directory name).
|
||||
func (b *Local) Location() string {
|
||||
return b.p
|
||||
}
|
||||
|
||||
// Return temp directory in correct directory for this backend.
|
||||
func (b *Local) tempFile() (*os.File, error) {
|
||||
return ioutil.TempFile(filepath.Join(b.p, backend.Paths.Temp), "temp-")
|
||||
}
|
||||
|
||||
type localBlob struct {
|
||||
f *os.File
|
||||
size uint
|
||||
final bool
|
||||
basedir string
|
||||
}
|
||||
|
||||
func (lb *localBlob) Write(p []byte) (int, error) {
|
||||
if lb.final {
|
||||
return 0, errors.New("blob already closed")
|
||||
}
|
||||
|
||||
n, err := lb.f.Write(p)
|
||||
lb.size += uint(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (lb *localBlob) Size() uint {
|
||||
return lb.size
|
||||
}
|
||||
|
||||
func (lb *localBlob) Finalize(t backend.Type, name string) error {
|
||||
if lb.final {
|
||||
return errors.New("Already finalized")
|
||||
}
|
||||
|
||||
lb.final = true
|
||||
|
||||
err := lb.f.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("local: file.Close: %v", err)
|
||||
}
|
||||
|
||||
f := filename(lb.basedir, t, name)
|
||||
|
||||
// create directories if necessary, ignore errors
|
||||
if t == backend.Data || t == backend.Tree {
|
||||
os.MkdirAll(filepath.Dir(f), backend.Modes.Dir)
|
||||
}
|
||||
|
||||
// test if new path already exists
|
||||
if _, err := os.Stat(f); err == nil {
|
||||
return fmt.Errorf("Close(): file %v already exists", f)
|
||||
}
|
||||
|
||||
if err := os.Rename(lb.f.Name(), f); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// set mode to read-only
|
||||
fi, err := os.Stat(f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.Chmod(f, fi.Mode()&os.FileMode(^uint32(0222)))
|
||||
}
|
||||
|
||||
// Create creates a new Blob. The data is available only after Finalize()
|
||||
// has been called on the returned Blob.
|
||||
func (b *Local) Create() (backend.Blob, error) {
|
||||
// TODO: make sure that tempfile is removed upon error
|
||||
|
||||
// create tempfile in backend
|
||||
file, err := b.tempFile()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
blob := localBlob{
|
||||
f: file,
|
||||
basedir: b.p,
|
||||
}
|
||||
|
||||
return &blob, nil
|
||||
}
|
||||
|
||||
// Construct path for given Type and name.
|
||||
func filename(base string, t backend.Type, name string) string {
|
||||
return filepath.Join(dirname(base, t, name), name)
|
||||
}
|
||||
|
||||
// Construct directory for given Type.
|
||||
func dirname(base string, t backend.Type, name string) string {
|
||||
var n string
|
||||
switch t {
|
||||
case backend.Data:
|
||||
n = backend.Paths.Data
|
||||
if len(name) > 2 {
|
||||
n = filepath.Join(n, name[:2])
|
||||
}
|
||||
case backend.Snapshot:
|
||||
n = backend.Paths.Snapshots
|
||||
case backend.Tree:
|
||||
n = backend.Paths.Trees
|
||||
if len(name) > 2 {
|
||||
n = filepath.Join(n, name[:2])
|
||||
}
|
||||
case backend.Lock:
|
||||
n = backend.Paths.Locks
|
||||
case backend.Key:
|
||||
n = backend.Paths.Keys
|
||||
}
|
||||
return filepath.Join(base, n)
|
||||
}
|
||||
|
||||
// Get returns a reader that yields the content stored under the given
|
||||
// name. The reader should be closed after draining it.
|
||||
func (b *Local) Get(t backend.Type, name string) (io.ReadCloser, error) {
|
||||
return os.Open(filename(b.p, t, name))
|
||||
}
|
||||
|
||||
// Test returns true if a blob of the given type and name exists in the backend.
|
||||
func (b *Local) Test(t backend.Type, name string) (bool, error) {
|
||||
_, err := os.Stat(filename(b.p, t, name))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Remove removes the blob with the given name and type.
|
||||
func (b *Local) Remove(t backend.Type, name string) error {
|
||||
return os.Remove(filename(b.p, t, name))
|
||||
}
|
||||
|
||||
// List returns a channel that yields all names of blobs of type t. A
|
||||
// goroutine ist started for this. If the channel done is closed, sending
|
||||
// stops.
|
||||
func (b *Local) List(t backend.Type, done <-chan struct{}) <-chan string {
|
||||
// TODO: use os.Open() and d.Readdirnames() instead of Glob()
|
||||
var pattern string
|
||||
if t == backend.Data || t == backend.Tree {
|
||||
pattern = filepath.Join(dirname(b.p, t, ""), "*", "*")
|
||||
} else {
|
||||
pattern = filepath.Join(dirname(b.p, t, ""), "*")
|
||||
}
|
||||
|
||||
ch := make(chan string)
|
||||
matches, err := filepath.Glob(pattern)
|
||||
if err != nil {
|
||||
close(ch)
|
||||
return ch
|
||||
}
|
||||
|
||||
for i := range matches {
|
||||
matches[i] = filepath.Base(matches[i])
|
||||
}
|
||||
|
||||
sort.Strings(matches)
|
||||
|
||||
go func() {
|
||||
defer close(ch)
|
||||
for _, m := range matches {
|
||||
if m == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case ch <- m:
|
||||
case <-done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return ch
|
||||
}
|
||||
|
||||
// Version returns the version of this local backend.
|
||||
func (b *Local) Version() uint {
|
||||
return b.ver
|
||||
}
|
||||
|
||||
// ID returns the ID of this local backend.
|
||||
func (b *Local) ID() string {
|
||||
return b.id
|
||||
}
|
||||
|
||||
// Delete removes the repository and all files.
|
||||
func (b *Local) Delete() error { return os.RemoveAll(b.p) }
|
||||
|
||||
// Close does nothing
|
||||
func (b *Local) Close() error { return nil }
|
||||
@@ -1,34 +1,21 @@
|
||||
package backend_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/backend"
|
||||
"github.com/restic/restic/backend/local"
|
||||
)
|
||||
|
||||
var testCleanup = flag.Bool("test.cleanup", true, "clean up after running tests (remove local backend directory with all content)")
|
||||
|
||||
var TestStrings = []struct {
|
||||
id string
|
||||
data string
|
||||
}{
|
||||
{"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", "foobar"},
|
||||
{"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"},
|
||||
{"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", "foo/bar"},
|
||||
{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"},
|
||||
}
|
||||
|
||||
func setupLocalBackend(t *testing.T) *backend.Local {
|
||||
func setupLocalBackend(t *testing.T) *local.Local {
|
||||
tempdir, err := ioutil.TempDir("", "restic-test-")
|
||||
ok(t, err)
|
||||
|
||||
b, err := backend.CreateLocal(tempdir)
|
||||
b, err := local.Create(tempdir)
|
||||
ok(t, err)
|
||||
|
||||
t.Logf("created local backend at %s", tempdir)
|
||||
@@ -36,7 +23,7 @@ func setupLocalBackend(t *testing.T) *backend.Local {
|
||||
return b
|
||||
}
|
||||
|
||||
func teardownLocalBackend(t *testing.T, b *backend.Local) {
|
||||
func teardownLocalBackend(t *testing.T, b *local.Local) {
|
||||
if !*testCleanup {
|
||||
t.Logf("leaving local backend at %s\n", b.Location())
|
||||
return
|
||||
@@ -45,137 +32,9 @@ func teardownLocalBackend(t *testing.T, b *backend.Local) {
|
||||
ok(t, b.Delete())
|
||||
}
|
||||
|
||||
func testBackend(b backend.Backend, t *testing.T) {
|
||||
for _, tpe := range []backend.Type{backend.Data, backend.Key, backend.Lock, backend.Snapshot, backend.Tree} {
|
||||
// detect non-existing files
|
||||
for _, test := range TestStrings {
|
||||
id, err := backend.ParseID(test.id)
|
||||
ok(t, err)
|
||||
|
||||
// test if blob is already in repository
|
||||
ret, err := b.Test(tpe, id)
|
||||
ok(t, err)
|
||||
assert(t, !ret, "blob was found to exist before creating")
|
||||
|
||||
// try to open not existing blob
|
||||
d, err := b.Get(tpe, id)
|
||||
assert(t, err != nil && d == nil, "blob data could be extracted befor creation")
|
||||
|
||||
// try to get string out, should fail
|
||||
ret, err = b.Test(tpe, id)
|
||||
ok(t, err)
|
||||
assert(t, !ret, fmt.Sprintf("id %q was found (but should not have)", test.id))
|
||||
}
|
||||
|
||||
// add files
|
||||
for _, test := range TestStrings {
|
||||
// store string in backend
|
||||
blob, err := b.Create(tpe)
|
||||
ok(t, err)
|
||||
|
||||
_, err = blob.Write([]byte(test.data))
|
||||
ok(t, err)
|
||||
ok(t, blob.Close())
|
||||
|
||||
id, err := blob.ID()
|
||||
ok(t, err)
|
||||
|
||||
equals(t, test.id, id.String())
|
||||
|
||||
// try to get it out again
|
||||
buf, err := b.Get(tpe, id)
|
||||
ok(t, err)
|
||||
assert(t, buf != nil, "Get() returned nil")
|
||||
|
||||
// compare content
|
||||
equals(t, test.data, string(buf))
|
||||
|
||||
// compare content again via stream function
|
||||
rd, err := b.GetReader(tpe, id)
|
||||
ok(t, err)
|
||||
buf, err = ioutil.ReadAll(rd)
|
||||
ok(t, err)
|
||||
equals(t, test.data, string(buf))
|
||||
}
|
||||
|
||||
// test adding the first file again
|
||||
test := TestStrings[0]
|
||||
id, err := backend.ParseID(test.id)
|
||||
ok(t, err)
|
||||
|
||||
// create blob
|
||||
blob, err := b.Create(tpe)
|
||||
ok(t, err)
|
||||
|
||||
_, err = io.Copy(blob, bytes.NewReader([]byte(test.data)))
|
||||
ok(t, err)
|
||||
err = blob.Close()
|
||||
assert(t, err == backend.ErrAlreadyPresent,
|
||||
"wrong error returned: expected %v, got %v",
|
||||
backend.ErrAlreadyPresent, err)
|
||||
|
||||
id2, err := blob.ID()
|
||||
ok(t, err)
|
||||
|
||||
assert(t, id.Equal(id2), "IDs do not match: expected %v, got %v", id, id2)
|
||||
|
||||
// remove and recreate
|
||||
err = b.Remove(tpe, id)
|
||||
ok(t, err)
|
||||
|
||||
// create blob
|
||||
blob, err = b.Create(tpe)
|
||||
ok(t, err)
|
||||
|
||||
_, err = io.Copy(blob, bytes.NewReader([]byte(test.data)))
|
||||
ok(t, err)
|
||||
err = blob.Close()
|
||||
ok(t, err)
|
||||
|
||||
id2, err = blob.ID()
|
||||
ok(t, err)
|
||||
assert(t, id.Equal(id2), "IDs do not match: expected %v, got %v", id, id2)
|
||||
|
||||
// list items
|
||||
IDs := backend.IDs{}
|
||||
|
||||
for _, test := range TestStrings {
|
||||
id, err := backend.ParseID(test.id)
|
||||
ok(t, err)
|
||||
IDs = append(IDs, id)
|
||||
}
|
||||
|
||||
ids, err := b.List(tpe)
|
||||
ok(t, err)
|
||||
|
||||
sort.Sort(ids)
|
||||
sort.Sort(IDs)
|
||||
equals(t, IDs, ids)
|
||||
|
||||
// remove content if requested
|
||||
if *testCleanup {
|
||||
for _, test := range TestStrings {
|
||||
id, err := backend.ParseID(test.id)
|
||||
ok(t, err)
|
||||
|
||||
found, err := b.Test(tpe, id)
|
||||
ok(t, err)
|
||||
assert(t, found, fmt.Sprintf("id %q was not found before removal", id))
|
||||
|
||||
ok(t, b.Remove(tpe, id))
|
||||
|
||||
found, err = b.Test(tpe, id)
|
||||
ok(t, err)
|
||||
assert(t, !found, fmt.Sprintf("id %q not found after removal", id))
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackend(t *testing.T) {
|
||||
func TestLocalBackend(t *testing.T) {
|
||||
// test for non-existing backend
|
||||
b, err := backend.OpenLocal("/invalid-restic-test")
|
||||
b, err := local.Open("/invalid-restic-test")
|
||||
assert(t, err != nil, "opening invalid repository at /invalid-restic-test should have failed, but err is nil")
|
||||
assert(t, b == nil, fmt.Sprintf("opening invalid repository at /invalid-restic-test should have failed, but b is not nil: %v", b))
|
||||
|
||||
@@ -190,10 +49,10 @@ func TestLocalBackendCreationFailures(t *testing.T) {
|
||||
defer teardownLocalBackend(t, b)
|
||||
|
||||
// test failure to create a new repository at the same location
|
||||
b2, err := backend.CreateLocal(b.Location())
|
||||
b2, err := local.Create(b.Location())
|
||||
assert(t, err != nil && b2 == nil, fmt.Sprintf("creating a repository at %s for the second time should have failed", b.Location()))
|
||||
|
||||
// test failure to create a new repository at the same location without a config file
|
||||
b2, err = backend.CreateLocal(b.Location())
|
||||
b2, err = local.Create(b.Location())
|
||||
assert(t, err != nil && b2 == nil, fmt.Sprintf("creating a repository at %s for the second time should have failed", b.Location()))
|
||||
}
|
||||
|
||||
27
backend/paths.go
Normal file
27
backend/paths.go
Normal file
@@ -0,0 +1,27 @@
|
||||
package backend
|
||||
|
||||
import "os"
|
||||
|
||||
// Default paths for file-based backends (e.g. local)
|
||||
var Paths = struct {
|
||||
Data string
|
||||
Snapshots string
|
||||
Trees string
|
||||
Locks string
|
||||
Keys string
|
||||
Temp string
|
||||
Version string
|
||||
ID string
|
||||
}{
|
||||
"data",
|
||||
"snapshots",
|
||||
"trees",
|
||||
"locks",
|
||||
"keys",
|
||||
"tmp",
|
||||
"version",
|
||||
"id",
|
||||
}
|
||||
|
||||
// Default modes for file-based backends
|
||||
var Modes = struct{ Dir, File os.FileMode }{0700, 0600}
|
||||
@@ -1,4 +1,4 @@
|
||||
package backend
|
||||
package sftp
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
@@ -12,9 +12,11 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/sftp"
|
||||
"github.com/restic/restic/backend"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -25,7 +27,7 @@ type SFTP struct {
|
||||
c *sftp.Client
|
||||
p string
|
||||
ver uint
|
||||
id ID
|
||||
id string
|
||||
|
||||
cmd *exec.Cmd
|
||||
}
|
||||
@@ -62,10 +64,10 @@ func start_client(program string, args ...string) (*SFTP, error) {
|
||||
return &SFTP{c: client, cmd: cmd}, nil
|
||||
}
|
||||
|
||||
// OpenSFTP opens an sftp backend. When the command is started via
|
||||
// Open opens an sftp backend. When the command is started via
|
||||
// exec.Command, it is expected to speak sftp on stdin/stdout. The backend
|
||||
// is expected at the given path.
|
||||
func OpenSFTP(dir string, program string, args ...string) (*SFTP, error) {
|
||||
func Open(dir string, program string, args ...string) (*SFTP, error) {
|
||||
sftp, err := start_client(program, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -74,12 +76,13 @@ func OpenSFTP(dir string, program string, args ...string) (*SFTP, error) {
|
||||
// test if all necessary dirs and files are there
|
||||
items := []string{
|
||||
dir,
|
||||
filepath.Join(dir, dataPath),
|
||||
filepath.Join(dir, snapshotPath),
|
||||
filepath.Join(dir, treePath),
|
||||
filepath.Join(dir, lockPath),
|
||||
filepath.Join(dir, keyPath),
|
||||
filepath.Join(dir, tempPath),
|
||||
filepath.Join(dir, backend.Paths.Data),
|
||||
filepath.Join(dir, backend.Paths.Snapshots),
|
||||
filepath.Join(dir, backend.Paths.Trees),
|
||||
filepath.Join(dir, backend.Paths.Locks),
|
||||
filepath.Join(dir, backend.Paths.Keys),
|
||||
filepath.Join(dir, backend.Paths.Version),
|
||||
filepath.Join(dir, backend.Paths.ID),
|
||||
}
|
||||
for _, d := range items {
|
||||
if _, err := sftp.c.Lstat(d); err != nil {
|
||||
@@ -88,7 +91,7 @@ func OpenSFTP(dir string, program string, args ...string) (*SFTP, error) {
|
||||
}
|
||||
|
||||
// read version file
|
||||
f, err := sftp.c.Open(filepath.Join(dir, versionFileName))
|
||||
f, err := sftp.c.Open(filepath.Join(dir, backend.Paths.Version))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to read version file: %v\n", err)
|
||||
}
|
||||
@@ -109,12 +112,12 @@ func OpenSFTP(dir string, program string, args ...string) (*SFTP, error) {
|
||||
}
|
||||
|
||||
// check version
|
||||
if version != BackendVersion {
|
||||
if version != backend.Version {
|
||||
return nil, fmt.Errorf("wrong version %d", version)
|
||||
}
|
||||
|
||||
// read ID
|
||||
f, err = sftp.c.Open(filepath.Join(dir, idFileName))
|
||||
f, err = sftp.c.Open(filepath.Join(dir, backend.Paths.ID))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -129,35 +132,30 @@ func OpenSFTP(dir string, program string, args ...string) (*SFTP, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
id, err := ParseID(strings.TrimSpace(string(buf)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sftp.id = id
|
||||
sftp.id = strings.TrimSpace(string(buf))
|
||||
sftp.p = dir
|
||||
|
||||
return sftp, nil
|
||||
}
|
||||
|
||||
// CreateSFTP creates all the necessary files and directories for a new sftp
|
||||
// Create creates all the necessary files and directories for a new sftp
|
||||
// backend at dir.
|
||||
func CreateSFTP(dir string, program string, args ...string) (*SFTP, error) {
|
||||
func Create(dir string, program string, args ...string) (*SFTP, error) {
|
||||
sftp, err := start_client(program, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
versionFile := filepath.Join(dir, versionFileName)
|
||||
idFile := filepath.Join(dir, idFileName)
|
||||
versionFile := filepath.Join(dir, backend.Paths.Version)
|
||||
idFile := filepath.Join(dir, backend.Paths.ID)
|
||||
dirs := []string{
|
||||
dir,
|
||||
filepath.Join(dir, dataPath),
|
||||
filepath.Join(dir, snapshotPath),
|
||||
filepath.Join(dir, treePath),
|
||||
filepath.Join(dir, lockPath),
|
||||
filepath.Join(dir, keyPath),
|
||||
filepath.Join(dir, tempPath),
|
||||
filepath.Join(dir, backend.Paths.Data),
|
||||
filepath.Join(dir, backend.Paths.Snapshots),
|
||||
filepath.Join(dir, backend.Paths.Trees),
|
||||
filepath.Join(dir, backend.Paths.Locks),
|
||||
filepath.Join(dir, backend.Paths.Keys),
|
||||
filepath.Join(dir, backend.Paths.Temp),
|
||||
}
|
||||
|
||||
// test if files already exist
|
||||
@@ -180,7 +178,7 @@ func CreateSFTP(dir string, program string, args ...string) (*SFTP, error) {
|
||||
|
||||
// create paths for data, refs and temp blobs
|
||||
for _, d := range dirs {
|
||||
err = sftp.mkdirAll(d, dirMode)
|
||||
err = sftp.mkdirAll(d, backend.Modes.Dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -192,7 +190,7 @@ func CreateSFTP(dir string, program string, args ...string) (*SFTP, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, err = fmt.Fprintf(f, "%d\n", BackendVersion)
|
||||
_, err = fmt.Fprintf(f, "%d\n", backend.Version)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -214,7 +212,7 @@ func CreateSFTP(dir string, program string, args ...string) (*SFTP, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, err = fmt.Fprintf(f, "%s\n", ID(id).String())
|
||||
_, err = fmt.Fprintln(f, hex.EncodeToString(id))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -235,7 +233,7 @@ func CreateSFTP(dir string, program string, args ...string) (*SFTP, error) {
|
||||
}
|
||||
|
||||
// open backend
|
||||
return OpenSFTP(dir, program, args...)
|
||||
return Open(dir, program, args...)
|
||||
}
|
||||
|
||||
// Location returns this backend's location (the directory name).
|
||||
@@ -257,7 +255,7 @@ func (r *SFTP) tempFile() (string, *sftp.File, error) {
|
||||
}
|
||||
|
||||
// construct tempfile name
|
||||
name := filepath.Join(r.p, tempPath, fmt.Sprintf("temp-%s", hex.EncodeToString(buf)))
|
||||
name := filepath.Join(r.p, backend.Paths.Temp, "temp-"+hex.EncodeToString(buf))
|
||||
|
||||
// create file in temp dir
|
||||
f, err := r.c.Create(name)
|
||||
@@ -280,7 +278,7 @@ func (r *SFTP) mkdirAll(dir string, mode os.FileMode) error {
|
||||
}
|
||||
|
||||
// create parent directories
|
||||
errMkdirAll := r.mkdirAll(filepath.Dir(dir), dirMode)
|
||||
errMkdirAll := r.mkdirAll(filepath.Dir(dir), backend.Modes.Dir)
|
||||
|
||||
// create directory
|
||||
errMkdir := r.c.Mkdir(dir)
|
||||
@@ -300,18 +298,23 @@ func (r *SFTP) mkdirAll(dir string, mode os.FileMode) error {
|
||||
return r.c.Chmod(dir, mode)
|
||||
}
|
||||
|
||||
// Rename temp file to final name according to type and ID.
|
||||
func (r *SFTP) renameFile(oldname string, t Type, id ID) error {
|
||||
filename := r.filename(t, id)
|
||||
// Rename temp file to final name according to type and name.
|
||||
func (r *SFTP) renameFile(oldname string, t backend.Type, name string) error {
|
||||
filename := r.filename(t, name)
|
||||
|
||||
// create directories if necessary
|
||||
if t == Data || t == Tree {
|
||||
err := r.mkdirAll(filepath.Dir(filename), dirMode)
|
||||
if t == backend.Data || t == backend.Tree {
|
||||
err := r.mkdirAll(filepath.Dir(filename), backend.Modes.Dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// test if new file exists
|
||||
if _, err := r.c.Lstat(filename); err == nil {
|
||||
return fmt.Errorf("Close(): file %v already exists", filename)
|
||||
}
|
||||
|
||||
err := r.c.Rename(oldname, filename)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -326,42 +329,15 @@ func (r *SFTP) renameFile(oldname string, t Type, id ID) error {
|
||||
return r.c.Chmod(filename, fi.Mode()&os.FileMode(^uint32(0222)))
|
||||
}
|
||||
|
||||
// Construct directory for given Type.
|
||||
func (r *SFTP) dirname(t Type, id ID) string {
|
||||
var n string
|
||||
switch t {
|
||||
case Data:
|
||||
n = dataPath
|
||||
if id != nil {
|
||||
n = filepath.Join(dataPath, fmt.Sprintf("%02x", id[0]))
|
||||
}
|
||||
case Snapshot:
|
||||
n = snapshotPath
|
||||
case Tree:
|
||||
n = treePath
|
||||
if id != nil {
|
||||
n = filepath.Join(treePath, fmt.Sprintf("%02x", id[0]))
|
||||
}
|
||||
case Lock:
|
||||
n = lockPath
|
||||
case Key:
|
||||
n = keyPath
|
||||
}
|
||||
return filepath.Join(r.p, n)
|
||||
}
|
||||
|
||||
type sftpBlob struct {
|
||||
f *sftp.File
|
||||
name string
|
||||
hw *HashingWriter
|
||||
backend *SFTP
|
||||
tpe Type
|
||||
id ID
|
||||
size uint
|
||||
closed bool
|
||||
f *sftp.File
|
||||
tempname string
|
||||
size uint
|
||||
closed bool
|
||||
backend *SFTP
|
||||
}
|
||||
|
||||
func (sb *sftpBlob) Close() error {
|
||||
func (sb *sftpBlob) Finalize(t backend.Type, name string) error {
|
||||
if sb.closed {
|
||||
return errors.New("Close() called on closed file")
|
||||
}
|
||||
@@ -372,30 +348,17 @@ func (sb *sftpBlob) Close() error {
|
||||
return fmt.Errorf("sftp: file.Close: %v", err)
|
||||
}
|
||||
|
||||
// get ID
|
||||
sb.id = ID(sb.hw.Sum(nil))
|
||||
|
||||
// check for duplicate ID
|
||||
res, err := sb.backend.Test(sb.tpe, sb.id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("testing presence of ID %v failed: %v", sb.id, err)
|
||||
}
|
||||
|
||||
if res {
|
||||
return ErrAlreadyPresent
|
||||
}
|
||||
|
||||
// rename file
|
||||
err = sb.backend.renameFile(sb.name, sb.tpe, sb.id)
|
||||
err = sb.backend.renameFile(sb.tempname, t, name)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("sftp: renameFile: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sb *sftpBlob) Write(p []byte) (int, error) {
|
||||
n, err := sb.hw.Write(p)
|
||||
n, err := sb.f.Write(p)
|
||||
sb.size += uint(n)
|
||||
return n, err
|
||||
}
|
||||
@@ -404,18 +367,9 @@ func (sb *sftpBlob) Size() uint {
|
||||
return sb.size
|
||||
}
|
||||
|
||||
func (sb *sftpBlob) ID() (ID, error) {
|
||||
if sb.id == nil {
|
||||
return nil, errors.New("blob is not closed, ID unavailable")
|
||||
}
|
||||
|
||||
return sb.id, nil
|
||||
}
|
||||
|
||||
// Create creates a new blob of type t. Blob implements io.WriteCloser. Once
|
||||
// Close() has been called, ID() can be used to retrieve the ID. If the blob is
|
||||
// already present, Close() returns ErrAlreadyPresent.
|
||||
func (r *SFTP) Create(t Type) (Blob, error) {
|
||||
// Create creates a new Blob. The data is available only after Finalize()
|
||||
// has been called on the returned Blob.
|
||||
func (r *SFTP) Create() (backend.Blob, error) {
|
||||
// TODO: make sure that tempfile is removed upon error
|
||||
|
||||
// create tempfile in backend
|
||||
@@ -425,64 +379,52 @@ func (r *SFTP) Create(t Type) (Blob, error) {
|
||||
}
|
||||
|
||||
blob := sftpBlob{
|
||||
hw: NewHashingWriter(file, newHash()),
|
||||
f: file,
|
||||
name: filename,
|
||||
backend: r,
|
||||
tpe: t,
|
||||
f: file,
|
||||
tempname: filename,
|
||||
backend: r,
|
||||
}
|
||||
|
||||
return &blob, nil
|
||||
}
|
||||
|
||||
// Construct path for given Type and ID.
|
||||
func (r *SFTP) filename(t Type, id ID) string {
|
||||
return filepath.Join(r.dirname(t, id), id.String())
|
||||
// Construct path for given backend.Type and name.
|
||||
func (r *SFTP) filename(t backend.Type, name string) string {
|
||||
return filepath.Join(r.dirname(t, name), name)
|
||||
}
|
||||
|
||||
// Get returns the content stored under the given ID. If the data doesn't match
|
||||
// the requested ID, ErrWrongData is returned.
|
||||
func (r *SFTP) Get(t Type, id ID) ([]byte, error) {
|
||||
if id == nil {
|
||||
return nil, errors.New("unable to load nil ID")
|
||||
}
|
||||
|
||||
// try to open file
|
||||
file, err := r.c.Open(r.filename(t, id))
|
||||
defer func() {
|
||||
// TODO: report bug against sftp client, ignore Close() for nil file
|
||||
if file != nil {
|
||||
file.Close()
|
||||
// Construct directory for given backend.Type.
|
||||
func (r *SFTP) dirname(t backend.Type, name string) string {
|
||||
var n string
|
||||
switch t {
|
||||
case backend.Data:
|
||||
n = backend.Paths.Data
|
||||
if len(name) > 2 {
|
||||
n = filepath.Join(n, name[:2])
|
||||
}
|
||||
}()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
case backend.Snapshot:
|
||||
n = backend.Paths.Snapshots
|
||||
case backend.Tree:
|
||||
n = backend.Paths.Trees
|
||||
if len(name) > 2 {
|
||||
n = filepath.Join(n, name[:2])
|
||||
}
|
||||
case backend.Lock:
|
||||
n = backend.Paths.Locks
|
||||
case backend.Key:
|
||||
n = backend.Paths.Keys
|
||||
}
|
||||
|
||||
// read all
|
||||
buf, err := ioutil.ReadAll(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// check id
|
||||
if !Hash(buf).Equal(id) {
|
||||
return nil, ErrWrongData
|
||||
}
|
||||
|
||||
return buf, nil
|
||||
return filepath.Join(r.p, n)
|
||||
}
|
||||
|
||||
// GetReader returns a reader that yields the content stored under the given
|
||||
// ID. The content is not verified. The reader should be closed after draining
|
||||
// it.
|
||||
func (r *SFTP) GetReader(t Type, id ID) (io.ReadCloser, error) {
|
||||
if id == nil {
|
||||
return nil, errors.New("unable to load nil ID")
|
||||
// Get returns a reader that yields the content stored under the given
|
||||
// name. The reader should be closed after draining it.
|
||||
func (r *SFTP) Get(t backend.Type, name string) (io.ReadCloser, error) {
|
||||
if name == "" {
|
||||
return nil, errors.New("unable to load empty name")
|
||||
}
|
||||
|
||||
// try to open file
|
||||
file, err := r.c.Open(r.filename(t, id))
|
||||
file, err := r.c.Open(r.filename(t, name))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -490,15 +432,9 @@ func (r *SFTP) GetReader(t Type, id ID) (io.ReadCloser, error) {
|
||||
return file, nil
|
||||
}
|
||||
|
||||
// Test returns true if a blob of the given type and ID exists in the backend.
|
||||
func (r *SFTP) Test(t Type, id ID) (bool, error) {
|
||||
file, err := r.c.Open(r.filename(t, id))
|
||||
defer func() {
|
||||
if file != nil {
|
||||
file.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
// Test returns true if a blob of the given type and name exists in the backend.
|
||||
func (r *SFTP) Test(t backend.Type, name string) (bool, error) {
|
||||
_, err := r.c.Lstat(r.filename(t, name))
|
||||
if err != nil {
|
||||
if _, ok := err.(*sftp.StatusError); ok {
|
||||
return false, nil
|
||||
@@ -510,54 +446,83 @@ func (r *SFTP) Test(t Type, id ID) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Remove removes the content stored at ID.
|
||||
func (r *SFTP) Remove(t Type, id ID) error {
|
||||
return r.c.Remove(r.filename(t, id))
|
||||
// Remove removes the content stored at name.
|
||||
func (r *SFTP) Remove(t backend.Type, name string) error {
|
||||
return r.c.Remove(r.filename(t, name))
|
||||
}
|
||||
|
||||
// List lists all objects of a given type.
|
||||
func (r *SFTP) List(t Type) (IDs, error) {
|
||||
list := []os.FileInfo{}
|
||||
var err error
|
||||
// List returns a channel that yields all names of blobs of type t. A
|
||||
// goroutine ist started for this. If the channel done is closed, sending
|
||||
// stops.
|
||||
func (r *SFTP) List(t backend.Type, done <-chan struct{}) <-chan string {
|
||||
ch := make(chan string)
|
||||
|
||||
if t == Data || t == Tree {
|
||||
// read first level
|
||||
basedir := r.dirname(t, nil)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
|
||||
list1, err := r.c.ReadDir(basedir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if t == backend.Data || t == backend.Tree {
|
||||
// read first level
|
||||
basedir := r.dirname(t, "")
|
||||
|
||||
// read files
|
||||
for _, dir := range list1 {
|
||||
entries, err := r.c.ReadDir(filepath.Join(basedir, dir.Name()))
|
||||
list1, err := r.c.ReadDir(basedir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return
|
||||
}
|
||||
|
||||
dirs := make([]string, 0, len(list1))
|
||||
for _, d := range list1 {
|
||||
dirs = append(dirs, d.Name())
|
||||
}
|
||||
|
||||
sort.Strings(dirs)
|
||||
|
||||
// read files
|
||||
for _, dir := range dirs {
|
||||
entries, err := r.c.ReadDir(filepath.Join(basedir, dir))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
items := make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
items = append(items, entry.Name())
|
||||
}
|
||||
|
||||
sort.Strings(items)
|
||||
|
||||
for _, file := range items {
|
||||
select {
|
||||
case ch <- file:
|
||||
case <-done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
entries, err := r.c.ReadDir(r.dirname(t, ""))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
items := make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
list = append(list, entry)
|
||||
items = append(items, entry.Name())
|
||||
}
|
||||
|
||||
sort.Strings(items)
|
||||
|
||||
for _, file := range items {
|
||||
select {
|
||||
case ch <- file:
|
||||
case <-done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
list, err = r.c.ReadDir(r.dirname(t, nil))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
ids := make(IDs, 0, len(list))
|
||||
for _, item := range list {
|
||||
id, err := ParseID(item.Name())
|
||||
// ignore everything that does not parse as an ID
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ch
|
||||
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
// Version returns the version of this local backend.
|
||||
@@ -566,7 +531,7 @@ func (r *SFTP) Version() uint {
|
||||
}
|
||||
|
||||
// ID returns the ID of this local backend.
|
||||
func (r *SFTP) ID() ID {
|
||||
func (r *SFTP) ID() string {
|
||||
return r.id
|
||||
}
|
||||
|
||||
@@ -6,16 +6,16 @@ import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/backend"
|
||||
"github.com/restic/restic/backend/sftp"
|
||||
)
|
||||
|
||||
var sftpPath = flag.String("test.sftppath", "", "sftp binary path (default: empty)")
|
||||
|
||||
func setupSFTPBackend(t *testing.T) *backend.SFTP {
|
||||
func setupSFTPBackend(t *testing.T) *sftp.SFTP {
|
||||
tempdir, err := ioutil.TempDir("", "restic-test-")
|
||||
ok(t, err)
|
||||
|
||||
b, err := backend.CreateSFTP(tempdir, *sftpPath)
|
||||
b, err := sftp.Create(tempdir, *sftpPath)
|
||||
ok(t, err)
|
||||
|
||||
t.Logf("created sftp backend locally at %s", tempdir)
|
||||
@@ -23,7 +23,7 @@ func setupSFTPBackend(t *testing.T) *backend.SFTP {
|
||||
return b
|
||||
}
|
||||
|
||||
func teardownSFTPBackend(t *testing.T, b *backend.SFTP) {
|
||||
func teardownSFTPBackend(t *testing.T, b *sftp.SFTP) {
|
||||
if !*testCleanup {
|
||||
t.Logf("leaving backend at %s\n", b.Location())
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user