Move Server and Key to new sub-package

This commit is contained in:
Alexander Neumann
2015-04-26 14:46:15 +02:00
parent 8498753eb7
commit d19b23d4f1
28 changed files with 317 additions and 356 deletions

48
server/blob.go Normal file
View File

@@ -0,0 +1,48 @@
package server
import (
"bytes"
"fmt"
"github.com/restic/restic/backend"
)
type Blob struct {
ID backend.ID `json:"id,omitempty"`
Size uint64 `json:"size,omitempty"`
Storage backend.ID `json:"sid,omitempty"` // encrypted ID
StorageSize uint64 `json:"ssize,omitempty"` // encrypted Size
}
type Blobs []Blob
func (b Blob) Valid() bool {
if b.ID == nil || b.Storage == nil || b.StorageSize == 0 {
return false
}
return true
}
func (b Blob) String() string {
return fmt.Sprintf("Blob<%s (%d) -> %s (%d)>",
b.ID.Str(), b.Size,
b.Storage.Str(), b.StorageSize)
}
// Compare compares two blobs by comparing the ID and the size. It returns -1,
// 0, or 1.
func (blob Blob) Compare(other Blob) int {
if res := bytes.Compare(other.ID, blob.ID); res != 0 {
return res
}
if blob.Size < other.Size {
return -1
}
if blob.Size > other.Size {
return 1
}
return 0
}

282
server/key.go Normal file
View File

@@ -0,0 +1,282 @@
package server
import (
"crypto/rand"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"os/user"
"time"
"github.com/restic/restic/backend"
"github.com/restic/restic/chunker"
"github.com/restic/restic/crypto"
"github.com/restic/restic/debug"
)
var (
// ErrNoKeyFound is returned when no key for the repository could be decrypted.
ErrNoKeyFound = errors.New("no key could be found")
)
// TODO: figure out scrypt values on the fly depending on the current
// hardware.
const (
scryptN = 65536
scryptR = 8
scryptP = 1
scryptSaltsize = 64
)
// Key represents an encrypted master key for a repository.
type Key struct {
Created time.Time `json:"created"`
Username string `json:"username"`
Hostname string `json:"hostname"`
KDF string `json:"kdf"`
N int `json:"N"`
R int `json:"r"`
P int `json:"p"`
Salt []byte `json:"salt"`
Data []byte `json:"data"`
user *crypto.Key
master *crypto.Key
name string
}
// CreateKey initializes a master key in the given backend and encrypts it with
// the password.
func CreateKey(s *Server, password string) (*Key, error) {
return AddKey(s, password, nil)
}
// OpenKey tries do decrypt the key specified by name with the given password.
func OpenKey(s *Server, name string, password string) (*Key, error) {
k, err := LoadKey(s, name)
if err != nil {
return nil, err
}
// check KDF
if k.KDF != "scrypt" {
return nil, errors.New("only supported KDF is scrypt()")
}
// derive user key
k.user, err = crypto.KDF(k.N, k.R, k.P, k.Salt, password)
if err != nil {
return nil, err
}
// decrypt master keys
buf, err := crypto.Decrypt(k.user, []byte{}, k.Data)
if err != nil {
return nil, err
}
// restore json
k.master = &crypto.Key{}
err = json.Unmarshal(buf, k.master)
if err != nil {
return nil, err
}
k.name = name
// test if polynomial is valid and irreducible
if k.master.ChunkerPolynomial == 0 {
return nil, errors.New("Polynomial for content defined chunking is zero")
}
if !k.master.ChunkerPolynomial.Irreducible() {
return nil, errors.New("Polynomial for content defined chunking is invalid")
}
debug.Log("OpenKey", "Master keys loaded, polynomial %v", k.master.ChunkerPolynomial)
return k, nil
}
// SearchKey tries to decrypt all keys in the backend with the given password.
// If none could be found, ErrNoKeyFound is returned.
func SearchKey(s *Server, password string) (*Key, error) {
// try all keys in repo
done := make(chan struct{})
defer close(done)
for name := range s.List(backend.Key, done) {
key, err := OpenKey(s, name, password)
if err != nil {
continue
}
return key, nil
}
return nil, ErrNoKeyFound
}
// LoadKey loads a key from the backend.
func LoadKey(s *Server, name string) (*Key, error) {
// extract data from repo
rd, err := s.be.Get(backend.Key, name)
if err != nil {
return nil, err
}
defer rd.Close()
// restore json
dec := json.NewDecoder(rd)
k := Key{}
err = dec.Decode(&k)
if err != nil {
return nil, err
}
return &k, nil
}
// AddKey adds a new key to an already existing repository.
func AddKey(s *Server, password string, template *Key) (*Key, error) {
// fill meta data about key
newkey := &Key{
Created: time.Now(),
KDF: "scrypt",
N: scryptN,
R: scryptR,
P: scryptP,
}
hn, err := os.Hostname()
if err == nil {
newkey.Hostname = hn
}
usr, err := user.Current()
if err == nil {
newkey.Username = usr.Username
}
// generate random salt
newkey.Salt = make([]byte, scryptSaltsize)
n, err := rand.Read(newkey.Salt)
if n != scryptSaltsize || err != nil {
panic("unable to read enough random bytes for salt")
}
// call KDF to derive user key
newkey.user, err = crypto.KDF(newkey.N, newkey.R, newkey.P, newkey.Salt, password)
if err != nil {
return nil, err
}
if template == nil {
// generate new random master keys
newkey.master = crypto.NewKey()
// generate random polynomial for cdc
p, err := chunker.RandomPolynomial()
if err != nil {
debug.Log("AddKey", "error generating new polynomial for cdc: %v", err)
return nil, err
}
debug.Log("AddKey", "generated new polynomial for cdc: %v", p)
newkey.master.ChunkerPolynomial = p
} else {
// copy master keys from old key
newkey.master = template.master
}
// encrypt master keys (as json) with user key
buf, err := json.Marshal(newkey.master)
if err != nil {
return nil, err
}
newkey.Data, err = crypto.Encrypt(newkey.user, nil, buf)
// dump as json
buf, err = json.Marshal(newkey)
if err != nil {
return nil, err
}
// store in repository and return
blob, err := s.be.Create()
if err != nil {
return nil, err
}
plainhw := backend.NewHashingWriter(blob, sha256.New())
_, err = plainhw.Write(buf)
if err != nil {
return nil, err
}
name := backend.ID(plainhw.Sum(nil)).String()
err = blob.Finalize(backend.Key, name)
if err != nil {
return nil, err
}
newkey.name = name
return newkey, nil
}
// Encrypt encrypts and signs data with the master key. Stored in ciphertext is
// IV || Ciphertext || MAC. Returns the ciphertext, which is extended if
// necessary.
func (k *Key) Encrypt(ciphertext, plaintext []byte) ([]byte, error) {
return crypto.Encrypt(k.master, ciphertext, plaintext)
}
// EncryptTo encrypts and signs data with the master key. The returned
// io.Writer writes IV || Ciphertext || HMAC. For the hash function, SHA256 is
// used.
func (k *Key) EncryptTo(wr io.Writer) io.WriteCloser {
return crypto.EncryptTo(k.master, wr)
}
// Decrypt verifes and decrypts the ciphertext with the master key. Ciphertext
// must be in the form IV || Ciphertext || MAC.
func (k *Key) Decrypt(plaintext, ciphertext []byte) ([]byte, error) {
return crypto.Decrypt(k.master, plaintext, ciphertext)
}
// DecryptFrom verifies and decrypts the ciphertext read from rd and makes it
// available on the returned Reader. Ciphertext must be in the form IV ||
// Ciphertext || MAC. In order to correctly verify the ciphertext, rd is
// drained, locally buffered and made available on the returned Reader
// afterwards. If an MAC verification failure is observed, it is returned
// immediately.
func (k *Key) DecryptFrom(rd io.Reader) (io.ReadCloser, error) {
return crypto.DecryptFrom(k.master, rd)
}
// Master() returns the master keys for this repository. Only included for
// debug purposes.
func (k *Key) Master() *crypto.Key {
return k.master
}
// User() returns the user keys for this key. Only included for debug purposes.
func (k *Key) User() *crypto.Key {
return k.user
}
func (k *Key) String() string {
if k == nil {
return "<Key nil>"
}
return fmt.Sprintf("<Key of %s@%s, created on %s>", k.Username, k.Hostname, k.Created)
}
func (k Key) Name() string {
return k.name
}

13
server/key_test.go Normal file
View File

@@ -0,0 +1,13 @@
package server_test
import (
"testing"
. "github.com/restic/restic/test"
)
func TestRepo(t *testing.T) {
s := SetupBackend(t)
defer TeardownBackend(t, s)
_ = SetupKey(t, s, TestPassword)
}

21
server/pool.go Normal file
View File

@@ -0,0 +1,21 @@
package server
import (
"sync"
"github.com/restic/restic/chunker"
)
var bufPool = sync.Pool{
New: func() interface{} {
return make([]byte, chunker.MinSize)
},
}
func getBuf() []byte {
return bufPool.Get().([]byte)
}
func freeBuf(data []byte) {
bufPool.Put(data)
}

352
server/server.go Normal file
View File

@@ -0,0 +1,352 @@
package server
import (
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"github.com/restic/restic/backend"
"github.com/restic/restic/chunker"
)
type Server struct {
be backend.Backend
key *Key
}
func NewServer(be backend.Backend) *Server {
return &Server{be: be}
}
func (s *Server) SetKey(k *Key) {
s.key = k
}
// ChunkerPolynomial returns the secret polynomial used for content defined chunking.
func (s *Server) ChunkerPolynomial() chunker.Pol {
return chunker.Pol(s.key.Master().ChunkerPolynomial)
}
// Find loads the list of all blobs of type t and searches for names which start
// with prefix. If none is found, nil and ErrNoIDPrefixFound is returned. If
// more than one is found, nil and ErrMultipleIDMatches is returned.
func (s *Server) Find(t backend.Type, prefix string) (string, error) {
return backend.Find(s.be, t, prefix)
}
// FindSnapshot takes a string and tries to find a snapshot whose ID matches
// the string as closely as possible.
func (s *Server) FindSnapshot(name string) (string, error) {
return backend.FindSnapshot(s.be, name)
}
// PrefixLength returns the number of bytes required so that all prefixes of
// all IDs of type t are unique.
func (s *Server) PrefixLength(t backend.Type) (int, error) {
return backend.PrefixLength(s.be, t)
}
// Load tries to load and decrypt content identified by t and blob from the
// backend. If the blob specifies an ID, the decrypted plaintext is checked
// against this ID. The same goes for blob.Size and blob.StorageSize: If they
// are set to a value > 0, this value is checked.
func (s *Server) Load(t backend.Type, blob Blob) ([]byte, error) {
// load data
rd, err := s.be.Get(t, blob.Storage.String())
if err != nil {
return nil, err
}
buf, err := ioutil.ReadAll(rd)
if err != nil {
return nil, err
}
// check hash
if !backend.Hash(buf).Equal(blob.Storage) {
return nil, errors.New("invalid data returned")
}
// check length
if blob.StorageSize > 0 && len(buf) != int(blob.StorageSize) {
return nil, errors.New("Invalid storage length")
}
// decrypt
buf, err = s.Decrypt(buf)
if err != nil {
return nil, err
}
// check length
if blob.Size > 0 && len(buf) != int(blob.Size) {
return nil, errors.New("Invalid length")
}
// check SHA256 sum
if blob.ID != nil {
id := backend.Hash(buf)
if !blob.ID.Equal(id) {
return nil, fmt.Errorf("load %v: expected plaintext hash %v, got %v", blob.Storage, blob.ID, id)
}
}
return buf, nil
}
// Load tries to load and decrypt content identified by t and id from the backend.
func (s *Server) LoadID(t backend.Type, storageID backend.ID) ([]byte, error) {
return s.Load(t, Blob{Storage: storageID})
}
// LoadJSON calls Load() to get content from the backend and afterwards calls
// json.Unmarshal on the item.
func (s *Server) LoadJSON(t backend.Type, blob Blob, item interface{}) error {
buf, err := s.Load(t, blob)
if err != nil {
return err
}
return json.Unmarshal(buf, item)
}
// LoadJSONID calls Load() to get content from the backend and afterwards calls
// json.Unmarshal on the item.
func (s *Server) LoadJSONID(t backend.Type, id backend.ID, item interface{}) error {
// read
rd, err := s.be.Get(t, id.String())
if err != nil {
return err
}
defer rd.Close()
// decrypt
decryptRd, err := s.key.DecryptFrom(rd)
defer decryptRd.Close()
if err != nil {
return err
}
// decode
decoder := json.NewDecoder(decryptRd)
err = decoder.Decode(item)
if err != nil {
return err
}
return nil
}
// Save encrypts data and stores it to the backend as type t.
func (s *Server) Save(t backend.Type, data []byte, id backend.ID) (Blob, error) {
if id == nil {
// compute plaintext hash
id = backend.Hash(data)
}
// create a new blob
blob := Blob{
ID: id,
Size: uint64(len(data)),
}
ciphertext := getBuf()
defer freeBuf(ciphertext)
// encrypt blob
ciphertext, err := s.Encrypt(ciphertext, data)
if err != nil {
return Blob{}, err
}
// compute ciphertext hash
sid := backend.Hash(ciphertext)
// save blob
backendBlob, err := s.be.Create()
if err != nil {
return Blob{}, err
}
_, err = backendBlob.Write(ciphertext)
if err != nil {
return Blob{}, err
}
err = backendBlob.Finalize(t, sid.String())
if err != nil {
return Blob{}, err
}
blob.Storage = sid
blob.StorageSize = uint64(len(ciphertext))
return blob, nil
}
// SaveFrom encrypts data read from rd and stores it to the backend as type t.
func (s *Server) SaveFrom(t backend.Type, id backend.ID, length uint, rd io.Reader) (Blob, error) {
if id == nil {
return Blob{}, errors.New("id is nil")
}
backendBlob, err := s.be.Create()
if err != nil {
return Blob{}, err
}
hw := backend.NewHashingWriter(backendBlob, sha256.New())
encWr := s.key.EncryptTo(hw)
_, err = io.Copy(encWr, rd)
if err != nil {
return Blob{}, err
}
// finish encryption
err = encWr.Close()
if err != nil {
return Blob{}, fmt.Errorf("EncryptedWriter.Close(): %v", err)
}
// finish backend blob
sid := backend.ID(hw.Sum(nil))
err = backendBlob.Finalize(t, sid.String())
if err != nil {
return Blob{}, fmt.Errorf("backend.Blob.Close(): %v", err)
}
return Blob{
ID: id,
Size: uint64(length),
Storage: sid,
StorageSize: uint64(backendBlob.Size()),
}, nil
}
// SaveJSON serialises item as JSON and encrypts and saves it in the backend as
// type t.
func (s *Server) SaveJSON(t backend.Type, item interface{}) (Blob, error) {
backendBlob, err := s.be.Create()
if err != nil {
return Blob{}, fmt.Errorf("Create: %v", err)
}
storagehw := backend.NewHashingWriter(backendBlob, sha256.New())
encWr := s.key.EncryptTo(storagehw)
plainhw := backend.NewHashingWriter(encWr, sha256.New())
enc := json.NewEncoder(plainhw)
err = enc.Encode(item)
if err != nil {
return Blob{}, fmt.Errorf("json.NewEncoder: %v", err)
}
// finish encryption
err = encWr.Close()
if err != nil {
return Blob{}, fmt.Errorf("EncryptedWriter.Close(): %v", err)
}
// finish backend blob
sid := backend.ID(storagehw.Sum(nil))
err = backendBlob.Finalize(t, sid.String())
if err != nil {
return Blob{}, fmt.Errorf("backend.Blob.Close(): %v", err)
}
id := backend.ID(plainhw.Sum(nil))
return Blob{
ID: id,
Size: uint64(plainhw.Size()),
Storage: sid,
StorageSize: uint64(backendBlob.Size()),
}, nil
}
// Returns the backend used for this server.
func (s *Server) Backend() backend.Backend {
return s.be
}
func (s *Server) SearchKey(password string) error {
key, err := SearchKey(s, password)
if err != nil {
return err
}
s.key = key
return nil
}
func (s *Server) Decrypt(ciphertext []byte) ([]byte, error) {
if s.key == nil {
return nil, errors.New("key for server not set")
}
return s.key.Decrypt([]byte{}, ciphertext)
}
func (s *Server) Encrypt(ciphertext, plaintext []byte) ([]byte, error) {
if s.key == nil {
return nil, errors.New("key for server not set")
}
return s.key.Encrypt(ciphertext, plaintext)
}
func (s *Server) Key() *Key {
return s.key
}
// Count returns the number of blobs of a given type in the backend.
func (s *Server) Count(t backend.Type) (n int) {
for _ = range s.List(t, nil) {
n++
}
return
}
// Proxy methods to backend
func (s *Server) Get(t backend.Type, name string) (io.ReadCloser, error) {
return s.be.Get(t, name)
}
func (s *Server) List(t backend.Type, done <-chan struct{}) <-chan string {
return s.be.List(t, done)
}
func (s *Server) Test(t backend.Type, name string) (bool, error) {
return s.be.Test(t, name)
}
func (s *Server) Remove(t backend.Type, name string) error {
return s.be.Remove(t, name)
}
func (s *Server) Close() error {
return s.be.Close()
}
func (s *Server) Delete() error {
if b, ok := s.be.(backend.Deleter); ok {
return b.Delete()
}
return errors.New("Delete() called for backend that does not implement this method")
}
func (s *Server) ID() string {
return s.be.ID()
}
func (s *Server) Location() string {
return s.be.Location()
}

183
server/server_test.go Normal file
View File

@@ -0,0 +1,183 @@
package server_test
import (
"bytes"
"crypto/rand"
"crypto/sha256"
"encoding/json"
"flag"
"io"
"testing"
"github.com/restic/restic"
"github.com/restic/restic/backend"
. "github.com/restic/restic/test"
)
var benchTestDir = flag.String("test.dir", ".", "dir used in benchmarks (default: .)")
type testJSONStruct struct {
Foo uint32
Bar string
Baz []byte
}
var serverTests = []testJSONStruct{
testJSONStruct{Foo: 23, Bar: "Teststring", Baz: []byte("xx")},
}
func TestSaveJSON(t *testing.T) {
server := SetupBackend(t)
defer TeardownBackend(t, server)
key := SetupKey(t, server, "geheim")
server.SetKey(key)
for _, obj := range serverTests {
data, err := json.Marshal(obj)
OK(t, err)
data = append(data, '\n')
h := sha256.Sum256(data)
blob, err := server.SaveJSON(backend.Tree, obj)
OK(t, err)
Assert(t, bytes.Equal(h[:], blob.ID),
"TestSaveJSON: wrong plaintext ID: expected %02x, got %02x",
h, blob.ID)
}
}
func BenchmarkSaveJSON(t *testing.B) {
server := SetupBackend(t)
defer TeardownBackend(t, server)
key := SetupKey(t, server, "geheim")
server.SetKey(key)
obj := serverTests[0]
data, err := json.Marshal(obj)
OK(t, err)
data = append(data, '\n')
h := sha256.Sum256(data)
t.ResetTimer()
for i := 0; i < t.N; i++ {
blob, err := server.SaveJSON(backend.Tree, obj)
OK(t, err)
Assert(t, bytes.Equal(h[:], blob.ID),
"TestSaveJSON: wrong plaintext ID: expected %02x, got %02x",
h, blob.ID)
}
}
var testSizes = []int{5, 23, 2<<18 + 23, 1 << 20}
func TestSaveFrom(t *testing.T) {
server := SetupBackend(t)
defer TeardownBackend(t, server)
key := SetupKey(t, server, "geheim")
server.SetKey(key)
for _, size := range testSizes {
data := make([]byte, size)
_, err := io.ReadFull(rand.Reader, data)
OK(t, err)
id := sha256.Sum256(data)
// save
blob, err := server.SaveFrom(backend.Data, id[:], uint(size), bytes.NewReader(data))
OK(t, err)
// read back
buf, err := server.Load(backend.Data, blob)
Assert(t, len(buf) == len(data),
"number of bytes read back does not match: expected %d, got %d",
len(data), len(buf))
Assert(t, bytes.Equal(buf, data),
"data does not match: expected %02x, got %02x",
data, buf)
}
}
func BenchmarkSaveFrom(t *testing.B) {
server := SetupBackend(t)
defer TeardownBackend(t, server)
key := SetupKey(t, server, "geheim")
server.SetKey(key)
size := 4 << 20 // 4MiB
data := make([]byte, size)
_, err := io.ReadFull(rand.Reader, data)
OK(t, err)
id := sha256.Sum256(data)
t.ResetTimer()
t.SetBytes(int64(size))
for i := 0; i < t.N; i++ {
// save
_, err := server.SaveFrom(backend.Data, id[:], uint(size), bytes.NewReader(data))
OK(t, err)
}
}
func TestLoadJSONID(t *testing.T) {
if *benchTestDir == "" {
t.Skip("benchdir not set, skipping TestServerStats")
}
server := SetupBackend(t)
defer TeardownBackend(t, server)
key := SetupKey(t, server, "geheim")
server.SetKey(key)
// archive a few files
sn := SnapshotDir(t, server, *benchTestDir, nil)
t.Logf("archived snapshot %v", sn.ID())
// benchmark loading first tree
done := make(chan struct{})
first, found := <-server.List(backend.Tree, done)
Assert(t, found, "no Trees in repository found")
close(done)
id, err := backend.ParseID(first)
OK(t, err)
tree := restic.NewTree()
err = server.LoadJSONID(backend.Tree, id, &tree)
OK(t, err)
}
func BenchmarkLoadJSONID(t *testing.B) {
if *benchTestDir == "" {
t.Skip("benchdir not set, skipping TestServerStats")
}
server := SetupBackend(t)
defer TeardownBackend(t, server)
key := SetupKey(t, server, "geheim")
server.SetKey(key)
// archive a few files
sn := SnapshotDir(t, server, *benchTestDir, nil)
t.Logf("archived snapshot %v", sn.ID())
t.ResetTimer()
tree := restic.NewTree()
for i := 0; i < t.N; i++ {
for name := range server.List(backend.Tree, nil) {
id, err := backend.ParseID(name)
OK(t, err)
OK(t, server.LoadJSONID(backend.Tree, id, &tree))
}
}
}