Moves files

This commit is contained in:
Alexander Neumann
2017-07-23 14:19:13 +02:00
parent d1bd160b0a
commit 83d1a46526
284 changed files with 0 additions and 0 deletions

331
internal/crypto/crypto.go Normal file
View File

@@ -0,0 +1,331 @@
package crypto
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"encoding/json"
"fmt"
"restic/errors"
"golang.org/x/crypto/poly1305"
)
const (
aesKeySize = 32 // for AES-256
macKeySizeK = 16 // for AES-128
macKeySizeR = 16 // for Poly1305
macKeySize = macKeySizeK + macKeySizeR // for Poly1305-AES128
ivSize = aes.BlockSize
macSize = poly1305.TagSize
// Extension is the number of bytes a plaintext is enlarged by encrypting it.
Extension = ivSize + macSize
)
var (
// ErrUnauthenticated is returned when ciphertext verification has failed.
ErrUnauthenticated = errors.New("ciphertext verification failed")
)
// Key holds encryption and message authentication keys for a repository. It is stored
// encrypted and authenticated as a JSON data structure in the Data field of the Key
// structure.
type Key struct {
MACKey `json:"mac"`
EncryptionKey `json:"encrypt"`
}
// EncryptionKey is key used for encryption
type EncryptionKey [32]byte
// MACKey is used to sign (authenticate) data.
type MACKey struct {
K [16]byte // for AES-128
R [16]byte // for Poly1305
masked bool // remember if the MAC key has already been masked
}
// mask for key, (cf. http://cr.yp.to/mac/poly1305-20050329.pdf)
var poly1305KeyMask = [16]byte{
0xff,
0xff,
0xff,
0x0f, // 3: top four bits zero
0xfc, // 4: bottom two bits zero
0xff,
0xff,
0x0f, // 7: top four bits zero
0xfc, // 8: bottom two bits zero
0xff,
0xff,
0x0f, // 11: top four bits zero
0xfc, // 12: bottom two bits zero
0xff,
0xff,
0x0f, // 15: top four bits zero
}
func poly1305MAC(msg []byte, nonce []byte, key *MACKey) []byte {
k := poly1305PrepareKey(nonce, key)
var out [16]byte
poly1305.Sum(&out, msg, &k)
return out[:]
}
// mask poly1305 key
func maskKey(k *MACKey) {
if k == nil || k.masked {
return
}
for i := 0; i < poly1305.TagSize; i++ {
k.R[i] = k.R[i] & poly1305KeyMask[i]
}
k.masked = true
}
// construct mac key from slice (k||r), with masking
func macKeyFromSlice(mk *MACKey, data []byte) {
copy(mk.K[:], data[:16])
copy(mk.R[:], data[16:32])
maskKey(mk)
}
// prepare key for low-level poly1305.Sum(): r||n
func poly1305PrepareKey(nonce []byte, key *MACKey) [32]byte {
var k [32]byte
maskKey(key)
cipher, err := aes.NewCipher(key.K[:])
if err != nil {
panic(err)
}
cipher.Encrypt(k[16:], nonce[:])
copy(k[:16], key.R[:])
return k
}
func poly1305Verify(msg []byte, nonce []byte, key *MACKey, mac []byte) bool {
k := poly1305PrepareKey(nonce, key)
var m [16]byte
copy(m[:], mac)
return poly1305.Verify(&m, msg, &k)
}
// NewRandomKey returns new encryption and message authentication keys.
func NewRandomKey() *Key {
k := &Key{}
n, err := rand.Read(k.EncryptionKey[:])
if n != aesKeySize || err != nil {
panic("unable to read enough random bytes for encryption key")
}
n, err = rand.Read(k.MACKey.K[:])
if n != macKeySizeK || err != nil {
panic("unable to read enough random bytes for MAC encryption key")
}
n, err = rand.Read(k.MACKey.R[:])
if n != macKeySizeR || err != nil {
panic("unable to read enough random bytes for MAC key")
}
maskKey(&k.MACKey)
return k
}
func newIV() []byte {
iv := make([]byte, ivSize)
n, err := rand.Read(iv)
if n != ivSize || err != nil {
panic("unable to read enough random bytes for iv")
}
return iv
}
type jsonMACKey struct {
K []byte `json:"k"`
R []byte `json:"r"`
}
// MarshalJSON converts the MACKey to JSON.
func (m *MACKey) MarshalJSON() ([]byte, error) {
return json.Marshal(jsonMACKey{K: m.K[:], R: m.R[:]})
}
// UnmarshalJSON fills the key m with data from the JSON representation.
func (m *MACKey) UnmarshalJSON(data []byte) error {
j := jsonMACKey{}
err := json.Unmarshal(data, &j)
if err != nil {
return errors.Wrap(err, "Unmarshal")
}
copy(m.K[:], j.K)
copy(m.R[:], j.R)
return nil
}
// Valid tests whether the key k is valid (i.e. not zero).
func (m *MACKey) Valid() bool {
nonzeroK := false
for i := 0; i < len(m.K); i++ {
if m.K[i] != 0 {
nonzeroK = true
}
}
if !nonzeroK {
return false
}
for i := 0; i < len(m.R); i++ {
if m.R[i] != 0 {
return true
}
}
return false
}
// MarshalJSON converts the EncryptionKey to JSON.
func (k *EncryptionKey) MarshalJSON() ([]byte, error) {
return json.Marshal(k[:])
}
// UnmarshalJSON fills the key k with data from the JSON representation.
func (k *EncryptionKey) UnmarshalJSON(data []byte) error {
d := make([]byte, aesKeySize)
err := json.Unmarshal(data, &d)
if err != nil {
return errors.Wrap(err, "Unmarshal")
}
copy(k[:], d)
return nil
}
// Valid tests whether the key k is valid (i.e. not zero).
func (k *EncryptionKey) Valid() bool {
for i := 0; i < len(k); i++ {
if k[i] != 0 {
return true
}
}
return false
}
// ErrInvalidCiphertext is returned when trying to encrypt into the slice that
// holds the plaintext.
var ErrInvalidCiphertext = errors.New("invalid ciphertext, same slice used for plaintext")
// Encrypt encrypts and authenticates data. Stored in ciphertext is IV || Ciphertext ||
// MAC. Encrypt returns the new ciphertext slice, which is extended when
// necessary. ciphertext and plaintext may not point to (exactly) the same
// slice or non-intersecting slices.
func (k *Key) Encrypt(ciphertext []byte, plaintext []byte) ([]byte, error) {
if !k.Valid() {
return nil, errors.New("invalid key")
}
ciphertext = ciphertext[:cap(ciphertext)]
// test for same slice, if possible
if len(plaintext) > 0 && len(ciphertext) > 0 && &plaintext[0] == &ciphertext[0] {
return nil, ErrInvalidCiphertext
}
// extend ciphertext slice if necessary
if len(ciphertext) < len(plaintext)+Extension {
ext := len(plaintext) + Extension - cap(ciphertext)
ciphertext = append(ciphertext, make([]byte, ext)...)
ciphertext = ciphertext[:cap(ciphertext)]
}
iv := newIV()
c, err := aes.NewCipher(k.EncryptionKey[:])
if err != nil {
panic(fmt.Sprintf("unable to create cipher: %v", err))
}
e := cipher.NewCTR(c, iv[:])
e.XORKeyStream(ciphertext[ivSize:], plaintext)
copy(ciphertext, iv[:])
// truncate to only cover iv and actual ciphertext
ciphertext = ciphertext[:ivSize+len(plaintext)]
mac := poly1305MAC(ciphertext[ivSize:], ciphertext[:ivSize], &k.MACKey)
ciphertext = append(ciphertext, mac...)
return ciphertext, nil
}
// Decrypt verifies and decrypts the ciphertext. Ciphertext must be in the form
// IV || Ciphertext || MAC. plaintext and ciphertext may point to (exactly) the
// same slice.
func (k *Key) Decrypt(plaintext []byte, ciphertextWithMac []byte) (int, error) {
if !k.Valid() {
return 0, errors.New("invalid key")
}
// check for plausible length
if len(ciphertextWithMac) < ivSize+macSize {
panic("trying to decrypt invalid data: ciphertext too small")
}
// check buffer length for plaintext
plaintextLength := len(ciphertextWithMac) - ivSize - macSize
if len(plaintext) < plaintextLength {
return 0, errors.Errorf("plaintext buffer too small, %d < %d", len(plaintext), plaintextLength)
}
// extract mac
l := len(ciphertextWithMac) - macSize
ciphertextWithIV, mac := ciphertextWithMac[:l], ciphertextWithMac[l:]
// verify mac
if !poly1305Verify(ciphertextWithIV[ivSize:], ciphertextWithIV[:ivSize], &k.MACKey, mac) {
return 0, ErrUnauthenticated
}
// extract iv
iv, ciphertext := ciphertextWithIV[:ivSize], ciphertextWithIV[ivSize:]
if len(ciphertext) != plaintextLength {
return 0, errors.Errorf("plaintext and ciphertext lengths do not match: %d != %d", len(ciphertext), plaintextLength)
}
// decrypt data
c, err := aes.NewCipher(k.EncryptionKey[:])
if err != nil {
panic(fmt.Sprintf("unable to create cipher: %v", err))
}
// decrypt
e := cipher.NewCTR(c, iv)
plaintext = plaintext[:len(ciphertext)]
e.XORKeyStream(plaintext, ciphertext)
return plaintextLength, nil
}
// Valid tests if the key is valid.
func (k *Key) Valid() bool {
return k.EncryptionKey.Valid() && k.MACKey.Valid()
}

View File

@@ -0,0 +1,139 @@
package crypto
import (
"bytes"
"encoding/hex"
"testing"
)
// test vectors from http://cr.yp.to/mac/poly1305-20050329.pdf
var poly1305Tests = []struct {
msg []byte
r []byte
k []byte
nonce []byte
mac []byte
}{
{
[]byte("\xf3\xf6"),
[]byte("\x85\x1f\xc4\x0c\x34\x67\xac\x0b\xe0\x5c\xc2\x04\x04\xf3\xf7\x00"),
[]byte("\xec\x07\x4c\x83\x55\x80\x74\x17\x01\x42\x5b\x62\x32\x35\xad\xd6"),
[]byte("\xfb\x44\x73\x50\xc4\xe8\x68\xc5\x2a\xc3\x27\x5c\xf9\xd4\x32\x7e"),
[]byte("\xf4\xc6\x33\xc3\x04\x4f\xc1\x45\xf8\x4f\x33\x5c\xb8\x19\x53\xde"),
},
{
[]byte(""),
[]byte("\xa0\xf3\x08\x00\x00\xf4\x64\x00\xd0\xc7\xe9\x07\x6c\x83\x44\x03"),
[]byte("\x75\xde\xaa\x25\xc0\x9f\x20\x8e\x1d\xc4\xce\x6b\x5c\xad\x3f\xbf"),
[]byte("\x61\xee\x09\x21\x8d\x29\xb0\xaa\xed\x7e\x15\x4a\x2c\x55\x09\xcc"),
[]byte("\xdd\x3f\xab\x22\x51\xf1\x1a\xc7\x59\xf0\x88\x71\x29\xcc\x2e\xe7"),
},
{
[]byte("\x66\x3c\xea\x19\x0f\xfb\x83\xd8\x95\x93\xf3\xf4\x76\xb6\xbc\x24\xd7\xe6\x79\x10\x7e\xa2\x6a\xdb\x8c\xaf\x66\x52\xd0\x65\x61\x36"),
[]byte("\x48\x44\x3d\x0b\xb0\xd2\x11\x09\xc8\x9a\x10\x0b\x5c\xe2\xc2\x08"),
[]byte("\x6a\xcb\x5f\x61\xa7\x17\x6d\xd3\x20\xc5\xc1\xeb\x2e\xdc\xdc\x74"),
[]byte("\xae\x21\x2a\x55\x39\x97\x29\x59\x5d\xea\x45\x8b\xc6\x21\xff\x0e"),
[]byte("\x0e\xe1\xc1\x6b\xb7\x3f\x0f\x4f\xd1\x98\x81\x75\x3c\x01\xcd\xbe"),
}, {
[]byte("\xab\x08\x12\x72\x4a\x7f\x1e\x34\x27\x42\xcb\xed\x37\x4d\x94\xd1\x36\xc6\xb8\x79\x5d\x45\xb3\x81\x98\x30\xf2\xc0\x44\x91\xfa\xf0\x99\x0c\x62\xe4\x8b\x80\x18\xb2\xc3\xe4\xa0\xfa\x31\x34\xcb\x67\xfa\x83\xe1\x58\xc9\x94\xd9\x61\xc4\xcb\x21\x09\x5c\x1b\xf9"),
[]byte("\x12\x97\x6a\x08\xc4\x42\x6d\x0c\xe8\xa8\x24\x07\xc4\xf4\x82\x07"),
[]byte("\xe1\xa5\x66\x8a\x4d\x5b\x66\xa5\xf6\x8c\xc5\x42\x4e\xd5\x98\x2d"),
[]byte("\x9a\xe8\x31\xe7\x43\x97\x8d\x3a\x23\x52\x7c\x71\x28\x14\x9e\x3a"),
[]byte("\x51\x54\xad\x0d\x2c\xb2\x6e\x01\x27\x4f\xc5\x11\x48\x49\x1f\x1b"),
},
}
func TestPoly1305(t *testing.T) {
for _, test := range poly1305Tests {
key := &MACKey{}
copy(key.K[:], test.k)
copy(key.R[:], test.r)
mac := poly1305MAC(test.msg, test.nonce, key)
if !bytes.Equal(mac, test.mac) {
t.Fatalf("wrong mac calculated, want: %02x, got: %02x", test.mac, mac)
}
if !poly1305Verify(test.msg, test.nonce, key, test.mac) {
t.Fatalf("mac does not verify: mac: %02x", test.mac)
}
}
}
var testValues = []struct {
ekey EncryptionKey
skey MACKey
ciphertext []byte
plaintext []byte
}{
{
ekey: EncryptionKey([...]byte{
0x30, 0x3e, 0x86, 0x87, 0xb1, 0xd7, 0xdb, 0x18, 0x42, 0x1b, 0xdc, 0x6b, 0xb8, 0x58, 0x8c, 0xca,
0xda, 0xc4, 0xd5, 0x9e, 0xe8, 0x7b, 0x8f, 0xf7, 0x0c, 0x44, 0xe6, 0x35, 0x79, 0x0c, 0xaf, 0xef,
}),
skey: MACKey{
K: [...]byte{0xef, 0x4d, 0x88, 0x24, 0xcb, 0x80, 0xb2, 0xbc, 0xc5, 0xfb, 0xff, 0x8a, 0x9b, 0x12, 0xa4, 0x2c},
R: [...]byte{0xcc, 0x8d, 0x4b, 0x94, 0x8e, 0xe0, 0xeb, 0xfe, 0x1d, 0x41, 0x5d, 0xe9, 0x21, 0xd1, 0x03, 0x53},
},
ciphertext: decodeHex("69fb41c62d12def4593bd71757138606338f621aeaeb39da0fe4f99233f8037a54ea63338a813bcf3f75d8c3cc75dddf8750"),
plaintext: []byte("Dies ist ein Test!"),
},
}
func decodeHex(s string) []byte {
d, _ := hex.DecodeString(s)
return d
}
func TestCrypto(t *testing.T) {
msg := make([]byte, 0, 8*1024*1024) // use 8MiB for now
for _, tv := range testValues {
// test encryption
k := &Key{
EncryptionKey: tv.ekey,
MACKey: tv.skey,
}
msg, err := k.Encrypt(msg, tv.plaintext)
if err != nil {
t.Fatal(err)
}
// decrypt message
buf := make([]byte, len(tv.plaintext))
n, err := k.Decrypt(buf, msg)
if err != nil {
t.Fatal(err)
}
buf = buf[:n]
// change mac, this must fail
msg[len(msg)-8] ^= 0x23
if _, err = k.Decrypt(buf, msg); err != ErrUnauthenticated {
t.Fatal("wrong MAC value not detected")
}
// reset mac
msg[len(msg)-8] ^= 0x23
// tamper with message, this must fail
msg[16+5] ^= 0x85
if _, err = k.Decrypt(buf, msg); err != ErrUnauthenticated {
t.Fatal("tampered message not detected")
}
// test decryption
p := make([]byte, len(tv.ciphertext))
n, err = k.Decrypt(p, tv.ciphertext)
if err != nil {
t.Fatal(err)
}
p = p[:n]
if !bytes.Equal(p, tv.plaintext) {
t.Fatalf("wrong plaintext: expected %q but got %q\n", tv.plaintext, p)
}
}
}

View File

@@ -0,0 +1,171 @@
package crypto_test
import (
"bytes"
"crypto/rand"
"io"
"testing"
"restic/crypto"
. "restic/test"
"github.com/restic/chunker"
)
const testLargeCrypto = false
func TestEncryptDecrypt(t *testing.T) {
k := crypto.NewRandomKey()
tests := []int{5, 23, 2<<18 + 23, 1 << 20}
if testLargeCrypto {
tests = append(tests, 7<<20+123)
}
for _, size := range tests {
data := Random(42, size)
buf := make([]byte, size+crypto.Extension)
ciphertext, err := k.Encrypt(buf, data)
OK(t, err)
Assert(t, len(ciphertext) == len(data)+crypto.Extension,
"ciphertext length does not match: want %d, got %d",
len(data)+crypto.Extension, len(ciphertext))
plaintext := make([]byte, len(ciphertext))
n, err := k.Decrypt(plaintext, ciphertext)
OK(t, err)
plaintext = plaintext[:n]
Assert(t, len(plaintext) == len(data),
"plaintext length does not match: want %d, got %d",
len(data), len(plaintext))
Equals(t, plaintext, data)
}
}
func TestSmallBuffer(t *testing.T) {
k := crypto.NewRandomKey()
size := 600
data := make([]byte, size)
_, err := io.ReadFull(rand.Reader, data)
OK(t, err)
ciphertext := make([]byte, size/2)
ciphertext, err = k.Encrypt(ciphertext, data)
// this must extend the slice
Assert(t, cap(ciphertext) > size/2,
"expected extended slice, but capacity is only %d bytes",
cap(ciphertext))
// check for the correct plaintext
plaintext := make([]byte, len(ciphertext))
n, err := k.Decrypt(plaintext, ciphertext)
OK(t, err)
plaintext = plaintext[:n]
Assert(t, bytes.Equal(plaintext, data),
"wrong plaintext returned")
}
func TestSameBuffer(t *testing.T) {
k := crypto.NewRandomKey()
size := 600
data := make([]byte, size)
_, err := io.ReadFull(rand.Reader, data)
OK(t, err)
ciphertext := make([]byte, 0, size+crypto.Extension)
ciphertext, err = k.Encrypt(ciphertext, data)
OK(t, err)
// use the same buffer for decryption
n, err := k.Decrypt(ciphertext, ciphertext)
OK(t, err)
ciphertext = ciphertext[:n]
Assert(t, bytes.Equal(ciphertext, data),
"wrong plaintext returned")
}
func TestCornerCases(t *testing.T) {
k := crypto.NewRandomKey()
// nil plaintext should encrypt to the empty string
// nil ciphertext should allocate a new slice for the ciphertext
c, err := k.Encrypt(nil, nil)
OK(t, err)
Assert(t, len(c) == crypto.Extension,
"wrong length returned for ciphertext, expected 0, got %d",
len(c))
// this should decrypt to nil
n, err := k.Decrypt(nil, c)
OK(t, err)
Equals(t, 0, n)
// test encryption for same slice, this should return an error
_, err = k.Encrypt(c, c)
Equals(t, crypto.ErrInvalidCiphertext, err)
}
func TestLargeEncrypt(t *testing.T) {
if !testLargeCrypto {
t.SkipNow()
}
k := crypto.NewRandomKey()
for _, size := range []int{chunker.MaxSize, chunker.MaxSize + 1, chunker.MaxSize + 1<<20} {
data := make([]byte, size)
_, err := io.ReadFull(rand.Reader, data)
OK(t, err)
ciphertext, err := k.Encrypt(make([]byte, size+crypto.Extension), data)
OK(t, err)
plaintext, err := k.Decrypt([]byte{}, ciphertext)
OK(t, err)
Equals(t, plaintext, data)
}
}
func BenchmarkEncrypt(b *testing.B) {
size := 8 << 20 // 8MiB
data := make([]byte, size)
k := crypto.NewRandomKey()
buf := make([]byte, len(data)+crypto.Extension)
b.ResetTimer()
b.SetBytes(int64(size))
for i := 0; i < b.N; i++ {
_, err := k.Encrypt(buf, data)
OK(b, err)
}
}
func BenchmarkDecrypt(b *testing.B) {
size := 8 << 20 // 8MiB
data := make([]byte, size)
k := crypto.NewRandomKey()
plaintext := make([]byte, size)
ciphertext := make([]byte, size+crypto.Extension)
ciphertext, err := k.Encrypt(ciphertext, data)
OK(b, err)
b.ResetTimer()
b.SetBytes(int64(size))
for i := 0; i < b.N; i++ {
_, err = k.Decrypt(plaintext, ciphertext)
OK(b, err)
}
}

2
internal/crypto/doc.go Normal file
View File

@@ -0,0 +1,2 @@
// Package crypto provides all cryptographic operations needed in restic.
package crypto

102
internal/crypto/kdf.go Normal file
View File

@@ -0,0 +1,102 @@
package crypto
import (
"crypto/rand"
"time"
"restic/errors"
sscrypt "github.com/elithrar/simple-scrypt"
"golang.org/x/crypto/scrypt"
)
const saltLength = 64
// KDFParams are the default parameters used for the key derivation function KDF().
type KDFParams struct {
N int
R int
P int
}
// DefaultKDFParams are the default parameters used for Calibrate and KDF().
var DefaultKDFParams = KDFParams{
N: sscrypt.DefaultParams.N,
R: sscrypt.DefaultParams.R,
P: sscrypt.DefaultParams.P,
}
// Calibrate determines new KDF parameters for the current hardware.
func Calibrate(timeout time.Duration, memory int) (KDFParams, error) {
defaultParams := sscrypt.Params{
N: DefaultKDFParams.N,
R: DefaultKDFParams.R,
P: DefaultKDFParams.P,
DKLen: sscrypt.DefaultParams.DKLen,
SaltLen: sscrypt.DefaultParams.SaltLen,
}
params, err := sscrypt.Calibrate(timeout, memory, defaultParams)
if err != nil {
return DefaultKDFParams, errors.Wrap(err, "scrypt.Calibrate")
}
return KDFParams{
N: params.N,
R: params.R,
P: params.P,
}, nil
}
// KDF derives encryption and message authentication keys from the password
// using the supplied parameters N, R and P and the Salt.
func KDF(p KDFParams, salt []byte, password string) (*Key, error) {
if len(salt) != saltLength {
return nil, errors.Errorf("scrypt() called with invalid salt bytes (len %d)", len(salt))
}
// make sure we have valid parameters
params := sscrypt.Params{
N: p.N,
R: p.R,
P: p.P,
DKLen: sscrypt.DefaultParams.DKLen,
SaltLen: len(salt),
}
if err := params.Check(); err != nil {
return nil, errors.Wrap(err, "Check")
}
derKeys := &Key{}
keybytes := macKeySize + aesKeySize
scryptKeys, err := scrypt.Key([]byte(password), salt, p.N, p.R, p.P, keybytes)
if err != nil {
return nil, errors.Wrap(err, "scrypt.Key")
}
if len(scryptKeys) != keybytes {
return nil, errors.Errorf("invalid numbers of bytes expanded from scrypt(): %d", len(scryptKeys))
}
// first 32 byte of scrypt output is the encryption key
copy(derKeys.EncryptionKey[:], scryptKeys[:aesKeySize])
// next 32 byte of scrypt output is the mac key, in the form k||r
macKeyFromSlice(&derKeys.MACKey, scryptKeys[aesKeySize:])
return derKeys, nil
}
// NewSalt returns new random salt bytes to use with KDF(). If NewSalt returns
// an error, this is a grave situation and the program must abort and terminate.
func NewSalt() ([]byte, error) {
buf := make([]byte, saltLength)
n, err := rand.Read(buf)
if n != saltLength || err != nil {
panic("unable to read enough random bytes for new salt")
}
return buf, nil
}

View File

@@ -0,0 +1,14 @@
package crypto
import (
"testing"
"time"
)
func TestCalibrate(t *testing.T) {
params, err := Calibrate(100*time.Millisecond, 50)
if err != nil {
t.Fatal(err)
}
t.Logf("testing calibrate, params after: %v", params)
}