fix(query): keys (#2755)

* fix: add keys to projections

* change to multiple tables

* query keys

* query keys

* fix race condition

* fix timer reset

* begin tests

* tests

* remove migration

* only send to keyChannel if not nil
This commit is contained in:
Livio Amstutz
2022-01-12 13:22:04 +01:00
committed by GitHub
parent ead61d240d
commit 9ab566fdeb
23 changed files with 927 additions and 419 deletions

330
internal/query/key.go Normal file
View File

@@ -0,0 +1,330 @@
package query
import (
"context"
"crypto/rsa"
"database/sql"
"time"
sq "github.com/Masterminds/squirrel"
"github.com/caos/zitadel/internal/crypto"
"github.com/caos/zitadel/internal/domain"
"github.com/caos/zitadel/internal/errors"
"github.com/caos/zitadel/internal/query/projection"
)
type Key interface {
ID() string
Algorithm() string
Use() domain.KeyUsage
Sequence() uint64
}
type PrivateKey interface {
Key
Expiry() time.Time
Key() *crypto.CryptoValue
}
type PublicKey interface {
Key
Expiry() time.Time
Key() interface{}
}
type PrivateKeys struct {
SearchResponse
Keys []PrivateKey
}
type PublicKeys struct {
SearchResponse
Keys []PublicKey
}
type key struct {
id string
creationDate time.Time
changeDate time.Time
sequence uint64
resourceOwner string
algorithm string
use domain.KeyUsage
}
func (k *key) ID() string {
return k.id
}
func (k *key) Algorithm() string {
return k.algorithm
}
func (k *key) Use() domain.KeyUsage {
return k.use
}
func (k *key) Sequence() uint64 {
return k.sequence
}
type privateKey struct {
key
expiry time.Time
privateKey *crypto.CryptoValue
}
func (k *privateKey) Expiry() time.Time {
return k.expiry
}
func (k *privateKey) Key() *crypto.CryptoValue {
return k.privateKey
}
type rsaPublicKey struct {
key
expiry time.Time
publicKey *rsa.PublicKey
}
func (r *rsaPublicKey) Expiry() time.Time {
return r.expiry
}
func (r *rsaPublicKey) Key() interface{} {
return r.publicKey
}
var (
keyTable = table{
name: projection.KeyProjectionTable,
}
KeyColID = Column{
name: projection.KeyColumnID,
table: keyTable,
}
KeyColCreationDate = Column{
name: projection.KeyColumnCreationDate,
table: keyTable,
}
KeyColChangeDate = Column{
name: projection.KeyColumnChangeDate,
table: keyTable,
}
KeyColResourceOwner = Column{
name: projection.KeyColumnResourceOwner,
table: keyTable,
}
KeyColSequence = Column{
name: projection.KeyColumnSequence,
table: keyTable,
}
KeyColAlgorithm = Column{
name: projection.KeyColumnAlgorithm,
table: keyTable,
}
KeyColUse = Column{
name: projection.KeyColumnUse,
table: keyTable,
}
)
var (
keyPrivateTable = table{
name: projection.KeyPrivateTable,
}
KeyPrivateColID = Column{
name: projection.KeyPrivateColumnID,
table: keyPrivateTable,
}
KeyPrivateColExpiry = Column{
name: projection.KeyPrivateColumnExpiry,
table: keyPrivateTable,
}
KeyPrivateColKey = Column{
name: projection.KeyPrivateColumnKey,
table: keyPrivateTable,
}
)
var (
keyPublicTable = table{
name: projection.KeyPublicTable,
}
KeyPublicColID = Column{
name: projection.KeyPublicColumnID,
table: keyPublicTable,
}
KeyPublicColExpiry = Column{
name: projection.KeyPublicColumnExpiry,
table: keyPublicTable,
}
KeyPublicColKey = Column{
name: projection.KeyPublicColumnKey,
table: keyPublicTable,
}
)
func (q *Queries) ActivePublicKeys(ctx context.Context, t time.Time) (*PublicKeys, error) {
query, scan := preparePublicKeysQuery()
if t.IsZero() {
t = time.Now()
}
stmt, args, err := query.Where(
sq.Gt{
KeyPublicColExpiry.identifier(): t,
}).ToSql()
if err != nil {
return nil, errors.ThrowInternal(err, "QUERY-SDFfg", "Errors.Query.SQLStatement")
}
rows, err := q.client.QueryContext(ctx, stmt, args...)
if err != nil {
return nil, errors.ThrowInternal(err, "QUERY-Sghn4", "Errors.Internal")
}
keys, err := scan(rows)
if err != nil {
return nil, err
}
keys.LatestSequence, err = q.latestSequence(ctx, keyTable)
return keys, err
}
func (q *Queries) ActivePrivateSigningKey(ctx context.Context, t time.Time) (*PrivateKeys, error) {
stmt, scan := preparePrivateKeysQuery()
if t.IsZero() {
t = time.Now()
}
query, args, err := stmt.Where(
sq.And{
sq.Eq{
KeyColUse.identifier(): domain.KeyUsageSigning,
},
sq.Gt{
KeyPrivateColExpiry.identifier(): t,
},
}).OrderBy(KeyPrivateColExpiry.identifier()).ToSql()
if err != nil {
return nil, errors.ThrowInternal(err, "QUERY-SDff2", "Errors.Query.SQLStatement")
}
rows, err := q.client.QueryContext(ctx, query, args...)
if err != nil {
return nil, errors.ThrowInternal(err, "QUERY-WRFG4", "Errors.Internal")
}
keys, err := scan(rows)
if err != nil {
return nil, err
}
keys.LatestSequence, err = q.latestSequence(ctx, keyTable)
return keys, err
}
func preparePublicKeysQuery() (sq.SelectBuilder, func(*sql.Rows) (*PublicKeys, error)) {
return sq.Select(
KeyColID.identifier(),
KeyColCreationDate.identifier(),
KeyColChangeDate.identifier(),
KeyColSequence.identifier(),
KeyColResourceOwner.identifier(),
KeyColAlgorithm.identifier(),
KeyColUse.identifier(),
KeyPublicColExpiry.identifier(),
KeyPublicColKey.identifier(),
countColumn.identifier(),
).From(keyTable.identifier()).
LeftJoin(join(KeyPublicColID, KeyColID)).
PlaceholderFormat(sq.Dollar),
func(rows *sql.Rows) (*PublicKeys, error) {
keys := make([]PublicKey, 0)
var count uint64
for rows.Next() {
k := new(rsaPublicKey)
var keyValue []byte
err := rows.Scan(
&k.id,
&k.creationDate,
&k.changeDate,
&k.sequence,
&k.resourceOwner,
&k.algorithm,
&k.use,
&k.expiry,
&keyValue,
&count,
)
if err != nil {
return nil, err
}
k.publicKey, err = crypto.BytesToPublicKey(keyValue)
if err != nil {
return nil, err
}
keys = append(keys, k)
}
if err := rows.Close(); err != nil {
return nil, errors.ThrowInternal(err, "QUERY-rKd6k", "Errors.Query.CloseRows")
}
return &PublicKeys{
Keys: keys,
SearchResponse: SearchResponse{
Count: count,
},
}, nil
}
}
func preparePrivateKeysQuery() (sq.SelectBuilder, func(*sql.Rows) (*PrivateKeys, error)) {
return sq.Select(
KeyColID.identifier(),
KeyColCreationDate.identifier(),
KeyColChangeDate.identifier(),
KeyColSequence.identifier(),
KeyColResourceOwner.identifier(),
KeyColAlgorithm.identifier(),
KeyColUse.identifier(),
KeyPrivateColExpiry.identifier(),
KeyPrivateColKey.identifier(),
countColumn.identifier(),
).From(keyTable.identifier()).
LeftJoin(join(KeyPrivateColID, KeyColID)).
PlaceholderFormat(sq.Dollar),
func(rows *sql.Rows) (*PrivateKeys, error) {
keys := make([]PrivateKey, 0)
var count uint64
for rows.Next() {
k := new(privateKey)
err := rows.Scan(
&k.id,
&k.creationDate,
&k.changeDate,
&k.sequence,
&k.resourceOwner,
&k.algorithm,
&k.use,
&k.expiry,
&k.privateKey,
&count,
)
if err != nil {
return nil, err
}
keys = append(keys, k)
}
if err := rows.Close(); err != nil {
return nil, errors.ThrowInternal(err, "QUERY-rKd6k", "Errors.Query.CloseRows")
}
return &PrivateKeys{
Keys: keys,
SearchResponse: SearchResponse{
Count: count,
},
}, nil
}
}

295
internal/query/key_test.go Normal file
View File

@@ -0,0 +1,295 @@
package query
import (
"crypto/rsa"
"database/sql"
"database/sql/driver"
"errors"
"fmt"
"math/big"
"regexp"
"testing"
"github.com/caos/zitadel/internal/crypto"
"github.com/caos/zitadel/internal/domain"
errs "github.com/caos/zitadel/internal/errors"
)
func Test_KeyPrepares(t *testing.T) {
type want struct {
sqlExpectations sqlExpectation
err checkErr
}
tests := []struct {
name string
prepare interface{}
want want
object interface{}
}{
{
name: "preparePublicKeysQuery no result",
prepare: preparePublicKeysQuery,
want: want{
sqlExpectations: mockQueries(
regexp.QuoteMeta(`SELECT zitadel.projections.keys.id,`+
` zitadel.projections.keys.creation_date,`+
` zitadel.projections.keys.change_date,`+
` zitadel.projections.keys.sequence,`+
` zitadel.projections.keys.resource_owner,`+
` zitadel.projections.keys.algorithm,`+
` zitadel.projections.keys.use,`+
` zitadel.projections.keys_public.expiry,`+
` zitadel.projections.keys_public.key,`+
` COUNT(*) OVER ()`+
` FROM zitadel.projections.keys`+
` LEFT JOIN zitadel.projections.keys_public ON zitadel.projections.keys.id = zitadel.projections.keys_public.id`),
nil,
nil,
),
err: func(err error) (error, bool) {
if !errs.IsNotFound(err) {
return fmt.Errorf("err should be zitadel.NotFoundError got: %w", err), false
}
return nil, true
},
},
object: &PublicKeys{Keys: []PublicKey{}},
},
{
name: "preparePublicKeysQuery found",
prepare: preparePublicKeysQuery,
want: want{
sqlExpectations: mockQueries(
regexp.QuoteMeta(`SELECT zitadel.projections.keys.id,`+
` zitadel.projections.keys.creation_date,`+
` zitadel.projections.keys.change_date,`+
` zitadel.projections.keys.sequence,`+
` zitadel.projections.keys.resource_owner,`+
` zitadel.projections.keys.algorithm,`+
` zitadel.projections.keys.use,`+
` zitadel.projections.keys_public.expiry,`+
` zitadel.projections.keys_public.key,`+
` COUNT(*) OVER ()`+
` FROM zitadel.projections.keys`+
` LEFT JOIN zitadel.projections.keys_public ON zitadel.projections.keys.id = zitadel.projections.keys_public.id`),
[]string{
"id",
"creation_date",
"change_date",
"sequence",
"resource_owner",
"algorithm",
"use",
"expiry",
"key",
"count",
},
[][]driver.Value{
{
"key-id",
testNow,
testNow,
uint64(20211109),
"ro",
"RS256",
0,
testNow,
[]byte("-----BEGIN RSA PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsvX9P58JFxEs5C+L+H7W\nduFSWL5EPzber7C2m94klrSV6q0bAcrYQnGwFOlveThsY200hRbadKaKjHD7qIKH\nDEe0IY2PSRht33Jye52AwhkRw+M3xuQH/7R8LydnsNFk2KHpr5X2SBv42e37LjkE\nslKSaMRgJW+v0KZ30piY8QsdFRKKaVg5/Ajt1YToM1YVsdHXJ3vmXFMtypLdxwUD\ndIaLEX6pFUkU75KSuEQ/E2luT61Q3ta9kOWm9+0zvi7OMcbdekJT7mzcVnh93R1c\n13ZhQCLbh9A7si8jKFtaMWevjayrvqQABEcTN9N4Hoxcyg6l4neZtRDk75OMYcqm\nDQIDAQAB\n-----END RSA PUBLIC KEY-----\n"),
},
},
),
},
object: &PublicKeys{
SearchResponse: SearchResponse{
Count: 1,
},
Keys: []PublicKey{
&rsaPublicKey{
key: key{
id: "key-id",
creationDate: testNow,
changeDate: testNow,
sequence: 20211109,
resourceOwner: "ro",
algorithm: "RS256",
use: domain.KeyUsageSigning,
},
expiry: testNow,
publicKey: &rsa.PublicKey{
E: 65537,
N: fromBase16("b2f5fd3f9f0917112ce42f8bf87ed676e15258be443f36deafb0b69bde2496b495eaad1b01cad84271b014e96f79386c636d348516da74a68a8c70fba882870c47b4218d8f49186ddf72727b9d80c21911c3e337c6e407ffb47c2f2767b0d164d8a1e9af95f6481bf8d9edfb2e3904b2529268c460256fafd0a677d29898f10b1d15128a695839fc08edd584e8335615b1d1d7277be65c532dca92ddc7050374868b117ea9154914ef9292b8443f13696e4fad50ded6bd90e5a6f7ed33be2ece31c6dd7a4253ee6cdc56787ddd1d5cd776614022db87d03bb22f23285b5a3167af8dacabbea40004471337d3781e8c5cca0ea5e27799b510e4ef938c61caa60d"),
},
},
},
},
},
{
name: "preparePublicKeysQuery sql err",
prepare: preparePublicKeysQuery,
want: want{
sqlExpectations: mockQueryErr(
regexp.QuoteMeta(`SELECT zitadel.projections.keys.id,`+
` zitadel.projections.keys.creation_date,`+
` zitadel.projections.keys.change_date,`+
` zitadel.projections.keys.sequence,`+
` zitadel.projections.keys.resource_owner,`+
` zitadel.projections.keys.algorithm,`+
` zitadel.projections.keys.use,`+
` zitadel.projections.keys_public.expiry,`+
` zitadel.projections.keys_public.key,`+
` COUNT(*) OVER ()`+
` FROM zitadel.projections.keys`+
` LEFT JOIN zitadel.projections.keys_public ON zitadel.projections.keys.id = zitadel.projections.keys_public.id`),
sql.ErrConnDone,
),
err: func(err error) (error, bool) {
if !errors.Is(err, sql.ErrConnDone) {
return fmt.Errorf("err should be sql.ErrConnDone got: %w", err), false
}
return nil, true
},
},
object: nil,
},
{
name: "preparePrivateKeysQuery no result",
prepare: preparePrivateKeysQuery,
want: want{
sqlExpectations: mockQueries(
regexp.QuoteMeta(`SELECT zitadel.projections.keys.id,`+
` zitadel.projections.keys.creation_date,`+
` zitadel.projections.keys.change_date,`+
` zitadel.projections.keys.sequence,`+
` zitadel.projections.keys.resource_owner,`+
` zitadel.projections.keys.algorithm,`+
` zitadel.projections.keys.use,`+
` zitadel.projections.keys_private.expiry,`+
` zitadel.projections.keys_private.key,`+
` COUNT(*) OVER ()`+
` FROM zitadel.projections.keys`+
` LEFT JOIN zitadel.projections.keys_private ON zitadel.projections.keys.id = zitadel.projections.keys_private.id`),
nil,
nil,
),
err: func(err error) (error, bool) {
if !errs.IsNotFound(err) {
return fmt.Errorf("err should be zitadel.NotFoundError got: %w", err), false
}
return nil, true
},
},
object: &PrivateKeys{Keys: []PrivateKey{}},
},
{
name: "preparePrivateKeysQuery found",
prepare: preparePrivateKeysQuery,
want: want{
sqlExpectations: mockQueries(
regexp.QuoteMeta(`SELECT zitadel.projections.keys.id,`+
` zitadel.projections.keys.creation_date,`+
` zitadel.projections.keys.change_date,`+
` zitadel.projections.keys.sequence,`+
` zitadel.projections.keys.resource_owner,`+
` zitadel.projections.keys.algorithm,`+
` zitadel.projections.keys.use,`+
` zitadel.projections.keys_private.expiry,`+
` zitadel.projections.keys_private.key,`+
` COUNT(*) OVER ()`+
` FROM zitadel.projections.keys`+
` LEFT JOIN zitadel.projections.keys_private ON zitadel.projections.keys.id = zitadel.projections.keys_private.id`),
[]string{
"id",
"creation_date",
"change_date",
"sequence",
"resource_owner",
"algorithm",
"use",
"expiry",
"key",
"count",
},
[][]driver.Value{
{
"key-id",
testNow,
testNow,
uint64(20211109),
"ro",
"RS256",
0,
testNow,
[]byte(`{"Algorithm": "enc", "Crypted": "cHJpdmF0ZUtleQ==", "CryptoType": 0, "KeyID": "id"}`),
},
},
),
},
object: &PrivateKeys{
SearchResponse: SearchResponse{
Count: 1,
},
Keys: []PrivateKey{
&privateKey{
key: key{
id: "key-id",
creationDate: testNow,
changeDate: testNow,
sequence: 20211109,
resourceOwner: "ro",
algorithm: "RS256",
use: domain.KeyUsageSigning,
},
expiry: testNow,
privateKey: &crypto.CryptoValue{
CryptoType: crypto.TypeEncryption,
Algorithm: "enc",
KeyID: "id",
Crypted: []byte("privateKey"),
},
},
},
},
},
{
name: "preparePrivateKeysQuery sql err",
prepare: preparePrivateKeysQuery,
want: want{
sqlExpectations: mockQueryErr(
regexp.QuoteMeta(`SELECT zitadel.projections.keys.id,`+
` zitadel.projections.keys.creation_date,`+
` zitadel.projections.keys.change_date,`+
` zitadel.projections.keys.sequence,`+
` zitadel.projections.keys.resource_owner,`+
` zitadel.projections.keys.algorithm,`+
` zitadel.projections.keys.use,`+
` zitadel.projections.keys_private.expiry,`+
` zitadel.projections.keys_private.key,`+
` COUNT(*) OVER ()`+
` FROM zitadel.projections.keys`+
` LEFT JOIN zitadel.projections.keys_private ON zitadel.projections.keys.id = zitadel.projections.keys_private.id`),
sql.ErrConnDone,
),
err: func(err error) (error, bool) {
if !errors.Is(err, sql.ErrConnDone) {
return fmt.Errorf("err should be sql.ErrConnDone got: %w", err), false
}
return nil, true
},
},
object: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assertPrepare(t, tt.prepare, tt.object, tt.want.sqlExpectations, tt.want.err)
})
}
}
func fromBase16(base16 string) *big.Int {
i, ok := new(big.Int).SetString(base16, 16)
if !ok {
panic("bad number: " + base16)
}
return i
}

View File

@@ -3,7 +3,6 @@ package query
import (
"database/sql"
"database/sql/driver"
"encoding/json"
"errors"
"fmt"
"log"
@@ -13,7 +12,7 @@ import (
"github.com/DATA-DOG/go-sqlmock"
sq "github.com/Masterminds/squirrel"
"github.com/nsf/jsondiff"
"github.com/stretchr/testify/assert"
)
var (
@@ -58,8 +57,7 @@ func assertPrepare(t *testing.T, prepareFunc, expectedObject interface{}, sqlExp
return false
}
if !reflect.DeepEqual(object, expectedObject) {
prettyPrintDiff(t, expectedObject, object)
if !assert.Equal(t, expectedObject, object) {
return false
}
@@ -315,19 +313,3 @@ func TestValidatePrepare(t *testing.T) {
})
}
}
func prettyPrintDiff(t *testing.T, expected, gotten interface{}) {
t.Helper()
expectedMarshalled, _ := json.Marshal(expected)
objectMarshalled, _ := json.Marshal(gotten)
_, diff := jsondiff.Compare(
expectedMarshalled,
objectMarshalled,
&jsondiff.Options{
SkipMatches: true,
Indent: " ",
ChangedSeparator: " is expected, got ",
})
t.Errorf("unexpected object: want %T, got %T, difference:\n%s", expected, gotten, diff)
}

View File

@@ -18,6 +18,7 @@ import (
type KeyProjection struct {
crdb.StatementHandler
encryptionAlgorithm crypto.EncryptionAlgorithm
keyChan chan<- interface{}
}
const (
@@ -26,11 +27,12 @@ const (
KeyPublicTable = KeyProjectionTable + "_" + publicKeyTableSuffix
)
func NewKeyProjection(ctx context.Context, config crdb.StatementHandlerConfig, keyConfig systemdefaults.KeyConfig) (_ *KeyProjection, err error) {
func NewKeyProjection(ctx context.Context, config crdb.StatementHandlerConfig, keyConfig systemdefaults.KeyConfig, keyChan chan<- interface{}) (_ *KeyProjection, err error) {
p := &KeyProjection{}
config.ProjectionName = KeyProjectionTable
config.Reducers = p.reducers()
p.StatementHandler = crdb.NewStatementHandler(ctx, config)
p.keyChan = keyChan
p.encryptionAlgorithm, err = crypto.NewAESCrypto(keyConfig.EncryptionConfig)
if err != nil {
return nil, err
@@ -103,6 +105,9 @@ func (p *KeyProjection) reduceKeyPairAdded(event eventstore.Event) (*handler.Sta
},
crdb.WithTableSuffix(privateKeyTableSuffix),
))
if p.keyChan != nil {
p.keyChan <- true
}
}
if e.PublicKey.Expiry.After(time.Now()) {
publicKey, err := crypto.Decrypt(e.PublicKey.Key, p.encryptionAlgorithm)

View File

@@ -17,7 +17,7 @@ const (
FailedEventsTable = "projections.failed_events"
)
func Start(ctx context.Context, sqlClient *sql.DB, es *eventstore.Eventstore, config Config, defaults systemdefaults.SystemDefaults) error {
func Start(ctx context.Context, sqlClient *sql.DB, es *eventstore.Eventstore, config Config, defaults systemdefaults.SystemDefaults, keyChan chan<- interface{}) error {
projectionConfig := crdb.StatementHandlerConfig{
ProjectionHandlerConfig: handler.ProjectionHandlerConfig{
HandlerConfig: handler.HandlerConfig{
@@ -63,8 +63,8 @@ func Start(ctx context.Context, sqlClient *sql.DB, es *eventstore.Eventstore, co
NewIAMMemberProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["iam_members"]))
NewProjectMemberProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["project_members"]))
NewProjectGrantMemberProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["project_grant_members"]))
_, err := NewKeyProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["keys"]), defaults.KeyConfig)
NewAuthNKeyProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["authn_keys"]))
_, err := NewKeyProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["keys"]), defaults.KeyConfig, keyChan)
return err
}

View File

@@ -40,7 +40,7 @@ type Config struct {
Eventstore types.SQLUser
}
func StartQueries(ctx context.Context, es *eventstore.Eventstore, projections projection.Config, defaults sd.SystemDefaults) (repo *Queries, err error) {
func StartQueries(ctx context.Context, es *eventstore.Eventstore, projections projection.Config, defaults sd.SystemDefaults, keyChan chan<- interface{}) (repo *Queries, err error) {
sqlClient, err := projections.CRDB.Start()
if err != nil {
return nil, err
@@ -69,7 +69,7 @@ func StartQueries(ctx context.Context, es *eventstore.Eventstore, projections pr
action.RegisterEventMappers(repo.eventstore)
keypair.RegisterEventMappers(repo.eventstore)
err = projection.Start(ctx, sqlClient, es, projections, defaults)
err = projection.Start(ctx, sqlClient, es, projections, defaults, keyChan)
if err != nil {
return nil, err
}