TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-11 21:47:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(query): keys (#2755)

Browse Source 
* fix: add keys to projections

* change to multiple tables

* query keys

* query keys

* fix race condition

* fix timer reset

* begin tests

* tests

* remove migration

* only send to keyChannel if not nil
This commit is contained in:
Livio Amstutz
2022-01-12 13:22:04 +01:00
committed by GitHub
parent ead61d240d
commit 9ab566fdeb
23 changed files with 927 additions and 419 deletions
Download Patch File Download Diff File Expand all files Collapse all files

175
internal/auth/repository/eventsourcing/eventstore/key.go
View File

@@ -1,175 +0,0 @@
package eventstore
import (
"context"
"os"
"time"
"github.com/caos/logging"
"gopkg.in/square/go-jose.v2"
"github.com/caos/zitadel/internal/auth/repository/eventsourcing/view"
"github.com/caos/zitadel/internal/command"
"github.com/caos/zitadel/internal/crypto"
"github.com/caos/zitadel/internal/errors"
"github.com/caos/zitadel/internal/eventstore"
"github.com/caos/zitadel/internal/eventstore/v1/spooler"
"github.com/caos/zitadel/internal/id"
"github.com/caos/zitadel/internal/key/model"
key_view "github.com/caos/zitadel/internal/key/repository/view"
)
type KeyRepository struct {
Commands *command.Commands
Eventstore *eventstore.Eventstore
View *view.View
SigningKeyRotationCheck time.Duration
SigningKeyGracefulPeriod time.Duration
KeyAlgorithm crypto.EncryptionAlgorithm
KeyChan <-chan *model.KeyView
Locker spooler.Locker
lockID string
currentKeyID string
currentKeyExpiration time.Time
}
const (
signingKey = "signing_key"
)
func (k *KeyRepository) GetSigningKey(ctx context.Context, keyCh chan<- jose.SigningKey, algorithm string) {
renewTimer := time.After(0)
go func() {
for {
select {
case <-ctx.Done():
return
case key := <-k.KeyChan:
refreshed, err := k.refreshSigningKey(ctx, key, keyCh, algorithm)
logging.Log("KEY-asd5g").OnError(err).Error("could not refresh signing key on key channel push")
renewTimer = time.After(k.getRenewTimer(refreshed))
case <-renewTimer:
key, err := k.latestSigningKey()
logging.Log("KEY-DAfh4-1").OnError(err).Error("could not check for latest signing key")
refreshed, err := k.refreshSigningKey(ctx, key, keyCh, algorithm)
logging.Log("KEY-DAfh4-2").OnError(err).Error("could not refresh signing key when ensuring key")
renewTimer = time.After(k.getRenewTimer(refreshed))
}
}
}()
}
func (k *KeyRepository) GetKeySet(ctx context.Context) (*jose.JSONWebKeySet, error) {
keys, err := k.View.GetActiveKeySet()
if err != nil {
return nil, err
}
webKeys := make([]jose.JSONWebKey, len(keys))
for i, key := range keys {
webKeys[i] = jose.JSONWebKey{KeyID: key.ID, Algorithm: key.Algorithm, Use: key.Usage.String(), Key: key.Key}
}
return &jose.JSONWebKeySet{Keys: webKeys}, nil
}
func (k *KeyRepository) getRenewTimer(refreshed bool) time.Duration {
duration := k.SigningKeyRotationCheck
if refreshed {
duration = k.currentKeyExpiration.Sub(time.Now().Add(k.SigningKeyGracefulPeriod + k.SigningKeyRotationCheck*2))
}
logging.LogWithFields("EVENT-dK432", "in", duration).Info("next signing key check")
return duration
}
func (k *KeyRepository) latestSigningKey() (shortRefresh *model.KeyView, err error) {
key, errView := k.View.GetActivePrivateKeyForSigning(time.Now().UTC().Add(k.SigningKeyGracefulPeriod))
if errView != nil && !errors.IsNotFound(errView) {
logging.Log("EVENT-GEd4h").WithError(errView).Warn("could not get signing key")
return nil, errView
}
return key, nil
}
func (k *KeyRepository) ensureIsLatestKey(ctx context.Context) (bool, error) {
sequence, err := k.View.GetLatestKeySequence()
if err != nil {
return false, err
}
events, err := k.getKeyEvents(ctx, sequence.CurrentSequence)
if err != nil {
logging.Log("EVENT-der5g").WithError(err).Warn("error retrieving new events")
return false, err
}
if len(events) > 0 {
logging.Log("EVENT-GBD23").Warn("view not up to date, retrying later")
return false, nil
}
return true, nil
}
func (k *KeyRepository) refreshSigningKey(ctx context.Context, key *model.KeyView, keyCh chan<- jose.SigningKey, algorithm string) (refreshed bool, err error) {
if key == nil {
if k.currentKeyExpiration.Before(time.Now().UTC()) {
logging.Log("EVENT-ADg26").Info("unset current signing key")
keyCh <- jose.SigningKey{}
}
if ok, err := k.ensureIsLatestKey(ctx); !ok && err == nil {
return false, err
}
logging.Log("EVENT-sdz53").Info("lock and generate signing key pair")
err = k.lockAndGenerateSigningKeyPair(ctx, algorithm)
logging.Log("EVENT-B4d21").OnError(err).Warn("could not create signing key")
return false, err
}
if k.currentKeyID == key.ID {
logging.Log("EVENT-Abb3e").Info("no new signing key")
return false, nil
}
if ok, err := k.ensureIsLatestKey(ctx); !ok && err == nil {
logging.Log("EVENT-HJd92").Info("signing key in view is not latest key")
return false, err
}
signingKey, err := model.SigningKeyFromKeyView(key, k.KeyAlgorithm)
if err != nil {
logging.Log("EVENT-HJd92").WithError(err).Error("signing key cannot be decrypted -> immediate refresh")
return k.refreshSigningKey(ctx, nil, keyCh, algorithm)
}
k.currentKeyID = signingKey.ID
k.currentKeyExpiration = key.Expiry
keyCh <- jose.SigningKey{
Algorithm: jose.SignatureAlgorithm(signingKey.Algorithm),
Key: jose.JSONWebKey{
KeyID: signingKey.ID,
Key: signingKey.Key,
},
}
logging.LogWithFields("EVENT-dsg54", "keyID", signingKey.ID).Info("refreshed signing key")
return true, nil
}
func (k *KeyRepository) lockAndGenerateSigningKeyPair(ctx context.Context, algorithm string) error {
err := k.Locker.Renew(k.lockerID(), signingKey, k.SigningKeyRotationCheck*2)
if err != nil {
if errors.IsErrorAlreadyExists(err) {
return nil
}
return err
}
return k.Commands.GenerateSigningKeyPair(ctx, algorithm)
}
func (k *KeyRepository) lockerID() string {
if k.lockID == "" {
var err error
k.lockID, err = os.Hostname()
if err != nil || k.lockID == "" {
k.lockID, err = id.SonyFlakeGenerator.Next()
logging.Log("EVENT-bsdf6").OnError(err).Panic("unable to generate lockID")
}
}
return k.lockID
}
func (k *KeyRepository) getKeyEvents(ctx context.Context, sequence uint64) ([]eventstore.Event, error) {
return k.Eventstore.Filter(ctx, key_view.KeyPairQuery(sequence))
}

6
internal/auth/repository/eventsourcing/handler/handler.go
View File

@@ -8,7 +8,6 @@ import (
"github.com/caos/zitadel/internal/config/types"
v1 "github.com/caos/zitadel/internal/eventstore/v1"
"github.com/caos/zitadel/internal/eventstore/v1/query"
key_model "github.com/caos/zitadel/internal/key/model"
)
type Configs map[string]*Config
@@ -30,7 +29,7 @@ func (h *handler) Eventstore() v1.Eventstore {
return h.es
}
func Register(configs Configs, bulkLimit, errorCount uint64, view *view.View, es v1.Eventstore, systemDefaults sd.SystemDefaults, keyChan chan<- *key_model.KeyView) []query.Handler {
func Register(configs Configs, bulkLimit, errorCount uint64, view *view.View, es v1.Eventstore, systemDefaults sd.SystemDefaults) []query.Handler {
return []query.Handler{
newUser(
handler{view, bulkLimit, configs.cycleDuration("User"), errorCount, es},
@@ -41,9 +40,6 @@ func Register(configs Configs, bulkLimit, errorCount uint64, view *view.View, es
handler{view, bulkLimit, configs.cycleDuration("UserMembership"), errorCount, es}),
newToken(
handler{view, bulkLimit, configs.cycleDuration("Token"), errorCount, es}),
newKey(
handler{view, bulkLimit, configs.cycleDuration("Key"), errorCount, es},
keyChan),
newUserGrant(
handler{view, bulkLimit, configs.cycleDuration("UserGrant"), errorCount, es},
systemDefaults.IamID),

106
internal/auth/repository/eventsourcing/handler/key.go
View File

@@ -1,106 +0,0 @@
package handler
import (
"github.com/caos/zitadel/internal/eventstore/v1"
"time"
"github.com/caos/logging"
"github.com/caos/zitadel/internal/eventstore/v1/models"
"github.com/caos/zitadel/internal/eventstore/v1/query"
"github.com/caos/zitadel/internal/eventstore/v1/spooler"
"github.com/caos/zitadel/internal/key/model"
"github.com/caos/zitadel/internal/key/repository/eventsourcing"
es_model "github.com/caos/zitadel/internal/key/repository/eventsourcing/model"
view_model "github.com/caos/zitadel/internal/key/repository/view/model"
)
const (
keyTable = "auth.keys"
)
type Key struct {
handler
subscription *v1.Subscription
keyChan chan<- *model.KeyView
}
func newKey(handler handler, keyChan chan<- *model.KeyView) *Key {
h := &Key{
handler: handler,
keyChan: keyChan,
}
h.subscribe()
return h
}
func (k *Key) subscribe() {
k.subscription = k.es.Subscribe(k.AggregateTypes()...)
go func() {
for event := range k.subscription.Events {
query.ReduceEvent(k, event)
}
}()
}
func (k *Key) ViewModel() string {
return keyTable
}
func (k *Key) Subscription() *v1.Subscription {
return k.subscription
}
func (_ *Key) AggregateTypes() []models.AggregateType {
return []models.AggregateType{es_model.KeyPairAggregate}
}
func (k *Key) CurrentSequence() (uint64, error) {
sequence, err := k.view.GetLatestKeySequence()
if err != nil {
return 0, err
}
return sequence.CurrentSequence, nil
}
func (k *Key) EventQuery() (*models.SearchQuery, error) {
sequence, err := k.view.GetLatestKeySequence()
if err != nil {
return nil, err
}
return eventsourcing.KeyPairQuery(sequence.CurrentSequence), nil
}
func (k *Key) Reduce(event *models.Event) error {
switch event.Type {
case es_model.KeyPairAdded:
privateKey, publicKey, err := view_model.KeysFromPairEvent(event)
if err != nil {
return err
}
if privateKey.Expiry.Before(time.Now()) && publicKey.Expiry.Before(time.Now()) {
return k.view.ProcessedKeySequence(event)
}
err = k.view.PutKeys(privateKey, publicKey, event)
if err != nil {
return err
}
k.keyChan <- view_model.KeyViewToModel(privateKey)
return nil
default:
return k.view.ProcessedKeySequence(event)
}
}
func (k *Key) OnError(event *models.Event, err error) error {
logging.LogWithFields("SPOOL-GHa3a", "id", event.AggregateID).WithError(err).Warn("something went wrong in key handler")
return spooler.HandleError(event, err, k.view.GetLatestKeyFailedEvent, k.view.ProcessedKeyFailedEvent, k.view.ProcessedKeySequence, k.errorCountUntilSkip)
}
func (k *Key) OnSuccess() error {
err := spooler.HandleSuccess(k.view.UpdateKeySpoolerRunTimestamp)
logging.LogWithFields("SPOOL-vM9sd", "table", keyTable).OnError(err).Warn("could not process on success func")
return err
}

16
internal/auth/repository/eventsourcing/repository.go
View File

@@ -20,7 +20,6 @@ import (
v1 "github.com/caos/zitadel/internal/eventstore/v1"
es_spol "github.com/caos/zitadel/internal/eventstore/v1/spooler"
"github.com/caos/zitadel/internal/id"
key_model "github.com/caos/zitadel/internal/key/model"
"github.com/caos/zitadel/internal/query"
)
@@ -41,7 +40,6 @@ type EsRepository struct {
eventstore.AuthRequestRepo
eventstore.TokenRepo
eventstore.RefreshTokenRepo
eventstore.KeyRepository
eventstore.ApplicationRepo
eventstore.UserSessionRepo
eventstore.UserGrantRepo
@@ -81,9 +79,7 @@ func Start(conf Config, authZ authz.Config, systemDefaults sd.SystemDefaults, co
statikLoginFS, err := fs.NewWithNamespace("login")
logging.Log("CONFI-20opp").OnError(err).Panic("unable to start login statik dir")
keyChan := make(chan *key_model.KeyView)
spool := spooler.StartSpooler(conf.Spooler, es, view, sqlClient, systemDefaults, keyChan)
locker := spooler.NewLocker(sqlClient)
spool := spooler.StartSpooler(conf.Spooler, es, view, sqlClient, systemDefaults)
userRepo := eventstore.UserRepo{
SearchLimit: conf.SearchLimit,
@@ -141,16 +137,6 @@ func Start(conf Config, authZ authz.Config, systemDefaults sd.SystemDefaults, co
SearchLimit: conf.SearchLimit,
KeyAlgorithm: keyAlgorithm,
},
eventstore.KeyRepository{
View: view,
Commands: command,
Eventstore: esV2,
SigningKeyRotationCheck: systemDefaults.KeyConfig.SigningKeyRotationCheck.Duration,
SigningKeyGracefulPeriod: systemDefaults.KeyConfig.SigningKeyGracefulPeriod.Duration,
KeyAlgorithm: keyAlgorithm,
Locker: locker,
KeyChan: keyChan,
},
eventstore.ApplicationRepo{
Commands: command,
Query: queries,

8
internal/auth/repository/eventsourcing/spooler/spooler.go
View File

@@ -2,13 +2,13 @@ package spooler
import (
"database/sql"
"github.com/caos/zitadel/internal/eventstore/v1"
v1 "github.com/caos/zitadel/internal/eventstore/v1"
"github.com/caos/zitadel/internal/auth/repository/eventsourcing/handler"
"github.com/caos/zitadel/internal/auth/repository/eventsourcing/view"
sd "github.com/caos/zitadel/internal/config/systemdefaults"
"github.com/caos/zitadel/internal/eventstore/v1/spooler"
key_model "github.com/caos/zitadel/internal/key/model"
)
type SpoolerConfig struct {
@@ -18,12 +18,12 @@ type SpoolerConfig struct {
Handlers handler.Configs
}
func StartSpooler(c SpoolerConfig, es v1.Eventstore, view *view.View, client *sql.DB, systemDefaults sd.SystemDefaults, keyChan chan<- *key_model.KeyView) *spooler.Spooler {
func StartSpooler(c SpoolerConfig, es v1.Eventstore, view *view.View, client *sql.DB, systemDefaults sd.SystemDefaults) *spooler.Spooler {
spoolerConfig := spooler.Config{
Eventstore: es,
Locker: &locker{dbClient: client},
ConcurrentWorkers: c.ConcurrentWorkers,
ViewHandlers: handler.Register(c.Handlers, c.BulkLimit, c.FailureCountUntilSkip, view, es, systemDefaults, keyChan),
ViewHandlers: handler.Register(c.Handlers, c.BulkLimit, c.FailureCountUntilSkip, view, es, systemDefaults),
}
spool := spoolerConfig.New()
spool.Start()

12
internal/auth/repository/key.go
View File

@@ -1,12 +0,0 @@
package repository
import (
"context"
"gopkg.in/square/go-jose.v2"
)
type KeyRepository interface {
GetSigningKey(ctx context.Context, keyCh chan<- jose.SigningKey, algorithm string)
GetKeySet(ctx context.Context) (*jose.JSONWebKeySet, error)
}

1
internal/auth/repository/repository.go
View File

@@ -10,7 +10,6 @@ type Repository interface {
AuthRequestRepository
TokenRepository
ApplicationRepository
KeyRepository
UserSessionRepository
UserGrantRepository
OrgRepository