TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-11 21:47:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(setup): init projections (#7194)

Browse Source 
Even though this is a feature it's released as fix so that we can back port to earlier revisions.

As reported by multiple users startup of ZITADEL after leaded to downtime and worst case rollbacks to the previously deployed version.

The problem starts rising when there are too many events to process after the start of ZITADEL. The root cause are changes on projections (database tables) which must be recomputed. This PR solves this problem by adding a new step to the setup phase which prefills the projections. The step can be enabled by adding the `--init-projections`-flag to `setup`, `start-from-init` and `start-from-setup`. Setting this flag results in potentially longer duration of the setup phase but reduces the risk of the problems mentioned in the paragraph above.
This commit is contained in:
Silvan
2024-01-25 17:28:20 +01:00
committed by GitHub
parent d590da7c7d
commit 17953e9040
80 changed files with 1296 additions and 962 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/setup/01.go
View File

@@ -4,6 +4,8 @@ import (
"context"
"database/sql"
_ "embed"
"github.com/zitadel/zitadel/internal/eventstore"
)
var (
@@ -19,7 +21,7 @@ type ProjectionTable struct {
dbClient *sql.DB
}
func (mig *ProjectionTable) Execute(ctx context.Context) error {
func (mig *ProjectionTable) Execute(ctx context.Context, _ eventstore.Event) error {
stmt := createAdminViews + createAuthViews + createProjections
_, err := mig.dbClient.ExecContext(ctx, stmt)
return err

4
cmd/setup/02.go
View File

@@ -3,6 +3,8 @@ package setup
import (
"context"
"database/sql"
"github.com/zitadel/zitadel/internal/eventstore"
)
const (
@@ -26,7 +28,7 @@ type AssetTable struct {
dbClient *sql.DB
}
func (mig *AssetTable) Execute(ctx context.Context) error {
func (mig *AssetTable) Execute(ctx context.Context, _ eventstore.Event) error {
_, err := mig.dbClient.ExecContext(ctx, createAssets)
return err
}

35
cmd/setup/03.go
View File

@@ -41,30 +41,19 @@ type FirstInstance struct {
domain string
}
func (mig *FirstInstance) Execute(ctx context.Context) error {
keyStorage, err := crypto_db.NewKeyStorage(mig.db, mig.masterKey)
func (mig *FirstInstance) Execute(ctx context.Context, _ eventstore.Event) error {
keyStorage, err := mig.verifyEncryptionKeys(ctx)
if err != nil {
return fmt.Errorf("cannot start key storage: %w", err)
}
if err = verifyKey(ctx, mig.userEncryptionKey, keyStorage); err != nil {
return err
}
userAlg, err := crypto.NewAESCrypto(mig.userEncryptionKey, keyStorage)
if err != nil {
return err
}
if err = verifyKey(ctx, mig.smtpEncryptionKey, keyStorage); err != nil {
return err
}
smtpEncryption, err := crypto.NewAESCrypto(mig.smtpEncryptionKey, keyStorage)
if err != nil {
return err
}
if err = verifyKey(ctx, mig.oidcEncryptionKey, keyStorage); err != nil {
return err
}
oidcEncryption, err := crypto.NewAESCrypto(mig.oidcEncryptionKey, keyStorage)
if err != nil {
return err
@@ -132,7 +121,27 @@ func (mig *FirstInstance) Execute(ctx context.Context) error {
(mig.instanceSetup.Org.Machine.MachineKey != nil && key == nil)) {
return err
}
return mig.outputMachineAuthentication(key, token)
}
func (mig *FirstInstance) verifyEncryptionKeys(ctx context.Context) (*crypto_db.Database, error) {
keyStorage, err := crypto_db.NewKeyStorage(mig.db, mig.masterKey)
if err != nil {
return nil, fmt.Errorf("cannot start key storage: %w", err)
}
if err = verifyKey(ctx, mig.userEncryptionKey, keyStorage); err != nil {
return nil, err
}
if err = verifyKey(ctx, mig.smtpEncryptionKey, keyStorage); err != nil {
return nil, err
}
if err = verifyKey(ctx, mig.oidcEncryptionKey, keyStorage); err != nil {
return nil, err
}
return keyStorage, nil
}
func (mig *FirstInstance) outputMachineAuthentication(key *command.MachineKey, token string) error {
if key != nil {
keyDetails, err := key.Detail()
if err != nil {

4
cmd/setup/05.go
View File

@@ -4,6 +4,8 @@ import (
"context"
"database/sql"
_ "embed"
"github.com/zitadel/zitadel/internal/eventstore"
)
var (
@@ -15,7 +17,7 @@ type LastFailed struct {
dbClient *sql.DB
}
func (mig *LastFailed) Execute(ctx context.Context) error {
func (mig *LastFailed) Execute(ctx context.Context, _ eventstore.Event) error {
_, err := mig.dbClient.ExecContext(ctx, lastFailedStmts)
return err
}

4
cmd/setup/06.go
View File

@@ -4,6 +4,8 @@ import (
"context"
"database/sql"
_ "embed"
"github.com/zitadel/zitadel/internal/eventstore"
)
var (
@@ -17,7 +19,7 @@ type OwnerRemoveColumns struct {
dbClient *sql.DB
}
func (mig *OwnerRemoveColumns) Execute(ctx context.Context) error {
func (mig *OwnerRemoveColumns) Execute(ctx context.Context, _ eventstore.Event) error {
stmt := createAdminViews06 + createAuthViews06
_, err := mig.dbClient.ExecContext(ctx, stmt)
return err

4
cmd/setup/07.go
View File

@@ -5,6 +5,8 @@ import (
"database/sql"
"embed"
"strings"
"github.com/zitadel/zitadel/internal/eventstore"
)
var (
@@ -24,7 +26,7 @@ type LogstoreTables struct {
dbType string
}
func (mig *LogstoreTables) Execute(ctx context.Context) error {
func (mig *LogstoreTables) Execute(ctx context.Context, _ eventstore.Event) error {
accessStmt, err := readStmt(createAccessLogsTable07, "07", mig.dbType, "access.sql")
if err != nil {
return err

3
cmd/setup/08.go
View File

@@ -5,6 +5,7 @@ import (
"embed"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore"
)
var (
@@ -17,7 +18,7 @@ type AuthTokenIndexes struct {
dbClient *database.DB
}
func (mig *AuthTokenIndexes) Execute(ctx context.Context) error {
func (mig *AuthTokenIndexes) Execute(ctx context.Context, _ eventstore.Event) error {
stmt, err := readStmt(tokenIndexes08, "08", mig.dbClient.Type(), "08.sql")
if err != nil {
return err

3
cmd/setup/10.go
View File

@@ -10,6 +10,7 @@ import (
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore"
)
var (
@@ -31,7 +32,7 @@ type CorrectCreationDate struct {
FailAfter time.Duration
}
func (mig *CorrectCreationDate) Execute(ctx context.Context) (err error) {
func (mig *CorrectCreationDate) Execute(ctx context.Context, _ eventstore.Event) (err error) {
ctx, cancel := context.WithTimeout(ctx, mig.FailAfter)
defer cancel()

3
cmd/setup/12.go
View File

@@ -5,6 +5,7 @@ import (
_ "embed"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore"
)
var (
@@ -16,7 +17,7 @@ type AddOTPColumns struct {
dbClient *database.DB
}
func (mig *AddOTPColumns) Execute(ctx context.Context) error {
func (mig *AddOTPColumns) Execute(ctx context.Context, _ eventstore.Event) error {
_, err := mig.dbClient.ExecContext(ctx, addOTPColumns)
return err
}

3
cmd/setup/13.go
View File

@@ -5,6 +5,7 @@ import (
_ "embed"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore"
)
var (
@@ -16,7 +17,7 @@ type FixQuotaConstraints struct {
dbClient *database.DB
}
func (mig *FixQuotaConstraints) Execute(ctx context.Context) error {
func (mig *FixQuotaConstraints) Execute(ctx context.Context, _ eventstore.Event) error {
_, err := mig.dbClient.ExecContext(ctx, fixQuotaConstraints)
return err
}

3
cmd/setup/14.go
View File

@@ -11,6 +11,7 @@ import (
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore"
)
var (
@@ -23,7 +24,7 @@ type NewEventsTable struct {
dbClient *database.DB
}
func (mig *NewEventsTable) Execute(ctx context.Context) error {
func (mig *NewEventsTable) Execute(ctx context.Context, _ eventstore.Event) error {
migrations, err := newEventsTable.ReadDir("14/" + mig.dbClient.Type())
if err != nil {
return err

3
cmd/setup/15.go
View File

@@ -7,6 +7,7 @@ import (
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore"
)
var (
@@ -19,7 +20,7 @@ type CurrentProjectionState struct {
dbClient *database.DB
}
func (mig *CurrentProjectionState) Execute(ctx context.Context) error {
func (mig *CurrentProjectionState) Execute(ctx context.Context, _ eventstore.Event) error {
migrations, err := currentProjectionState.ReadDir("15/" + mig.dbClient.Type())
if err != nil {
return err

3
cmd/setup/16.go
View File

@@ -7,6 +7,7 @@ import (
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore"
)
var (
@@ -18,7 +19,7 @@ type UniqueConstraintToLower struct {
dbClient *database.DB
}
func (mig *UniqueConstraintToLower) Execute(ctx context.Context) error {
func (mig *UniqueConstraintToLower) Execute(ctx context.Context, _ eventstore.Event) error {
res, err := mig.dbClient.ExecContext(ctx, uniqueConstraintLower)
if err != nil {
return err

3
cmd/setup/17.go
View File

@@ -5,6 +5,7 @@ import (
_ "embed"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore"
)
var (
@@ -16,7 +17,7 @@ type AddOffsetToCurrentStates struct {
dbClient *database.DB
}
func (mig *AddOffsetToCurrentStates) Execute(ctx context.Context) error {
func (mig *AddOffsetToCurrentStates) Execute(ctx context.Context, _ eventstore.Event) error {
_, err := mig.dbClient.ExecContext(ctx, addOffsetField)
return err
}

3
cmd/setup/18.go
View File

@@ -5,6 +5,7 @@ import (
_ "embed"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore"
)
var (
@@ -16,7 +17,7 @@ type AddLowerFieldsToLoginNames struct {
dbClient *database.DB
}
func (mig *AddLowerFieldsToLoginNames) Execute(ctx context.Context) error {
func (mig *AddLowerFieldsToLoginNames) Execute(ctx context.Context, _ eventstore.Event) error {
_, err := mig.dbClient.ExecContext(ctx, addLowerFieldsToLoginNames)
return err
}

3
cmd/setup/19.go
View File

@@ -5,6 +5,7 @@ import (
_ "embed"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore"
)
var (
@@ -16,7 +17,7 @@ type AddCurrentSequencesIndex struct {
dbClient *database.DB
}
func (mig *AddCurrentSequencesIndex) Execute(ctx context.Context) error {
func (mig *AddCurrentSequencesIndex) Execute(ctx context.Context, _ eventstore.Event) error {
_, err := mig.dbClient.ExecContext(ctx, addCurrentSequencesIndex)
return err
}

3
cmd/setup/20.go
View File

@@ -5,6 +5,7 @@ import (
_ "embed"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore"
)
var (
@@ -16,7 +17,7 @@ type AddByUserIndexToSession struct {
dbClient *database.DB
}
func (mig *AddByUserIndexToSession) Execute(ctx context.Context) error {
func (mig *AddByUserIndexToSession) Execute(ctx context.Context, _ eventstore.Event) error {
_, err := mig.dbClient.ExecContext(ctx, addByUserIndexToSession)
return err
}

3
cmd/setup/21.go
View File

@@ -5,6 +5,7 @@ import (
_ "embed"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore"
)
var (
@@ -16,7 +17,7 @@ type AddBlockFieldToLimits struct {
dbClient *database.DB
}
func (mig *AddBlockFieldToLimits) Execute(ctx context.Context) error {
func (mig *AddBlockFieldToLimits) Execute(ctx context.Context, _ eventstore.Event) error {
_, err := mig.dbClient.ExecContext(ctx, addBlockFieldToLimits)
return err
}

27
cmd/setup/22.go Normal file
View File

@@ -0,0 +1,27 @@
package setup
import (
"context"
_ "embed"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore"
)
var (
//go:embed 22.sql
activeInstanceEvents string
)
type ActiveInstanceEvents struct {
dbClient *database.DB
}
func (mig *ActiveInstanceEvents) Execute(ctx context.Context, _ eventstore.Event) error {
_, err := mig.dbClient.ExecContext(ctx, activeInstanceEvents)
return err
}
func (mig *ActiveInstanceEvents) String() string {
return "22_active_instance_events_index"
}

1
cmd/setup/22.sql Normal file
View File

@@ -0,0 +1 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS active_instances_events ON eventstore.events2 (aggregate_type, event_type) WHERE aggregate_type = 'instance' AND event_type IN ('instance.added', 'instance.removed');

1
cmd/setup/cleanup.go
View File

@@ -40,7 +40,6 @@ func Cleanup(config *Config) {
config.Eventstore.Pusher = new_es.NewEventstore(esPusherDBClient)
config.Eventstore.Querier = old_es.NewCRDB(queryDBClient)
es := eventstore.NewEventstore(config.Eventstore)
migration.RegisterMappers(es)
step, err := migration.LastStuckStep(ctx, es)
logging.OnError(err).Fatal("unable to query latest migration")

34
cmd/setup/config.go
View File

@@ -9,16 +9,22 @@ import (
"github.com/spf13/viper"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/cmd/encryption"
"github.com/zitadel/zitadel/cmd/systemapi"
"github.com/zitadel/zitadel/internal/actions"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/api/oidc"
"github.com/zitadel/zitadel/internal/api/ui/login"
"github.com/zitadel/zitadel/internal/command"
"github.com/zitadel/zitadel/internal/config/hook"
"github.com/zitadel/zitadel/internal/config/systemdefaults"
"github.com/zitadel/zitadel/internal/crypto"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/domain"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/id"
"github.com/zitadel/zitadel/internal/notification/handlers"
"github.com/zitadel/zitadel/internal/query/projection"
static_config "github.com/zitadel/zitadel/internal/static/config"
)
type Config struct {
@@ -29,11 +35,26 @@ type Config struct {
ExternalPort uint16
ExternalSecure bool
Log *logging.Config
EncryptionKeys *encryptionKeyConfig
EncryptionKeys *encryption.EncryptionKeyConfig
DefaultInstance command.InstanceSetup
Machine *id.Config
Projections projection.Config
Eventstore *eventstore.Config
InitProjections InitProjections
AssetStorage static_config.AssetStorageConfig
OIDC oidc.Config
Login login.Config
WebAuthNName string
Telemetry *handlers.TelemetryPusherConfig
SystemAPIUsers systemapi.Users
}
type InitProjections struct {
Enabled bool
RetryFailedAfter time.Duration
MaxFailureCount uint8
BulkLimit uint64
}
func MustNewConfig(v *viper.Viper) *Config {
@@ -48,6 +69,7 @@ func MustNewConfig(v *viper.Viper) *Config {
database.DecodeHook,
hook.EnumHookFunc(domain.FeatureString),
hook.EnumHookFunc(authz.MemberTypeString),
actions.HTTPConfigDecodeHook,
)),
)
logging.OnError(err).Fatal("unable to read default config")
@@ -79,12 +101,7 @@ type Steps struct {
s19AddCurrentStatesIndex *AddCurrentSequencesIndex
s20AddByUserSessionIndex *AddByUserIndexToSession
s21AddBlockFieldToLimits *AddBlockFieldToLimits
}
type encryptionKeyConfig struct {
User *crypto.KeyConfig
SMTP *crypto.KeyConfig
OIDC *crypto.KeyConfig
s22ActiveInstancesIndex *ActiveInstanceEvents
}
func MustNewSteps(v *viper.Viper) *Steps {
@@ -110,6 +127,7 @@ func MustNewSteps(v *viper.Viper) *Steps {
mapstructure.StringToTimeHookFunc(time.RFC3339),
mapstructure.StringToSliceHookFunc(","),
hook.EnumHookFunc(domain.FeatureString),
systemapi.UsersDecodeHook,
)),
)
logging.OnError(err).Fatal("unable to read steps")

2
cmd/setup/config_change.go
View File

@@ -30,7 +30,7 @@ func (mig *externalConfigChange) Check(lastRun map[string]interface{}) bool {
mig.currentExternalDomain != mig.ExternalDomain
}
func (mig *externalConfigChange) Execute(ctx context.Context) error {
func (mig *externalConfigChange) Execute(ctx context.Context, _ eventstore.Event) error {
cmd, err := command.StartCommands(
mig.es,
mig.defaults,

2
cmd/setup/projections.go
View File

@@ -18,7 +18,7 @@ func (mig *projectionTables) Check(lastRun map[string]interface{}) bool {
return currentVersion != mig.Version
}
func (mig *projectionTables) Execute(ctx context.Context) error {
func (mig *projectionTables) Execute(ctx context.Context, _ eventstore.Event) error {
return projection.Init(ctx)
}

194
cmd/setup/setup.go
View File

@@ -4,22 +4,37 @@ import (
"context"
"embed"
_ "embed"
"net/http"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/cmd/build"
"github.com/zitadel/zitadel/cmd/encryption"
"github.com/zitadel/zitadel/cmd/key"
"github.com/zitadel/zitadel/cmd/tls"
admin_handler "github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/handler"
admin_view "github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/view"
internal_authz "github.com/zitadel/zitadel/internal/api/authz"
auth_handler "github.com/zitadel/zitadel/internal/auth/repository/eventsourcing/handler"
auth_view "github.com/zitadel/zitadel/internal/auth/repository/eventsourcing/view"
"github.com/zitadel/zitadel/internal/authz"
authz_es "github.com/zitadel/zitadel/internal/authz/repository/eventsourcing/eventstore"
"github.com/zitadel/zitadel/internal/command"
cryptoDB "github.com/zitadel/zitadel/internal/crypto/database"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/database/dialect"
"github.com/zitadel/zitadel/internal/domain"
"github.com/zitadel/zitadel/internal/eventstore"
old_es "github.com/zitadel/zitadel/internal/eventstore/repository/sql"
new_es "github.com/zitadel/zitadel/internal/eventstore/v3"
"github.com/zitadel/zitadel/internal/i18n"
"github.com/zitadel/zitadel/internal/migration"
notify_handler "github.com/zitadel/zitadel/internal/notification"
"github.com/zitadel/zitadel/internal/query"
"github.com/zitadel/zitadel/internal/query/projection"
"github.com/zitadel/zitadel/internal/webauthn"
)
var (
@@ -39,6 +54,9 @@ Requirements:
err := tls.ModeFromFlag(cmd)
logging.OnError(err).Fatal("invalid tlsMode")
err = BindInitProjections(cmd)
logging.OnError(err).Fatal("unable to bind \"init-projections\" flag")
config := MustNewConfig(viper.GetViper())
steps := MustNewSteps(viper.New())
@@ -58,10 +76,15 @@ Requirements:
func Flags(cmd *cobra.Command) {
cmd.PersistentFlags().StringArrayVar(&stepFiles, "steps", nil, "paths to step files to overwrite default steps")
cmd.Flags().Bool("init-projections", viper.GetBool("InitProjections"), "beta feature: initializes projections after they are created, allows smooth start as projections are up to date")
key.AddMasterKeyFlag(cmd)
tls.AddTLSModeFlag(cmd)
}
func BindInitProjections(cmd *cobra.Command) error {
return viper.BindPFlag("InitProjections.Enabled", cmd.Flags().Lookup("init-projections"))
}
func Setup(config *Config, steps *Steps, masterKey string) {
ctx := context.Background()
logging.Info("setup started")
@@ -79,7 +102,6 @@ func Setup(config *Config, steps *Steps, masterKey string) {
config.Eventstore.Pusher = new_es.NewEventstore(esPusherDBClient)
eventstoreClient := eventstore.NewEventstore(config.Eventstore)
logging.OnError(err).Fatal("unable to start eventstore")
migration.RegisterMappers(eventstoreClient)
steps.s1ProjectionTable = &ProjectionTable{dbClient: queryDBClient.DB}
steps.s2AssetsTable = &AssetTable{dbClient: queryDBClient.DB}
@@ -112,6 +134,7 @@ func Setup(config *Config, steps *Steps, masterKey string) {
steps.s19AddCurrentStatesIndex = &AddCurrentSequencesIndex{dbClient: queryDBClient}
steps.s20AddByUserSessionIndex = &AddByUserIndexToSession{dbClient: queryDBClient}
steps.s21AddBlockFieldToLimits = &AddBlockFieldToLimits{dbClient: queryDBClient}
steps.s22ActiveInstancesIndex = &ActiveInstanceEvents{dbClient: queryDBClient}
err = projection.Create(ctx, projectionDBClient, eventstoreClient, config.Projections, nil, nil, nil)
logging.OnError(err).Fatal("unable to start projections")
@@ -160,6 +183,8 @@ func Setup(config *Config, steps *Steps, masterKey string) {
logging.WithFields("name", steps.s19AddCurrentStatesIndex.String()).OnError(err).Fatal("migration failed")
err = migration.Migrate(ctx, eventstoreClient, steps.s20AddByUserSessionIndex)
logging.WithFields("name", steps.s20AddByUserSessionIndex.String()).OnError(err).Fatal("migration failed")
err = migration.Migrate(ctx, eventstoreClient, steps.s22ActiveInstancesIndex)
logging.WithFields("name", steps.s22ActiveInstancesIndex.String()).OnError(err).Fatal("migration failed")
for _, repeatableStep := range repeatableSteps {
err = migration.Migrate(ctx, eventstoreClient, repeatableStep)
@@ -171,9 +196,176 @@ func Setup(config *Config, steps *Steps, masterKey string) {
logging.WithFields("name", steps.s18AddLowerFieldsToLoginNames.String()).OnError(err).Fatal("migration failed")
err = migration.Migrate(ctx, eventstoreClient, steps.s21AddBlockFieldToLimits)
logging.WithFields("name", steps.s21AddBlockFieldToLimits.String()).OnError(err).Fatal("migration failed")
// projection initialization must be done last, since the steps above might add required columns to the projections
if config.InitProjections.Enabled {
initProjections(
ctx,
eventstoreClient,
queryDBClient,
projectionDBClient,
masterKey,
config,
)
}
}
func readStmt(fs embed.FS, folder, typ, filename string) (string, error) {
stmt, err := fs.ReadFile(folder + "/" + typ + "/" + filename)
return string(stmt), err
}
func initProjections(
ctx context.Context,
eventstoreClient *eventstore.Eventstore,
queryDBClient,
projectionDBClient *database.DB,
masterKey string,
config *Config,
) {
logging.Info("init-projections is currently in beta")
keyStorage, err := cryptoDB.NewKeyStorage(queryDBClient, masterKey)
logging.OnError(err).Fatal("unable to start key storage")
keys, err := encryption.EnsureEncryptionKeys(ctx, config.EncryptionKeys, keyStorage)
logging.OnError(err).Fatal("unable to ensure encryption keys")
err = projection.Create(
ctx,
queryDBClient,
eventstoreClient,
projection.Config{
RetryFailedAfter: config.InitProjections.RetryFailedAfter,
MaxFailureCount: config.InitProjections.MaxFailureCount,
BulkLimit: config.InitProjections.BulkLimit,
},
keys.OIDC,
keys.SAML,
config.SystemAPIUsers,
)
logging.OnError(err).Fatal("unable to start projections")
for _, p := range projection.Projections() {
err := migration.Migrate(ctx, eventstoreClient, p)
logging.WithFields("name", p.String()).OnError(err).Fatal("migration failed")
}
staticStorage, err := config.AssetStorage.NewStorage(queryDBClient.DB)
logging.OnError(err).Fatal("unable to start asset storage")
adminView, err := admin_view.StartView(queryDBClient)
logging.OnError(err).Fatal("unable to start admin view")
admin_handler.Register(ctx,
admin_handler.Config{
Client: queryDBClient,
Eventstore: eventstoreClient,
BulkLimit: config.InitProjections.BulkLimit,
FailureCountUntilSkip: uint64(config.InitProjections.MaxFailureCount),
},
adminView,
staticStorage,
)
for _, p := range admin_handler.Projections() {
err := migration.Migrate(ctx, eventstoreClient, p)
logging.WithFields("name", p.String()).OnError(err).Fatal("migration failed")
}
sessionTokenVerifier := internal_authz.SessionTokenVerifier(keys.OIDC)
queries, err := query.StartQueries(
ctx,
eventstoreClient,
queryDBClient,
projectionDBClient,
config.Projections,
config.SystemDefaults,
keys.IDPConfig,
keys.OTP,
keys.OIDC,
keys.SAML,
config.InternalAuthZ.RolePermissionMappings,
sessionTokenVerifier,
func(q *query.Queries) domain.PermissionCheck {
return func(ctx context.Context, permission, orgID, resourceID string) (err error) {
return internal_authz.CheckPermission(ctx, &authz_es.UserMembershipRepo{Queries: q}, config.InternalAuthZ.RolePermissionMappings, permission, orgID, resourceID)
}
},
0, // not needed for projections
nil, // not needed for projections
false,
)
logging.OnError(err).Fatal("unable to start queries")
authView, err := auth_view.StartView(queryDBClient, keys.OIDC, queries, eventstoreClient)
logging.OnError(err).Fatal("unable to start admin view")
auth_handler.Register(ctx,
auth_handler.Config{
Client: queryDBClient,
Eventstore: eventstoreClient,
BulkLimit: config.InitProjections.BulkLimit,
FailureCountUntilSkip: uint64(config.InitProjections.MaxFailureCount),
},
authView,
queries,
)
for _, p := range auth_handler.Projections() {
err := migration.Migrate(ctx, eventstoreClient, p)
logging.WithFields("name", p.String()).OnError(err).Fatal("migration failed")
}
authZRepo, err := authz.Start(queries, eventstoreClient, queryDBClient, keys.OIDC, config.ExternalSecure)
logging.OnError(err).Fatal("unable to start authz repo")
permissionCheck := func(ctx context.Context, permission, orgID, resourceID string) (err error) {
return internal_authz.CheckPermission(ctx, authZRepo, config.InternalAuthZ.RolePermissionMappings, permission, orgID, resourceID)
}
commands, err := command.StartCommands(
eventstoreClient,
config.SystemDefaults,
config.InternalAuthZ.RolePermissionMappings,
staticStorage,
&webauthn.Config{
DisplayName: config.WebAuthNName,
ExternalSecure: config.ExternalSecure,
},
config.ExternalDomain,
config.ExternalSecure,
config.ExternalPort,
keys.IDPConfig,
keys.OTP,
keys.SMTP,
keys.SMS,
keys.User,
keys.DomainVerification,
keys.OIDC,
keys.SAML,
&http.Client{},
permissionCheck,
sessionTokenVerifier,
config.OIDC.DefaultAccessTokenLifetime,
config.OIDC.DefaultRefreshTokenExpiration,
config.OIDC.DefaultRefreshTokenIdleExpiration,
config.DefaultInstance.SecretGenerators,
)
logging.OnError(err).Fatal("unable to start commands")
notify_handler.Register(
ctx,
config.Projections.Customizations["notifications"],
config.Projections.Customizations["notificationsquotas"],
config.Projections.Customizations["telemetry"],
*config.Telemetry,
config.ExternalDomain,
config.ExternalPort,
config.ExternalSecure,
commands,
queries,
eventstoreClient,
config.Login.DefaultOTPEmailURLV2,
config.SystemDefaults.Notifications.FileSystemPath,
keys.User,
keys.SMTP,
keys.SMS,
)
for _, p := range notify_handler.Projections() {
err := migration.Migrate(ctx, eventstoreClient, p)
logging.WithFields("name", p.String()).OnError(err).Fatal("migration failed")
}
}