TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-11 14:47:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore!: Introduce ZITADEL v3 (#9645)

Browse Source 
This PR summarizes multiple changes specifically only available with
ZITADEL v3:

- feat: Web Keys management
(https://github.com/zitadel/zitadel/pull/9526)
- fix(cmd): ensure proper working of mirror
(https://github.com/zitadel/zitadel/pull/9509)
- feat(Authz): system user support for permission check v2
(https://github.com/zitadel/zitadel/pull/9640)
- chore(license): change from Apache to AGPL
(https://github.com/zitadel/zitadel/pull/9597)
- feat(console): list v2 sessions
(https://github.com/zitadel/zitadel/pull/9539)
- fix(console): add loginV2 feature flag
(https://github.com/zitadel/zitadel/pull/9682)
- fix(feature flags): allow reading "own" flags
(https://github.com/zitadel/zitadel/pull/9649)
- feat(console): add Actions V2 UI
(https://github.com/zitadel/zitadel/pull/9591)

BREAKING CHANGE
- feat(webkey): migrate to v2beta API
(https://github.com/zitadel/zitadel/pull/9445)
- chore!: remove CockroachDB Support
(https://github.com/zitadel/zitadel/pull/9444)
- feat(actions): migrate to v2beta API
(https://github.com/zitadel/zitadel/pull/9489)

---------

Co-authored-by: Livio Spring <livio.a@gmail.com>
Co-authored-by: Stefan Benz <46600784+stebenz@users.noreply.github.com>
Co-authored-by: Silvan <27845747+adlerhurst@users.noreply.github.com>
Co-authored-by: Ramon <mail@conblem.me>
Co-authored-by: Elio Bischof <elio@zitadel.com>
Co-authored-by: Kenta Yamaguchi <56732734+KEY60228@users.noreply.github.com>
Co-authored-by: Harsha Reddy <harsha.reddy@klaviyo.com>
Co-authored-by: Livio Spring <livio@zitadel.com>
Co-authored-by: Max Peintner <max@caos.ch>
Co-authored-by: Iraq <66622793+kkrime@users.noreply.github.com>
Co-authored-by: Florian Forster <florian@zitadel.com>
Co-authored-by: Tim Möhlmann <tim+github@zitadel.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Max Peintner <peintnerm@gmail.com>
This commit is contained in:
Fabienne Bühler
2025-04-02 16:53:06 +02:00
committed by GitHub
parent d14a23ae7e
commit 07ce3b6905
559 changed files with 14578 additions and 7622 deletions
Download Patch File Download Diff File Expand all files Collapse all files

388
cmd/defaults.yaml
View File

@@ -113,67 +113,36 @@ PublicHostHeaders: # ZITADEL_PUBLICHOSTHEADERS
WebAuthNName: ZITADEL # ZITADEL_WEBAUTHNNAME
Database:
# CockroachDB is the default database of ZITADEL
cockroach:
Host: localhost # ZITADEL_DATABASE_COCKROACH_HOST
Port: 26257 # ZITADEL_DATABASE_COCKROACH_PORT
Database: zitadel # ZITADEL_DATABASE_COCKROACH_DATABASE
MaxOpenConns: 5 # ZITADEL_DATABASE_COCKROACH_MAXOPENCONNS
MaxIdleConns: 2 # ZITADEL_DATABASE_COCKROACH_MAXIDLECONNS
MaxConnLifetime: 30m # ZITADEL_DATABASE_COCKROACH_MAXCONNLIFETIME
MaxConnIdleTime: 5m # ZITADEL_DATABASE_COCKROACH_MAXCONNIDLETIME
Options: "" # ZITADEL_DATABASE_COCKROACH_OPTIONS
# Postgres is the default database of ZITADEL
postgres:
Host: localhost # ZITADEL_DATABASE_POSTGRES_HOST
Port: 5432 # ZITADEL_DATABASE_POSTGRES_PORT
Database: zitadel # ZITADEL_DATABASE_POSTGRES_DATABASE
MaxOpenConns: 10 # ZITADEL_DATABASE_POSTGRES_MAXOPENCONNS
MaxIdleConns: 5 # ZITADEL_DATABASE_POSTGRES_MAXIDLECONNS
MaxConnLifetime: 30m # ZITADEL_DATABASE_POSTGRES_MAXCONNLIFETIME
MaxConnIdleTime: 5m # ZITADEL_DATABASE_POSTGRES_MAXCONNIDLETIME
Options: "" # ZITADEL_DATABASE_POSTGRES_OPTIONS
User:
Username: zitadel # ZITADEL_DATABASE_COCKROACH_USER_USERNAME
Password: "" # ZITADEL_DATABASE_COCKROACH_USER_PASSWORD
Username: zitadel # ZITADEL_DATABASE_POSTGRES_USER_USERNAME
Password: "" # ZITADEL_DATABASE_POSTGRES_USER_PASSWORD
SSL:
Mode: disable # ZITADEL_DATABASE_COCKROACH_USER_SSL_MODE
RootCert: "" # ZITADEL_DATABASE_COCKROACH_USER_SSL_ROOTCERT
Cert: "" # ZITADEL_DATABASE_COCKROACH_USER_SSL_CERT
Key: "" # ZITADEL_DATABASE_COCKROACH_USER_SSL_KEY
Mode: disable # ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE
RootCert: "" # ZITADEL_DATABASE_POSTGRES_USER_SSL_ROOTCERT
Cert: "" # ZITADEL_DATABASE_POSTGRES_USER_SSL_CERT
Key: "" # ZITADEL_DATABASE_POSTGRES_USER_SSL_KEY
Admin:
# By default, ExistingDatabase is not specified in the connection string
# If the connection resolves to a database that is not existing in your system, configure an existing one here
# It is used in zitadel init to connect to cockroach and create a dedicated database for ZITADEL.
ExistingDatabase: # ZITADEL_DATABASE_COCKROACH_ADMIN_EXISTINGDATABASE
Username: root # ZITADEL_DATABASE_COCKROACH_ADMIN_USERNAME
Password: "" # ZITADEL_DATABASE_COCKROACH_ADMIN_PASSWORD
SSL:
Mode: disable # ZITADEL_DATABASE_COCKROACH_ADMIN_SSL_MODE
RootCert: "" # ZITADEL_DATABASE_COCKROACH_ADMIN_SSL_ROOTCERT
Cert: "" # ZITADEL_DATABASE_COCKROACH_ADMIN_SSL_CERT
Key: "" # ZITADEL_DATABASE_COCKROACH_ADMIN_SSL_KEY
# Postgres is used as soon as a value is set
# The values describe the possible fields to set values
postgres:
Host: # ZITADEL_DATABASE_POSTGRES_HOST
Port: # ZITADEL_DATABASE_POSTGRES_PORT
Database: # ZITADEL_DATABASE_POSTGRES_DATABASE
MaxOpenConns: # ZITADEL_DATABASE_POSTGRES_MAXOPENCONNS
MaxIdleConns: # ZITADEL_DATABASE_POSTGRES_MAXIDLECONNS
MaxConnLifetime: # ZITADEL_DATABASE_POSTGRES_MAXCONNLIFETIME
MaxConnIdleTime: # ZITADEL_DATABASE_POSTGRES_MAXCONNIDLETIME
Options: # ZITADEL_DATABASE_POSTGRES_OPTIONS
User:
Username: # ZITADEL_DATABASE_POSTGRES_USER_USERNAME
Password: # ZITADEL_DATABASE_POSTGRES_USER_PASSWORD
SSL:
Mode: # ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE
RootCert: # ZITADEL_DATABASE_POSTGRES_USER_SSL_ROOTCERT
Cert: # ZITADEL_DATABASE_POSTGRES_USER_SSL_CERT
Key: # ZITADEL_DATABASE_POSTGRES_USER_SSL_KEY
Admin:
# The default ExistingDatabase is postgres
# If your db system doesn't have a database named postgres, configure an existing database here
# It is used in zitadel init to connect to postgres and create a dedicated database for ZITADEL.
ExistingDatabase: # ZITADEL_DATABASE_POSTGRES_ADMIN_EXISTINGDATABASE
Username: # ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME
Password: # ZITADEL_DATABASE_POSTGRES_ADMIN_PASSWORD
Username: postgres # ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME
Password: postgres # ZITADEL_DATABASE_POSTGRES_ADMIN_PASSWORD
SSL:
Mode: # ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_MODE
RootCert: # ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_ROOTCERT
Cert: # ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_CERT
Key: # ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_KEY
Mode: disable # ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_MODE
RootCert: "" # ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_ROOTCERT
Cert: "" # ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_CERT
Key: "" # ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_KEY
# Caches are EXPERIMENTAL. The following config may have breaking changes in the future.
# If no config is provided, caching is disabled by default.
@@ -447,19 +416,30 @@ Projections:
Notifications:
# Notifications can be processed by either a sequential mode (legacy) or a new parallel mode.
# The parallel mode is currently only recommended for Postgres databases.
# For CockroachDB, the sequential mode is recommended, see: https://github.com/zitadel/zitadel/issues/9002
# If legacy mode is enabled, the worker config below is ignored.
LegacyEnabled: true # ZITADEL_NOTIFICATIONS_LEGACYENABLED
# The amount of workers processing the notification request events.
# If set to 0, no notification request events will be handled. This can be useful when running in
# multi binary / pod setup and allowing only certain executables to process the events.
Workers: 1 # ZITADEL_NOTIFIACATIONS_WORKERS
Workers: 1 # ZITADEL_NOTIFICATIONS_WORKERS
# The maximum duration a job can do it's work before it is considered as failed.
TransactionDuration: 10s # ZITADEL_NOTIFIACATIONS_TRANSACTIONDURATION
TransactionDuration: 10s # ZITADEL_NOTIFICATIONS_TRANSACTIONDURATION
# Automatically cancel the notification after the amount of failed attempts
MaxAttempts: 3 # ZITADEL_NOTIFIACATIONS_MAXATTEMPTS
MaxAttempts: 3 # ZITADEL_NOTIFICATIONS_MAXATTEMPTS
# Automatically cancel the notification if it cannot be handled within a specific time
MaxTtl: 5m # ZITADEL_NOTIFIACATIONS_MAXTTL
MaxTtl: 5m # ZITADEL_NOTIFICATIONS_MAXTTL
Executions:
# The amount of workers processing the execution request events.
# If set to 0, no execution request events will be handled. This can be useful when running in
# multi binary / pod setup and allowing only certain executables to process the events.
Workers: 1 # ZITADEL_EXECUTIONS_WORKERS
# The maximum duration a job can do it's work before it is considered as failed.
# This maximum duration is prioritized in case that the sum of the target's timeouts is higher,
# to limit the runtime of a singular execution.
TransactionDuration: 10s # ZITADEL_EXECUTIONS_TRANSACTIONDURATION
# Automatically cancel the notification if it cannot be handled within a specific time
MaxTtl: 5m # ZITADEL_EXECUTIONS_MAXTTL
Auth:
# See Projections.BulkLimit
@@ -1733,6 +1713,298 @@ InternalAuthZ:
- "user.grant.read"
- "user.membership.read"
SystemAuthZ:
RolePermissionMappings:
- Role: "SYSTEM_OWNER"
Permissions:
- "system.instance.read"
- "system.instance.write"
- "system.instance.delete"
- "system.domain.read"
- "system.domain.write"
- "system.domain.delete"
- "system.debug.read"
- "system.debug.write"
- "system.debug.delete"
- "system.feature.read"
- "system.feature.write"
- "system.feature.delete"
- "system.limits.write"
- "system.limits.delete"
- "system.quota.write"
- "system.quota.delete"
- "system.iam.member.read"
- Role: "SYSTEM_OWNER_VIEWER"
Permissions:
- "system.instance.read"
- "system.domain.read"
- "system.debug.read"
- "system.feature.read"
- "system.iam.member.read"
- Role: "IAM_OWNER"
Permissions:
- "iam.read"
- "iam.write"
- "iam.policy.read"
- "iam.policy.write"
- "iam.policy.delete"
- "iam.member.read"
- "iam.member.write"
- "iam.member.delete"
- "iam.idp.read"
- "iam.idp.write"
- "iam.idp.delete"
- "iam.action.read"
- "iam.action.write"
- "iam.action.delete"
- "iam.flow.read"
- "iam.flow.write"
- "iam.flow.delete"
- "iam.feature.read"
- "iam.feature.write"
- "iam.feature.delete"
- "iam.restrictions.read"
- "iam.restrictions.write"
- "iam.web_key.write"
- "iam.web_key.delete"
- "iam.web_key.read"
- "iam.debug.write"
- "iam.debug.read"
- "org.read"
- "org.global.read"
- "org.create"
- "org.write"
- "org.delete"
- "org.member.read"
- "org.member.write"
- "org.member.delete"
- "org.idp.read"
- "org.idp.write"
- "org.idp.delete"
- "org.action.read"
- "org.action.write"
- "org.action.delete"
- "org.flow.read"
- "org.flow.write"
- "org.flow.delete"
- "org.feature.read"
- "org.feature.write"
- "org.feature.delete"
- "user.read"
- "user.global.read"
- "user.write"
- "user.delete"
- "user.grant.read"
- "user.grant.write"
- "user.grant.delete"
- "user.membership.read"
- "user.credential.write"
- "user.passkey.write"
- "user.feature.read"
- "user.feature.write"
- "user.feature.delete"
- "policy.read"
- "policy.write"
- "policy.delete"
- "project.read"
- "project.create"
- "project.write"
- "project.delete"
- "project.member.read"
- "project.member.write"
- "project.member.delete"
- "project.role.read"
- "project.role.write"
- "project.role.delete"
- "project.app.read"
- "project.app.write"
- "project.app.delete"
- "project.grant.read"
- "project.grant.write"
- "project.grant.delete"
- "project.grant.member.read"
- "project.grant.member.write"
- "project.grant.member.delete"
- "events.read"
- "milestones.read"
- "session.read"
- "session.delete"
- "action.target.read"
- "action.target.write"
- "action.target.delete"
- "action.execution.read"
- "action.execution.write"
- "userschema.read"
- "userschema.write"
- "userschema.delete"
- "session.read"
- "session.delete"
- Role: "IAM_OWNER_VIEWER"
Permissions:
- "iam.read"
- "iam.policy.read"
- "iam.member.read"
- "iam.idp.read"
- "iam.action.read"
- "iam.flow.read"
- "iam.restrictions.read"
- "iam.feature.read"
- "iam.web_key.read"
- "iam.debug.read"
- "org.read"
- "org.member.read"
- "org.idp.read"
- "org.action.read"
- "org.flow.read"
- "org.feature.read"
- "user.read"
- "user.global.read"
- "user.grant.read"
- "user.membership.read"
- "user.feature.read"
- "policy.read"
- "project.read"
- "project.member.read"
- "project.role.read"
- "project.app.read"
- "project.grant.read"
- "project.grant.member.read"
- "events.read"
- "milestones.read"
- "action.target.read"
- "action.execution.read"
- "userschema.read"
- "session.read"
- Role: "IAM_ORG_MANAGER"
Permissions:
- "org.read"
- "org.global.read"
- "org.create"
- "org.write"
- "org.delete"
- "org.member.read"
- "org.member.write"
- "org.member.delete"
- "org.idp.read"
- "org.idp.write"
- "org.idp.delete"
- "org.action.read"
- "org.action.write"
- "org.action.delete"
- "org.flow.read"
- "org.flow.write"
- "org.flow.delete"
- "org.feature.read"
- "org.feature.write"
- "org.feature.delete"
- "user.read"
- "user.global.read"
- "user.write"
- "user.delete"
- "user.grant.read"
- "user.grant.write"
- "user.grant.delete"
- "user.membership.read"
- "user.credential.write"
- "user.passkey.write"
- "user.feature.read"
- "user.feature.write"
- "user.feature.delete"
- "policy.read"
- "policy.write"
- "policy.delete"
- "project.read"
- "project.create"
- "project.write"
- "project.delete"
- "project.member.read"
- "project.member.write"
- "project.member.delete"
- "project.role.read"
- "project.role.write"
- "project.role.delete"
- "project.app.read"
- "project.app.write"
- "project.app.delete"
- "project.grant.read"
- "project.grant.write"
- "project.grant.delete"
- "project.grant.member.read"
- "project.grant.member.write"
- "project.grant.member.delete"
- "session.delete"
- Role: "IAM_USER_MANAGER"
Permissions:
- "org.read"
- "org.global.read"
- "org.member.read"
- "org.member.delete"
- "user.read"
- "user.global.read"
- "user.write"
- "user.delete"
- "user.grant.read"
- "user.grant.write"
- "user.grant.delete"
- "user.membership.read"
- "user.passkey.write"
- "user.feature.read"
- "user.feature.write"
- "user.feature.delete"
- "project.read"
- "project.member.read"
- "project.role.read"
- "project.app.read"
- "project.grant.read"
- "project.grant.write"
- "project.grant.delete"
- "project.grant.member.read"
- "session.delete"
- Role: "IAM_ADMIN_IMPERSONATOR"
Permissions:
- "admin.impersonation"
- "impersonation"
- Role: "IAM_END_USER_IMPERSONATOR"
Permissions:
- "impersonation"
- Role: "IAM_LOGIN_CLIENT"
Permissions:
- "iam.read"
- "iam.policy.read"
- "iam.member.read"
- "iam.member.write"
- "iam.idp.read"
- "iam.feature.read"
- "iam.restrictions.read"
- "org.read"
- "org.member.read"
- "org.member.write"
- "org.idp.read"
- "org.feature.read"
- "user.read"
- "user.write"
- "user.grant.read"
- "user.grant.write"
- "user.membership.read"
- "user.credential.write"
- "user.passkey.write"
- "user.feature.read"
- "policy.read"
- "project.read"
- "project.member.read"
- "project.member.write"
- "project.role.read"
- "project.app.read"
- "project.member.read"
- "project.member.write"
- "project.grant.read"
- "project.grant.member.read"
- "project.grant.member.write"
- "session.read"
- "session.link"
- "session.delete"
- "userschema.read"
# If a new projection is introduced it will be prefilled during the setup process (if enabled)
# This can prevent serving outdated data after a version upgrade, but might require a longer setup / upgrade process:
# https://zitadel.com/docs/self-hosting/manage/updating_scaling

2
cmd/initialise/config.go
View File

@@ -19,7 +19,7 @@ func MustNewConfig(v *viper.Viper) *Config {
config := new(Config)
err := v.Unmarshal(config,
viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
database.DecodeHook,
database.DecodeHook(false),
mapstructure.TextUnmarshallerHookFunc(),
)),
)

46
cmd/initialise/init.go
View File

@@ -12,20 +12,17 @@ import (
)
var (
//go:embed sql/cockroach/*
//go:embed sql/postgres/*
//go:embed sql/*.sql
stmts embed.FS
createUserStmt string
grantStmt string
settingsStmt string
databaseStmt string
createEventstoreStmt string
createProjectionsStmt string
createSystemStmt string
createEncryptionKeysStmt string
createEventsStmt string
createSystemSequenceStmt string
createUniqueConstraints string
roleAlreadyExistsCode = "42710"
@@ -39,7 +36,7 @@ func New() *cobra.Command {
Long: `Sets up the minimum requirements to start ZITADEL.
Prerequisites:
- database (PostgreSql or cockroachdb)
- PostgreSql database
The user provided by flags needs privileges to
- create the database if it does not exist
@@ -53,7 +50,7 @@ The user provided by flags needs privileges to
},
}
cmd.AddCommand(newZitadel(), newDatabase(), newUser(), newGrant(), newSettings())
cmd.AddCommand(newZitadel(), newDatabase(), newUser(), newGrant())
return cmd
}
@@ -62,7 +59,6 @@ func InitAll(ctx context.Context, config *Config) {
VerifyUser(config.Database.Username(), config.Database.Password()),
VerifyDatabase(config.Database.DatabaseName()),
VerifyGrant(config.Database.DatabaseName(), config.Database.Username()),
VerifySettings(config.Database.DatabaseName(), config.Database.Username()),
)
logging.OnError(err).Fatal("unable to initialize the database")
@@ -73,7 +69,7 @@ func InitAll(ctx context.Context, config *Config) {
func initialise(ctx context.Context, config database.Config, steps ...func(context.Context, *database.DB) error) error {
logging.Info("initialization started")
err := ReadStmts(config.Type())
err := ReadStmts()
if err != nil {
return err
}
@@ -97,58 +93,48 @@ func Init(ctx context.Context, db *database.DB, steps ...func(context.Context, *
return nil
}
func ReadStmts(typ string) (err error) {
createUserStmt, err = readStmt(typ, "01_user")
func ReadStmts() (err error) {
createUserStmt, err = readStmt("01_user")
if err != nil {
return err
}
databaseStmt, err = readStmt(typ, "02_database")
databaseStmt, err = readStmt("02_database")
if err != nil {
return err
}
grantStmt, err = readStmt(typ, "03_grant_user")
grantStmt, err = readStmt("03_grant_user")
if err != nil {
return err
}
createEventstoreStmt, err = readStmt(typ, "04_eventstore")
createEventstoreStmt, err = readStmt("04_eventstore")
if err != nil {
return err
}
createProjectionsStmt, err = readStmt(typ, "05_projections")
createProjectionsStmt, err = readStmt("05_projections")
if err != nil {
return err
}
createSystemStmt, err = readStmt(typ, "06_system")
createSystemStmt, err = readStmt("06_system")
if err != nil {
return err
}
createEncryptionKeysStmt, err = readStmt(typ, "07_encryption_keys_table")
createEncryptionKeysStmt, err = readStmt("07_encryption_keys_table")
if err != nil {
return err
}
createEventsStmt, err = readStmt(typ, "08_events_table")
createEventsStmt, err = readStmt("08_events_table")
if err != nil {
return err
}
createSystemSequenceStmt, err = readStmt(typ, "09_system_sequence")
if err != nil {
return err
}
createUniqueConstraints, err = readStmt(typ, "10_unique_constraints_table")
if err != nil {
return err
}
settingsStmt, err = readStmt(typ, "11_settings")
createUniqueConstraints, err = readStmt("10_unique_constraints_table")
if err != nil {
return err
}
@@ -156,7 +142,7 @@ func ReadStmts(typ string) (err error) {
return nil
}
func readStmt(typ, step string) (string, error) {
stmt, err := stmts.ReadFile("sql/" + typ + "/" + step + ".sql")
func readStmt(step string) (string, error) {
stmt, err := stmts.ReadFile("sql/" + step + ".sql")
return string(stmt), err
}

2
cmd/initialise/sql/cockroach/01_user.sql → cmd/initialise/sql/01_user.sql
View File

@@ -1,2 +1,2 @@
-- replace %[1]s with the name of the user
CREATE USER IF NOT EXISTS "%[1]s"
CREATE USER "%[1]s"

2
cmd/initialise/sql/cockroach/02_database.sql → cmd/initialise/sql/02_database.sql
View File

@@ -1,2 +1,2 @@
-- replace %[1]s with the name of the database
CREATE DATABASE IF NOT EXISTS "%[1]s";
CREATE DATABASE "%[1]s"

0
cmd/initialise/sql/postgres/03_grant_user.sql → cmd/initialise/sql/03_grant_user.sql
View File

0
cmd/initialise/sql/cockroach/04_eventstore.sql → cmd/initialise/sql/04_eventstore.sql
View File

0
cmd/initialise/sql/cockroach/05_projections.sql → cmd/initialise/sql/05_projections.sql
View File

0
cmd/initialise/sql/cockroach/06_system.sql → cmd/initialise/sql/06_system.sql
View File

0
cmd/initialise/sql/cockroach/07_encryption_keys_table.sql → cmd/initialise/sql/07_encryption_keys_table.sql
View File

0
cmd/initialise/sql/postgres/08_events_table.sql → cmd/initialise/sql/08_events_table.sql
View File

0
cmd/initialise/sql/postgres/10_unique_constraints_table.sql → cmd/initialise/sql/10_unique_constraints_table.sql
View File

5
cmd/initialise/sql/README.md
View File

@@ -11,6 +11,5 @@ The sql-files in this folder initialize the ZITADEL database and user. These obj
- 05_projections.sql: creates the schema needed to read the data
- 06_system.sql: creates the schema needed for ZITADEL itself
- 07_encryption_keys_table.sql: creates the table for encryption keys (for event data)
- files 08_enable_hash_sharded_indexes.sql and 09_events_table.sql must run in the same session
- 08_enable_hash_sharded_indexes.sql enables the [hash sharded index](https://www.cockroachlabs.com/docs/stable/hash-sharded-indexes.html) feature for this session
- 09_events_table.sql creates the table for eventsourcing
- 08_events_table.sql creates the table for eventsourcing
- 10_unique_constraints_table.sql creates the table to check unique constraints for events

4
cmd/initialise/sql/cockroach/03_grant_user.sql
View File

@@ -1,4 +0,0 @@
-- replace the first %[1]s with the database
-- replace the second \%[2]s with the user
GRANT ALL ON DATABASE "%[1]s" TO "%[2]s";
GRANT SYSTEM VIEWACTIVITY TO "%[2]s";

116
cmd/initialise/sql/cockroach/08_events_table.sql
View File

@@ -1,116 +0,0 @@
CREATE TABLE IF NOT EXISTS eventstore.events2 (
instance_id TEXT NOT NULL
, aggregate_type TEXT NOT NULL
, aggregate_id TEXT NOT NULL
, event_type TEXT NOT NULL
, "sequence" BIGINT NOT NULL
, revision SMALLINT NOT NULL
, created_at TIMESTAMPTZ NOT NULL
, payload JSONB
, creator TEXT NOT NULL
, "owner" TEXT NOT NULL
, "position" DECIMAL NOT NULL
, in_tx_order INTEGER NOT NULL
, PRIMARY KEY (instance_id, aggregate_type, aggregate_id, "sequence")
, INDEX es_active_instances (created_at DESC) STORING ("position")
, INDEX es_wm (aggregate_id, instance_id, aggregate_type, event_type)
, INDEX es_projection (instance_id, aggregate_type, event_type, "position" DESC)
);
-- represents an event to be created.
CREATE TYPE IF NOT EXISTS eventstore.command AS (
instance_id TEXT
, aggregate_type TEXT
, aggregate_id TEXT
, command_type TEXT
, revision INT2
, payload JSONB
, creator TEXT
, owner TEXT
);
CREATE OR REPLACE FUNCTION eventstore.commands_to_events(commands eventstore.command[]) RETURNS SETOF eventstore.events2 VOLATILE AS $$
SELECT
("c").instance_id
, ("c").aggregate_type
, ("c").aggregate_id
, ("c").command_type AS event_type
, cs.sequence + ROW_NUMBER() OVER (PARTITION BY ("c").instance_id, ("c").aggregate_type, ("c").aggregate_id ORDER BY ("c").in_tx_order) AS sequence
, ("c").revision
, hlc_to_timestamp(cluster_logical_timestamp()) AS created_at
, ("c").payload
, ("c").creator
, cs.owner
, cluster_logical_timestamp() AS position
, ("c").in_tx_order
FROM (
SELECT
("c").instance_id
, ("c").aggregate_type
, ("c").aggregate_id
, ("c").command_type
, ("c").revision
, ("c").payload
, ("c").creator
, ("c").owner
, ROW_NUMBER() OVER () AS in_tx_order
FROM
UNNEST(commands) AS "c"
) AS "c"
JOIN (
SELECT
cmds.instance_id
, cmds.aggregate_type
, cmds.aggregate_id
, CASE WHEN (e.owner IS NOT NULL OR e.owner <> '') THEN e.owner ELSE command_owners.owner END AS owner
, COALESCE(MAX(e.sequence), 0) AS sequence
FROM (
SELECT DISTINCT
("cmds").instance_id
, ("cmds").aggregate_type
, ("cmds").aggregate_id
, ("cmds").owner
FROM UNNEST(commands) AS "cmds"
) AS cmds
LEFT JOIN eventstore.events2 AS e
ON cmds.instance_id = e.instance_id
AND cmds.aggregate_type = e.aggregate_type
AND cmds.aggregate_id = e.aggregate_id
JOIN (
SELECT
DISTINCT ON (
("c").instance_id
, ("c").aggregate_type
, ("c").aggregate_id
)
("c").instance_id
, ("c").aggregate_type
, ("c").aggregate_id
, ("c").owner
FROM
UNNEST(commands) AS "c"
) AS command_owners ON
cmds.instance_id = command_owners.instance_id
AND cmds.aggregate_type = command_owners.aggregate_type
AND cmds.aggregate_id = command_owners.aggregate_id
GROUP BY
cmds.instance_id
, cmds.aggregate_type
, cmds.aggregate_id
, 4 -- owner
) AS cs
ON ("c").instance_id = cs.instance_id
AND ("c").aggregate_type = cs.aggregate_type
AND ("c").aggregate_id = cs.aggregate_id
ORDER BY
in_tx_order
$$ LANGUAGE SQL;
CREATE OR REPLACE FUNCTION eventstore.push(commands eventstore.command[]) RETURNS SETOF eventstore.events2 AS $$
INSERT INTO eventstore.events2
SELECT * FROM eventstore.commands_to_events(commands)
RETURNING *
$$ LANGUAGE SQL;

1
cmd/initialise/sql/cockroach/09_system_sequence.sql
View File

@@ -1 +0,0 @@
CREATE SEQUENCE IF NOT EXISTS eventstore.system_seq

6
cmd/initialise/sql/cockroach/10_unique_constraints_table.sql
View File

@@ -1,6 +0,0 @@
CREATE TABLE IF NOT EXISTS eventstore.unique_constraints (
instance_id TEXT,
unique_type TEXT,
unique_field TEXT,
PRIMARY KEY (instance_id, unique_type, unique_field)
)

4
cmd/initialise/sql/cockroach/11_settings.sql
View File

@@ -1,4 +0,0 @@
-- replace the first %[1]q with the database in double quotes
-- replace the second \%[2]q with the user in double quotes$
-- For more information see technical advisory 10009 (https://zitadel.com/docs/support/advisory/a10009)
ALTER ROLE %[2]q IN DATABASE %[1]q SET enable_durable_locking_for_serializable = on;

1
cmd/initialise/sql/postgres/01_user.sql
View File

@@ -1 +0,0 @@
CREATE USER "%[1]s"

1
cmd/initialise/sql/postgres/02_database.sql
View File

@@ -1 +0,0 @@
CREATE DATABASE "%[1]s"

3
cmd/initialise/sql/postgres/04_eventstore.sql
View File

@@ -1,3 +0,0 @@
CREATE SCHEMA IF NOT EXISTS eventstore;
GRANT ALL ON ALL TABLES IN SCHEMA eventstore TO "%[1]s";

3
cmd/initialise/sql/postgres/05_projections.sql
View File

@@ -1,3 +0,0 @@
CREATE SCHEMA IF NOT EXISTS projections;
GRANT ALL ON ALL TABLES IN SCHEMA projections TO "%[1]s";

3
cmd/initialise/sql/postgres/06_system.sql
View File

@@ -1,3 +0,0 @@
CREATE SCHEMA IF NOT EXISTS system;
GRANT ALL ON ALL TABLES IN SCHEMA system TO "%[1]s";

6
cmd/initialise/sql/postgres/07_encryption_keys_table.sql
View File

@@ -1,6 +0,0 @@
CREATE TABLE IF NOT EXISTS system.encryption_keys (
id TEXT NOT NULL
, key TEXT NOT NULL
, PRIMARY KEY (id)
);

1
cmd/initialise/sql/postgres/09_system_sequence.sql
View File

@@ -1 +0,0 @@
CREATE SEQUENCE IF NOT EXISTS eventstore.system_seq;

0
cmd/initialise/sql/postgres/11_settings.sql
View File

2
cmd/initialise/verify_database.go
View File

@@ -19,7 +19,7 @@ func newDatabase() *cobra.Command {
Long: `Sets up the ZITADEL database.
Prerequisites:
- cockroachDB or postgreSQL
- postgreSQL
The user provided by flags needs privileges to
- create the database if it does not exist

8
cmd/initialise/verify_database_test.go
View File

@@ -8,7 +8,7 @@ import (
)
func Test_verifyDB(t *testing.T) {
err := ReadStmts("cockroach") //TODO: check all dialects
err := ReadStmts()
if err != nil {
t.Errorf("unable to read stmts: %v", err)
t.FailNow()
@@ -27,7 +27,7 @@ func Test_verifyDB(t *testing.T) {
name: "doesn't exists, create fails",
args: args{
db: prepareDB(t,
expectExec("-- replace zitadel with the name of the database\nCREATE DATABASE IF NOT EXISTS \"zitadel\"", sql.ErrTxDone),
expectExec("-- replace zitadel with the name of the database\nCREATE DATABASE \"zitadel\"", sql.ErrTxDone),
),
database: "zitadel",
},
@@ -37,7 +37,7 @@ func Test_verifyDB(t *testing.T) {
name: "doesn't exists, create successful",
args: args{
db: prepareDB(t,
expectExec("-- replace zitadel with the name of the database\nCREATE DATABASE IF NOT EXISTS \"zitadel\"", nil),
expectExec("-- replace zitadel with the name of the database\nCREATE DATABASE \"zitadel\"", nil),
),
database: "zitadel",
},
@@ -47,7 +47,7 @@ func Test_verifyDB(t *testing.T) {
name: "already exists",
args: args{
db: prepareDB(t,
expectExec("-- replace zitadel with the name of the database\nCREATE DATABASE IF NOT EXISTS \"zitadel\"", nil),
expectExec("-- replace zitadel with the name of the database\nCREATE DATABASE \"zitadel\"", nil),
),
database: "zitadel",
},

2
cmd/initialise/verify_grant.go
View File

@@ -19,7 +19,7 @@ func newGrant() *cobra.Command {
Long: `Sets ALL grant to the database user.
Prerequisites:
- cockroachDB or postgreSQL
- postgreSQL
`,
Run: func(cmd *cobra.Command, args []string) {
config := MustNewConfig(viper.GetViper())

45
cmd/initialise/verify_settings.go
View File

@@ -1,45 +0,0 @@
package initialise
import (
"context"
_ "embed"
"fmt"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/database"
)
func newSettings() *cobra.Command {
return &cobra.Command{
Use: "settings",
Short: "Ensures proper settings on the database",
Long: `Ensures proper settings on the database.
Prerequisites:
- cockroachDB or postgreSQL
Cockroach
- Sets enable_durable_locking_for_serializable to on for the zitadel user and database
`,
Run: func(cmd *cobra.Command, args []string) {
config := MustNewConfig(viper.GetViper())
err := initialise(cmd.Context(), config.Database, VerifySettings(config.Database.DatabaseName(), config.Database.Username()))
logging.OnError(err).Fatal("unable to set settings")
},
}
}
func VerifySettings(databaseName, username string) func(context.Context, *database.DB) error {
return func(ctx context.Context, db *database.DB) error {
if db.Type() == "postgres" {
return nil
}
logging.WithFields("user", username, "database", databaseName).Info("verify settings")
return exec(ctx, db, fmt.Sprintf(settingsStmt, databaseName, username), nil)
}
}

2
cmd/initialise/verify_user.go
View File

@@ -19,7 +19,7 @@ func newUser() *cobra.Command {
Long: `Sets up the ZITADEL database user.
Prerequisites:
- cockroachDB or postgreSQL
- postgreSQL
The user provided by flags needs privileges to
- create the database if it does not exist

10
cmd/initialise/verify_user_test.go
View File

@@ -8,7 +8,7 @@ import (
)
func Test_verifyUser(t *testing.T) {
err := ReadStmts("cockroach") //TODO: check all dialects
err := ReadStmts()
if err != nil {
t.Errorf("unable to read stmts: %v", err)
t.FailNow()
@@ -28,7 +28,7 @@ func Test_verifyUser(t *testing.T) {
name: "doesn't exists, create fails",
args: args{
db: prepareDB(t,
expectExec("-- replace zitadel-user with the name of the user\nCREATE USER IF NOT EXISTS \"zitadel-user\"", sql.ErrTxDone),
expectExec("-- replace zitadel-user with the name of the user\nCREATE USER \"zitadel-user\"", sql.ErrTxDone),
),
username: "zitadel-user",
password: "",
@@ -39,7 +39,7 @@ func Test_verifyUser(t *testing.T) {
name: "correct without password",
args: args{
db: prepareDB(t,
expectExec("-- replace zitadel-user with the name of the user\nCREATE USER IF NOT EXISTS \"zitadel-user\"", nil),
expectExec("-- replace zitadel-user with the name of the user\nCREATE USER \"zitadel-user\"", nil),
),
username: "zitadel-user",
password: "",
@@ -50,7 +50,7 @@ func Test_verifyUser(t *testing.T) {
name: "correct with password",
args: args{
db: prepareDB(t,
expectExec("-- replace zitadel-user with the name of the user\nCREATE USER IF NOT EXISTS \"zitadel-user\" WITH PASSWORD 'password'", nil),
expectExec("-- replace zitadel-user with the name of the user\nCREATE USER \"zitadel-user\" WITH PASSWORD 'password'", nil),
),
username: "zitadel-user",
password: "password",
@@ -61,7 +61,7 @@ func Test_verifyUser(t *testing.T) {
name: "already exists",
args: args{
db: prepareDB(t,
expectExec("-- replace zitadel-user with the name of the user\nCREATE USER IF NOT EXISTS \"zitadel-user\" WITH PASSWORD 'password'", nil),
expectExec("-- replace zitadel-user with the name of the user\nCREATE USER \"zitadel-user\" WITH PASSWORD 'password'", nil),
),
username: "zitadel-user",
password: "",

9
cmd/initialise/verify_zitadel.go
View File

@@ -21,7 +21,7 @@ func newZitadel() *cobra.Command {
Long: `initialize ZITADEL internals.
Prerequisites:
- cockroachDB or postgreSQL with user and database
- postgreSQL with user and database
`,
Run: func(cmd *cobra.Command, args []string) {
config := MustNewConfig(viper.GetViper())
@@ -32,7 +32,7 @@ Prerequisites:
}
func VerifyZitadel(ctx context.Context, db *database.DB, config database.Config) error {
err := ReadStmts(config.Type())
err := ReadStmts()
if err != nil {
return err
}
@@ -68,11 +68,6 @@ func VerifyZitadel(ctx context.Context, db *database.DB, config database.Config)
return err
}
logging.WithFields().Info("verify system sequence")
if err := exec(ctx, conn, createSystemSequenceStmt, nil); err != nil {
return err
}
logging.WithFields().Info("verify unique constraints")
if err := exec(ctx, conn, createUniqueConstraints, nil); err != nil {
return err

2
cmd/initialise/verify_zitadel_test.go
View File

@@ -9,7 +9,7 @@ import (
)
func Test_verifyEvents(t *testing.T) {
err := ReadStmts("cockroach") //TODO: check all dialects
err := ReadStmts()
if err != nil {
t.Errorf("unable to read stmts: %v", err)
t.FailNow()

2
cmd/key/key.go
View File

@@ -40,7 +40,7 @@ func newKey() *cobra.Command {
Long: `create new encryption key(s) (encrypted by the provided master key)
provide key(s) by YAML file and/or by argument
Requirements:
- cockroachdb`,
- postgreSQL`,
Example: `new -f keys.yaml
new key1=somekey key2=anotherkey
new -f keys.yaml key2=anotherkey`,

2
cmd/mirror/config.go
View File

@@ -71,7 +71,7 @@ func mustNewConfig(v *viper.Viper, config any) {
mapstructure.StringToTimeDurationHookFunc(),
mapstructure.StringToTimeHookFunc(time.RFC3339),
mapstructure.StringToSliceHookFunc(","),
database.DecodeHook,
database.DecodeHook(true),
actions.HTTPConfigDecodeHook,
hook.EnumHookFunc(internal_authz.MemberTypeString),
mapstructure.TextUnmarshallerHookFunc(),

51
cmd/mirror/defaults.yaml
View File

@@ -5,8 +5,6 @@ Source:
Database: zitadel # ZITADEL_DATABASE_COCKROACH_DATABASE
MaxOpenConns: 6 # ZITADEL_DATABASE_COCKROACH_MAXOPENCONNS
MaxIdleConns: 6 # ZITADEL_DATABASE_COCKROACH_MAXIDLECONNS
EventPushConnRatio: 0.33 # ZITADEL_DATABASE_COCKROACH_EVENTPUSHCONNRATIO
ProjectionSpoolerConnRatio: 0.33 # ZITADEL_DATABASE_COCKROACH_PROJECTIONSPOOLERCONNRATIO
MaxConnLifetime: 30m # ZITADEL_DATABASE_COCKROACH_MAXCONNLIFETIME
MaxConnIdleTime: 5m # ZITADEL_DATABASE_COCKROACH_MAXCONNIDLETIME
Options: "" # ZITADEL_DATABASE_COCKROACH_OPTIONS
@@ -39,44 +37,23 @@ Source:
Key: # ZITADEL_DATABASE_POSTGRES_USER_SSL_KEY
Destination:
cockroach:
Host: localhost # ZITADEL_DATABASE_COCKROACH_HOST
Port: 26257 # ZITADEL_DATABASE_COCKROACH_PORT
Database: zitadel # ZITADEL_DATABASE_COCKROACH_DATABASE
MaxOpenConns: 0 # ZITADEL_DATABASE_COCKROACH_MAXOPENCONNS
MaxIdleConns: 0 # ZITADEL_DATABASE_COCKROACH_MAXIDLECONNS
MaxConnLifetime: 30m # ZITADEL_DATABASE_COCKROACH_MAXCONNLIFETIME
MaxConnIdleTime: 5m # ZITADEL_DATABASE_COCKROACH_MAXCONNIDLETIME
EventPushConnRatio: 0.01 # ZITADEL_DATABASE_COCKROACH_EVENTPUSHCONNRATIO
ProjectionSpoolerConnRatio: 0.5 # ZITADEL_DATABASE_COCKROACH_PROJECTIONSPOOLERCONNRATIO
Options: "" # ZITADEL_DATABASE_COCKROACH_OPTIONS
User:
Username: zitadel # ZITADEL_DATABASE_COCKROACH_USER_USERNAME
Password: "" # ZITADEL_DATABASE_COCKROACH_USER_PASSWORD
SSL:
Mode: disable # ZITADEL_DATABASE_COCKROACH_USER_SSL_MODE
RootCert: "" # ZITADEL_DATABASE_COCKROACH_USER_SSL_ROOTCERT
Cert: "" # ZITADEL_DATABASE_COCKROACH_USER_SSL_CERT
Key: "" # ZITADEL_DATABASE_COCKROACH_USER_SSL_KEY
# Postgres is used as soon as a value is set
# The values describe the possible fields to set values
postgres:
Host: # ZITADEL_DATABASE_POSTGRES_HOST
Port: # ZITADEL_DATABASE_POSTGRES_PORT
Database: # ZITADEL_DATABASE_POSTGRES_DATABASE
MaxOpenConns: # ZITADEL_DATABASE_POSTGRES_MAXOPENCONNS
MaxIdleConns: # ZITADEL_DATABASE_POSTGRES_MAXIDLECONNS
MaxConnLifetime: # ZITADEL_DATABASE_POSTGRES_MAXCONNLIFETIME
MaxConnIdleTime: # ZITADEL_DATABASE_POSTGRES_MAXCONNIDLETIME
Options: # ZITADEL_DATABASE_POSTGRES_OPTIONS
Host: localhost # ZITADEL_DATABASE_POSTGRES_HOST
Port: 5432 # ZITADEL_DATABASE_POSTGRES_PORT
Database: zitadel # ZITADEL_DATABASE_POSTGRES_DATABASE
MaxOpenConns: 5 # ZITADEL_DATABASE_POSTGRES_MAXOPENCONNS
MaxIdleConns: 2 # ZITADEL_DATABASE_POSTGRES_MAXIDLECONNS
MaxConnLifetime: 30m # ZITADEL_DATABASE_POSTGRES_MAXCONNLIFETIME
MaxConnIdleTime: 5m # ZITADEL_DATABASE_POSTGRES_MAXCONNIDLETIME
Options: "" # ZITADEL_DATABASE_POSTGRES_OPTIONS
User:
Username: # ZITADEL_DATABASE_POSTGRES_USER_USERNAME
Password: # ZITADEL_DATABASE_POSTGRES_USER_PASSWORD
Username: zitadel # ZITADEL_DATABASE_POSTGRES_USER_USERNAME
Password: "" # ZITADEL_DATABASE_POSTGRES_USER_PASSWORD
SSL:
Mode: # ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE
RootCert: # ZITADEL_DATABASE_POSTGRES_USER_SSL_ROOTCERT
Cert: # ZITADEL_DATABASE_POSTGRES_USER_SSL_CERT
Key: # ZITADEL_DATABASE_POSTGRES_USER_SSL_KEY
Mode: disable # ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE
RootCert: "" # ZITADEL_DATABASE_POSTGRES_USER_SSL_ROOTCERT
Cert: "" # ZITADEL_DATABASE_POSTGRES_USER_SSL_CERT
Key: "" # ZITADEL_DATABASE_POSTGRES_USER_SSL_KEY
EventBulkSize: 10000

34
cmd/mirror/event.go
View File

@@ -4,7 +4,6 @@ import (
"context"
"github.com/zitadel/zitadel/internal/v2/eventstore"
"github.com/zitadel/zitadel/internal/v2/projection"
"github.com/zitadel/zitadel/internal/v2/readmodel"
"github.com/zitadel/zitadel/internal/v2/system"
mirror_event "github.com/zitadel/zitadel/internal/v2/system/mirror"
@@ -30,39 +29,6 @@ func queryLastSuccessfulMigration(ctx context.Context, destinationES *eventstore
return lastSuccess, nil
}
func writeMigrationStart(ctx context.Context, sourceES *eventstore.EventStore, id string, destination string) (_ float64, err error) {
var cmd *eventstore.Command
if len(instanceIDs) > 0 {
cmd, err = mirror_event.NewStartedInstancesCommand(destination, instanceIDs)
if err != nil {
return 0, err
}
} else {
cmd = mirror_event.NewStartedSystemCommand(destination)
}
var position projection.HighestPosition
err = sourceES.Push(
ctx,
eventstore.NewPushIntent(
system.AggregateInstance,
eventstore.AppendAggregate(
system.AggregateOwner,
system.AggregateType,
id,
eventstore.CurrentSequenceMatches(0),
eventstore.AppendCommands(cmd),
),
eventstore.PushReducer(&position),
),
)
if err != nil {
return 0, err
}
return position.Position, nil
}
func writeMigrationSucceeded(ctx context.Context, destinationES *eventstore.EventStore, id, source string, position float64) error {
return destinationES.Push(
ctx,

18
cmd/mirror/event_store.go
View File

@@ -14,6 +14,7 @@ import (
"github.com/zitadel/logging"
db "github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/database/dialect"
"github.com/zitadel/zitadel/internal/id"
"github.com/zitadel/zitadel/internal/v2/database"
"github.com/zitadel/zitadel/internal/v2/eventstore"
@@ -57,9 +58,9 @@ func copyEventstore(ctx context.Context, config *Migration) {
func positionQuery(db *db.DB) string {
switch db.Type() {
case "postgres":
case dialect.DatabaseTypePostgres:
return "SELECT EXTRACT(EPOCH FROM clock_timestamp())"
case "cockroach":
case dialect.DatabaseTypeCockroach:
return "SELECT cluster_logical_timestamp()"
default:
logging.WithFields("db_type", db.Type()).Fatal("database type not recognized")
@@ -80,9 +81,6 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
destConn, err := dest.Conn(ctx)
logging.OnError(err).Fatal("unable to acquire dest connection")
sourceES := eventstore.NewEventstoreFromOne(postgres.New(source, &postgres.Config{
MaxRetries: 3,
}))
destinationES := eventstore.NewEventstoreFromOne(postgres.New(dest, &postgres.Config{
MaxRetries: 3,
}))
@@ -90,8 +88,14 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
previousMigration, err := queryLastSuccessfulMigration(ctx, destinationES, source.DatabaseName())
logging.OnError(err).Fatal("unable to query latest successful migration")
maxPosition, err := writeMigrationStart(ctx, sourceES, migrationID, dest.DatabaseName())
logging.OnError(err).Fatal("unable to write migration started event")
var maxPosition float64
err = source.QueryRowContext(ctx,
func(row *sql.Row) error {
return row.Scan(&maxPosition)
},
"SELECT MAX(position) FROM eventstore.events2 "+instanceClause(),
)
logging.OnError(err).Fatal("unable to query max position from source")
logging.WithFields("from", previousMigration.Position, "to", maxPosition).Info("start event migration")

1
cmd/mirror/mirror.go
View File

@@ -56,7 +56,6 @@ Order of execution:
copyEventstore(cmd.Context(), config)
projections(cmd.Context(), projectionConfig, masterKey)
verifyMigration(cmd.Context(), config)
},
}

31
cmd/mirror/projections.go
View File

@@ -84,6 +84,7 @@ type ProjectionsConfig struct {
ExternalDomain string
ExternalSecure bool
InternalAuthZ internal_authz.Config
SystemAuthZ internal_authz.Config
SystemDefaults systemdefaults.SystemDefaults
Telemetry *handlers.TelemetryPusherConfig
Login login.Config
@@ -117,8 +118,11 @@ func projections(
staticStorage, err := config.AssetStorage.NewStorage(client.DB)
logging.OnError(err).Fatal("unable create static storage")
config.Eventstore.Querier = old_es.NewCRDB(client)
config.Eventstore.Pusher = new_es.NewEventstore(client)
newEventstore := new_es.NewEventstore(client)
config.Eventstore.Querier = old_es.NewPostgres(client)
config.Eventstore.Pusher = newEventstore
config.Eventstore.Searcher = newEventstore
es := eventstore.NewEventstore(config.Eventstore)
esV4 := es_v4.NewEventstoreFromOne(es_v4_pg.New(client, &es_v4_pg.Config{
MaxRetries: config.Eventstore.MaxRetries,
@@ -147,7 +151,7 @@ func projections(
sessionTokenVerifier,
func(q *query.Queries) domain.PermissionCheck {
return func(ctx context.Context, permission, orgID, resourceID string) (err error) {
return internal_authz.CheckPermission(ctx, &authz_es.UserMembershipRepo{Queries: q}, config.InternalAuthZ.RolePermissionMappings, permission, orgID, resourceID)
return internal_authz.CheckPermission(ctx, &authz_es.UserMembershipRepo{Queries: q}, config.SystemAuthZ.RolePermissionMappings, config.InternalAuthZ.RolePermissionMappings, permission, orgID, resourceID)
}
},
0,
@@ -184,7 +188,7 @@ func projections(
keys.Target,
&http.Client{},
func(ctx context.Context, permission, orgID, resourceID string) (err error) {
return internal_authz.CheckPermission(ctx, authZRepo, config.InternalAuthZ.RolePermissionMappings, permission, orgID, resourceID)
return internal_authz.CheckPermission(ctx, authZRepo, config.SystemAuthZ.RolePermissionMappings, config.InternalAuthZ.RolePermissionMappings, permission, orgID, resourceID)
},
sessionTokenVerifier,
config.OIDC.DefaultAccessTokenLifetime,
@@ -220,7 +224,6 @@ func projections(
keys.SMS,
keys.OIDC,
config.OIDC.DefaultBackChannelLogoutLifetime,
client,
nil,
)
@@ -248,7 +251,7 @@ func projections(
}
}()
for i := 0; i < int(config.Projections.ConcurrentInstances); i++ {
for range int(config.Projections.ConcurrentInstances) {
go execProjections(ctx, instances, failedInstances, &wg)
}
@@ -270,31 +273,39 @@ func execProjections(ctx context.Context, instances <-chan string, failedInstanc
err := projection.ProjectInstance(ctx)
if err != nil {
logging.WithFields("instance", instance).OnError(err).Info("trigger failed")
logging.WithFields("instance", instance).WithError(err).Info("trigger failed")
failedInstances <- instance
continue
}
err = projection.ProjectInstanceFields(ctx)
if err != nil {
logging.WithFields("instance", instance).WithError(err).Info("trigger fields failed")
failedInstances <- instance
continue
}
err = admin_handler.ProjectInstance(ctx)
if err != nil {
logging.WithFields("instance", instance).OnError(err).Info("trigger admin handler failed")
logging.WithFields("instance", instance).WithError(err).Info("trigger admin handler failed")
failedInstances <- instance
continue
}
err = auth_handler.ProjectInstance(ctx)
if err != nil {
logging.WithFields("instance", instance).OnError(err).Info("trigger auth handler failed")
logging.WithFields("instance", instance).WithError(err).Info("trigger auth handler failed")
failedInstances <- instance
continue
}
err = notification.ProjectInstance(ctx)
if err != nil {
logging.WithFields("instance", instance).OnError(err).Info("trigger notification failed")
logging.WithFields("instance", instance).WithError(err).Info("trigger notification failed")
failedInstances <- instance
continue
}
logging.WithFields("instance", instance).Info("projections done")
}
wg.Done()

25
cmd/setup/07.go
View File

@@ -3,7 +3,7 @@ package setup
import (
"context"
"database/sql"
"embed"
_ "embed"
"strings"
"github.com/zitadel/zitadel/internal/eventstore"
@@ -12,31 +12,20 @@ import (
var (
//go:embed 07/logstore.sql
createLogstoreSchema07 string
//go:embed 07/cockroach/access.sql
//go:embed 07/postgres/access.sql
createAccessLogsTable07 embed.FS
//go:embed 07/cockroach/execution.sql
//go:embed 07/postgres/execution.sql
createExecutionLogsTable07 embed.FS
//go:embed 07/access.sql
createAccessLogsTable07 string
//go:embed 07/execution.sql
createExecutionLogsTable07 string
)
type LogstoreTables struct {
dbClient *sql.DB
username string
dbType string
}
func (mig *LogstoreTables) Execute(ctx context.Context, _ eventstore.Event) error {
accessStmt, err := readStmt(createAccessLogsTable07, "07", mig.dbType, "access.sql")
if err != nil {
return err
}
executionStmt, err := readStmt(createExecutionLogsTable07, "07", mig.dbType, "execution.sql")
if err != nil {
return err
}
stmt := strings.ReplaceAll(createLogstoreSchema07, "%[1]s", mig.username) + accessStmt + executionStmt
_, err = mig.dbClient.ExecContext(ctx, stmt)
stmt := strings.ReplaceAll(createLogstoreSchema07, "%[1]s", mig.username) + createAccessLogsTable07 + createExecutionLogsTable07
_, err := mig.dbClient.ExecContext(ctx, stmt)
return err
}

0
cmd/setup/07/postgres/access.sql → cmd/setup/07/access.sql
View File

14
cmd/setup/07/cockroach/access.sql
View File

@@ -1,14 +0,0 @@
CREATE TABLE IF NOT EXISTS logstore.access (
log_date TIMESTAMPTZ NOT NULL
, protocol INT NOT NULL
, request_url TEXT NOT NULL
, response_status INT NOT NULL
, request_headers JSONB
, response_headers JSONB
, instance_id TEXT NOT NULL
, project_id TEXT NOT NULL
, requested_domain TEXT
, requested_host TEXT
, INDEX protocol_date_desc (instance_id, protocol, log_date DESC) STORING (request_url, response_status, request_headers)
);

11
cmd/setup/07/cockroach/execution.sql
View File

@@ -1,11 +0,0 @@
CREATE TABLE IF NOT EXISTS logstore.execution (
log_date TIMESTAMPTZ NOT NULL
, took INTERVAL
, message TEXT NOT NULL
, loglevel INT NOT NULL
, instance_id TEXT NOT NULL
, action_id TEXT NOT NULL
, metadata JSONB
, INDEX log_date_desc (instance_id, log_date DESC) STORING (took)
);

0
cmd/setup/07/postgres/execution.sql → cmd/setup/07/execution.sql
View File

13
cmd/setup/08.go
View File

@@ -2,16 +2,15 @@ package setup
import (
"context"
"embed"
_ "embed"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore"
)
var (
//go:embed 08/cockroach/08.sql
//go:embed 08/postgres/08.sql
tokenIndexes08 embed.FS
//go:embed 08/08.sql
tokenIndexes08 string
)
type AuthTokenIndexes struct {
@@ -19,11 +18,7 @@ type AuthTokenIndexes struct {
}
func (mig *AuthTokenIndexes) Execute(ctx context.Context, _ eventstore.Event) error {
stmt, err := readStmt(tokenIndexes08, "08", mig.dbClient.Type(), "08.sql")
if err != nil {
return err
}
_, err = mig.dbClient.ExecContext(ctx, stmt)
_, err := mig.dbClient.ExecContext(ctx, tokenIndexes08)
return err
}

0
cmd/setup/08/postgres/08.sql → cmd/setup/08/08.sql
View File

5
cmd/setup/08/cockroach/08.sql
View File

@@ -1,5 +0,0 @@
CREATE INDEX IF NOT EXISTS inst_refresh_tkn_idx ON auth.tokens(instance_id, refresh_token_id);
CREATE INDEX IF NOT EXISTS inst_app_tkn_idx ON auth.tokens(instance_id, application_id);
CREATE INDEX IF NOT EXISTS inst_ro_tkn_idx ON auth.tokens(instance_id, resource_owner);
DROP INDEX IF EXISTS auth.tokens@user_user_agent_idx;
CREATE INDEX IF NOT EXISTS inst_usr_agnt_tkn_idx ON auth.tokens(instance_id, user_id, user_agent_id);

18
cmd/setup/10.go
View File

@@ -3,7 +3,7 @@ package setup
import (
"context"
"database/sql"
"embed"
_ "embed"
"time"
"github.com/cockroachdb/cockroach-go/v2/crdb"
@@ -18,9 +18,8 @@ var (
correctCreationDate10CreateTable string
//go:embed 10/10_fill_table.sql
correctCreationDate10FillTable string
//go:embed 10/cockroach/10_update.sql
//go:embed 10/postgres/10_update.sql
correctCreationDate10Update embed.FS
//go:embed 10/10_update.sql
correctCreationDate10Update string
//go:embed 10/10_count_wrong_events.sql
correctCreationDate10CountWrongEvents string
//go:embed 10/10_empty_table.sql
@@ -40,11 +39,6 @@ func (mig *CorrectCreationDate) Execute(ctx context.Context, _ eventstore.Event)
logging.WithFields("mig", mig.String(), "iteration", i).Debug("start iteration")
var affected int64
err = crdb.ExecuteTx(ctx, mig.dbClient.DB, nil, func(tx *sql.Tx) error {
if mig.dbClient.Type() == "cockroach" {
if _, err := tx.Exec("SET experimental_enable_temp_tables=on"); err != nil {
return err
}
}
_, err := tx.ExecContext(ctx, correctCreationDate10CreateTable)
if err != nil {
return err
@@ -66,11 +60,7 @@ func (mig *CorrectCreationDate) Execute(ctx context.Context, _ eventstore.Event)
return err
}
updateStmt, err := readStmt(correctCreationDate10Update, "10", mig.dbClient.Type(), "10_update.sql")
if err != nil {
return err
}
_, err = tx.ExecContext(ctx, updateStmt)
_, err = tx.ExecContext(ctx, correctCreationDate10Update)
if err != nil {
return err
}

0
cmd/setup/10/postgres/10_update.sql → cmd/setup/10/10_update.sql
View File

1
cmd/setup/10/cockroach/10_update.sql
View File

@@ -1 +0,0 @@
UPDATE eventstore.events e SET (creation_date, "position") = (we.next_cd, we.next_cd::DECIMAL) FROM wrong_events we WHERE e.event_sequence = we.event_sequence AND e.instance_id = we.instance_id;

5
cmd/setup/14.go
View File

@@ -15,8 +15,7 @@ import (
)
var (
//go:embed 14/cockroach/*.sql
//go:embed 14/postgres/*.sql
//go:embed 14/*.sql
newEventsTable embed.FS
)
@@ -40,7 +39,7 @@ func (mig *NewEventsTable) Execute(ctx context.Context, _ eventstore.Event) erro
return err
}
statements, err := readStatements(newEventsTable, "14", mig.dbClient.Type())
statements, err := readStatements(newEventsTable, "14")
if err != nil {
return err
}

0
cmd/setup/14/cockroach/01_disable_inserts.sql → cmd/setup/14/01_disable_inserts.sql
View File

0
cmd/setup/14/postgres/02_create_and_fill_events2.sql → cmd/setup/14/02_create_and_fill_events2.sql
View File

0
cmd/setup/14/postgres/03_events2_pk.sql → cmd/setup/14/03_events2_pk.sql
View File

0
cmd/setup/14/postgres/04_constraints.sql → cmd/setup/14/04_constraints.sql
View File

0
cmd/setup/14/postgres/05_indexes.sql → cmd/setup/14/05_indexes.sql
View File

33
cmd/setup/14/cockroach/02_create_and_fill_events2.sql
View File

@@ -1,33 +0,0 @@
CREATE TABLE eventstore.events2 (
instance_id,
aggregate_type,
aggregate_id,
event_type,
"sequence",
revision,
created_at,
payload,
creator,
"owner",
"position",
in_tx_order,
PRIMARY KEY (instance_id, aggregate_type, aggregate_id, "sequence")
) AS SELECT
instance_id,
aggregate_type,
aggregate_id,
event_type,
event_sequence,
substr(aggregate_version, 2)::SMALLINT,
creation_date,
event_data,
editor_user,
resource_owner,
creation_date::DECIMAL,
event_sequence
FROM eventstore.events_old;

7
cmd/setup/14/cockroach/03_constraints.sql
View File

@@ -1,7 +0,0 @@
ALTER TABLE eventstore.events2 ALTER COLUMN event_type SET NOT NULL;
ALTER TABLE eventstore.events2 ALTER COLUMN revision SET NOT NULL;
ALTER TABLE eventstore.events2 ALTER COLUMN created_at SET NOT NULL;
ALTER TABLE eventstore.events2 ALTER COLUMN creator SET NOT NULL;
ALTER TABLE eventstore.events2 ALTER COLUMN "owner" SET NOT NULL;
ALTER TABLE eventstore.events2 ALTER COLUMN "position" SET NOT NULL;
ALTER TABLE eventstore.events2 ALTER COLUMN in_tx_order SET NOT NULL;

3
cmd/setup/14/cockroach/04_indexes.sql
View File

@@ -1,3 +0,0 @@
CREATE INDEX IF NOT EXISTS es_active_instances ON eventstore.events2 (created_at DESC) STORING ("position");
CREATE INDEX IF NOT EXISTS es_wm ON eventstore.events2 (aggregate_id, instance_id, aggregate_type, event_type);
CREATE INDEX IF NOT EXISTS es_projection ON eventstore.events2 (instance_id, aggregate_type, event_type, "position");

1
cmd/setup/14/postgres/01_disable_inserts.sql
View File

@@ -1 +0,0 @@
ALTER TABLE eventstore.events RENAME TO events_old;

5
cmd/setup/15.go
View File

@@ -11,8 +11,7 @@ import (
)
var (
//go:embed 15/cockroach/*.sql
//go:embed 15/postgres/*.sql
//go:embed 15/*.sql
currentProjectionState embed.FS
)
@@ -21,7 +20,7 @@ type CurrentProjectionState struct {
}
func (mig *CurrentProjectionState) Execute(ctx context.Context, _ eventstore.Event) error {
statements, err := readStatements(currentProjectionState, "15", mig.dbClient.Type())
statements, err := readStatements(currentProjectionState, "15")
if err != nil {
return err
}

0
cmd/setup/15/cockroach/01_new_failed_events.sql → cmd/setup/15/01_new_failed_events.sql
View File

0
cmd/setup/15/postgres/02_fe_from_projections.sql → cmd/setup/15/02_fe_from_projections.sql
View File

0
cmd/setup/15/cockroach/03_fe_from_adminapi.sql → cmd/setup/15/03_fe_from_adminapi.sql
View File

0
cmd/setup/15/cockroach/04_fe_from_auth.sql → cmd/setup/15/04_fe_from_auth.sql
View File

0
cmd/setup/15/cockroach/05_current_states.sql → cmd/setup/15/05_current_states.sql
View File

0
cmd/setup/15/postgres/06_cs_from_projections.sql → cmd/setup/15/06_cs_from_projections.sql
View File

0
cmd/setup/15/postgres/07_cs_from_adminapi.sql → cmd/setup/15/07_cs_from_adminapi.sql
View File

0
cmd/setup/15/postgres/08_cs_from_auth.sql → cmd/setup/15/08_cs_from_auth.sql
View File

26
cmd/setup/15/cockroach/02_fe_from_projections.sql
View File

@@ -1,26 +0,0 @@
INSERT INTO projections.failed_events2 (
projection_name
, instance_id
, aggregate_type
, aggregate_id
, event_creation_date
, failed_sequence
, failure_count
, error
, last_failed
) SELECT
fe.projection_name
, fe.instance_id
, e.aggregate_type
, e.aggregate_id
, e.created_at
, e.sequence
, fe.failure_count
, fe.error
, fe.last_failed
FROM
projections.failed_events fe
JOIN eventstore.events2 e ON
e.instance_id = fe.instance_id
AND e.sequence = fe.failed_sequence
ON CONFLICT DO NOTHING;

29
cmd/setup/15/cockroach/06_cs_from_projections.sql
View File

@@ -1,29 +0,0 @@
INSERT INTO projections.current_states (
projection_name
, instance_id
, event_date
, "position"
, last_updated
) (SELECT
cs.projection_name
, cs.instance_id
, e.created_at
, e.position
, cs.timestamp
FROM
projections.current_sequences cs
JOIN eventstore.events2 e ON
e.instance_id = cs.instance_id
AND e.aggregate_type = cs.aggregate_type
AND e.sequence = cs.current_sequence
AND cs.current_sequence = (
SELECT
MAX(cs2.current_sequence)
FROM
projections.current_sequences cs2
WHERE
cs.projection_name = cs2.projection_name
AND cs.instance_id = cs2.instance_id
)
)
ON CONFLICT DO NOTHING;

28
cmd/setup/15/cockroach/07_cs_from_adminapi.sql
View File

@@ -1,28 +0,0 @@
INSERT INTO projections.current_states (
projection_name
, instance_id
, event_date
, "position"
, last_updated
) (SELECT
cs.view_name
, cs.instance_id
, e.created_at
, e.position
, cs.last_successful_spooler_run
FROM
adminapi.current_sequences cs
JOIN eventstore.events2 e ON
e.instance_id = cs.instance_id
AND e.sequence = cs.current_sequence
AND cs.current_sequence = (
SELECT
MAX(cs2.current_sequence)
FROM
adminapi.current_sequences cs2
WHERE
cs.view_name = cs2.view_name
AND cs.instance_id = cs2.instance_id
)
)
ON CONFLICT DO NOTHING;

28
cmd/setup/15/cockroach/08_cs_from_auth.sql
View File

@@ -1,28 +0,0 @@
INSERT INTO projections.current_states (
projection_name
, instance_id
, event_date
, "position"
, last_updated
) (SELECT
cs.view_name
, cs.instance_id
, e.created_at
, e.position
, cs.last_successful_spooler_run
FROM
auth.current_sequences cs
JOIN eventstore.events2 e ON
e.instance_id = cs.instance_id
AND e.sequence = cs.current_sequence
AND cs.current_sequence = (
SELECT
MAX(cs2.current_sequence)
FROM
auth.current_sequences cs2
WHERE
cs.view_name = cs2.view_name
AND cs.instance_id = cs2.instance_id
)
)
ON CONFLICT DO NOTHING;

16
cmd/setup/15/postgres/01_new_failed_events.sql
View File

@@ -1,16 +0,0 @@
CREATE TABLE IF NOT EXISTS projections.failed_events2 (
projection_name TEXT NOT NULL
, instance_id TEXT NOT NULL
, aggregate_type TEXT NOT NULL
, aggregate_id TEXT NOT NULL
, event_creation_date TIMESTAMPTZ NOT NULL
, failed_sequence INT8 NOT NULL
, failure_count INT2 NULL DEFAULT 0
, error TEXT
, last_failed TIMESTAMPTZ
, PRIMARY KEY (projection_name, instance_id, aggregate_type, aggregate_id, failed_sequence)
);
CREATE INDEX IF NOT EXISTS fe2_instance_id_idx on projections.failed_events2 (instance_id);

26
cmd/setup/15/postgres/03_fe_from_adminapi.sql
View File

@@ -1,26 +0,0 @@
INSERT INTO projections.failed_events2 (
projection_name
, instance_id
, aggregate_type
, aggregate_id
, event_creation_date
, failed_sequence
, failure_count
, error
, last_failed
) SELECT
fe.view_name
, fe.instance_id
, e.aggregate_type
, e.aggregate_id
, e.created_at
, e.sequence
, fe.failure_count
, fe.err_msg
, fe.last_failed
FROM
adminapi.failed_events fe
JOIN eventstore.events2 e ON
e.instance_id = fe.instance_id
AND e.sequence = fe.failed_sequence
ON CONFLICT DO NOTHING;

26
cmd/setup/15/postgres/04_fe_from_auth.sql
View File

@@ -1,26 +0,0 @@
INSERT INTO projections.failed_events2 (
projection_name
, instance_id
, aggregate_type
, aggregate_id
, event_creation_date
, failed_sequence
, failure_count
, error
, last_failed
) SELECT
fe.view_name
, fe.instance_id
, e.aggregate_type
, e.aggregate_id
, e.created_at
, e.sequence
, fe.failure_count
, fe.err_msg
, fe.last_failed
FROM
auth.failed_events fe
JOIN eventstore.events2 e ON
e.instance_id = fe.instance_id
AND e.sequence = fe.failed_sequence
ON CONFLICT DO NOTHING;

15
cmd/setup/15/postgres/05_current_states.sql
View File

@@ -1,15 +0,0 @@
CREATE TABLE IF NOT EXISTS projections.current_states (
projection_name TEXT NOT NULL
, instance_id TEXT NOT NULL
, last_updated TIMESTAMPTZ
, aggregate_id TEXT
, aggregate_type TEXT
, "sequence" INT8
, event_date TIMESTAMPTZ
, "position" DECIMAL
, PRIMARY KEY (projection_name, instance_id)
);
CREATE INDEX IF NOT EXISTS cs_instance_id_idx ON projections.current_states (instance_id);

16
cmd/setup/34.go
View File

@@ -3,17 +3,14 @@ package setup
import (
"context"
_ "embed"
"fmt"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore"
)
var (
//go:embed 34/cockroach/34_cache_schema.sql
addCacheSchemaCockroach string
//go:embed 34/postgres/34_cache_schema.sql
addCacheSchemaPostgres string
//go:embed 34/34_cache_schema.sql
addCacheSchema string
)
type AddCacheSchema struct {
@@ -21,14 +18,7 @@ type AddCacheSchema struct {
}
func (mig *AddCacheSchema) Execute(ctx context.Context, _ eventstore.Event) (err error) {
switch mig.dbClient.Type() {
case "cockroach":
_, err = mig.dbClient.ExecContext(ctx, addCacheSchemaCockroach)
case "postgres":
_, err = mig.dbClient.ExecContext(ctx, addCacheSchemaPostgres)
default:
err = fmt.Errorf("add cache schema: unsupported db type %q", mig.dbClient.Type())
}
_, err = mig.dbClient.ExecContext(ctx, addCacheSchema)
return err
}

0
cmd/setup/34/postgres/34_cache_schema.sql → cmd/setup/34/34_cache_schema.sql
View File

27
cmd/setup/34/cockroach/34_cache_schema.sql
View File

@@ -1,27 +0,0 @@
create schema if not exists cache;
create table if not exists cache.objects (
cache_name varchar not null,
id uuid not null default gen_random_uuid(),
created_at timestamptz not null default now(),
last_used_at timestamptz not null default now(),
payload jsonb not null,
primary key(cache_name, id)
);
create table if not exists cache.string_keys(
cache_name varchar not null check (cache_name <> ''),
index_id integer not null check (index_id > 0),
index_key varchar not null check (index_key <> ''),
object_id uuid not null,
primary key (cache_name, index_id, index_key),
constraint fk_object
foreign key(cache_name, object_id)
references cache.objects(cache_name, id)
on delete cascade
);
create index if not exists string_keys_object_id_idx
on cache.string_keys (cache_name, object_id); -- for delete cascade

2
cmd/setup/35.go
View File

@@ -21,7 +21,7 @@ type AddPositionToIndexEsWm struct {
}
func (mig *AddPositionToIndexEsWm) Execute(ctx context.Context, _ eventstore.Event) error {
statements, err := readStatements(addPositionToEsWmIndex, "35", "")
statements, err := readStatements(addPositionToEsWmIndex, "35")
if err != nil {
return err
}

5
cmd/setup/40.go
View File

@@ -24,8 +24,7 @@ const (
)
var (
//go:embed 40/cockroach/*.sql
//go:embed 40/postgres/*.sql
//go:embed 40/*.sql
initPushFunc embed.FS
)
@@ -112,5 +111,5 @@ func (mig *InitPushFunc) inTxOrderType(ctx context.Context) (typeName string, er
}
func (mig *InitPushFunc) filePath(fileName string) string {
return path.Join("40", mig.dbClient.Type(), fileName)
return path.Join("40", fileName)
}

0
cmd/setup/40/cockroach/00_in_tx_order_type.sql → cmd/setup/40/00_in_tx_order_type.sql
View File

0
cmd/setup/40/postgres/01_type.sql → cmd/setup/40/01_type.sql
View File

0
cmd/setup/40/postgres/02_func.sql → cmd/setup/40/02_func.sql
View File

10
cmd/setup/40/cockroach/01_type.sql
View File

@@ -1,10 +0,0 @@
CREATE TYPE IF NOT EXISTS eventstore.command AS (
instance_id TEXT
, aggregate_type TEXT
, aggregate_id TEXT
, command_type TEXT
, revision INT2
, payload JSONB
, creator TEXT
, owner TEXT
);

137
cmd/setup/40/cockroach/02_func.sql
View File

@@ -1,137 +0,0 @@
CREATE OR REPLACE FUNCTION eventstore.latest_aggregate_state(
instance_id TEXT
, aggregate_type TEXT
, aggregate_id TEXT
, sequence OUT BIGINT
, owner OUT TEXT
)
LANGUAGE 'plpgsql'
AS $$
BEGIN
SELECT
COALESCE(e.sequence, 0) AS sequence
, e.owner
INTO
sequence
, owner
FROM
eventstore.events2 e
WHERE
e.instance_id = $1
AND e.aggregate_type = $2
AND e.aggregate_id = $3
ORDER BY
e.sequence DESC
LIMIT 1;
RETURN;
END;
$$;
CREATE OR REPLACE FUNCTION eventstore.commands_to_events2(commands eventstore.command[])
RETURNS eventstore.events2[]
LANGUAGE 'plpgsql'
AS $$
DECLARE
current_sequence BIGINT;
current_owner TEXT;
instance_id TEXT;
aggregate_type TEXT;
aggregate_id TEXT;
_events eventstore.events2[];
_aggregates CURSOR FOR
select
DISTINCT ("c").instance_id
, ("c").aggregate_type
, ("c").aggregate_id
FROM
UNNEST(commands) AS c;
BEGIN
OPEN _aggregates;
LOOP
FETCH NEXT IN _aggregates INTO instance_id, aggregate_type, aggregate_id;
-- crdb does not support EXIT WHEN NOT FOUND
EXIT WHEN instance_id IS NULL;
SELECT
*
INTO
current_sequence
, current_owner
FROM eventstore.latest_aggregate_state(
instance_id
, aggregate_type
, aggregate_id
);
-- RETURN QUERY is not supported by crdb: https://github.com/cockroachdb/cockroach/issues/105240
SELECT
ARRAY_CAT(_events, ARRAY_AGG(e))
INTO
_events
FROM (
SELECT
("c").instance_id
, ("c").aggregate_type
, ("c").aggregate_id
, ("c").command_type -- AS event_type
, COALESCE(current_sequence, 0) + ROW_NUMBER() OVER () -- AS sequence
, ("c").revision
, NOW() -- AS created_at
, ("c").payload
, ("c").creator
, COALESCE(current_owner, ("c").owner) -- AS owner
, cluster_logical_timestamp() -- AS position
, ordinality::{{ .InTxOrderType }} -- AS in_tx_order
FROM
UNNEST(commands) WITH ORDINALITY AS c
WHERE
("c").instance_id = instance_id
AND ("c").aggregate_type = aggregate_type
AND ("c").aggregate_id = aggregate_id
) AS e;
END LOOP;
CLOSE _aggregates;
RETURN _events;
END;
$$;
CREATE OR REPLACE FUNCTION eventstore.push(commands eventstore.command[]) RETURNS SETOF eventstore.events2 AS $$
INSERT INTO eventstore.events2
SELECT
("e").instance_id
, ("e").aggregate_type
, ("e").aggregate_id
, ("e").event_type
, ("e").sequence
, ("e").revision
, ("e").created_at
, ("e").payload
, ("e").creator
, ("e").owner
, ("e")."position"
, ("e").in_tx_order
FROM
UNNEST(eventstore.commands_to_events2(commands)) e
ORDER BY
in_tx_order
RETURNING *
$$ LANGUAGE SQL;
/*
select (c).* from UNNEST(eventstore.commands_to_events2(
ARRAY[
ROW('', 'system', 'SYSTEM', 'ct1', 1, '{"key": "value"}', 'c1', 'SYSTEM')
, ROW('', 'system', 'SYSTEM', 'ct2', 1, '{"key": "value"}', 'c1', 'SYSTEM')
, ROW('289525561255060732', 'org', '289575074711790844', 'ct3', 1, '{"key": "value"}', 'c1', '289575074711790844')
, ROW('289525561255060732', 'user', '289575075164906748', 'ct3', 1, '{"key": "value"}', 'c1', '289575074711790844')
, ROW('289525561255060732', 'oidc_session', 'V2_289575178579535100', 'ct3', 1, '{"key": "value"}', 'c1', '289575074711790844')
, ROW('', 'system', 'SYSTEM', 'ct3', 1, '{"key": "value"}', 'c1', 'SYSTEM')
]::eventstore.command[]
) )c;
*/

5
cmd/setup/40/postgres/00_in_tx_order_type.sql
View File

@@ -1,5 +0,0 @@
SELECT data_type
FROM information_schema.columns
WHERE table_schema = 'eventstore'
AND table_name = 'events2'
AND column_name = 'in_tx_order';

5
cmd/setup/43.go
View File

@@ -12,8 +12,7 @@ import (
)
var (
//go:embed 43/cockroach/*.sql
//go:embed 43/postgres/*.sql
//go:embed 43/*.sql
createFieldsDomainIndex embed.FS
)
@@ -22,7 +21,7 @@ type CreateFieldsDomainIndex struct {
}
func (mig *CreateFieldsDomainIndex) Execute(ctx context.Context, _ eventstore.Event) error {
statements, err := readStatements(createFieldsDomainIndex, "43", mig.dbClient.Type())
statements, err := readStatements(createFieldsDomainIndex, "43")
if err != nil {
return err
}

0
cmd/setup/43/postgres/43.sql → cmd/setup/43/43.sql
View File

3
cmd/setup/43/cockroach/43.sql
View File

@@ -1,3 +0,0 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS fields_instance_domains_idx
ON eventstore.fields (object_id)
WHERE object_type = 'instance_domain' AND field_name = 'domain';

2
cmd/setup/44.go
View File

@@ -21,7 +21,7 @@ type ReplaceCurrentSequencesIndex struct {
}
func (mig *ReplaceCurrentSequencesIndex) Execute(ctx context.Context, _ eventstore.Event) error {
statements, err := readStatements(replaceCurrentSequencesIndex, "44", "")
statements, err := readStatements(replaceCurrentSequencesIndex, "44")
if err != nil {
return err
}

2
cmd/setup/46.go
View File

@@ -21,7 +21,7 @@ var (
)
func (mig *InitPermissionFunctions) Execute(ctx context.Context, _ eventstore.Event) error {
statements, err := readStatements(permissionFunctions, "46", "")
statements, err := readStatements(permissionFunctions, "46")
if err != nil {
return err
}

2
cmd/setup/49.go
View File

@@ -21,7 +21,7 @@ var (
)
func (mig *InitPermittedOrgsFunction) Execute(ctx context.Context, _ eventstore.Event) error {
statements, err := readStatements(permittedOrgsFunction, "49", "")
statements, err := readStatements(permittedOrgsFunction, "49")
if err != nil {
return err
}

37
cmd/setup/53.go Normal file
View File

@@ -0,0 +1,37 @@
package setup
import (
"context"
"embed"
"fmt"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore"
)
type InitPermittedOrgsFunction53 struct {
dbClient *database.DB
}
//go:embed 53/*.sql
var permittedOrgsFunction53 embed.FS
func (mig *InitPermittedOrgsFunction53) Execute(ctx context.Context, _ eventstore.Event) error {
statements, err := readStatements(permittedOrgsFunction53, "53")
if err != nil {
return err
}
for _, stmt := range statements {
logging.WithFields("file", stmt.file, "migration", mig.String()).Info("execute statement")
if _, err := mig.dbClient.ExecContext(ctx, stmt.query); err != nil {
return fmt.Errorf("%s %s: %w", mig.String(), stmt.file, err)
}
}
return nil
}
func (*InitPermittedOrgsFunction53) String() string {
return "53_init_permitted_orgs_function"
}

43
cmd/setup/53/01-get-permissions-from-JSON.sql Normal file
View File

@@ -0,0 +1,43 @@
DROP FUNCTION IF EXISTS eventstore.get_system_permissions;
CREATE OR REPLACE FUNCTION eventstore.get_system_permissions(
permissions_json JSONB
/*
[
{
"member_type": "System",
"aggregate_id": "",
"object_id": "",
"permissions": ["iam.read", "iam.write", "iam.polic.read"]
},
{
"member_type": "IAM",
"aggregate_id": "310716990375453665",
"object_id": "",
"permissions": ["iam.read", "iam.write", "iam.polic.read"]
}
]
*/
, permm TEXT
)
RETURNS TABLE (
member_type TEXT,
aggregate_id TEXT,
object_id TEXT
)
LANGUAGE 'plpgsql'
AS $$
BEGIN
RETURN QUERY
SELECT res.member_type, res.aggregate_id, res.object_id FROM (
SELECT
(perm)->>'member_type' AS member_type,
(perm)->>'aggregate_id' AS aggregate_id,
(perm)->>'object_id' AS object_id,
permission
FROM jsonb_array_elements(permissions_json) AS perm
CROSS JOIN jsonb_array_elements_text(perm->'permissions') AS permission) AS res
WHERE res. permission= permm;
END;
$$;

144
cmd/setup/53/02-permitted_orgs_function.sql Normal file
View File

@@ -0,0 +1,144 @@
DROP FUNCTION IF EXISTS eventstore.check_system_user_perms;
CREATE OR REPLACE FUNCTION eventstore.check_system_user_perms(
system_user_perms JSONB
, perm TEXT
, filter_orgs TEXT
, org_ids OUT TEXT[]
)
LANGUAGE 'plpgsql'
AS $$
BEGIN
WITH found_permissions(member_type, aggregate_id, object_id ) AS (
SELECT * FROM eventstore.get_system_permissions(
system_user_perms,
perm)
)
SELECT array_agg(DISTINCT o.org_id) INTO org_ids
FROM eventstore.instance_orgs o, found_permissions
WHERE
CASE WHEN (SELECT TRUE WHERE found_permissions.member_type = 'System' LIMIT 1) THEN
TRUE
WHEN (SELECT TRUE WHERE found_permissions.member_type = 'IAM' LIMIT 1) THEN
-- aggregate_id not present
CASE WHEN (SELECT TRUE WHERE '' = ANY (
(
SELECT array_agg(found_permissions.aggregate_id)
FROM found_permissions
WHERE member_type = 'IAM'
GROUP BY member_type
LIMIT 1
)::TEXT[])) THEN
TRUE
-- aggregate_id is present
ELSE
o.instance_id = ANY (
(
SELECT array_agg(found_permissions.aggregate_id)
FROM found_permissions
WHERE member_type = 'IAM'
GROUP BY member_type
LIMIT 1
)::TEXT[])
END
WHEN (SELECT TRUE WHERE found_permissions.member_type = 'Organization' LIMIT 1) THEN
-- aggregate_id not present
CASE WHEN (SELECT TRUE WHERE '' = ANY (
(
SELECT array_agg(found_permissions.aggregate_id)
FROM found_permissions
WHERE member_type = 'Organization'
GROUP BY member_type
LIMIT 1
)::TEXT[])) THEN
TRUE
-- aggregate_id is present
ELSE
o.org_id = ANY (
(
SELECT array_agg(found_permissions.aggregate_id)
FROM found_permissions
WHERE member_type = 'Organization'
GROUP BY member_type
LIMIT 1
)::TEXT[])
END
END
AND
CASE WHEN filter_orgs != ''
THEN o.org_id IN (filter_orgs)
ELSE TRUE END
LIMIT 1;
END;
$$;
DROP FUNCTION IF EXISTS eventstore.permitted_orgs;
CREATE OR REPLACE FUNCTION eventstore.permitted_orgs(
instanceId TEXT
, userId TEXT
, system_user_perms JSONB
, perm TEXT
, filter_orgs TEXT
, org_ids OUT TEXT[]
)
LANGUAGE 'plpgsql'
AS $$
BEGIN
-- if system user
IF system_user_perms IS NOT NULL THEN
org_ids := eventstore.check_system_user_perms(system_user_perms, perm, filter_orgs);
-- if human/machine user
ELSE
DECLARE
matched_roles TEXT[]; -- roles containing permission
BEGIN
SELECT array_agg(rp.role) INTO matched_roles
FROM eventstore.role_permissions rp
WHERE rp.instance_id = instanceId
AND rp.permission = perm;
-- First try if the permission was granted thru an instance-level role
DECLARE
has_instance_permission bool;
BEGIN
SELECT true INTO has_instance_permission
FROM eventstore.instance_members im
WHERE im.role = ANY(matched_roles)
AND im.instance_id = instanceId
AND im.user_id = userId
LIMIT 1;
IF has_instance_permission THEN
-- Return all organizations or only those in filter_orgs
SELECT array_agg(o.org_id) INTO org_ids
FROM eventstore.instance_orgs o
WHERE o.instance_id = instanceId
AND CASE WHEN filter_orgs != ''
THEN o.org_id IN (filter_orgs)
ELSE TRUE END;
RETURN;
END IF;
END;
-- Return the organizations where permission were granted thru org-level roles
SELECT array_agg(sub.org_id) INTO org_ids
FROM (
SELECT DISTINCT om.org_id
FROM eventstore.org_members om
WHERE om.role = ANY(matched_roles)
AND om.instance_id = instanceID
AND om.user_id = userId
) AS sub;
END;
END IF;
END;
$$;

2
cmd/setup/cleanup.go
View File

@@ -35,7 +35,7 @@ func Cleanup(config *Config) {
logging.OnError(err).Fatal("unable to connect to database")
config.Eventstore.Pusher = new_es.NewEventstore(dbClient)
config.Eventstore.Querier = old_es.NewCRDB(dbClient)
config.Eventstore.Querier = old_es.NewPostgres(dbClient)
es := eventstore.NewEventstore(config.Eventstore)
step, err := migration.LastStuckStep(ctx, es)

Some files were not shown because too many files have changed in this diff Show More