From cff0f73e24f938c0f492726f1ac75c73246c9ee5 Mon Sep 17 00:00:00 2001 From: Silvan Date: Mon, 27 May 2024 11:03:34 +0200 Subject: [PATCH] fix(init): add setting to enable durable locks on crdb (#7982) feat(init): add setting to enable durable locks on crdb --- cmd/initialise/init.go | 9 +++- cmd/initialise/sql/cockroach/11_settings.sql | 4 ++ cmd/initialise/sql/postgres/11_settings.sql | 0 cmd/initialise/verify_settings.go | 44 +++++++++++++++++++ .../manage/database/_cockroachdb.mdx | 2 + docs/docs/support/advisory/a10009.md | 27 ++++++++++++ docs/docs/support/technical_advisory.mdx | 14 +++++- internal/eventstore/local_crdb_test.go | 3 +- .../repository/sql/local_crdb_test.go | 3 +- 9 files changed, 102 insertions(+), 4 deletions(-) create mode 100644 cmd/initialise/sql/cockroach/11_settings.sql create mode 100644 cmd/initialise/sql/postgres/11_settings.sql create mode 100644 cmd/initialise/verify_settings.go create mode 100644 docs/docs/support/advisory/a10009.md diff --git a/cmd/initialise/init.go b/cmd/initialise/init.go index 7451f73ad3..d45bacfc17 100644 --- a/cmd/initialise/init.go +++ b/cmd/initialise/init.go @@ -19,6 +19,7 @@ var ( createUserStmt string grantStmt string + settingsStmt string databaseStmt string createEventstoreStmt string createProjectionsStmt string @@ -53,7 +54,7 @@ The user provided by flags needs privileges to }, } - cmd.AddCommand(newZitadel(), newDatabase(), newUser(), newGrant()) + cmd.AddCommand(newZitadel(), newDatabase(), newUser(), newGrant(), newSettings()) return cmd } @@ -62,6 +63,7 @@ func InitAll(ctx context.Context, config *Config) { VerifyUser(config.Database.Username(), config.Database.Password()), VerifyDatabase(config.Database.DatabaseName()), VerifyGrant(config.Database.DatabaseName(), config.Database.Username()), + VerifySettings(config.Database.DatabaseName(), config.Database.Username()), ) logging.OnError(err).Fatal("unable to initialize the database") @@ -147,6 +149,11 @@ func ReadStmts(typ string) (err error) { return err } + settingsStmt, err = readStmt(typ, "11_settings") + if err != nil { + return err + } + return nil } diff --git a/cmd/initialise/sql/cockroach/11_settings.sql b/cmd/initialise/sql/cockroach/11_settings.sql new file mode 100644 index 0000000000..5fa9dd72f6 --- /dev/null +++ b/cmd/initialise/sql/cockroach/11_settings.sql @@ -0,0 +1,4 @@ +-- replace the first %[1]q with the database in double quotes +-- replace the second \%[2]q with the user in double quotes$ +-- For more information see technical advisory 10009 (https://zitadel.com/docs/support/advisory/a10009) +ALTER ROLE %[2]q IN DATABASE %[1]q SET enable_durable_locking_for_serializable = on; \ No newline at end of file diff --git a/cmd/initialise/sql/postgres/11_settings.sql b/cmd/initialise/sql/postgres/11_settings.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/initialise/verify_settings.go b/cmd/initialise/verify_settings.go new file mode 100644 index 0000000000..75e811663f --- /dev/null +++ b/cmd/initialise/verify_settings.go @@ -0,0 +1,44 @@ +package initialise + +import ( + _ "embed" + "fmt" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/zitadel/logging" + + "github.com/zitadel/zitadel/internal/database" +) + +func newSettings() *cobra.Command { + return &cobra.Command{ + Use: "settings", + Short: "Ensures proper settings on the database", + Long: `Ensures proper settings on the database. + +Prerequisites: +- cockroachDB or postgreSQL + +Cockroach +- Sets enable_durable_locking_for_serializable to on for the zitadel user and database +`, + Run: func(cmd *cobra.Command, args []string) { + config := MustNewConfig(viper.GetViper()) + + err := initialise(config.Database, VerifySettings(config.Database.DatabaseName(), config.Database.Username())) + logging.OnError(err).Fatal("unable to set settings") + }, + } +} + +func VerifySettings(databaseName, username string) func(*database.DB) error { + return func(db *database.DB) error { + if db.Type() == "postgres" { + return nil + } + logging.WithFields("user", username, "database", databaseName).Info("verify settings") + + return exec(db, fmt.Sprintf(settingsStmt, databaseName, username), nil) + } +} diff --git a/docs/docs/self-hosting/manage/database/_cockroachdb.mdx b/docs/docs/self-hosting/manage/database/_cockroachdb.mdx index 2836640669..edc3f139fd 100644 --- a/docs/docs/self-hosting/manage/database/_cockroachdb.mdx +++ b/docs/docs/self-hosting/manage/database/_cockroachdb.mdx @@ -2,6 +2,8 @@ The default database of ZITADEL is [CockroachDB](https://www.cockroachlabs.com). The SQL database provides a bunch of features like horizontal scalability, data regionality and many more. +Currently versions >= 23.2 are supported. + The default configuration of the database looks like this: ```yaml diff --git a/docs/docs/support/advisory/a10009.md b/docs/docs/support/advisory/a10009.md new file mode 100644 index 0000000000..7137fdceaf --- /dev/null +++ b/docs/docs/support/advisory/a10009.md @@ -0,0 +1,27 @@ +--- +title: Technical Advisory 10009 +--- + +## Date and Version + +Version: 2.53.0 + +Date: Calendar week 23/24 2024 + +## Description + +There were rare cases where Cockroachdb got blocked during runtime of ZITADEL and returned `WRITE_TOO_OLD`-errors to ZITADEL. The root cause of the problem is described in [this github issue of the database](https://github.com/cockroachdb/cockroach/issues/77119). The workaround provided by the database is enabling the `enable_durable_locking_for_serializable`-flag as described [here](https://github.com/cockroachdb/cockroach/issues/75456#issuecomment-1936277716). + +Because enabling flags requires admin privileges the statement must be executed manually or by executing `zitadel init` command. + +## Statement + +Ensure lock distribution for `FOR UPDATE`-statements on Cockroachdb. + +## Mitigation + +Cockroachdb version >= 23.2. + +## Impact + +Adding additional raft queries to `FOR UPDATE`-statements can impact performance slightly but ensures availability of the system. diff --git a/docs/docs/support/technical_advisory.mdx b/docs/docs/support/technical_advisory.mdx index d1a865f88d..565dae3904 100644 --- a/docs/docs/support/technical_advisory.mdx +++ b/docs/docs/support/technical_advisory.mdx @@ -149,11 +149,23 @@ We understand that these advisories may include breaking changes, and we aim to New flag to prefill projections during setup instead of after start Feature description - new flag `--init-projections` introduced to `zitadel setup` commands (`setup`, `start-from-setup`, `start-from-init`) + New flag `--init-projections` introduced to `zitadel setup` commands (`setup`, `start-from-setup`, `start-from-init`) 2.44.0, 2.43.6, 2.42.12 2024-01-25 + + + A-10009 + + Ensure lock distribution for `FOR UPDATE`-statements on Cockroachdb + Feature description + + Fixes rare cases where updating projections was blocked by a `WRITE_TOO_OLD`-error when using cockroachdb. + + 2.53.0 + 2024-05-27 + ## Subscribe to our Mailing List diff --git a/internal/eventstore/local_crdb_test.go b/internal/eventstore/local_crdb_test.go index fc7a35daf9..6df9e9fd29 100644 --- a/internal/eventstore/local_crdb_test.go +++ b/internal/eventstore/local_crdb_test.go @@ -88,7 +88,8 @@ func initDB(db *database.DB) error { err := initialise.Init(db, initialise.VerifyUser(config.Username(), ""), initialise.VerifyDatabase(config.DatabaseName()), - initialise.VerifyGrant(config.DatabaseName(), config.Username())) + initialise.VerifyGrant(config.DatabaseName(), config.Username()), + initialise.VerifySettings(config.DatabaseName(), config.Username())) if err != nil { return err } diff --git a/internal/eventstore/repository/sql/local_crdb_test.go b/internal/eventstore/repository/sql/local_crdb_test.go index ca5128aaf7..fccb169341 100644 --- a/internal/eventstore/repository/sql/local_crdb_test.go +++ b/internal/eventstore/repository/sql/local_crdb_test.go @@ -60,7 +60,8 @@ func initDB(db *database.DB) error { err := initialise.Init(db, initialise.VerifyUser(config.Username(), ""), initialise.VerifyDatabase(config.DatabaseName()), - initialise.VerifyGrant(config.DatabaseName(), config.Username())) + initialise.VerifyGrant(config.DatabaseName(), config.Username()), + initialise.VerifySettings(config.DatabaseName(), config.Username())) if err != nil { return err }