fix(init): add setting to enable durable locks on crdb (#7982)

feat(init): add setting to enable durable locks on crdb

cmd/initialise/init.go

@@ -19,6 +19,7 @@ var (
 
 	createUserStmt           string
 	grantStmt                string
+	settingsStmt             string
 	databaseStmt             string
 	createEventstoreStmt     string
 	createProjectionsStmt    string
@@ -53,7 +54,7 @@ The user provided by flags needs privileges to
 		},
 	}
 
-	cmd.AddCommand(newZitadel(), newDatabase(), newUser(), newGrant())
+	cmd.AddCommand(newZitadel(), newDatabase(), newUser(), newGrant(), newSettings())
 	return cmd
 }
 
@@ -62,6 +63,7 @@ func InitAll(ctx context.Context, config *Config) {
 		VerifyUser(config.Database.Username(), config.Database.Password()),
 		VerifyDatabase(config.Database.DatabaseName()),
 		VerifyGrant(config.Database.DatabaseName(), config.Database.Username()),
+		VerifySettings(config.Database.DatabaseName(), config.Database.Username()),
 	)
 	logging.OnError(err).Fatal("unable to initialize the database")
 
@@ -147,6 +149,11 @@ func ReadStmts(typ string) (err error) {
 		return err
 	}
 
+	settingsStmt, err = readStmt(typ, "11_settings")
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 

cmd/initialise/sql/cockroach/11_settings.sql

@@ -0,0 +1,4 @@
+-- replace the first %[1]q with the database in double quotes
+-- replace the second \%[2]q with the user in double quotes$
+-- For more information see technical advisory 10009 (https://zitadel.com/docs/support/advisory/a10009)
+ALTER ROLE %[2]q IN DATABASE %[1]q SET enable_durable_locking_for_serializable = on;

cmd/initialise/verify_settings.go

@@ -0,0 +1,44 @@
+package initialise
+
+import (
+	_ "embed"
+	"fmt"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"github.com/zitadel/logging"
+
+	"github.com/zitadel/zitadel/internal/database"
+)
+
+func newSettings() *cobra.Command {
+	return &cobra.Command{
+		Use:   "settings",
+		Short: "Ensures proper settings on the database",
+		Long: `Ensures proper settings on the database.
+
+Prerequisites:
+- cockroachDB or postgreSQL
+
+Cockroach
+- Sets enable_durable_locking_for_serializable to on for the zitadel user and database
+`,
+		Run: func(cmd *cobra.Command, args []string) {
+			config := MustNewConfig(viper.GetViper())
+
+			err := initialise(config.Database, VerifySettings(config.Database.DatabaseName(), config.Database.Username()))
+			logging.OnError(err).Fatal("unable to set settings")
+		},
+	}
+}
+
+func VerifySettings(databaseName, username string) func(*database.DB) error {
+	return func(db *database.DB) error {
+		if db.Type() == "postgres" {
+			return nil
+		}
+		logging.WithFields("user", username, "database", databaseName).Info("verify settings")
+
+		return exec(db, fmt.Sprintf(settingsStmt, databaseName, username), nil)
+	}
+}

docs/docs/self-hosting/manage/database/_cockroachdb.mdx

@@ -2,6 +2,8 @@
 
 The default database of ZITADEL is [CockroachDB](https://www.cockroachlabs.com). The SQL database provides a bunch of features like horizontal scalability, data regionality and many more.
 
+Currently versions >= 23.2 are supported.
+
 The default configuration of the database looks like this:
 
 ```yaml

docs/docs/support/advisory/a10009.md

@@ -0,0 +1,27 @@
+---
+title: Technical Advisory 10009
+---
+
+## Date and Version
+
+Version: 2.53.0
+
+Date: Calendar week 23/24 2024
+
+## Description
+
+There were rare cases where Cockroachdb got blocked during runtime of ZITADEL and returned `WRITE_TOO_OLD`-errors to ZITADEL. The root cause of the problem is described in [this github issue of the database](https://github.com/cockroachdb/cockroach/issues/77119). The workaround provided by the database is enabling the `enable_durable_locking_for_serializable`-flag as described [here](https://github.com/cockroachdb/cockroach/issues/75456#issuecomment-1936277716).
+
+Because enabling flags requires admin privileges the statement must be executed manually or by executing `zitadel init` command.
+
+## Statement
+
+Ensure lock distribution for `FOR UPDATE`-statements on Cockroachdb.
+
+## Mitigation
+
+Cockroachdb version >= 23.2.
+
+## Impact
+
+Adding additional raft queries to `FOR UPDATE`-statements can impact performance slightly but ensures availability of the system.

docs/docs/support/technical_advisory.mdx

@@ -149,11 +149,23 @@ We understand that these advisories may include breaking changes, and we aim to
     <td>New flag to prefill projections during setup instead of after start</td>
     <td>Feature description</td>
     <td>
-      new flag `--init-projections` introduced to `zitadel setup` commands (`setup`, `start-from-setup`, `start-from-init`)
+      New flag `--init-projections` introduced to `zitadel setup` commands (`setup`, `start-from-setup`, `start-from-init`)
     </td>
     <td>2.44.0, 2.43.6, 2.42.12</td>
     <td>2024-01-25</td>
   </tr>
+  <tr>
+    <td>
+      <a href="./advisory/a10009">A-10009</a>
+    </td>
+    <td>Ensure lock distribution for `FOR UPDATE`-statements on Cockroachdb</td>
+    <td>Feature description</td>
+    <td>
+      Fixes rare cases where updating projections was blocked by a `WRITE_TOO_OLD`-error when using cockroachdb.
+    </td>
+    <td>2.53.0</td>
+    <td>2024-05-27</td>
+  </tr>
 </table>
 
 ## Subscribe to our Mailing List

internal/eventstore/local_crdb_test.go

@@ -88,7 +88,8 @@ func initDB(db *database.DB) error {
 	err := initialise.Init(db,
 		initialise.VerifyUser(config.Username(), ""),
 		initialise.VerifyDatabase(config.DatabaseName()),
-		initialise.VerifyGrant(config.DatabaseName(), config.Username()))
+		initialise.VerifyGrant(config.DatabaseName(), config.Username()),
+		initialise.VerifySettings(config.DatabaseName(), config.Username()))
 	if err != nil {
 		return err
 	}

internal/eventstore/repository/sql/local_crdb_test.go

@@ -60,7 +60,8 @@ func initDB(db *database.DB) error {
 	err := initialise.Init(db,
 		initialise.VerifyUser(config.Username(), ""),
 		initialise.VerifyDatabase(config.DatabaseName()),
-		initialise.VerifyGrant(config.DatabaseName(), config.Username()))
+		initialise.VerifyGrant(config.DatabaseName(), config.Username()),
+		initialise.VerifySettings(config.DatabaseName(), config.Username()))
 	if err != nil {
 		return err
 	}