TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-11 18:17:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(eventstore): increase parallel write capabilities (#5940)

Browse Source 
This implementation increases parallel write capabilities of the eventstore.
Please have a look at the technical advisories: [05](https://zitadel.com/docs/support/advisory/a10005) and  [06](https://zitadel.com/docs/support/advisory/a10006).
The implementation of eventstore.push is rewritten and stored events are migrated to a new table `eventstore.events2`.
If you are using cockroach: make sure that the database user of ZITADEL has `VIEWACTIVITY` grant. This is used to query events.
This commit is contained in:
Silvan
2023-10-19 12:19:10 +02:00
committed by GitHub
parent 259faba3f0
commit b5564572bc
791 changed files with 30326 additions and 43202 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/initialise/helper.go
View File

@@ -1,13 +1,13 @@
package initialise
import (
"database/sql"
"errors"
"github.com/jackc/pgconn"
"github.com/zitadel/zitadel/internal/database"
)
func exec(db *sql.DB, stmt string, possibleErrCodes []string, args ...interface{}) error {
func exec(db *database.DB, stmt string, possibleErrCodes []string, args ...interface{}) error {
_, err := db.Exec(stmt, args...)
pgErr := new(pgconn.PgError)
if errors.As(err, &pgErr) {

9
cmd/initialise/init.go
View File

@@ -1,7 +1,6 @@
package initialise
import (
"database/sql"
"embed"
"github.com/spf13/cobra"
@@ -68,7 +67,7 @@ func InitAll(config *Config) {
logging.OnError(err).Fatal("unable to initialize ZITADEL")
}
func initialise(config database.Config, steps ...func(*sql.DB) error) error {
func initialise(config database.Config, steps ...func(*database.DB) error) error {
logging.Info("initialization started")
err := ReadStmts(config.Type())
@@ -76,16 +75,16 @@ func initialise(config database.Config, steps ...func(*sql.DB) error) error {
return err
}
db, err := database.Connect(config, true)
db, err := database.Connect(config, true, false)
if err != nil {
return err
}
defer db.Close()
return Init(db.DB, steps...)
return Init(db, steps...)
}
func Init(db *sql.DB, steps ...func(*sql.DB) error) error {
func Init(db *database.DB, steps ...func(*database.DB) error) error {
for _, step := range steps {
if err := step(db); err != nil {
return err

20
cmd/initialise/init_test.go
View File

@@ -1,17 +1,17 @@
package initialise
import (
"database/sql"
"database/sql/driver"
"regexp"
"testing"
"github.com/DATA-DOG/go-sqlmock"
"github.com/zitadel/zitadel/internal/database"
)
type db struct {
mock sqlmock.Sqlmock
db *sql.DB
db *database.DB
}
func prepareDB(t *testing.T, expectations ...expectation) db {
@@ -25,7 +25,7 @@ func prepareDB(t *testing.T, expectations ...expectation) db {
}
return db{
mock: mock,
db: client,
db: &database.DB{DB: client},
}
}
@@ -42,6 +42,20 @@ func expectExec(stmt string, err error, args ...driver.Value) expectation {
}
}
func expectQuery(stmt string, err error, columns []string, rows [][]driver.Value, args ...driver.Value) expectation {
return func(m sqlmock.Sqlmock) {
res := sqlmock.NewRows(columns)
for _, row := range rows {
res.AddRow(row...)
}
query := m.ExpectQuery(regexp.QuoteMeta(stmt)).WithArgs(args...).WillReturnRows(res)
if err != nil {
query.WillReturnError(err)
return
}
}
}
func expectBegin(err error) expectation {
return func(m sqlmock.Sqlmock) {
query := m.ExpectBegin()

3
cmd/initialise/sql/cockroach/03_grant_user.sql
View File

@@ -1,3 +1,4 @@
-- replace the first %[1]s with the database
-- replace the second \%[2]s with the user
GRANT ALL ON DATABASE %[1]s TO %[2]s
GRANT ALL ON DATABASE %[1]s TO %[2]s;
GRANT SYSTEM VIEWACTIVITY TO %[2]s;

46
cmd/initialise/sql/cockroach/08_events_table.sql
View File

@@ -1,27 +1,21 @@
SET experimental_enable_hash_sharded_indexes = on;
CREATE TABLE IF NOT EXISTS eventstore.events2 (
instance_id TEXT NOT NULL
, aggregate_type TEXT NOT NULL
, aggregate_id TEXT NOT NULL
, event_type TEXT NOT NULL
, "sequence" BIGINT NOT NULL
, revision SMALLINT NOT NULL
, created_at TIMESTAMPTZ NOT NULL
, payload JSONB
, creator TEXT NOT NULL
, "owner" TEXT NOT NULL
, "position" DECIMAL NOT NULL
, in_tx_order INTEGER NOT NULL
CREATE TABLE IF NOT EXISTS eventstore.events (
id UUID DEFAULT gen_random_uuid()
, event_type TEXT NOT NULL
, aggregate_type TEXT NOT NULL
, aggregate_id TEXT NOT NULL
, aggregate_version TEXT NOT NULL
, event_sequence BIGINT NOT NULL
, previous_aggregate_sequence BIGINT
, previous_aggregate_type_sequence INT8
, creation_date TIMESTAMPTZ NOT NULL DEFAULT now()
, event_data JSONB
, editor_user TEXT NOT NULL
, editor_service TEXT NOT NULL
, resource_owner TEXT NOT NULL
, instance_id TEXT NOT NULL
, PRIMARY KEY (event_sequence DESC, instance_id) USING HASH WITH BUCKET_COUNT = 10
, INDEX agg_type_agg_id (aggregate_type, aggregate_id, instance_id)
, INDEX agg_type (aggregate_type, instance_id)
, INDEX agg_type_seq (aggregate_type, event_sequence DESC, instance_id)
STORING (id, event_type, aggregate_id, aggregate_version, previous_aggregate_sequence, creation_date, event_data, editor_user, editor_service, resource_owner, previous_aggregate_type_sequence)
, INDEX max_sequence (aggregate_type, aggregate_id, event_sequence DESC, instance_id)
, CONSTRAINT previous_sequence_unique UNIQUE (previous_aggregate_sequence DESC, instance_id)
, CONSTRAINT prev_agg_type_seq_unique UNIQUE(previous_aggregate_type_sequence, instance_id)
);
, PRIMARY KEY (instance_id, aggregate_type, aggregate_id, "sequence")
, INDEX es_active_instances (created_at DESC) STORING ("position")
, INDEX es_wm (aggregate_id, instance_id, aggregate_type, event_type)
, INDEX es_projection (instance_id, aggregate_type, event_type, "position" DESC)
);

41
cmd/initialise/sql/postgres/08_events_table.sql
View File

@@ -1,25 +1,22 @@
CREATE TABLE IF NOT EXISTS eventstore.events (
id UUID DEFAULT gen_random_uuid()
, event_type TEXT NOT NULL
, aggregate_type TEXT NOT NULL
, aggregate_id TEXT NOT NULL
, aggregate_version TEXT NOT NULL
, event_sequence BIGINT NOT NULL
, previous_aggregate_sequence BIGINT
, previous_aggregate_type_sequence INT8
, creation_date TIMESTAMPTZ NOT NULL DEFAULT now()
, event_data JSONB
, editor_user TEXT NOT NULL
, editor_service TEXT NOT NULL
, resource_owner TEXT NOT NULL
, instance_id TEXT NOT NULL
CREATE TABLE IF NOT EXISTS eventstore.events2 (
instance_id TEXT NOT NULL
, aggregate_type TEXT NOT NULL
, aggregate_id TEXT NOT NULL
, event_type TEXT NOT NULL
, "sequence" BIGINT NOT NULL
, revision SMALLINT NOT NULL
, created_at TIMESTAMPTZ NOT NULL
, payload JSONB
, creator TEXT NOT NULL
, "owner" TEXT NOT NULL
, "position" DECIMAL NOT NULL
, in_tx_order INTEGER NOT NULL
, PRIMARY KEY (event_sequence, instance_id)
, CONSTRAINT previous_sequence_unique UNIQUE(previous_aggregate_sequence, instance_id)
, CONSTRAINT prev_agg_type_seq_unique UNIQUE(previous_aggregate_type_sequence, instance_id)
, PRIMARY KEY (instance_id, aggregate_type, aggregate_id, "sequence")
);
CREATE INDEX IF NOT EXISTS agg_type_agg_id ON eventstore.events (aggregate_type, aggregate_id, instance_id);
CREATE INDEX IF NOT EXISTS agg_type ON eventstore.events (aggregate_type, instance_id);
CREATE INDEX IF NOT EXISTS agg_type_seq ON eventstore.events (aggregate_type, event_sequence DESC, instance_id);
CREATE INDEX IF NOT EXISTS max_sequence ON eventstore.events (aggregate_type, aggregate_id, event_sequence DESC, instance_id);
CREATE INDEX IF NOT EXISTS es_active_instances ON eventstore.events2 (created_at DESC, instance_id);
CREATE INDEX IF NOT EXISTS es_wm ON eventstore.events2 (aggregate_id, instance_id, aggregate_type, event_type);
CREATE INDEX IF NOT EXISTS es_projection ON eventstore.events2 (instance_id, aggregate_type, event_type, "position");

7
cmd/initialise/verify_database.go
View File

@@ -1,13 +1,14 @@
package initialise
import (
"database/sql"
_ "embed"
"fmt"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/database"
)
func newDatabase() *cobra.Command {
@@ -33,8 +34,8 @@ The user provided by flags needs priviledge to
}
}
func VerifyDatabase(databaseName string) func(*sql.DB) error {
return func(db *sql.DB) error {
func VerifyDatabase(databaseName string) func(*database.DB) error {
return func(db *database.DB) error {
logging.WithFields("database", databaseName).Info("verify database")
return exec(db, fmt.Sprintf(string(databaseStmt), databaseName), []string{dbAlreadyExistsCode})

7
cmd/initialise/verify_grant.go
View File

@@ -1,13 +1,14 @@
package initialise
import (
"database/sql"
_ "embed"
"fmt"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/database"
)
func newGrant() *cobra.Command {
@@ -28,8 +29,8 @@ Prereqesits:
}
}
func VerifyGrant(databaseName, username string) func(*sql.DB) error {
return func(db *sql.DB) error {
func VerifyGrant(databaseName, username string) func(*database.DB) error {
return func(db *database.DB) error {
logging.WithFields("user", username, "database", databaseName).Info("verify grant")
return exec(db, fmt.Sprintf(grantStmt, databaseName, username), nil)

7
cmd/initialise/verify_user.go
View File

@@ -1,13 +1,14 @@
package initialise
import (
"database/sql"
_ "embed"
"fmt"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/database"
)
func newUser() *cobra.Command {
@@ -33,8 +34,8 @@ The user provided by flags needs priviledge to
}
}
func VerifyUser(username, password string) func(*sql.DB) error {
return func(db *sql.DB) error {
func VerifyUser(username, password string) func(*database.DB) error {
return func(db *database.DB) error {
logging.WithFields("username", username).Info("verify user")
if password != "" {

42
cmd/initialise/verify_zitadel.go
View File

@@ -1,14 +1,13 @@
package initialise
import (
"database/sql"
_ "embed"
"fmt"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/database"
)
@@ -29,58 +28,66 @@ Prereqesits:
}
}
func VerifyZitadel(db *sql.DB, config database.Config) error {
func VerifyZitadel(db *database.DB, config database.Config) error {
err := ReadStmts(config.Type())
if err != nil {
return err
}
logging.WithFields().Info("verify system")
if err := exec(db, fmt.Sprintf(createSystemStmt, config.Username()), nil); err != nil {
return err
}
logging.WithFields().Info("verify encryption keys")
if err := createEncryptionKeys(db); err != nil {
return err
}
logging.WithFields().Info("verify projections")
if err := exec(db, fmt.Sprintf(createProjectionsStmt, config.Username()), nil); err != nil {
return err
}
logging.WithFields().Info("verify eventstore")
if err := exec(db, fmt.Sprintf(createEventstoreStmt, config.Username()), nil); err != nil {
return err
}
logging.WithFields().Info("verify events tables")
if err := createEvents(db); err != nil {
return err
}
logging.WithFields().Info("verify system sequence")
if err := exec(db, createSystemSequenceStmt, nil); err != nil {
return err
}
logging.WithFields().Info("verify unique constraints")
if err := exec(db, createUniqueConstraints, nil); err != nil {
return err
}
return nil
}
func verifyZitadel(config database.Config) error {
logging.WithFields("database", config.DatabaseName()).Info("verify zitadel")
db, err := database.Connect(config, false)
db, err := database.Connect(config, false, false)
if err != nil {
return err
}
if err := VerifyZitadel(db.DB, config); err != nil {
if err := VerifyZitadel(db, config); err != nil {
return err
}
return db.Close()
}
func createEncryptionKeys(db *sql.DB) error {
func createEncryptionKeys(db *database.DB) error {
tx, err := db.Begin()
if err != nil {
return err
@@ -93,16 +100,29 @@ func createEncryptionKeys(db *sql.DB) error {
return tx.Commit()
}
func createEvents(db *sql.DB) error {
func createEvents(db *database.DB) (err error) {
tx, err := db.Begin()
if err != nil {
return err
}
defer func() {
if err != nil {
rollbackErr := tx.Rollback()
logging.OnError(rollbackErr).Debug("rollback failed")
return
}
err = tx.Commit()
}()
if _, err = tx.Exec(createEventsStmt); err != nil {
tx.Rollback()
// if events already exists events2 is created during a setup job
var count int
row := tx.QueryRow("SELECT count(*) FROM information_schema.tables WHERE table_schema = 'eventstore' AND table_name like 'events%'")
if err = row.Scan(&count); err != nil {
return err
}
return tx.Commit()
if row.Err() != nil || count >= 1 {
return row.Err()
}
_, err = tx.Exec(createEventsStmt)
return err
}

51
cmd/initialise/verify_zitadel_test.go
View File

@@ -2,6 +2,7 @@ package initialise
import (
"database/sql"
"database/sql/driver"
"errors"
"testing"
)
@@ -30,11 +31,53 @@ func Test_verifyEvents(t *testing.T) {
},
targetErr: sql.ErrConnDone,
},
{
name: "events already exists",
args: args{
db: prepareDB(t,
expectBegin(nil),
expectQuery(
"SELECT count(*) FROM information_schema.tables WHERE table_schema = 'eventstore' AND table_name like 'events%'",
nil,
[]string{"count"},
[][]driver.Value{
{1},
},
),
expectCommit(nil),
),
},
},
{
name: "events and events2 already exists",
args: args{
db: prepareDB(t,
expectBegin(nil),
expectQuery(
"SELECT count(*) FROM information_schema.tables WHERE table_schema = 'eventstore' AND table_name like 'events%'",
nil,
[]string{"count"},
[][]driver.Value{
{2},
},
),
expectCommit(nil),
),
},
},
{
name: "create table fails",
args: args{
db: prepareDB(t,
expectBegin(nil),
expectQuery(
"SELECT count(*) FROM information_schema.tables WHERE table_schema = 'eventstore' AND table_name like 'events%'",
nil,
[]string{"count"},
[][]driver.Value{
{0},
},
),
expectExec(createEventsStmt, sql.ErrNoRows),
expectRollback(nil),
),
@@ -46,6 +89,14 @@ func Test_verifyEvents(t *testing.T) {
args: args{
db: prepareDB(t,
expectBegin(nil),
expectQuery(
"SELECT count(*) FROM information_schema.tables WHERE table_schema = 'eventstore' AND table_name like 'events%'",
nil,
[]string{"count"},
[][]driver.Value{
{0},
},
),
expectExec(createEventsStmt, nil),
expectCommit(nil),
),