chore!: Introduce ZITADEL v3 (#9645)

This PR summarizes multiple changes specifically only available with ZITADEL v3: - feat: Web Keys management (https://github.com/zitadel/zitadel/pull/9526) - fix(cmd): ensure proper working of mirror (https://github.com/zitadel/zitadel/pull/9509) - feat(Authz): system user support for permission check v2 (https://github.com/zitadel/zitadel/pull/9640) - chore(license): change from Apache to AGPL (https://github.com/zitadel/zitadel/pull/9597) - feat(console): list v2 sessions (https://github.com/zitadel/zitadel/pull/9539) - fix(console): add loginV2 feature flag (https://github.com/zitadel/zitadel/pull/9682) - fix(feature flags): allow reading "own" flags (https://github.com/zitadel/zitadel/pull/9649) - feat(console): add Actions V2 UI (https://github.com/zitadel/zitadel/pull/9591) BREAKING CHANGE - feat(webkey): migrate to v2beta API (https://github.com/zitadel/zitadel/pull/9445) - chore!: remove CockroachDB Support (https://github.com/zitadel/zitadel/pull/9444) - feat(actions): migrate to v2beta API (https://github.com/zitadel/zitadel/pull/9489) --------- Co-authored-by: Livio Spring <livio.a@gmail.com> Co-authored-by: Stefan Benz <46600784+stebenz@users.noreply.github.com> Co-authored-by: Silvan <27845747+adlerhurst@users.noreply.github.com> Co-authored-by: Ramon <mail@conblem.me> Co-authored-by: Elio Bischof <elio@zitadel.com> Co-authored-by: Kenta Yamaguchi <56732734+KEY60228@users.noreply.github.com> Co-authored-by: Harsha Reddy <harsha.reddy@klaviyo.com> Co-authored-by: Livio Spring <livio@zitadel.com> Co-authored-by: Max Peintner <max@caos.ch> Co-authored-by: Iraq <66622793+kkrime@users.noreply.github.com> Co-authored-by: Florian Forster <florian@zitadel.com> Co-authored-by: Tim Möhlmann <tim+github@zitadel.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Max Peintner <peintnerm@gmail.com>
2025-08-11 18:07:31 +00:00 · 2025-04-02 16:53:06 +02:00
parent d14a23ae7e
commit 07ce3b6905
559 changed files with 14578 additions and 7622 deletions
--- a/cmd/mirror/config.go
+++ b/cmd/mirror/config.go
@@ -71,7 +71,7 @@ func mustNewConfig(v *viper.Viper, config any) {
 			mapstructure.StringToTimeDurationHookFunc(),
 			mapstructure.StringToTimeHookFunc(time.RFC3339),
 			mapstructure.StringToSliceHookFunc(","),
-			database.DecodeHook,
+			database.DecodeHook(true),
 			actions.HTTPConfigDecodeHook,
 			hook.EnumHookFunc(internal_authz.MemberTypeString),
 			mapstructure.TextUnmarshallerHookFunc(),
--- a/cmd/mirror/defaults.yaml
+++ b/cmd/mirror/defaults.yaml
@@ -5,8 +5,6 @@ Source:
    Database: zitadel # ZITADEL_DATABASE_COCKROACH_DATABASE
    MaxOpenConns: 6 # ZITADEL_DATABASE_COCKROACH_MAXOPENCONNS
    MaxIdleConns: 6 # ZITADEL_DATABASE_COCKROACH_MAXIDLECONNS
-    EventPushConnRatio: 0.33 # ZITADEL_DATABASE_COCKROACH_EVENTPUSHCONNRATIO
-    ProjectionSpoolerConnRatio: 0.33 # ZITADEL_DATABASE_COCKROACH_PROJECTIONSPOOLERCONNRATIO
    MaxConnLifetime: 30m # ZITADEL_DATABASE_COCKROACH_MAXCONNLIFETIME
    MaxConnIdleTime: 5m # ZITADEL_DATABASE_COCKROACH_MAXCONNIDLETIME
    Options: "" # ZITADEL_DATABASE_COCKROACH_OPTIONS
@@ -39,44 +37,23 @@ Source:
        Key: # ZITADEL_DATABASE_POSTGRES_USER_SSL_KEY

 Destination:
-  cockroach:
-    Host: localhost # ZITADEL_DATABASE_COCKROACH_HOST
-    Port: 26257 # ZITADEL_DATABASE_COCKROACH_PORT
-    Database: zitadel # ZITADEL_DATABASE_COCKROACH_DATABASE
-    MaxOpenConns: 0 # ZITADEL_DATABASE_COCKROACH_MAXOPENCONNS
-    MaxIdleConns: 0 # ZITADEL_DATABASE_COCKROACH_MAXIDLECONNS
-    MaxConnLifetime: 30m # ZITADEL_DATABASE_COCKROACH_MAXCONNLIFETIME
-    MaxConnIdleTime: 5m # ZITADEL_DATABASE_COCKROACH_MAXCONNIDLETIME
-    EventPushConnRatio: 0.01 # ZITADEL_DATABASE_COCKROACH_EVENTPUSHCONNRATIO
-    ProjectionSpoolerConnRatio: 0.5 # ZITADEL_DATABASE_COCKROACH_PROJECTIONSPOOLERCONNRATIO
-    Options: "" # ZITADEL_DATABASE_COCKROACH_OPTIONS
-    User:
-      Username: zitadel # ZITADEL_DATABASE_COCKROACH_USER_USERNAME
-      Password: "" # ZITADEL_DATABASE_COCKROACH_USER_PASSWORD
-      SSL:
-        Mode: disable # ZITADEL_DATABASE_COCKROACH_USER_SSL_MODE
-        RootCert: "" # ZITADEL_DATABASE_COCKROACH_USER_SSL_ROOTCERT
-        Cert: "" # ZITADEL_DATABASE_COCKROACH_USER_SSL_CERT
-        Key: "" # ZITADEL_DATABASE_COCKROACH_USER_SSL_KEY
-  # Postgres is used as soon as a value is set
-  # The values describe the possible fields to set values
  postgres:
-    Host: # ZITADEL_DATABASE_POSTGRES_HOST
-    Port: # ZITADEL_DATABASE_POSTGRES_PORT
-    Database: # ZITADEL_DATABASE_POSTGRES_DATABASE
-    MaxOpenConns: # ZITADEL_DATABASE_POSTGRES_MAXOPENCONNS
-    MaxIdleConns: # ZITADEL_DATABASE_POSTGRES_MAXIDLECONNS
-    MaxConnLifetime: # ZITADEL_DATABASE_POSTGRES_MAXCONNLIFETIME
-    MaxConnIdleTime: # ZITADEL_DATABASE_POSTGRES_MAXCONNIDLETIME
-    Options: # ZITADEL_DATABASE_POSTGRES_OPTIONS
+    Host: localhost # ZITADEL_DATABASE_POSTGRES_HOST
+    Port: 5432 # ZITADEL_DATABASE_POSTGRES_PORT
+    Database: zitadel # ZITADEL_DATABASE_POSTGRES_DATABASE
+    MaxOpenConns: 5 # ZITADEL_DATABASE_POSTGRES_MAXOPENCONNS
+    MaxIdleConns: 2 # ZITADEL_DATABASE_POSTGRES_MAXIDLECONNS
+    MaxConnLifetime: 30m # ZITADEL_DATABASE_POSTGRES_MAXCONNLIFETIME
+    MaxConnIdleTime: 5m # ZITADEL_DATABASE_POSTGRES_MAXCONNIDLETIME
+    Options: "" # ZITADEL_DATABASE_POSTGRES_OPTIONS
    User:
-      Username: # ZITADEL_DATABASE_POSTGRES_USER_USERNAME
-      Password: # ZITADEL_DATABASE_POSTGRES_USER_PASSWORD
+      Username: zitadel # ZITADEL_DATABASE_POSTGRES_USER_USERNAME
+      Password: "" # ZITADEL_DATABASE_POSTGRES_USER_PASSWORD
      SSL:
-        Mode: # ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE
-        RootCert: # ZITADEL_DATABASE_POSTGRES_USER_SSL_ROOTCERT
-        Cert: # ZITADEL_DATABASE_POSTGRES_USER_SSL_CERT
-        Key: # ZITADEL_DATABASE_POSTGRES_USER_SSL_KEY
+        Mode: disable # ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE
+        RootCert: "" # ZITADEL_DATABASE_POSTGRES_USER_SSL_ROOTCERT
+        Cert: "" # ZITADEL_DATABASE_POSTGRES_USER_SSL_CERT
+        Key: "" # ZITADEL_DATABASE_POSTGRES_USER_SSL_KEY

 EventBulkSize: 10000

--- a/cmd/mirror/event.go
+++ b/cmd/mirror/event.go
@@ -4,7 +4,6 @@ import (
 	"context"

 	"github.com/zitadel/zitadel/internal/v2/eventstore"
-	"github.com/zitadel/zitadel/internal/v2/projection"
 	"github.com/zitadel/zitadel/internal/v2/readmodel"
 	"github.com/zitadel/zitadel/internal/v2/system"
 	mirror_event "github.com/zitadel/zitadel/internal/v2/system/mirror"
@@ -30,39 +29,6 @@ func queryLastSuccessfulMigration(ctx context.Context, destinationES *eventstore
 	return lastSuccess, nil
 }

-func writeMigrationStart(ctx context.Context, sourceES *eventstore.EventStore, id string, destination string) (_ float64, err error) {
-	var cmd *eventstore.Command
-	if len(instanceIDs) > 0 {
-		cmd, err = mirror_event.NewStartedInstancesCommand(destination, instanceIDs)
-		if err != nil {
-			return 0, err
-		}
-	} else {
-		cmd = mirror_event.NewStartedSystemCommand(destination)
-	}
-
-	var position projection.HighestPosition
-
-	err = sourceES.Push(
-		ctx,
-		eventstore.NewPushIntent(
-			system.AggregateInstance,
-			eventstore.AppendAggregate(
-				system.AggregateOwner,
-				system.AggregateType,
-				id,
-				eventstore.CurrentSequenceMatches(0),
-				eventstore.AppendCommands(cmd),
-			),
-			eventstore.PushReducer(&position),
-		),
-	)
-	if err != nil {
-		return 0, err
-	}
-	return position.Position, nil
-}
-
 func writeMigrationSucceeded(ctx context.Context, destinationES *eventstore.EventStore, id, source string, position float64) error {
 	return destinationES.Push(
 		ctx,
--- a/cmd/mirror/event_store.go
+++ b/cmd/mirror/event_store.go
@@ -14,6 +14,7 @@ import (
 	"github.com/zitadel/logging"

 	db "github.com/zitadel/zitadel/internal/database"
+	"github.com/zitadel/zitadel/internal/database/dialect"
 	"github.com/zitadel/zitadel/internal/id"
 	"github.com/zitadel/zitadel/internal/v2/database"
 	"github.com/zitadel/zitadel/internal/v2/eventstore"
@@ -57,9 +58,9 @@ func copyEventstore(ctx context.Context, config *Migration) {

 func positionQuery(db *db.DB) string {
 	switch db.Type() {
-	case "postgres":
+	case dialect.DatabaseTypePostgres:
 		return "SELECT EXTRACT(EPOCH FROM clock_timestamp())"
-	case "cockroach":
+	case dialect.DatabaseTypeCockroach:
 		return "SELECT cluster_logical_timestamp()"
 	default:
 		logging.WithFields("db_type", db.Type()).Fatal("database type not recognized")
@@ -80,9 +81,6 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
 	destConn, err := dest.Conn(ctx)
 	logging.OnError(err).Fatal("unable to acquire dest connection")

-	sourceES := eventstore.NewEventstoreFromOne(postgres.New(source, &postgres.Config{
-		MaxRetries: 3,
-	}))
 	destinationES := eventstore.NewEventstoreFromOne(postgres.New(dest, &postgres.Config{
 		MaxRetries: 3,
 	}))
@@ -90,8 +88,14 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
 	previousMigration, err := queryLastSuccessfulMigration(ctx, destinationES, source.DatabaseName())
 	logging.OnError(err).Fatal("unable to query latest successful migration")

-	maxPosition, err := writeMigrationStart(ctx, sourceES, migrationID, dest.DatabaseName())
-	logging.OnError(err).Fatal("unable to write migration started event")
+	var maxPosition float64
+	err = source.QueryRowContext(ctx,
+		func(row *sql.Row) error {
+			return row.Scan(&maxPosition)
+		},
+		"SELECT MAX(position) FROM eventstore.events2 "+instanceClause(),
+	)
+	logging.OnError(err).Fatal("unable to query max position from source")

 	logging.WithFields("from", previousMigration.Position, "to", maxPosition).Info("start event migration")

--- a/cmd/mirror/mirror.go
+++ b/cmd/mirror/mirror.go
@@ -56,7 +56,6 @@ Order of execution:
 			copyEventstore(cmd.Context(), config)

 			projections(cmd.Context(), projectionConfig, masterKey)
-			verifyMigration(cmd.Context(), config)
 		},
 	}

--- a/cmd/mirror/projections.go
+++ b/cmd/mirror/projections.go
@@ -84,6 +84,7 @@ type ProjectionsConfig struct {
 	ExternalDomain  string
 	ExternalSecure  bool
 	InternalAuthZ   internal_authz.Config
+	SystemAuthZ     internal_authz.Config
 	SystemDefaults  systemdefaults.SystemDefaults
 	Telemetry       *handlers.TelemetryPusherConfig
 	Login           login.Config
@@ -117,8 +118,11 @@ func projections(
 	staticStorage, err := config.AssetStorage.NewStorage(client.DB)
 	logging.OnError(err).Fatal("unable create static storage")

-	config.Eventstore.Querier = old_es.NewCRDB(client)
-	config.Eventstore.Pusher = new_es.NewEventstore(client)
+	newEventstore := new_es.NewEventstore(client)
+	config.Eventstore.Querier = old_es.NewPostgres(client)
+	config.Eventstore.Pusher = newEventstore
+	config.Eventstore.Searcher = newEventstore
+
 	es := eventstore.NewEventstore(config.Eventstore)
 	esV4 := es_v4.NewEventstoreFromOne(es_v4_pg.New(client, &es_v4_pg.Config{
 		MaxRetries: config.Eventstore.MaxRetries,
@@ -147,7 +151,7 @@ func projections(
 		sessionTokenVerifier,
 		func(q *query.Queries) domain.PermissionCheck {
 			return func(ctx context.Context, permission, orgID, resourceID string) (err error) {
-				return internal_authz.CheckPermission(ctx, &authz_es.UserMembershipRepo{Queries: q}, config.InternalAuthZ.RolePermissionMappings, permission, orgID, resourceID)
+				return internal_authz.CheckPermission(ctx, &authz_es.UserMembershipRepo{Queries: q}, config.SystemAuthZ.RolePermissionMappings, config.InternalAuthZ.RolePermissionMappings, permission, orgID, resourceID)
 			}
 		},
 		0,
@@ -184,7 +188,7 @@ func projections(
 		keys.Target,
 		&http.Client{},
 		func(ctx context.Context, permission, orgID, resourceID string) (err error) {
-			return internal_authz.CheckPermission(ctx, authZRepo, config.InternalAuthZ.RolePermissionMappings, permission, orgID, resourceID)
+			return internal_authz.CheckPermission(ctx, authZRepo, config.SystemAuthZ.RolePermissionMappings, config.InternalAuthZ.RolePermissionMappings, permission, orgID, resourceID)
 		},
 		sessionTokenVerifier,
 		config.OIDC.DefaultAccessTokenLifetime,
@@ -220,7 +224,6 @@ func projections(
 		keys.SMS,
 		keys.OIDC,
 		config.OIDC.DefaultBackChannelLogoutLifetime,
-		client,
 		nil,
 	)

@@ -248,7 +251,7 @@ func projections(
 		}
 	}()

-	for i := 0; i < int(config.Projections.ConcurrentInstances); i++ {
+	for range int(config.Projections.ConcurrentInstances) {
 		go execProjections(ctx, instances, failedInstances, &wg)
 	}

@@ -270,31 +273,39 @@ func execProjections(ctx context.Context, instances <-chan string, failedInstanc

 		err := projection.ProjectInstance(ctx)
 		if err != nil {
-			logging.WithFields("instance", instance).OnError(err).Info("trigger failed")
+			logging.WithFields("instance", instance).WithError(err).Info("trigger failed")
+			failedInstances <- instance
+			continue
+		}
+
+		err = projection.ProjectInstanceFields(ctx)
+		if err != nil {
+			logging.WithFields("instance", instance).WithError(err).Info("trigger fields failed")
 			failedInstances <- instance
 			continue
 		}

 		err = admin_handler.ProjectInstance(ctx)
 		if err != nil {
-			logging.WithFields("instance", instance).OnError(err).Info("trigger admin handler failed")
+			logging.WithFields("instance", instance).WithError(err).Info("trigger admin handler failed")
 			failedInstances <- instance
 			continue
 		}

 		err = auth_handler.ProjectInstance(ctx)
 		if err != nil {
-			logging.WithFields("instance", instance).OnError(err).Info("trigger auth handler failed")
+			logging.WithFields("instance", instance).WithError(err).Info("trigger auth handler failed")
 			failedInstances <- instance
 			continue
 		}

 		err = notification.ProjectInstance(ctx)
 		if err != nil {
-			logging.WithFields("instance", instance).OnError(err).Info("trigger notification failed")
+			logging.WithFields("instance", instance).WithError(err).Info("trigger notification failed")
 			failedInstances <- instance
 			continue
 		}
+
 		logging.WithFields("instance", instance).Info("projections done")
 	}
 	wg.Done()