fix(setup): init projections (#7194)

Even though this is a feature it's released as fix so that we can back port to earlier revisions.

As reported by multiple users startup of ZITADEL after leaded to downtime and worst case rollbacks to the previously deployed version.

The problem starts rising when there are too many events to process after the start of ZITADEL. The root cause are changes on projections (database tables) which must be recomputed. This PR solves this problem by adding a new step to the setup phase which prefills the projections. The step can be enabled by adding the `--init-projections`-flag to `setup`, `start-from-init` and `start-from-setup`. Setting this flag results in potentially longer duration of the setup phase but reduces the risk of the problems mentioned in the paragraph above.

internal/query/projection/eventstore_mock_test.go

@@ -38,6 +38,13 @@ func (m *mockEventStore) Filter(ctx context.Context, queryFactory *eventstore.Se
 	return m.filterResponse[m.filterCounter-1], nil
 }
 
+func (m *mockEventStore) FilterToQueryReducer(ctx context.Context, reducer eventstore.QueryReducer) error {
+	m.filterCounter++
+	events := m.filterResponse[m.filterCounter-1]
+	reducer.AppendEvents(events...)
+	return reducer.Reduce()
+}
+
 func (m *mockEventStore) Push(ctx context.Context, cmds ...eventstore.Command) ([]eventstore.Event, error) {
 	m.pushCounter++
 	return m.pushResponse[m.pushCounter-1], nil

internal/query/projection/main_test.go

@@ -8,15 +8,6 @@ import (
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/eventstore/repository"
 	"github.com/zitadel/zitadel/internal/eventstore/repository/mock"
-	action_repo "github.com/zitadel/zitadel/internal/repository/action"
-	iam_repo "github.com/zitadel/zitadel/internal/repository/instance"
-	key_repo "github.com/zitadel/zitadel/internal/repository/keypair"
-	"github.com/zitadel/zitadel/internal/repository/limits"
-	"github.com/zitadel/zitadel/internal/repository/org"
-	proj_repo "github.com/zitadel/zitadel/internal/repository/project"
-	quota_repo "github.com/zitadel/zitadel/internal/repository/quota"
-	usr_repo "github.com/zitadel/zitadel/internal/repository/user"
-	"github.com/zitadel/zitadel/internal/repository/usergrant"
 )
 
 type expect func(mockRepository *mock.MockRepository)
@@ -32,15 +23,6 @@ func eventstoreExpect(t *testing.T, expects ...expect) *eventstore.Eventstore {
 			Pusher:  m.MockPusher,
 		},
 	)
-	iam_repo.RegisterEventMappers(es)
-	org.RegisterEventMappers(es)
-	usr_repo.RegisterEventMappers(es)
-	proj_repo.RegisterEventMappers(es)
-	quota_repo.RegisterEventMappers(es)
-	limits.RegisterEventMappers(es)
-	usergrant.RegisterEventMappers(es)
-	key_repo.RegisterEventMappers(es)
-	action_repo.RegisterEventMappers(es)
 	return es
 }
 

internal/query/projection/projection.go

@@ -7,6 +7,7 @@ import (
 	"github.com/zitadel/zitadel/internal/crypto"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/eventstore/handler/v2"
+	"github.com/zitadel/zitadel/internal/migration"
 )
 
 const (
@@ -76,6 +77,8 @@ var (
 type projection interface {
 	Start(ctx context.Context)
 	Init(ctx context.Context) error
+	Trigger(ctx context.Context, opts ...handler.TriggerOpt) (_ context.Context, err error)
+	migration.Migration
 }
 
 var (
@@ -149,6 +152,10 @@ func Create(ctx context.Context, sqlClient *database.DB, es handler.EventStore,
 	return nil
 }
 
+func Projections() []projection {
+	return projections
+}
+
 func Init(ctx context.Context) error {
 	for _, p := range projections {
 		if err := p.Init(ctx); err != nil {

internal/query/query.go

@@ -18,21 +18,6 @@ import (
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/eventstore/handler/v2"
 	"github.com/zitadel/zitadel/internal/query/projection"
-	"github.com/zitadel/zitadel/internal/repository/action"
-	"github.com/zitadel/zitadel/internal/repository/authrequest"
-	"github.com/zitadel/zitadel/internal/repository/deviceauth"
-	"github.com/zitadel/zitadel/internal/repository/idpintent"
-	iam_repo "github.com/zitadel/zitadel/internal/repository/instance"
-	"github.com/zitadel/zitadel/internal/repository/keypair"
-	"github.com/zitadel/zitadel/internal/repository/limits"
-	"github.com/zitadel/zitadel/internal/repository/oidcsession"
-	"github.com/zitadel/zitadel/internal/repository/org"
-	"github.com/zitadel/zitadel/internal/repository/project"
-	"github.com/zitadel/zitadel/internal/repository/quota"
-	"github.com/zitadel/zitadel/internal/repository/restrictions"
-	"github.com/zitadel/zitadel/internal/repository/session"
-	usr_repo "github.com/zitadel/zitadel/internal/repository/user"
-	"github.com/zitadel/zitadel/internal/repository/usergrant"
 	"github.com/zitadel/zitadel/internal/telemetry/tracing"
 )
 
@@ -67,6 +52,7 @@ func StartQueries(
 	permissionCheck func(q *Queries) domain.PermissionCheck,
 	defaultAuditLogRetention time.Duration,
 	systemAPIUsers map[string]*authz.SystemAPIUser,
+	startProjections bool,
 ) (repo *Queries, err error) {
 	repo = &Queries{
 		eventstore:                          es,
@@ -86,21 +72,6 @@ func StartQueries(
 		},
 		defaultAuditLogRetention: defaultAuditLogRetention,
 	}
-	iam_repo.RegisterEventMappers(repo.eventstore)
-	usr_repo.RegisterEventMappers(repo.eventstore)
-	org.RegisterEventMappers(repo.eventstore)
-	project.RegisterEventMappers(repo.eventstore)
-	action.RegisterEventMappers(repo.eventstore)
-	keypair.RegisterEventMappers(repo.eventstore)
-	usergrant.RegisterEventMappers(repo.eventstore)
-	session.RegisterEventMappers(repo.eventstore)
-	idpintent.RegisterEventMappers(repo.eventstore)
-	authrequest.RegisterEventMappers(repo.eventstore)
-	oidcsession.RegisterEventMappers(repo.eventstore)
-	quota.RegisterEventMappers(repo.eventstore)
-	limits.RegisterEventMappers(repo.eventstore)
-	restrictions.RegisterEventMappers(repo.eventstore)
-	deviceauth.RegisterEventMappers(repo.eventstore)
 
 	repo.checkPermission = permissionCheck(repo)
 
@@ -108,7 +79,9 @@ func StartQueries(
 	if err != nil {
 		return nil, err
 	}
-	projection.Start(ctx)
+	if startProjections {
+		projection.Start(ctx)
+	}
 
 	return repo, nil
 }

internal/query/query_test.go

@@ -10,21 +10,6 @@ import (
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/eventstore/repository"
 	"github.com/zitadel/zitadel/internal/eventstore/repository/mock"
-	action_repo "github.com/zitadel/zitadel/internal/repository/action"
-	"github.com/zitadel/zitadel/internal/repository/authrequest"
-	"github.com/zitadel/zitadel/internal/repository/deviceauth"
-	"github.com/zitadel/zitadel/internal/repository/feature"
-	"github.com/zitadel/zitadel/internal/repository/idpintent"
-	iam_repo "github.com/zitadel/zitadel/internal/repository/instance"
-	key_repo "github.com/zitadel/zitadel/internal/repository/keypair"
-	"github.com/zitadel/zitadel/internal/repository/limits"
-	"github.com/zitadel/zitadel/internal/repository/oidcsession"
-	"github.com/zitadel/zitadel/internal/repository/org"
-	proj_repo "github.com/zitadel/zitadel/internal/repository/project"
-	quota_repo "github.com/zitadel/zitadel/internal/repository/quota"
-	"github.com/zitadel/zitadel/internal/repository/session"
-	usr_repo "github.com/zitadel/zitadel/internal/repository/user"
-	"github.com/zitadel/zitadel/internal/repository/usergrant"
 )
 
 type expect func(mockRepository *mock.MockRepository)
@@ -41,21 +26,6 @@ func expectEventstore(expects ...expect) func(*testing.T) *eventstore.Eventstore
 				Pusher:  m.MockPusher,
 			},
 		)
-		iam_repo.RegisterEventMappers(es)
-		org.RegisterEventMappers(es)
-		usr_repo.RegisterEventMappers(es)
-		proj_repo.RegisterEventMappers(es)
-		usergrant.RegisterEventMappers(es)
-		key_repo.RegisterEventMappers(es)
-		action_repo.RegisterEventMappers(es)
-		session.RegisterEventMappers(es)
-		idpintent.RegisterEventMappers(es)
-		authrequest.RegisterEventMappers(es)
-		oidcsession.RegisterEventMappers(es)
-		quota_repo.RegisterEventMappers(es)
-		limits.RegisterEventMappers(es)
-		feature.RegisterEventMappers(es)
-		deviceauth.RegisterEventMappers(es)
 		return es
 	}
 }