fix: scheduling (#3978)

* fix: improve scheduling

* build pre-release

* fix: locker

* fix: user handler and print stack in case of panic in reducer

* chore: remove sentry

* fix: improve handler projection and implement tests

* more tests

* fix: race condition in tests

* Update internal/eventstore/repository/sql/query.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* fix: implemented suggested changes

* fix: lock statement

Co-authored-by: Silvan <silvan.reusser@gmail.com>

internal/query/app.go

@@ -208,7 +208,7 @@ var (
 
 func (q *Queries) AppByProjectAndAppID(ctx context.Context, shouldTriggerBulk bool, projectID, appID string) (*App, error) {
 	if shouldTriggerBulk {
-		projection.AppProjection.TriggerBulk(ctx)
+		projection.AppProjection.Trigger(ctx)
 	}
 
 	stmt, scan := prepareAppQuery()

internal/query/authn_key.go

@@ -124,7 +124,7 @@ func (q *Queries) SearchAuthNKeys(ctx context.Context, queries *AuthNKeySearchQu
 
 func (q *Queries) GetAuthNKeyByID(ctx context.Context, shouldTriggerBulk bool, id string, queries ...SearchQuery) (*AuthNKey, error) {
 	if shouldTriggerBulk {
-		projection.AuthNKeyProjection.TriggerBulk(ctx)
+		projection.AuthNKeyProjection.Trigger(ctx)
 	}
 
 	query, scan := prepareAuthNKeyQuery()

internal/query/domain_policy.go

@@ -82,7 +82,7 @@ var (
 
 func (q *Queries) DomainPolicyByOrg(ctx context.Context, shouldTriggerBulk bool, orgID string) (*DomainPolicy, error) {
 	if shouldTriggerBulk {
-		projection.DomainPolicyProjection.TriggerBulk(ctx)
+		projection.DomainPolicyProjection.Trigger(ctx)
 	}
 
 	stmt, scan := prepareDomainPolicyQuery()

internal/query/failed_events.go

@@ -115,7 +115,7 @@ func (q *FailedEventSearchQueries) toQuery(query sq.SelectBuilder) sq.SelectBuil
 	return query
 }
 
-func prepareFailedEventQuery() (sq.SelectBuilder, func(*sql.Row) (*FailedEvent, error)) {
+func prepareFailedEventQuery(instanceIDs ...string) (sq.SelectBuilder, func(*sql.Row) (*FailedEvent, error)) {
 	return sq.Select(
 			FailedEventsColumnProjectionName.identifier(),
 			FailedEventsColumnFailedSequence.identifier(),

internal/query/idp.go

@@ -182,7 +182,7 @@ var (
 //IDPByIDAndResourceOwner searches for the requested id in the context of the resource owner and IAM
 func (q *Queries) IDPByIDAndResourceOwner(ctx context.Context, shouldTriggerBulk bool, id, resourceOwner string) (*IDP, error) {
 	if shouldTriggerBulk {
-		projection.IDPProjection.TriggerBulk(ctx)
+		projection.IDPProjection.Trigger(ctx)
 	}
 
 	stmt, scan := prepareIDPByIDQuery()

internal/query/instance.go

@@ -159,7 +159,7 @@ func (q *Queries) SearchInstances(ctx context.Context, queries *InstanceSearchQu
 
 func (q *Queries) Instance(ctx context.Context, shouldTriggerBulk bool) (*Instance, error) {
 	if shouldTriggerBulk {
-		projection.InstanceProjection.TriggerBulk(ctx)
+		projection.InstanceProjection.Trigger(ctx)
 	}
 
 	stmt, scan := prepareInstanceDomainQuery(authz.GetInstance(ctx).RequestedDomain())

internal/query/lockout_policy.go

@@ -77,7 +77,7 @@ var (
 
 func (q *Queries) LockoutPolicyByOrg(ctx context.Context, shouldTriggerBulk bool, orgID string) (*LockoutPolicy, error) {
 	if shouldTriggerBulk {
-		projection.LockoutPolicyProjection.TriggerBulk(ctx)
+		projection.LockoutPolicyProjection.Trigger(ctx)
 	}
 
 	stmt, scan := prepareLockoutPolicyQuery()

internal/query/login_policy.go

@@ -141,7 +141,7 @@ var (
 
 func (q *Queries) LoginPolicyByID(ctx context.Context, shouldTriggerBulk bool, orgID string) (*LoginPolicy, error) {
 	if shouldTriggerBulk {
-		projection.LoginPolicyProjection.TriggerBulk(ctx)
+		projection.LoginPolicyProjection.Trigger(ctx)
 	}
 
 	query, scan := prepareLoginPolicyQuery()

internal/query/org.go

@@ -88,7 +88,7 @@ func (q *OrgSearchQueries) toQuery(query sq.SelectBuilder) sq.SelectBuilder {
 
 func (q *Queries) OrgByID(ctx context.Context, shouldTriggerBulk bool, id string) (*Org, error) {
 	if shouldTriggerBulk {
-		projection.OrgProjection.TriggerBulk(ctx)
+		projection.OrgProjection.Trigger(ctx)
 	}
 
 	stmt, scan := prepareOrgQuery()

internal/query/password_age_policy.go

@@ -76,7 +76,7 @@ var (
 
 func (q *Queries) PasswordAgePolicyByOrg(ctx context.Context, shouldTriggerBulk bool, orgID string) (*PasswordAgePolicy, error) {
 	if shouldTriggerBulk {
-		projection.PasswordAgeProjection.TriggerBulk(ctx)
+		projection.PasswordAgeProjection.Trigger(ctx)
 	}
 
 	stmt, scan := preparePasswordAgePolicyQuery()
@@ -106,7 +106,7 @@ func (q *Queries) PasswordAgePolicyByOrg(ctx context.Context, shouldTriggerBulk
 
 func (q *Queries) DefaultPasswordAgePolicy(ctx context.Context, shouldTriggerBulk bool) (*PasswordAgePolicy, error) {
 	if shouldTriggerBulk {
-		projection.PasswordAgeProjection.TriggerBulk(ctx)
+		projection.PasswordAgeProjection.Trigger(ctx)
 	}
 
 	stmt, scan := preparePasswordAgePolicyQuery()

internal/query/password_complexity_policy.go

@@ -33,7 +33,7 @@ type PasswordComplexityPolicy struct {
 
 func (q *Queries) PasswordComplexityPolicyByOrg(ctx context.Context, shouldTriggerBulk bool, orgID string) (*PasswordComplexityPolicy, error) {
 	if shouldTriggerBulk {
-		projection.PasswordComplexityProjection.TriggerBulk(ctx)
+		projection.PasswordComplexityProjection.Trigger(ctx)
 	}
 
 	stmt, scan := preparePasswordComplexityPolicyQuery()
@@ -63,7 +63,7 @@ func (q *Queries) PasswordComplexityPolicyByOrg(ctx context.Context, shouldTrigg
 
 func (q *Queries) DefaultPasswordComplexityPolicy(ctx context.Context, shouldTriggerBulk bool) (*PasswordComplexityPolicy, error) {
 	if shouldTriggerBulk {
-		projection.PasswordComplexityProjection.TriggerBulk(ctx)
+		projection.PasswordComplexityProjection.Trigger(ctx)
 	}
 
 	stmt, scan := preparePasswordComplexityPolicyQuery()

internal/query/privacy_policy.go

@@ -81,7 +81,7 @@ var (
 
 func (q *Queries) PrivacyPolicyByOrg(ctx context.Context, shouldTriggerBulk bool, orgID string) (*PrivacyPolicy, error) {
 	if shouldTriggerBulk {
-		projection.PrivacyPolicyProjection.TriggerBulk(ctx)
+		projection.PrivacyPolicyProjection.Trigger(ctx)
 	}
 
 	stmt, scan := preparePrivacyPolicyQuery()
@@ -111,7 +111,7 @@ func (q *Queries) PrivacyPolicyByOrg(ctx context.Context, shouldTriggerBulk bool
 
 func (q *Queries) DefaultPrivacyPolicy(ctx context.Context, shouldTriggerBulk bool) (*PrivacyPolicy, error) {
 	if shouldTriggerBulk {
-		projection.PrivacyPolicyProjection.TriggerBulk(ctx)
+		projection.PrivacyPolicyProjection.Trigger(ctx)
 	}
 
 	stmt, scan := preparePrivacyPolicyQuery()

internal/query/project.go

@@ -96,7 +96,7 @@ type ProjectSearchQueries struct {
 
 func (q *Queries) ProjectByID(ctx context.Context, shouldTriggerBulk bool, id string) (*Project, error) {
 	if shouldTriggerBulk {
-		projection.ProjectProjection.TriggerBulk(ctx)
+		projection.ProjectProjection.Trigger(ctx)
 	}
 
 	stmt, scan := prepareProjectQuery()

internal/query/project_grant.go

@@ -103,7 +103,7 @@ type ProjectGrantSearchQueries struct {
 
 func (q *Queries) ProjectGrantByID(ctx context.Context, shouldTriggerBulk bool, id string) (*ProjectGrant, error) {
 	if shouldTriggerBulk {
-		projection.ProjectGrantProjection.TriggerBulk(ctx)
+		projection.ProjectGrantProjection.Trigger(ctx)
 	}
 
 	stmt, scan := prepareProjectGrantQuery()

internal/query/project_role.go

@@ -78,7 +78,7 @@ type ProjectRoleSearchQueries struct {
 
 func (q *Queries) SearchProjectRoles(ctx context.Context, shouldTriggerBulk bool, queries *ProjectRoleSearchQueries) (projects *ProjectRoles, err error) {
 	if shouldTriggerBulk {
-		projection.ProjectRoleProjection.TriggerBulk(ctx)
+		projection.ProjectRoleProjection.Trigger(ctx)
 	}
 
 	query, scan := prepareProjectRolesQuery()

internal/query/projection/config.go

@@ -5,17 +5,19 @@ import (
 )
 
 type Config struct {
-	RequeueEvery     time.Duration
-	RetryFailedAfter time.Duration
-	MaxFailureCount  uint
-	BulkLimit        uint64
-	Customizations   map[string]CustomConfig
-	MaxIterators     int
+	RequeueEvery        time.Duration
+	RetryFailedAfter    time.Duration
+	MaxFailureCount     uint
+	ConcurrentInstances uint
+	BulkLimit           uint64
+	Customizations      map[string]CustomConfig
+	MaxIterators        int
 }
 
 type CustomConfig struct {
-	RequeueEvery     *time.Duration
-	RetryFailedAfter *time.Duration
-	MaxFailureCount  *uint
-	BulkLimit        *uint64
+	RequeueEvery        *time.Duration
+	RetryFailedAfter    *time.Duration
+	MaxFailureCount     *uint
+	ConcurrentInstances *uint
+	BulkLimit           *uint64
 }

internal/query/projection/key.go

@@ -83,8 +83,8 @@ func newKeyProjection(ctx context.Context, config crdb.StatementHandlerConfig, k
 			crdb.WithForeignKey(crdb.NewForeignKeyOfPublicKeys("fk_public_ref_keys")),
 		),
 	)
-	p.StatementHandler = crdb.NewStatementHandler(ctx, config)
 	p.encryptionAlgorithm = keyEncryptionAlgorithm
+	p.StatementHandler = crdb.NewStatementHandler(ctx, config)
 
 	return p
 }

internal/query/projection/projection.go

@@ -68,8 +68,10 @@ func Start(ctx context.Context, sqlClient *sql.DB, es *eventstore.Eventstore, co
 			HandlerConfig: handler.HandlerConfig{
 				Eventstore: es,
 			},
-			RequeueEvery:     config.RequeueEvery,
-			RetryFailedAfter: config.RetryFailedAfter,
+			RequeueEvery:        config.RequeueEvery,
+			RetryFailedAfter:    config.RetryFailedAfter,
+			Retries:             config.MaxFailureCount,
+			ConcurrentInstances: config.ConcurrentInstances,
 		},
 		Client:            sqlClient,
 		SequenceTable:     CurrentSeqTable,

internal/query/user.go

@@ -294,8 +294,8 @@ var (
 
 func (q *Queries) GetUserByID(ctx context.Context, shouldTriggerBulk bool, userID string, queries ...SearchQuery) (*User, error) {
 	if shouldTriggerBulk {
-		projection.UserProjection.TriggerBulk(ctx)
-		projection.LoginNameProjection.TriggerBulk(ctx)
+		projection.UserProjection.Trigger(ctx)
+		projection.LoginNameProjection.Trigger(ctx)
 	}
 
 	instanceID := authz.GetInstance(ctx).InstanceID()
@@ -317,8 +317,8 @@ func (q *Queries) GetUserByID(ctx context.Context, shouldTriggerBulk bool, userI
 
 func (q *Queries) GetUser(ctx context.Context, shouldTriggerBulk bool, queries ...SearchQuery) (*User, error) {
 	if shouldTriggerBulk {
-		projection.UserProjection.TriggerBulk(ctx)
-		projection.LoginNameProjection.TriggerBulk(ctx)
+		projection.UserProjection.Trigger(ctx)
+		projection.LoginNameProjection.Trigger(ctx)
 	}
 
 	instanceID := authz.GetInstance(ctx).InstanceID()
@@ -390,8 +390,8 @@ func (q *Queries) GetHumanPhone(ctx context.Context, userID string, queries ...S
 
 func (q *Queries) GeNotifyUser(ctx context.Context, shouldTriggered bool, userID string, queries ...SearchQuery) (*NotifyUser, error) {
 	if shouldTriggered {
-		projection.UserProjection.TriggerBulk(ctx)
-		projection.LoginNameProjection.TriggerBulk(ctx)
+		projection.UserProjection.Trigger(ctx)
+		projection.LoginNameProjection.Trigger(ctx)
 	}
 
 	instanceID := authz.GetInstance(ctx).InstanceID()

internal/query/user_grant.go

@@ -193,7 +193,7 @@ var (
 
 func (q *Queries) UserGrant(ctx context.Context, shouldTriggerBulk bool, queries ...SearchQuery) (*UserGrant, error) {
 	if shouldTriggerBulk {
-		projection.UserGrantProjection.TriggerBulk(ctx)
+		projection.UserGrantProjection.Trigger(ctx)
 	}
 
 	query, scan := prepareUserGrantQuery()

internal/query/user_metadata.go

@@ -73,7 +73,7 @@ var (
 
 func (q *Queries) GetUserMetadataByKey(ctx context.Context, shouldTriggerBulk bool, userID, key string, queries ...SearchQuery) (*UserMetadata, error) {
 	if shouldTriggerBulk {
-		projection.UserMetadataProjection.TriggerBulk(ctx)
+		projection.UserMetadataProjection.Trigger(ctx)
 	}
 
 	query, scan := prepareUserMetadataQuery()
@@ -96,7 +96,7 @@ func (q *Queries) GetUserMetadataByKey(ctx context.Context, shouldTriggerBulk bo
 
 func (q *Queries) SearchUserMetadata(ctx context.Context, shouldTriggerBulk bool, userID string, queries *UserMetadataSearchQueries) (*UserMetadataList, error) {
 	if shouldTriggerBulk {
-		projection.UserMetadataProjection.TriggerBulk(ctx)
+		projection.UserMetadataProjection.Trigger(ctx)
 	}
 
 	query, scan := prepareUserMetadataListQuery()

internal/query/user_personal_access_token.go

@@ -82,7 +82,7 @@ type PersonalAccessTokenSearchQueries struct {
 
 func (q *Queries) PersonalAccessTokenByID(ctx context.Context, shouldTriggerBulk bool, id string, queries ...SearchQuery) (*PersonalAccessToken, error) {
 	if shouldTriggerBulk {
-		projection.PersonalAccessTokenProjection.TriggerBulk(ctx)
+		projection.PersonalAccessTokenProjection.Trigger(ctx)
 	}
 
 	query, scan := preparePersonalAccessTokenQuery()