TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-03-01 00:07:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: use fixed active instances duration (#5567)

Browse Source 
* fix: use fixed active instances duration

* fix active instances tests

* fix syntax error

* run pipeline

---------

Co-authored-by: Silvan <silvan.reusser@gmail.com>
This commit is contained in:
Elio Bischof 2023-03-30 13:01:27 +02:00 committed by GitHub
parent 9b30d6ad83
commit 887e2f474d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 107 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
cmd/defaults.yaml
View File

@ -159,10 +159,10 @@ Projections:
ConcurrentInstances: 1
# Limit of returned events per query
BulkLimit: 200
# If HandleInactiveInstances this is false, only instances are projected,
# for which at least a projection relevant event exists withing the timeframe
# from twice the RequeueEvery time in the past until the projections current time
HandleInactiveInstances: false
# Only instance are projected, for which at least a projection relevant event exists withing the timeframe
# from HandleActiveInstances duration in the past until the projections current time
# Defaults to twice the RequeueEvery duration
HandleActiveInstances: 120s
# In the Customizations section, all settings from above can be overwritten for each specific projection
Customizations:
Projects:
@ -174,7 +174,8 @@ Projections:
# The NotificationsQuotas projection is used for calling quota webhooks
NotificationsQuotas:
# Delivery guarantee requirements are probably higher for quota webhooks
HandleInactiveInstances: true
# Defaults to 45 days
HandleActiveInstances: 1080h
# As quota notification projections don't result in database statements, retries don't have an effect
MaxFailureCount: 0
# Quota notifications are not so time critical. Setting RequeueEvery every five minutes doesn't annoy the db too much.

13
docs/docs/self-hosting/manage/production.md
View File

@ -84,14 +84,14 @@ Projections:
RetryFailedAfter: 1s
# Retried execution number of database statements resulting from projected events
MaxFailureCount: 5
# Number of concurrent projection routines
# Number of concurrent projection routines. Values of 0 and below are overwritten to 1
ConcurrentInstances: 1
# Limit of returned events per query
BulkLimit: 200
# If HandleInactiveInstances this is false, only instances are projected,
# for which at least a projection relevant event exists withing the timeframe
# from twice the RequeueEvery time in the past until the projections current time
HandleInactiveInstances: false
# Only instance are projected, for which at least a projection relevant event exists withing the timeframe
# from HandleActiveInstances duration in the past until the projections current time
# Defaults to twice the RequeueEvery duration
HandleActiveInstances: 120s
# In the Customizations section, all settings from above can be overwritten for each specific projection
Customizations:
Projects:
@ -103,7 +103,8 @@ Projections:
# The NotificationsQuotas projection is used for calling quota webhooks
NotificationsQuotas:
# Delivery guarantee requirements are probably higher for quota webhooks
HandleInactiveInstances: true
# Defaults to 45 days
HandleActiveInstances: 1080h
# As quota notification projections don't result in database statements, retries don't have an effect
MaxFailureCount: 0
# Quota notifications are not so time critical. Setting RequeueEvery every five minutes doesn't annoy the db too much.

12
internal/eventstore/handler/handler_projection.go
View File

@ -25,7 +25,7 @@ type ProjectionHandlerConfig struct {
RetryFailedAfter time.Duration
Retries uint
ConcurrentInstances uint
HandleInactiveInstances bool
HandleActiveInstances time.Duration
}
// Update updates the projection with the given statements
@ -60,7 +60,7 @@ type ProjectionHandler struct {
retryFailedAfter time.Duration
retries int
concurrentInstances int
handleInactiveInstances bool
handleActiveInstances time.Duration
nowFunc NowFunc
}
@ -91,7 +91,7 @@ func NewProjectionHandler(
retryFailedAfter: config.RetryFailedAfter,
retries: int(config.Retries),
concurrentInstances: concurrentInstances,
handleInactiveInstances: config.HandleInactiveInstances,
handleActiveInstances: config.HandleActiveInstances,
nowFunc: time.Now,
}
@ -229,11 +229,11 @@ func (h *ProjectionHandler) schedule(ctx context.Context) {
}
go h.cancelOnErr(lockCtx, errs, cancelLock)
}
if succeededOnce && !h.handleInactiveInstances {
if succeededOnce {
// since we have at least one successful run, we can restrict it to events not older than
// twice the requeue time (just to be sure not to miss an event)
// h.handleActiveInstances (just to be sure not to miss an event)
// This ensures that only instances with recent events on the handler are projected
query = query.CreationDateAfter(h.nowFunc().Add(-2 * h.requeueAfter))
query = query.CreationDateAfter(h.nowFunc().Add(-1 * h.handleActiveInstances))
}
ids, err := h.Eventstore.InstanceIDs(ctx, query.Builder())
if err != nil {

22
internal/eventstore/handler/handler_projection_test.go
View File

@ -674,7 +674,7 @@ func TestProjection_schedule(t *testing.T) {
lock *lockMock
unlock *unlockMock
query SearchQuery
handleInactiveInstances bool
handleActiveInstances time.Duration
}
type want struct {
locksCount int
@ -711,7 +711,7 @@ func TestProjection_schedule(t *testing.T) {
),
)
},
handleInactiveInstances: false,
handleActiveInstances: 2 * time.Minute,
},
want{
locksCount: 0,
@ -739,7 +739,7 @@ func TestProjection_schedule(t *testing.T) {
),
)
},
handleInactiveInstances: false,
handleActiveInstances: 2 * time.Minute,
},
want{
locksCount: 0,
@ -771,7 +771,7 @@ func TestProjection_schedule(t *testing.T) {
firstErr: ErrLock,
canceled: make(chan bool, 1),
},
handleInactiveInstances: false,
handleActiveInstances: 2 * time.Minute,
},
want{
locksCount: 1,
@ -805,7 +805,7 @@ func TestProjection_schedule(t *testing.T) {
},
unlock: &unlockMock{},
query: testQuery(nil, 0, ErrQuery),
handleInactiveInstances: false,
handleActiveInstances: 2 * time.Minute,
},
want{
locksCount: 1,
@ -837,7 +837,7 @@ func TestProjection_schedule(t *testing.T) {
}, {
Field: repository.FieldCreationDate,
Operation: repository.OperationGreater,
Value: now().Add(-2 * time.Hour),
Value: now().Add(-2 * time.Minute),
}},
"206626268110651755",
).
@ -856,7 +856,7 @@ func TestProjection_schedule(t *testing.T) {
errWait: 100 * time.Millisecond,
},
unlock: &unlockMock{},
handleInactiveInstances: false,
handleActiveInstances: 2 * time.Minute,
reduce: testReduce(newTestStatement("aggregate1", 1, 0)),
update: testUpdate(t, 1, 1, nil),
query: testQuery(
@ -894,6 +894,10 @@ func TestProjection_schedule(t *testing.T) {
Field: repository.FieldInstanceID,
Operation: repository.OperationNotIn,
Value: database.StringArray{""},
}, {
Field: repository.FieldCreationDate,
Operation: repository.OperationGreater,
Value: now().Add(-45 * time.Hour),
}}, "206626268110651755").
ExpectFilterEvents(&repository.Event{
AggregateType: "quota",
@ -910,7 +914,7 @@ func TestProjection_schedule(t *testing.T) {
errWait: 100 * time.Millisecond,
},
unlock: &unlockMock{},
handleInactiveInstances: true,
handleActiveInstances: 45 * time.Hour,
reduce: testReduce(newTestStatement("aggregate1", 1, 0)),
update: testUpdate(t, 1, 1, nil),
query: testQuery(
@ -944,7 +948,7 @@ func TestProjection_schedule(t *testing.T) {
triggerProjection: time.NewTimer(0), // immediately run an iteration
requeueAfter: time.Hour, // run only one iteration
concurrentInstances: 1,
handleInactiveInstances: tt.fields.handleInactiveInstances,
handleActiveInstances: tt.fields.handleActiveInstances,
retries: 0,
nowFunc: now,
}

2
internal/logstore/service_test.go
View File

@ -252,7 +252,7 @@ func given(t *testing.T, args args, want want) (context.Context, *clock.Mock, *e
svc := logstore.New(
quotaqueriermock.NewNoopQuerier(&args.config, periodStart),
logstore.UsageReporterFunc(func(context.Context, []*quota.NotifiedEvent) error { return nil }),
logstore.UsageReporterFunc(func(context.Context, []*quota.NotificationDueEvent) error { return nil }),
mainEmitter,
secondaryEmitter)

4
internal/query/projection/config.go
View File

@ -11,7 +11,7 @@ type Config struct {
ConcurrentInstances uint
BulkLimit uint64
Customizations map[string]CustomConfig
HandleInactiveInstances bool
HandleActiveInstances time.Duration
}
type CustomConfig struct {
@ -20,5 +20,5 @@ type CustomConfig struct {
MaxFailureCount *uint
ConcurrentInstances *uint
BulkLimit *uint64
HandleInactiveInstances *bool
HandleActiveInstances *time.Duration
}

6
internal/query/projection/projection.go
View File

@ -85,7 +85,7 @@ func Create(ctx context.Context, sqlClient *database.DB, es *eventstore.Eventsto
RetryFailedAfter: config.RetryFailedAfter,
Retries: config.MaxFailureCount,
ConcurrentInstances: config.ConcurrentInstances,
HandleInactiveInstances: config.HandleInactiveInstances,
HandleActiveInstances: config.HandleActiveInstances,
},
Client: sqlClient,
SequenceTable: CurrentSeqTable,
@ -175,8 +175,8 @@ func applyCustomConfig(config crdb.StatementHandlerConfig, customConfig CustomCo
if customConfig.RetryFailedAfter != nil {
config.RetryFailedAfter = *customConfig.RetryFailedAfter
}
if customConfig.HandleInactiveInstances != nil {
config.HandleInactiveInstances = *customConfig.HandleInactiveInstances
if customConfig.HandleActiveInstances != nil {
config.HandleActiveInstances = *customConfig.HandleActiveInstances
}
return config