feat(eventstore): increase parallel write capabilities (#5940)

This implementation increases parallel write capabilities of the eventstore.
Please have a look at the technical advisories: [05](https://zitadel.com/docs/support/advisory/a10005) and  [06](https://zitadel.com/docs/support/advisory/a10006).
The implementation of eventstore.push is rewritten and stored events are migrated to a new table `eventstore.events2`.
If you are using cockroach: make sure that the database user of ZITADEL has `VIEWACTIVITY` grant. This is used to query events.
This commit is contained in:
Silvan
2023-10-19 12:19:10 +02:00
committed by GitHub
parent 259faba3f0
commit b5564572bc
791 changed files with 30326 additions and 43202 deletions

View File

@@ -5,56 +5,55 @@ import (
"time"
"github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/view"
v1 "github.com/zitadel/zitadel/internal/eventstore/v1"
"github.com/zitadel/zitadel/internal/eventstore/v1/query"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore"
handler2 "github.com/zitadel/zitadel/internal/eventstore/handler/v2"
"github.com/zitadel/zitadel/internal/static"
)
type Configs map[string]*Config
type Config struct {
Client *database.DB
Eventstore *eventstore.Eventstore
BulkLimit uint64
FailureCountUntilSkip uint64
HandleActiveInstances time.Duration
TransactionDuration time.Duration
Handlers map[string]*ConfigOverwrites
}
type ConfigOverwrites struct {
MinimumCycleDuration time.Duration
}
type handler struct {
view *view.View
bulkLimit uint64
cycleDuration time.Duration
errorCountUntilSkip uint64
es v1.Eventstore
}
func (h *handler) Eventstore() v1.Eventstore {
return h.es
}
func Register(ctx context.Context, configs Configs, bulkLimit, errorCount uint64, view *view.View, es v1.Eventstore, static static.Storage) []query.Handler {
handlers := []query.Handler{}
if static != nil {
handlers = append(handlers, newStyling(ctx,
handler{view, bulkLimit, configs.cycleDuration("Styling"), errorCount, es},
static))
func Register(ctx context.Context, config Config, view *view.View, static static.Storage) {
if static == nil {
return
}
return handlers
newStyling(ctx,
config.overwrite("Styling"),
static,
view,
).Start(ctx)
}
func (configs Configs) cycleDuration(viewModel string) time.Duration {
c, ok := configs[viewModel]
func (config Config) overwrite(viewModel string) handler2.Config {
c := handler2.Config{
Client: config.Client,
Eventstore: config.Eventstore,
BulkLimit: uint16(config.BulkLimit),
RequeueEvery: 3 * time.Minute,
HandleActiveInstances: config.HandleActiveInstances,
MaxFailureCount: uint8(config.FailureCountUntilSkip),
TransactionDuration: config.TransactionDuration,
}
overwrite, ok := config.Handlers[viewModel]
if !ok {
return 3 * time.Minute
return c
}
return c.MinimumCycleDuration
}
func (h *handler) MinimumCycleDuration() time.Duration {
return h.cycleDuration
}
func (h *handler) LockDuration() time.Duration {
return h.cycleDuration / 3
}
func (h *handler) QueryLimit() uint64 {
return h.bulkLimit
if overwrite.MinimumCycleDuration > 0 {
c.RequeueEvery = overwrite.MinimumCycleDuration
}
return c
}