mirror of
https://github.com/zitadel/zitadel.git
synced 2025-08-11 19:17:32 +00:00
feat(eventstore): increase parallel write capabilities (#5940)
This implementation increases parallel write capabilities of the eventstore. Please have a look at the technical advisories: [05](https://zitadel.com/docs/support/advisory/a10005) and [06](https://zitadel.com/docs/support/advisory/a10006). The implementation of eventstore.push is rewritten and stored events are migrated to a new table `eventstore.events2`. If you are using cockroach: make sure that the database user of ZITADEL has `VIEWACTIVITY` grant. This is used to query events.
This commit is contained in:
Silvan
@@ -5,56 +5,55 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/view"
|
||||
v1 "github.com/zitadel/zitadel/internal/eventstore/v1"
|
||||
"github.com/zitadel/zitadel/internal/eventstore/v1/query"
|
||||
"github.com/zitadel/zitadel/internal/database"
|
||||
"github.com/zitadel/zitadel/internal/eventstore"
|
||||
handler2 "github.com/zitadel/zitadel/internal/eventstore/handler/v2"
|
||||
"github.com/zitadel/zitadel/internal/static"
|
||||
)
|
||||
|
||||
type Configs map[string]*Config
|
||||
|
||||
type Config struct {
|
||||
Client *database.DB
|
||||
Eventstore *eventstore.Eventstore
|
||||
|
||||
BulkLimit uint64
|
||||
FailureCountUntilSkip uint64
|
||||
HandleActiveInstances time.Duration
|
||||
TransactionDuration time.Duration
|
||||
Handlers map[string]*ConfigOverwrites
|
||||
}
|
||||
|
||||
type ConfigOverwrites struct {
|
||||
MinimumCycleDuration time.Duration
|
||||
}
|
||||
|
||||
type handler struct {
|
||||
view *view.View
|
||||
bulkLimit uint64
|
||||
cycleDuration time.Duration
|
||||
errorCountUntilSkip uint64
|
||||
|
||||
es v1.Eventstore
|
||||
}
|
||||
|
||||
func (h *handler) Eventstore() v1.Eventstore {
|
||||
return h.es
|
||||
}
|
||||
|
||||
func Register(ctx context.Context, configs Configs, bulkLimit, errorCount uint64, view *view.View, es v1.Eventstore, static static.Storage) []query.Handler {
|
||||
handlers := []query.Handler{}
|
||||
if static != nil {
|
||||
handlers = append(handlers, newStyling(ctx,
|
||||
handler{view, bulkLimit, configs.cycleDuration("Styling"), errorCount, es},
|
||||
static))
|
||||
func Register(ctx context.Context, config Config, view *view.View, static static.Storage) {
|
||||
if static == nil {
|
||||
return
|
||||
}
|
||||
return handlers
|
||||
|
||||
newStyling(ctx,
|
||||
config.overwrite("Styling"),
|
||||
static,
|
||||
view,
|
||||
).Start(ctx)
|
||||
}
|
||||
|
||||
func (configs Configs) cycleDuration(viewModel string) time.Duration {
|
||||
c, ok := configs[viewModel]
|
||||
func (config Config) overwrite(viewModel string) handler2.Config {
|
||||
c := handler2.Config{
|
||||
Client: config.Client,
|
||||
Eventstore: config.Eventstore,
|
||||
BulkLimit: uint16(config.BulkLimit),
|
||||
RequeueEvery: 3 * time.Minute,
|
||||
HandleActiveInstances: config.HandleActiveInstances,
|
||||
MaxFailureCount: uint8(config.FailureCountUntilSkip),
|
||||
TransactionDuration: config.TransactionDuration,
|
||||
}
|
||||
overwrite, ok := config.Handlers[viewModel]
|
||||
if !ok {
|
||||
return 3 * time.Minute
|
||||
return c
|
||||
}
|
||||
return c.MinimumCycleDuration
|
||||
}
|
||||
|
||||
func (h *handler) MinimumCycleDuration() time.Duration {
|
||||
return h.cycleDuration
|
||||
}
|
||||
|
||||
func (h *handler) LockDuration() time.Duration {
|
||||
return h.cycleDuration / 3
|
||||
}
|
||||
|
||||
func (h *handler) QueryLimit() uint64 {
|
||||
return h.bulkLimit
|
||||
if overwrite.MinimumCycleDuration > 0 {
|
||||
c.RequeueEvery = overwrite.MinimumCycleDuration
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -9,15 +9,12 @@ import (
|
||||
|
||||
"github.com/lucasb-eyer/go-colorful"
|
||||
"github.com/muesli/gamut"
|
||||
"github.com/zitadel/logging"
|
||||
|
||||
admin_view "github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/view"
|
||||
"github.com/zitadel/zitadel/internal/api/ui/login"
|
||||
"github.com/zitadel/zitadel/internal/domain"
|
||||
"github.com/zitadel/zitadel/internal/eventstore"
|
||||
v1 "github.com/zitadel/zitadel/internal/eventstore/v1"
|
||||
"github.com/zitadel/zitadel/internal/eventstore/v1/models"
|
||||
"github.com/zitadel/zitadel/internal/eventstore/v1/query"
|
||||
"github.com/zitadel/zitadel/internal/eventstore/v1/spooler"
|
||||
"github.com/zitadel/zitadel/internal/eventstore/handler/v2"
|
||||
iam_model "github.com/zitadel/zitadel/internal/iam/repository/view/model"
|
||||
"github.com/zitadel/zitadel/internal/repository/instance"
|
||||
"github.com/zitadel/zitadel/internal/repository/org"
|
||||
@@ -28,152 +25,231 @@ const (
|
||||
stylingTable = "adminapi.styling2"
|
||||
)
|
||||
|
||||
var _ handler.Projection = (*Styling)(nil)
|
||||
|
||||
type Styling struct {
|
||||
handler
|
||||
static static.Storage
|
||||
subscription *v1.Subscription
|
||||
static static.Storage
|
||||
view *admin_view.View
|
||||
}
|
||||
|
||||
func newStyling(ctx context.Context, handler handler, static static.Storage) *Styling {
|
||||
h := &Styling{
|
||||
handler: handler,
|
||||
static: static,
|
||||
}
|
||||
h.subscribe(ctx)
|
||||
|
||||
return h
|
||||
func newStyling(ctx context.Context, config handler.Config, static static.Storage, view *admin_view.View) *handler.Handler {
|
||||
return handler.NewHandler(
|
||||
ctx,
|
||||
&config,
|
||||
&Styling{
|
||||
static: static,
|
||||
view: view,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func (m *Styling) subscribe(ctx context.Context) {
|
||||
m.subscription = m.es.Subscribe(m.AggregateTypes()...)
|
||||
go func() {
|
||||
for event := range m.subscription.Events {
|
||||
query.ReduceEvent(ctx, m, event)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (m *Styling) ViewModel() string {
|
||||
// Name implements [handler.Projection]
|
||||
func (*Styling) Name() string {
|
||||
return stylingTable
|
||||
}
|
||||
|
||||
func (m *Styling) Subscription() *v1.Subscription {
|
||||
return m.subscription
|
||||
}
|
||||
|
||||
func (_ *Styling) AggregateTypes() []models.AggregateType {
|
||||
return []models.AggregateType{org.AggregateType, instance.AggregateType}
|
||||
}
|
||||
|
||||
func (m *Styling) CurrentSequence(ctx context.Context, instanceID string) (uint64, error) {
|
||||
sequence, err := m.view.GetLatestStylingSequence(ctx, instanceID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
// Reducers implements [handler.Projection]
|
||||
func (s *Styling) Reducers() []handler.AggregateReducer {
|
||||
return []handler.AggregateReducer{
|
||||
{
|
||||
Aggregate: org.AggregateType,
|
||||
EventReducers: []handler.EventReducer{
|
||||
{
|
||||
Event: org.LabelPolicyAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyChangedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyLogoAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyLogoRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyIconAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyIconRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyLogoDarkAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyLogoDarkRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyIconDarkAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyIconDarkRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyFontAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyFontRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyAssetsRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyActivatedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.OrgRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Aggregate: instance.AggregateType,
|
||||
EventReducers: []handler.EventReducer{
|
||||
{
|
||||
Event: instance.LabelPolicyAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyChangedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyLogoAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyLogoRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyIconAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyIconRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyLogoDarkAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyLogoDarkRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyIconDarkAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyIconDarkRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyFontAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyFontRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyAssetsRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyActivatedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.InstanceRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
return sequence.CurrentSequence, nil
|
||||
}
|
||||
|
||||
func (m *Styling) EventQuery(ctx context.Context, instanceIDs []string) (*models.SearchQuery, error) {
|
||||
sequences, err := m.view.GetLatestStylingSequences(ctx, instanceIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
searchQuery := models.NewSearchQuery()
|
||||
for _, instanceID := range instanceIDs {
|
||||
var seq uint64
|
||||
for _, sequence := range sequences {
|
||||
if sequence.InstanceID == instanceID {
|
||||
seq = sequence.CurrentSequence
|
||||
break
|
||||
func (m *Styling) processLabelPolicy(event eventstore.Event) (_ *handler.Statement, err error) {
|
||||
return handler.NewStatement(event, func(ex handler.Executer, projectionName string) error {
|
||||
policy := new(iam_model.LabelPolicyView)
|
||||
switch event.Type() {
|
||||
case instance.LabelPolicyAddedEventType,
|
||||
org.LabelPolicyAddedEventType:
|
||||
err = policy.AppendEvent(event)
|
||||
case instance.LabelPolicyChangedEventType,
|
||||
org.LabelPolicyChangedEventType,
|
||||
instance.LabelPolicyLogoAddedEventType,
|
||||
org.LabelPolicyLogoAddedEventType,
|
||||
instance.LabelPolicyLogoRemovedEventType,
|
||||
org.LabelPolicyLogoRemovedEventType,
|
||||
instance.LabelPolicyIconAddedEventType,
|
||||
org.LabelPolicyIconAddedEventType,
|
||||
instance.LabelPolicyIconRemovedEventType,
|
||||
org.LabelPolicyIconRemovedEventType,
|
||||
instance.LabelPolicyLogoDarkAddedEventType,
|
||||
org.LabelPolicyLogoDarkAddedEventType,
|
||||
instance.LabelPolicyLogoDarkRemovedEventType,
|
||||
org.LabelPolicyLogoDarkRemovedEventType,
|
||||
instance.LabelPolicyIconDarkAddedEventType,
|
||||
org.LabelPolicyIconDarkAddedEventType,
|
||||
instance.LabelPolicyIconDarkRemovedEventType,
|
||||
org.LabelPolicyIconDarkRemovedEventType,
|
||||
instance.LabelPolicyFontAddedEventType,
|
||||
org.LabelPolicyFontAddedEventType,
|
||||
instance.LabelPolicyFontRemovedEventType,
|
||||
org.LabelPolicyFontRemovedEventType,
|
||||
instance.LabelPolicyAssetsRemovedEventType,
|
||||
org.LabelPolicyAssetsRemovedEventType:
|
||||
|
||||
policy, err = m.view.StylingByAggregateIDAndState(event.Aggregate().ID, event.Aggregate().InstanceID, int32(domain.LabelPolicyStatePreview))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = policy.AppendEvent(event)
|
||||
case instance.LabelPolicyActivatedEventType,
|
||||
org.LabelPolicyActivatedEventType:
|
||||
|
||||
policy, err = m.view.StylingByAggregateIDAndState(event.Aggregate().ID, event.Aggregate().InstanceID, int32(domain.LabelPolicyStatePreview))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = policy.AppendEvent(event)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = m.generateStylingFile(policy)
|
||||
case instance.InstanceRemovedEventType:
|
||||
err = m.deleteInstanceFilesFromStorage(event.Aggregate().InstanceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return m.view.DeleteInstanceStyling(event)
|
||||
case org.OrgRemovedEventType:
|
||||
return m.view.UpdateOrgOwnerRemovedStyling(event)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
searchQuery.AddQuery().
|
||||
AggregateTypeFilter(m.AggregateTypes()...).
|
||||
LatestSequenceFilter(seq).
|
||||
InstanceIDFilter(instanceID)
|
||||
}
|
||||
return searchQuery, nil
|
||||
}
|
||||
|
||||
func (m *Styling) Reduce(event *models.Event) (err error) {
|
||||
switch event.AggregateType {
|
||||
case org.AggregateType, instance.AggregateType:
|
||||
err = m.processLabelPolicy(event)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (m *Styling) processLabelPolicy(event *models.Event) (err error) {
|
||||
policy := new(iam_model.LabelPolicyView)
|
||||
switch eventstore.EventType(event.Type) {
|
||||
case instance.LabelPolicyAddedEventType,
|
||||
org.LabelPolicyAddedEventType:
|
||||
err = policy.AppendEvent(event)
|
||||
case instance.LabelPolicyChangedEventType,
|
||||
org.LabelPolicyChangedEventType,
|
||||
instance.LabelPolicyLogoAddedEventType,
|
||||
org.LabelPolicyLogoAddedEventType,
|
||||
instance.LabelPolicyLogoRemovedEventType,
|
||||
org.LabelPolicyLogoRemovedEventType,
|
||||
instance.LabelPolicyIconAddedEventType,
|
||||
org.LabelPolicyIconAddedEventType,
|
||||
instance.LabelPolicyIconRemovedEventType,
|
||||
org.LabelPolicyIconRemovedEventType,
|
||||
instance.LabelPolicyLogoDarkAddedEventType,
|
||||
org.LabelPolicyLogoDarkAddedEventType,
|
||||
instance.LabelPolicyLogoDarkRemovedEventType,
|
||||
org.LabelPolicyLogoDarkRemovedEventType,
|
||||
instance.LabelPolicyIconDarkAddedEventType,
|
||||
org.LabelPolicyIconDarkAddedEventType,
|
||||
instance.LabelPolicyIconDarkRemovedEventType,
|
||||
org.LabelPolicyIconDarkRemovedEventType,
|
||||
instance.LabelPolicyFontAddedEventType,
|
||||
org.LabelPolicyFontAddedEventType,
|
||||
instance.LabelPolicyFontRemovedEventType,
|
||||
org.LabelPolicyFontRemovedEventType,
|
||||
instance.LabelPolicyAssetsRemovedEventType,
|
||||
org.LabelPolicyAssetsRemovedEventType:
|
||||
policy, err = m.view.StylingByAggregateIDAndState(event.AggregateID, event.InstanceID, int32(domain.LabelPolicyStatePreview))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = policy.AppendEvent(event)
|
||||
|
||||
case instance.LabelPolicyActivatedEventType,
|
||||
org.LabelPolicyActivatedEventType:
|
||||
policy, err = m.view.StylingByAggregateIDAndState(event.AggregateID, event.InstanceID, int32(domain.LabelPolicyStatePreview))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = policy.AppendEvent(event)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = m.generateStylingFile(policy)
|
||||
case instance.InstanceRemovedEventType:
|
||||
err = m.deleteInstanceFilesFromStorage(event.InstanceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return m.view.DeleteInstanceStyling(event)
|
||||
case org.OrgRemovedEventType:
|
||||
return m.view.UpdateOrgOwnerRemovedStyling(event)
|
||||
default:
|
||||
return m.view.ProcessedStylingSequence(event)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return m.view.PutStyling(policy, event)
|
||||
}
|
||||
|
||||
func (m *Styling) OnError(event *models.Event, err error) error {
|
||||
logging.WithFields("id", event.AggregateID).WithError(err).Warn("something went wrong in label policy handler")
|
||||
return spooler.HandleError(event, err, m.view.GetLatestStylingFailedEvent, m.view.ProcessedStylingFailedEvent, m.view.ProcessedStylingSequence, m.errorCountUntilSkip)
|
||||
}
|
||||
|
||||
func (m *Styling) OnSuccess(instanceIDs []string) error {
|
||||
return spooler.HandleSuccess(m.view.UpdateStylingSpoolerRunTimestamp, instanceIDs)
|
||||
return m.view.PutStyling(policy, event)
|
||||
}), nil
|
||||
}
|
||||
|
||||
func (m *Styling) generateStylingFile(policy *iam_model.LabelPolicyView) error {
|
||||
|
||||
Reference in New Issue
Block a user