mirror of
https://github.com/zitadel/zitadel.git
synced 2025-08-11 21:17:32 +00:00
feat(eventstore): increase parallel write capabilities (#5940)
This implementation increases parallel write capabilities of the eventstore. Please have a look at the technical advisories: [05](https://zitadel.com/docs/support/advisory/a10005) and [06](https://zitadel.com/docs/support/advisory/a10006). The implementation of eventstore.push is rewritten and stored events are migrated to a new table `eventstore.events2`. If you are using cockroach: make sure that the database user of ZITADEL has `VIEWACTIVITY` grant. This is used to query events.
This commit is contained in:
Silvan
@@ -9,15 +9,12 @@ import (
|
||||
|
||||
"github.com/lucasb-eyer/go-colorful"
|
||||
"github.com/muesli/gamut"
|
||||
"github.com/zitadel/logging"
|
||||
|
||||
admin_view "github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/view"
|
||||
"github.com/zitadel/zitadel/internal/api/ui/login"
|
||||
"github.com/zitadel/zitadel/internal/domain"
|
||||
"github.com/zitadel/zitadel/internal/eventstore"
|
||||
v1 "github.com/zitadel/zitadel/internal/eventstore/v1"
|
||||
"github.com/zitadel/zitadel/internal/eventstore/v1/models"
|
||||
"github.com/zitadel/zitadel/internal/eventstore/v1/query"
|
||||
"github.com/zitadel/zitadel/internal/eventstore/v1/spooler"
|
||||
"github.com/zitadel/zitadel/internal/eventstore/handler/v2"
|
||||
iam_model "github.com/zitadel/zitadel/internal/iam/repository/view/model"
|
||||
"github.com/zitadel/zitadel/internal/repository/instance"
|
||||
"github.com/zitadel/zitadel/internal/repository/org"
|
||||
@@ -28,152 +25,231 @@ const (
|
||||
stylingTable = "adminapi.styling2"
|
||||
)
|
||||
|
||||
var _ handler.Projection = (*Styling)(nil)
|
||||
|
||||
type Styling struct {
|
||||
handler
|
||||
static static.Storage
|
||||
subscription *v1.Subscription
|
||||
static static.Storage
|
||||
view *admin_view.View
|
||||
}
|
||||
|
||||
func newStyling(ctx context.Context, handler handler, static static.Storage) *Styling {
|
||||
h := &Styling{
|
||||
handler: handler,
|
||||
static: static,
|
||||
}
|
||||
h.subscribe(ctx)
|
||||
|
||||
return h
|
||||
func newStyling(ctx context.Context, config handler.Config, static static.Storage, view *admin_view.View) *handler.Handler {
|
||||
return handler.NewHandler(
|
||||
ctx,
|
||||
&config,
|
||||
&Styling{
|
||||
static: static,
|
||||
view: view,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func (m *Styling) subscribe(ctx context.Context) {
|
||||
m.subscription = m.es.Subscribe(m.AggregateTypes()...)
|
||||
go func() {
|
||||
for event := range m.subscription.Events {
|
||||
query.ReduceEvent(ctx, m, event)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (m *Styling) ViewModel() string {
|
||||
// Name implements [handler.Projection]
|
||||
func (*Styling) Name() string {
|
||||
return stylingTable
|
||||
}
|
||||
|
||||
func (m *Styling) Subscription() *v1.Subscription {
|
||||
return m.subscription
|
||||
}
|
||||
|
||||
func (_ *Styling) AggregateTypes() []models.AggregateType {
|
||||
return []models.AggregateType{org.AggregateType, instance.AggregateType}
|
||||
}
|
||||
|
||||
func (m *Styling) CurrentSequence(ctx context.Context, instanceID string) (uint64, error) {
|
||||
sequence, err := m.view.GetLatestStylingSequence(ctx, instanceID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
// Reducers implements [handler.Projection]
|
||||
func (s *Styling) Reducers() []handler.AggregateReducer {
|
||||
return []handler.AggregateReducer{
|
||||
{
|
||||
Aggregate: org.AggregateType,
|
||||
EventReducers: []handler.EventReducer{
|
||||
{
|
||||
Event: org.LabelPolicyAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyChangedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyLogoAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyLogoRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyIconAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyIconRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyLogoDarkAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyLogoDarkRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyIconDarkAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyIconDarkRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyFontAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyFontRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyAssetsRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyActivatedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.OrgRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Aggregate: instance.AggregateType,
|
||||
EventReducers: []handler.EventReducer{
|
||||
{
|
||||
Event: instance.LabelPolicyAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyChangedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyLogoAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyLogoRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyIconAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyIconRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyLogoDarkAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyLogoDarkRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyIconDarkAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyIconDarkRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyFontAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyFontRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyAssetsRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyActivatedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.InstanceRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
return sequence.CurrentSequence, nil
|
||||
}
|
||||
|
||||
func (m *Styling) EventQuery(ctx context.Context, instanceIDs []string) (*models.SearchQuery, error) {
|
||||
sequences, err := m.view.GetLatestStylingSequences(ctx, instanceIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
searchQuery := models.NewSearchQuery()
|
||||
for _, instanceID := range instanceIDs {
|
||||
var seq uint64
|
||||
for _, sequence := range sequences {
|
||||
if sequence.InstanceID == instanceID {
|
||||
seq = sequence.CurrentSequence
|
||||
break
|
||||
func (m *Styling) processLabelPolicy(event eventstore.Event) (_ *handler.Statement, err error) {
|
||||
return handler.NewStatement(event, func(ex handler.Executer, projectionName string) error {
|
||||
policy := new(iam_model.LabelPolicyView)
|
||||
switch event.Type() {
|
||||
case instance.LabelPolicyAddedEventType,
|
||||
org.LabelPolicyAddedEventType:
|
||||
err = policy.AppendEvent(event)
|
||||
case instance.LabelPolicyChangedEventType,
|
||||
org.LabelPolicyChangedEventType,
|
||||
instance.LabelPolicyLogoAddedEventType,
|
||||
org.LabelPolicyLogoAddedEventType,
|
||||
instance.LabelPolicyLogoRemovedEventType,
|
||||
org.LabelPolicyLogoRemovedEventType,
|
||||
instance.LabelPolicyIconAddedEventType,
|
||||
org.LabelPolicyIconAddedEventType,
|
||||
instance.LabelPolicyIconRemovedEventType,
|
||||
org.LabelPolicyIconRemovedEventType,
|
||||
instance.LabelPolicyLogoDarkAddedEventType,
|
||||
org.LabelPolicyLogoDarkAddedEventType,
|
||||
instance.LabelPolicyLogoDarkRemovedEventType,
|
||||
org.LabelPolicyLogoDarkRemovedEventType,
|
||||
instance.LabelPolicyIconDarkAddedEventType,
|
||||
org.LabelPolicyIconDarkAddedEventType,
|
||||
instance.LabelPolicyIconDarkRemovedEventType,
|
||||
org.LabelPolicyIconDarkRemovedEventType,
|
||||
instance.LabelPolicyFontAddedEventType,
|
||||
org.LabelPolicyFontAddedEventType,
|
||||
instance.LabelPolicyFontRemovedEventType,
|
||||
org.LabelPolicyFontRemovedEventType,
|
||||
instance.LabelPolicyAssetsRemovedEventType,
|
||||
org.LabelPolicyAssetsRemovedEventType:
|
||||
|
||||
policy, err = m.view.StylingByAggregateIDAndState(event.Aggregate().ID, event.Aggregate().InstanceID, int32(domain.LabelPolicyStatePreview))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = policy.AppendEvent(event)
|
||||
case instance.LabelPolicyActivatedEventType,
|
||||
org.LabelPolicyActivatedEventType:
|
||||
|
||||
policy, err = m.view.StylingByAggregateIDAndState(event.Aggregate().ID, event.Aggregate().InstanceID, int32(domain.LabelPolicyStatePreview))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = policy.AppendEvent(event)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = m.generateStylingFile(policy)
|
||||
case instance.InstanceRemovedEventType:
|
||||
err = m.deleteInstanceFilesFromStorage(event.Aggregate().InstanceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return m.view.DeleteInstanceStyling(event)
|
||||
case org.OrgRemovedEventType:
|
||||
return m.view.UpdateOrgOwnerRemovedStyling(event)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
searchQuery.AddQuery().
|
||||
AggregateTypeFilter(m.AggregateTypes()...).
|
||||
LatestSequenceFilter(seq).
|
||||
InstanceIDFilter(instanceID)
|
||||
}
|
||||
return searchQuery, nil
|
||||
}
|
||||
|
||||
func (m *Styling) Reduce(event *models.Event) (err error) {
|
||||
switch event.AggregateType {
|
||||
case org.AggregateType, instance.AggregateType:
|
||||
err = m.processLabelPolicy(event)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (m *Styling) processLabelPolicy(event *models.Event) (err error) {
|
||||
policy := new(iam_model.LabelPolicyView)
|
||||
switch eventstore.EventType(event.Type) {
|
||||
case instance.LabelPolicyAddedEventType,
|
||||
org.LabelPolicyAddedEventType:
|
||||
err = policy.AppendEvent(event)
|
||||
case instance.LabelPolicyChangedEventType,
|
||||
org.LabelPolicyChangedEventType,
|
||||
instance.LabelPolicyLogoAddedEventType,
|
||||
org.LabelPolicyLogoAddedEventType,
|
||||
instance.LabelPolicyLogoRemovedEventType,
|
||||
org.LabelPolicyLogoRemovedEventType,
|
||||
instance.LabelPolicyIconAddedEventType,
|
||||
org.LabelPolicyIconAddedEventType,
|
||||
instance.LabelPolicyIconRemovedEventType,
|
||||
org.LabelPolicyIconRemovedEventType,
|
||||
instance.LabelPolicyLogoDarkAddedEventType,
|
||||
org.LabelPolicyLogoDarkAddedEventType,
|
||||
instance.LabelPolicyLogoDarkRemovedEventType,
|
||||
org.LabelPolicyLogoDarkRemovedEventType,
|
||||
instance.LabelPolicyIconDarkAddedEventType,
|
||||
org.LabelPolicyIconDarkAddedEventType,
|
||||
instance.LabelPolicyIconDarkRemovedEventType,
|
||||
org.LabelPolicyIconDarkRemovedEventType,
|
||||
instance.LabelPolicyFontAddedEventType,
|
||||
org.LabelPolicyFontAddedEventType,
|
||||
instance.LabelPolicyFontRemovedEventType,
|
||||
org.LabelPolicyFontRemovedEventType,
|
||||
instance.LabelPolicyAssetsRemovedEventType,
|
||||
org.LabelPolicyAssetsRemovedEventType:
|
||||
policy, err = m.view.StylingByAggregateIDAndState(event.AggregateID, event.InstanceID, int32(domain.LabelPolicyStatePreview))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = policy.AppendEvent(event)
|
||||
|
||||
case instance.LabelPolicyActivatedEventType,
|
||||
org.LabelPolicyActivatedEventType:
|
||||
policy, err = m.view.StylingByAggregateIDAndState(event.AggregateID, event.InstanceID, int32(domain.LabelPolicyStatePreview))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = policy.AppendEvent(event)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = m.generateStylingFile(policy)
|
||||
case instance.InstanceRemovedEventType:
|
||||
err = m.deleteInstanceFilesFromStorage(event.InstanceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return m.view.DeleteInstanceStyling(event)
|
||||
case org.OrgRemovedEventType:
|
||||
return m.view.UpdateOrgOwnerRemovedStyling(event)
|
||||
default:
|
||||
return m.view.ProcessedStylingSequence(event)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return m.view.PutStyling(policy, event)
|
||||
}
|
||||
|
||||
func (m *Styling) OnError(event *models.Event, err error) error {
|
||||
logging.WithFields("id", event.AggregateID).WithError(err).Warn("something went wrong in label policy handler")
|
||||
return spooler.HandleError(event, err, m.view.GetLatestStylingFailedEvent, m.view.ProcessedStylingFailedEvent, m.view.ProcessedStylingSequence, m.errorCountUntilSkip)
|
||||
}
|
||||
|
||||
func (m *Styling) OnSuccess(instanceIDs []string) error {
|
||||
return spooler.HandleSuccess(m.view.UpdateStylingSpoolerRunTimestamp, instanceIDs)
|
||||
return m.view.PutStyling(policy, event)
|
||||
}), nil
|
||||
}
|
||||
|
||||
func (m *Styling) generateStylingFile(policy *iam_model.LabelPolicyView) error {
|
||||
|
||||
Reference in New Issue
Block a user