mirror of
https://github.com/zitadel/zitadel.git
synced 2025-08-11 21:37:32 +00:00
feat(eventstore): increase parallel write capabilities (#5940)
This implementation increases parallel write capabilities of the eventstore. Please have a look at the technical advisories: [05](https://zitadel.com/docs/support/advisory/a10005) and [06](https://zitadel.com/docs/support/advisory/a10006). The implementation of eventstore.push is rewritten and stored events are migrated to a new table `eventstore.events2`. If you are using cockroach: make sure that the database user of ZITADEL has `VIEWACTIVITY` grant. This is used to query events.
This commit is contained in:
@@ -1,14 +0,0 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/zitadel/zitadel/internal/view/model"
|
||||
)
|
||||
|
||||
type AdministratorRepository interface {
|
||||
GetFailedEvents(ctx context.Context, instanceID string) ([]*model.FailedEvent, error)
|
||||
RemoveFailedEvent(context.Context, *model.FailedEvent) error
|
||||
GetViews(instanceID string) ([]*model.View, error)
|
||||
ClearView(ctx context.Context, db, viewName string) error
|
||||
}
|
@@ -1,51 +0,0 @@
|
||||
package eventstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/view"
|
||||
view_model "github.com/zitadel/zitadel/internal/view/model"
|
||||
"github.com/zitadel/zitadel/internal/view/repository"
|
||||
)
|
||||
|
||||
var dbList = []string{"auth", "adminapi"}
|
||||
|
||||
type AdministratorRepo struct {
|
||||
View *view.View
|
||||
}
|
||||
|
||||
func (repo *AdministratorRepo) GetFailedEvents(ctx context.Context, instanceID string) ([]*view_model.FailedEvent, error) {
|
||||
allFailedEvents := make([]*view_model.FailedEvent, 0)
|
||||
for _, db := range dbList {
|
||||
failedEvents, err := repo.View.AllFailedEvents(db, instanceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, failedEvent := range failedEvents {
|
||||
allFailedEvents = append(allFailedEvents, repository.FailedEventToModel(failedEvent))
|
||||
}
|
||||
}
|
||||
return allFailedEvents, nil
|
||||
}
|
||||
|
||||
func (repo *AdministratorRepo) RemoveFailedEvent(ctx context.Context, failedEvent *view_model.FailedEvent) error {
|
||||
return repo.View.RemoveFailedEvent(failedEvent.Database, repository.FailedEventFromModel(failedEvent))
|
||||
}
|
||||
|
||||
func (repo *AdministratorRepo) GetViews(instanceID string) ([]*view_model.View, error) {
|
||||
views := make([]*view_model.View, 0)
|
||||
for _, db := range dbList {
|
||||
sequences, err := repo.View.AllCurrentSequences(db, instanceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, sequence := range sequences {
|
||||
views = append(views, repository.CurrentSequenceToModel(sequence))
|
||||
}
|
||||
}
|
||||
return views, nil
|
||||
}
|
||||
|
||||
func (repo *AdministratorRepo) ClearView(ctx context.Context, database, view string) error {
|
||||
return repo.View.ClearView(database, view)
|
||||
}
|
@@ -5,56 +5,55 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/view"
|
||||
v1 "github.com/zitadel/zitadel/internal/eventstore/v1"
|
||||
"github.com/zitadel/zitadel/internal/eventstore/v1/query"
|
||||
"github.com/zitadel/zitadel/internal/database"
|
||||
"github.com/zitadel/zitadel/internal/eventstore"
|
||||
handler2 "github.com/zitadel/zitadel/internal/eventstore/handler/v2"
|
||||
"github.com/zitadel/zitadel/internal/static"
|
||||
)
|
||||
|
||||
type Configs map[string]*Config
|
||||
|
||||
type Config struct {
|
||||
Client *database.DB
|
||||
Eventstore *eventstore.Eventstore
|
||||
|
||||
BulkLimit uint64
|
||||
FailureCountUntilSkip uint64
|
||||
HandleActiveInstances time.Duration
|
||||
TransactionDuration time.Duration
|
||||
Handlers map[string]*ConfigOverwrites
|
||||
}
|
||||
|
||||
type ConfigOverwrites struct {
|
||||
MinimumCycleDuration time.Duration
|
||||
}
|
||||
|
||||
type handler struct {
|
||||
view *view.View
|
||||
bulkLimit uint64
|
||||
cycleDuration time.Duration
|
||||
errorCountUntilSkip uint64
|
||||
|
||||
es v1.Eventstore
|
||||
}
|
||||
|
||||
func (h *handler) Eventstore() v1.Eventstore {
|
||||
return h.es
|
||||
}
|
||||
|
||||
func Register(ctx context.Context, configs Configs, bulkLimit, errorCount uint64, view *view.View, es v1.Eventstore, static static.Storage) []query.Handler {
|
||||
handlers := []query.Handler{}
|
||||
if static != nil {
|
||||
handlers = append(handlers, newStyling(ctx,
|
||||
handler{view, bulkLimit, configs.cycleDuration("Styling"), errorCount, es},
|
||||
static))
|
||||
func Register(ctx context.Context, config Config, view *view.View, static static.Storage) {
|
||||
if static == nil {
|
||||
return
|
||||
}
|
||||
return handlers
|
||||
|
||||
newStyling(ctx,
|
||||
config.overwrite("Styling"),
|
||||
static,
|
||||
view,
|
||||
).Start(ctx)
|
||||
}
|
||||
|
||||
func (configs Configs) cycleDuration(viewModel string) time.Duration {
|
||||
c, ok := configs[viewModel]
|
||||
func (config Config) overwrite(viewModel string) handler2.Config {
|
||||
c := handler2.Config{
|
||||
Client: config.Client,
|
||||
Eventstore: config.Eventstore,
|
||||
BulkLimit: uint16(config.BulkLimit),
|
||||
RequeueEvery: 3 * time.Minute,
|
||||
HandleActiveInstances: config.HandleActiveInstances,
|
||||
MaxFailureCount: uint8(config.FailureCountUntilSkip),
|
||||
TransactionDuration: config.TransactionDuration,
|
||||
}
|
||||
overwrite, ok := config.Handlers[viewModel]
|
||||
if !ok {
|
||||
return 3 * time.Minute
|
||||
return c
|
||||
}
|
||||
return c.MinimumCycleDuration
|
||||
}
|
||||
|
||||
func (h *handler) MinimumCycleDuration() time.Duration {
|
||||
return h.cycleDuration
|
||||
}
|
||||
|
||||
func (h *handler) LockDuration() time.Duration {
|
||||
return h.cycleDuration / 3
|
||||
}
|
||||
|
||||
func (h *handler) QueryLimit() uint64 {
|
||||
return h.bulkLimit
|
||||
if overwrite.MinimumCycleDuration > 0 {
|
||||
c.RequeueEvery = overwrite.MinimumCycleDuration
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
@@ -9,15 +9,12 @@ import (
|
||||
|
||||
"github.com/lucasb-eyer/go-colorful"
|
||||
"github.com/muesli/gamut"
|
||||
"github.com/zitadel/logging"
|
||||
|
||||
admin_view "github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/view"
|
||||
"github.com/zitadel/zitadel/internal/api/ui/login"
|
||||
"github.com/zitadel/zitadel/internal/domain"
|
||||
"github.com/zitadel/zitadel/internal/eventstore"
|
||||
v1 "github.com/zitadel/zitadel/internal/eventstore/v1"
|
||||
"github.com/zitadel/zitadel/internal/eventstore/v1/models"
|
||||
"github.com/zitadel/zitadel/internal/eventstore/v1/query"
|
||||
"github.com/zitadel/zitadel/internal/eventstore/v1/spooler"
|
||||
"github.com/zitadel/zitadel/internal/eventstore/handler/v2"
|
||||
iam_model "github.com/zitadel/zitadel/internal/iam/repository/view/model"
|
||||
"github.com/zitadel/zitadel/internal/repository/instance"
|
||||
"github.com/zitadel/zitadel/internal/repository/org"
|
||||
@@ -28,152 +25,231 @@ const (
|
||||
stylingTable = "adminapi.styling2"
|
||||
)
|
||||
|
||||
var _ handler.Projection = (*Styling)(nil)
|
||||
|
||||
type Styling struct {
|
||||
handler
|
||||
static static.Storage
|
||||
subscription *v1.Subscription
|
||||
static static.Storage
|
||||
view *admin_view.View
|
||||
}
|
||||
|
||||
func newStyling(ctx context.Context, handler handler, static static.Storage) *Styling {
|
||||
h := &Styling{
|
||||
handler: handler,
|
||||
static: static,
|
||||
}
|
||||
h.subscribe(ctx)
|
||||
|
||||
return h
|
||||
func newStyling(ctx context.Context, config handler.Config, static static.Storage, view *admin_view.View) *handler.Handler {
|
||||
return handler.NewHandler(
|
||||
ctx,
|
||||
&config,
|
||||
&Styling{
|
||||
static: static,
|
||||
view: view,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func (m *Styling) subscribe(ctx context.Context) {
|
||||
m.subscription = m.es.Subscribe(m.AggregateTypes()...)
|
||||
go func() {
|
||||
for event := range m.subscription.Events {
|
||||
query.ReduceEvent(ctx, m, event)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (m *Styling) ViewModel() string {
|
||||
// Name implements [handler.Projection]
|
||||
func (*Styling) Name() string {
|
||||
return stylingTable
|
||||
}
|
||||
|
||||
func (m *Styling) Subscription() *v1.Subscription {
|
||||
return m.subscription
|
||||
}
|
||||
|
||||
func (_ *Styling) AggregateTypes() []models.AggregateType {
|
||||
return []models.AggregateType{org.AggregateType, instance.AggregateType}
|
||||
}
|
||||
|
||||
func (m *Styling) CurrentSequence(ctx context.Context, instanceID string) (uint64, error) {
|
||||
sequence, err := m.view.GetLatestStylingSequence(ctx, instanceID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
// Reducers implements [handler.Projection]
|
||||
func (s *Styling) Reducers() []handler.AggregateReducer {
|
||||
return []handler.AggregateReducer{
|
||||
{
|
||||
Aggregate: org.AggregateType,
|
||||
EventReducers: []handler.EventReducer{
|
||||
{
|
||||
Event: org.LabelPolicyAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyChangedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyLogoAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyLogoRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyIconAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyIconRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyLogoDarkAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyLogoDarkRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyIconDarkAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyIconDarkRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyFontAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyFontRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyAssetsRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.LabelPolicyActivatedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: org.OrgRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Aggregate: instance.AggregateType,
|
||||
EventReducers: []handler.EventReducer{
|
||||
{
|
||||
Event: instance.LabelPolicyAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyChangedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyLogoAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyLogoRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyIconAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyIconRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyLogoDarkAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyLogoDarkRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyIconDarkAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyIconDarkRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyFontAddedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyFontRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyAssetsRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.LabelPolicyActivatedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
{
|
||||
Event: instance.InstanceRemovedEventType,
|
||||
Reduce: s.processLabelPolicy,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
return sequence.CurrentSequence, nil
|
||||
}
|
||||
|
||||
func (m *Styling) EventQuery(ctx context.Context, instanceIDs []string) (*models.SearchQuery, error) {
|
||||
sequences, err := m.view.GetLatestStylingSequences(ctx, instanceIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
searchQuery := models.NewSearchQuery()
|
||||
for _, instanceID := range instanceIDs {
|
||||
var seq uint64
|
||||
for _, sequence := range sequences {
|
||||
if sequence.InstanceID == instanceID {
|
||||
seq = sequence.CurrentSequence
|
||||
break
|
||||
func (m *Styling) processLabelPolicy(event eventstore.Event) (_ *handler.Statement, err error) {
|
||||
return handler.NewStatement(event, func(ex handler.Executer, projectionName string) error {
|
||||
policy := new(iam_model.LabelPolicyView)
|
||||
switch event.Type() {
|
||||
case instance.LabelPolicyAddedEventType,
|
||||
org.LabelPolicyAddedEventType:
|
||||
err = policy.AppendEvent(event)
|
||||
case instance.LabelPolicyChangedEventType,
|
||||
org.LabelPolicyChangedEventType,
|
||||
instance.LabelPolicyLogoAddedEventType,
|
||||
org.LabelPolicyLogoAddedEventType,
|
||||
instance.LabelPolicyLogoRemovedEventType,
|
||||
org.LabelPolicyLogoRemovedEventType,
|
||||
instance.LabelPolicyIconAddedEventType,
|
||||
org.LabelPolicyIconAddedEventType,
|
||||
instance.LabelPolicyIconRemovedEventType,
|
||||
org.LabelPolicyIconRemovedEventType,
|
||||
instance.LabelPolicyLogoDarkAddedEventType,
|
||||
org.LabelPolicyLogoDarkAddedEventType,
|
||||
instance.LabelPolicyLogoDarkRemovedEventType,
|
||||
org.LabelPolicyLogoDarkRemovedEventType,
|
||||
instance.LabelPolicyIconDarkAddedEventType,
|
||||
org.LabelPolicyIconDarkAddedEventType,
|
||||
instance.LabelPolicyIconDarkRemovedEventType,
|
||||
org.LabelPolicyIconDarkRemovedEventType,
|
||||
instance.LabelPolicyFontAddedEventType,
|
||||
org.LabelPolicyFontAddedEventType,
|
||||
instance.LabelPolicyFontRemovedEventType,
|
||||
org.LabelPolicyFontRemovedEventType,
|
||||
instance.LabelPolicyAssetsRemovedEventType,
|
||||
org.LabelPolicyAssetsRemovedEventType:
|
||||
|
||||
policy, err = m.view.StylingByAggregateIDAndState(event.Aggregate().ID, event.Aggregate().InstanceID, int32(domain.LabelPolicyStatePreview))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = policy.AppendEvent(event)
|
||||
case instance.LabelPolicyActivatedEventType,
|
||||
org.LabelPolicyActivatedEventType:
|
||||
|
||||
policy, err = m.view.StylingByAggregateIDAndState(event.Aggregate().ID, event.Aggregate().InstanceID, int32(domain.LabelPolicyStatePreview))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = policy.AppendEvent(event)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = m.generateStylingFile(policy)
|
||||
case instance.InstanceRemovedEventType:
|
||||
err = m.deleteInstanceFilesFromStorage(event.Aggregate().InstanceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return m.view.DeleteInstanceStyling(event)
|
||||
case org.OrgRemovedEventType:
|
||||
return m.view.UpdateOrgOwnerRemovedStyling(event)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
searchQuery.AddQuery().
|
||||
AggregateTypeFilter(m.AggregateTypes()...).
|
||||
LatestSequenceFilter(seq).
|
||||
InstanceIDFilter(instanceID)
|
||||
}
|
||||
return searchQuery, nil
|
||||
}
|
||||
|
||||
func (m *Styling) Reduce(event *models.Event) (err error) {
|
||||
switch event.AggregateType {
|
||||
case org.AggregateType, instance.AggregateType:
|
||||
err = m.processLabelPolicy(event)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (m *Styling) processLabelPolicy(event *models.Event) (err error) {
|
||||
policy := new(iam_model.LabelPolicyView)
|
||||
switch eventstore.EventType(event.Type) {
|
||||
case instance.LabelPolicyAddedEventType,
|
||||
org.LabelPolicyAddedEventType:
|
||||
err = policy.AppendEvent(event)
|
||||
case instance.LabelPolicyChangedEventType,
|
||||
org.LabelPolicyChangedEventType,
|
||||
instance.LabelPolicyLogoAddedEventType,
|
||||
org.LabelPolicyLogoAddedEventType,
|
||||
instance.LabelPolicyLogoRemovedEventType,
|
||||
org.LabelPolicyLogoRemovedEventType,
|
||||
instance.LabelPolicyIconAddedEventType,
|
||||
org.LabelPolicyIconAddedEventType,
|
||||
instance.LabelPolicyIconRemovedEventType,
|
||||
org.LabelPolicyIconRemovedEventType,
|
||||
instance.LabelPolicyLogoDarkAddedEventType,
|
||||
org.LabelPolicyLogoDarkAddedEventType,
|
||||
instance.LabelPolicyLogoDarkRemovedEventType,
|
||||
org.LabelPolicyLogoDarkRemovedEventType,
|
||||
instance.LabelPolicyIconDarkAddedEventType,
|
||||
org.LabelPolicyIconDarkAddedEventType,
|
||||
instance.LabelPolicyIconDarkRemovedEventType,
|
||||
org.LabelPolicyIconDarkRemovedEventType,
|
||||
instance.LabelPolicyFontAddedEventType,
|
||||
org.LabelPolicyFontAddedEventType,
|
||||
instance.LabelPolicyFontRemovedEventType,
|
||||
org.LabelPolicyFontRemovedEventType,
|
||||
instance.LabelPolicyAssetsRemovedEventType,
|
||||
org.LabelPolicyAssetsRemovedEventType:
|
||||
policy, err = m.view.StylingByAggregateIDAndState(event.AggregateID, event.InstanceID, int32(domain.LabelPolicyStatePreview))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = policy.AppendEvent(event)
|
||||
|
||||
case instance.LabelPolicyActivatedEventType,
|
||||
org.LabelPolicyActivatedEventType:
|
||||
policy, err = m.view.StylingByAggregateIDAndState(event.AggregateID, event.InstanceID, int32(domain.LabelPolicyStatePreview))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = policy.AppendEvent(event)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = m.generateStylingFile(policy)
|
||||
case instance.InstanceRemovedEventType:
|
||||
err = m.deleteInstanceFilesFromStorage(event.InstanceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return m.view.DeleteInstanceStyling(event)
|
||||
case org.OrgRemovedEventType:
|
||||
return m.view.UpdateOrgOwnerRemovedStyling(event)
|
||||
default:
|
||||
return m.view.ProcessedStylingSequence(event)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return m.view.PutStyling(policy, event)
|
||||
}
|
||||
|
||||
func (m *Styling) OnError(event *models.Event, err error) error {
|
||||
logging.WithFields("id", event.AggregateID).WithError(err).Warn("something went wrong in label policy handler")
|
||||
return spooler.HandleError(event, err, m.view.GetLatestStylingFailedEvent, m.view.ProcessedStylingFailedEvent, m.view.ProcessedStylingSequence, m.errorCountUntilSkip)
|
||||
}
|
||||
|
||||
func (m *Styling) OnSuccess(instanceIDs []string) error {
|
||||
return spooler.HandleSuccess(m.view.UpdateStylingSpoolerRunTimestamp, instanceIDs)
|
||||
return m.view.PutStyling(policy, event)
|
||||
}), nil
|
||||
}
|
||||
|
||||
func (m *Styling) generateStylingFile(policy *iam_model.LabelPolicyView) error {
|
||||
|
@@ -3,46 +3,23 @@ package eventsourcing
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/eventstore"
|
||||
"github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/spooler"
|
||||
admin_handler "github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/handler"
|
||||
admin_view "github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/view"
|
||||
"github.com/zitadel/zitadel/internal/database"
|
||||
eventstore2 "github.com/zitadel/zitadel/internal/eventstore"
|
||||
v1 "github.com/zitadel/zitadel/internal/eventstore/v1"
|
||||
es_spol "github.com/zitadel/zitadel/internal/eventstore/v1/spooler"
|
||||
"github.com/zitadel/zitadel/internal/static"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
SearchLimit uint64
|
||||
Spooler spooler.SpoolerConfig
|
||||
Spooler admin_handler.Config
|
||||
}
|
||||
|
||||
type EsRepository struct {
|
||||
spooler *es_spol.Spooler
|
||||
eventstore.AdministratorRepo
|
||||
}
|
||||
|
||||
func Start(ctx context.Context, conf Config, static static.Storage, dbClient *database.DB, esV2 *eventstore2.Eventstore, allowOrderByCreationDate bool) (*EsRepository, error) {
|
||||
es, err := v1.Start(dbClient, allowOrderByCreationDate)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
func Start(ctx context.Context, conf Config, static static.Storage, dbClient *database.DB) error {
|
||||
view, err := admin_view.StartView(dbClient)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return err
|
||||
}
|
||||
|
||||
spool := spooler.StartSpooler(ctx, conf.Spooler, es, esV2, view, dbClient, static)
|
||||
admin_handler.Register(ctx, conf.Spooler, view, static)
|
||||
|
||||
return &EsRepository{
|
||||
spooler: spool,
|
||||
AdministratorRepo: eventstore.AdministratorRepo{
|
||||
View: view,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (repo *EsRepository) Health(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
@@ -1,20 +0,0 @@
|
||||
package spooler
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
es_locker "github.com/zitadel/zitadel/internal/eventstore/v1/locker"
|
||||
)
|
||||
|
||||
const (
|
||||
lockTable = "adminapi.locks"
|
||||
)
|
||||
|
||||
type locker struct {
|
||||
dbClient *sql.DB
|
||||
}
|
||||
|
||||
func (l *locker) Renew(lockerID, viewModel, instanceID string, waitTime time.Duration) error {
|
||||
return es_locker.Renew(l.dbClient, lockTable, lockerID, viewModel, instanceID, waitTime)
|
||||
}
|
@@ -1,35 +0,0 @@
|
||||
package spooler
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/handler"
|
||||
"github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/view"
|
||||
"github.com/zitadel/zitadel/internal/database"
|
||||
"github.com/zitadel/zitadel/internal/eventstore"
|
||||
v1 "github.com/zitadel/zitadel/internal/eventstore/v1"
|
||||
"github.com/zitadel/zitadel/internal/eventstore/v1/spooler"
|
||||
"github.com/zitadel/zitadel/internal/static"
|
||||
)
|
||||
|
||||
type SpoolerConfig struct {
|
||||
BulkLimit uint64
|
||||
FailureCountUntilSkip uint64
|
||||
ConcurrentWorkers int
|
||||
ConcurrentInstances int
|
||||
Handlers handler.Configs
|
||||
}
|
||||
|
||||
func StartSpooler(ctx context.Context, c SpoolerConfig, es v1.Eventstore, esV2 *eventstore.Eventstore, view *view.View, sql *database.DB, static static.Storage) *spooler.Spooler {
|
||||
spoolerConfig := spooler.Config{
|
||||
Eventstore: es,
|
||||
EventstoreV2: esV2,
|
||||
Locker: &locker{dbClient: sql.DB},
|
||||
ConcurrentWorkers: c.ConcurrentWorkers,
|
||||
ConcurrentInstances: c.ConcurrentInstances,
|
||||
ViewHandlers: handler.Register(ctx, c.Handlers, c.BulkLimit, c.FailureCountUntilSkip, view, es, static),
|
||||
}
|
||||
spool := spoolerConfig.New()
|
||||
spool.Start()
|
||||
return spool
|
||||
}
|
@@ -1,26 +0,0 @@
|
||||
package view
|
||||
|
||||
import (
|
||||
"github.com/zitadel/zitadel/internal/view/repository"
|
||||
)
|
||||
|
||||
const (
|
||||
errTable = "adminapi.failed_events"
|
||||
errColumn = "failed_events"
|
||||
)
|
||||
|
||||
func (v *View) saveFailedEvent(failedEvent *repository.FailedEvent) error {
|
||||
return repository.SaveFailedEvent(v.Db, errTable, failedEvent)
|
||||
}
|
||||
|
||||
func (v *View) RemoveFailedEvent(database string, failedEvent *repository.FailedEvent) error {
|
||||
return repository.RemoveFailedEvent(v.Db, database+"."+errColumn, failedEvent)
|
||||
}
|
||||
|
||||
func (v *View) latestFailedEvent(viewName, instanceID string, sequence uint64) (*repository.FailedEvent, error) {
|
||||
return repository.LatestFailedEvent(v.Db, errTable, viewName, instanceID, sequence)
|
||||
}
|
||||
|
||||
func (v *View) AllFailedEvents(db, instanceID string) ([]*repository.FailedEvent, error) {
|
||||
return repository.AllFailedEvents(v.Db, db+"."+errColumn, instanceID)
|
||||
}
|
@@ -1,49 +0,0 @@
|
||||
package view
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/zitadel/zitadel/internal/eventstore/v1/models"
|
||||
"github.com/zitadel/zitadel/internal/view/repository"
|
||||
)
|
||||
|
||||
const (
|
||||
sequencesTable = "adminapi.current_sequences"
|
||||
)
|
||||
|
||||
func (v *View) saveCurrentSequence(viewName string, event *models.Event) error {
|
||||
return repository.SaveCurrentSequence(v.Db, sequencesTable, viewName, event.InstanceID, event.Sequence, event.CreationDate)
|
||||
}
|
||||
|
||||
func (v *View) latestSequence(ctx context.Context, viewName, instanceID string) (*repository.CurrentSequence, error) {
|
||||
return repository.LatestSequence(v.Db, v.TimeTravel(ctx, sequencesTable), viewName, instanceID)
|
||||
}
|
||||
|
||||
func (v *View) latestSequences(ctx context.Context, viewName string, instanceIDs []string) ([]*repository.CurrentSequence, error) {
|
||||
return repository.LatestSequences(v.Db, v.TimeTravel(ctx, sequencesTable), viewName, instanceIDs)
|
||||
}
|
||||
|
||||
func (v *View) AllCurrentSequences(db, instanceID string) ([]*repository.CurrentSequence, error) {
|
||||
return repository.AllCurrentSequences(v.Db, db+".current_sequences", instanceID)
|
||||
}
|
||||
|
||||
func (v *View) updateSpoolerRunSequence(viewName string, instanceIDs []string) error {
|
||||
currentSequences, err := repository.LatestSequences(v.Db, sequencesTable, viewName, instanceIDs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, currentSequence := range currentSequences {
|
||||
if currentSequence.ViewName == "" {
|
||||
currentSequence.ViewName = viewName
|
||||
}
|
||||
currentSequence.LastSuccessfulSpoolerRun = time.Now()
|
||||
}
|
||||
return repository.UpdateCurrentSequences(v.Db, sequencesTable, currentSequences)
|
||||
}
|
||||
|
||||
func (v *View) ClearView(db, viewName string) error {
|
||||
truncateView := db + "." + viewName
|
||||
sequenceTable := db + ".current_sequences"
|
||||
return repository.ClearView(v.Db, truncateView, sequenceTable)
|
||||
}
|
@@ -1,12 +1,9 @@
|
||||
package view
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/zitadel/zitadel/internal/eventstore/v1/models"
|
||||
"github.com/zitadel/zitadel/internal/eventstore"
|
||||
"github.com/zitadel/zitadel/internal/iam/repository/view"
|
||||
"github.com/zitadel/zitadel/internal/iam/repository/view/model"
|
||||
global_view "github.com/zitadel/zitadel/internal/view/repository"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -17,50 +14,14 @@ func (v *View) StylingByAggregateIDAndState(aggregateID, instanceID string, stat
|
||||
return view.GetStylingByAggregateIDAndState(v.Db, stylingTyble, aggregateID, instanceID, state)
|
||||
}
|
||||
|
||||
func (v *View) PutStyling(policy *model.LabelPolicyView, event *models.Event) error {
|
||||
err := view.PutStyling(v.Db, stylingTyble, policy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return v.ProcessedStylingSequence(event)
|
||||
func (v *View) PutStyling(policy *model.LabelPolicyView, event eventstore.Event) error {
|
||||
return view.PutStyling(v.Db, stylingTyble, policy)
|
||||
}
|
||||
|
||||
func (v *View) DeleteInstanceStyling(event *models.Event) error {
|
||||
err := view.DeleteInstanceStyling(v.Db, stylingTyble, event.InstanceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return v.ProcessedStylingSequence(event)
|
||||
func (v *View) DeleteInstanceStyling(event eventstore.Event) error {
|
||||
return view.DeleteInstanceStyling(v.Db, stylingTyble, event.Aggregate().InstanceID)
|
||||
}
|
||||
|
||||
func (v *View) UpdateOrgOwnerRemovedStyling(event *models.Event) error {
|
||||
err := view.UpdateOrgOwnerRemovedStyling(v.Db, stylingTyble, event.InstanceID, event.AggregateID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return v.ProcessedStylingSequence(event)
|
||||
}
|
||||
|
||||
func (v *View) GetLatestStylingSequence(ctx context.Context, instanceID string) (*global_view.CurrentSequence, error) {
|
||||
return v.latestSequence(ctx, stylingTyble, instanceID)
|
||||
}
|
||||
|
||||
func (v *View) GetLatestStylingSequences(ctx context.Context, instanceIDs []string) ([]*global_view.CurrentSequence, error) {
|
||||
return v.latestSequences(ctx, stylingTyble, instanceIDs)
|
||||
}
|
||||
|
||||
func (v *View) ProcessedStylingSequence(event *models.Event) error {
|
||||
return v.saveCurrentSequence(stylingTyble, event)
|
||||
}
|
||||
|
||||
func (v *View) UpdateStylingSpoolerRunTimestamp(instanceIDs []string) error {
|
||||
return v.updateSpoolerRunSequence(stylingTyble, instanceIDs)
|
||||
}
|
||||
|
||||
func (v *View) GetLatestStylingFailedEvent(sequence uint64, instanceID string) (*global_view.FailedEvent, error) {
|
||||
return v.latestFailedEvent(stylingTyble, instanceID, sequence)
|
||||
}
|
||||
|
||||
func (v *View) ProcessedStylingFailedEvent(failedEvent *global_view.FailedEvent) error {
|
||||
return v.saveFailedEvent(failedEvent)
|
||||
func (v *View) UpdateOrgOwnerRemovedStyling(event eventstore.Event) error {
|
||||
return view.UpdateOrgOwnerRemovedStyling(v.Db, stylingTyble, event.Aggregate().InstanceID, event.Aggregate().ID)
|
||||
}
|
||||
|
@@ -1,8 +0,0 @@
|
||||
package repository
|
||||
|
||||
import "context"
|
||||
|
||||
type Repository interface {
|
||||
Health(ctx context.Context) error
|
||||
AdministratorRepository
|
||||
}
|
Reference in New Issue
Block a user