TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-11 18:17:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: reduce load on view tables (#4716)

Browse Source 
* fix: reduce load on view tables

* create prerelease

* linting: pass context to view handlers

* fix error handling of refresh token handler

* fix: improve processing of successful instanceIDs on views

* fix revert intended change in .golangci.yaml

* fix: set timeout for processInstances in spooler

* fix: reduce update to active tokens on profile change

* change token expiration query to db now()

* remove branch from .releaserc.js
This commit is contained in:
Livio Spring
2022-11-22 07:36:48 +01:00
committed by GitHub
parent 29441ce4b6
commit e8babf1048
37 changed files with 295 additions and 238 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
internal/admin/repository/eventsourcing/handler/handler.go
View File

@@ -1,6 +1,7 @@
package handler
import (
"context"
"time"
"github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/view"
@@ -28,10 +29,10 @@ func (h *handler) Eventstore() v1.Eventstore {
return h.es
}
func Register(configs Configs, bulkLimit, errorCount uint64, view *view.View, es v1.Eventstore, static static.Storage) []query.Handler {
func Register(ctx context.Context, configs Configs, bulkLimit, errorCount uint64, view *view.View, es v1.Eventstore, static static.Storage) []query.Handler {
handlers := []query.Handler{}
if static != nil {
handlers = append(handlers, newStyling(
handlers = append(handlers, newStyling(ctx,
handler{view, bulkLimit, configs.cycleDuration("Styling"), errorCount, es},
static))
}

24
internal/admin/repository/eventsourcing/handler/styling.go
View File

@@ -34,21 +34,21 @@ type Styling struct {
subscription *v1.Subscription
}
func newStyling(handler handler, static static.Storage) *Styling {
func newStyling(ctx context.Context, handler handler, static static.Storage) *Styling {
h := &Styling{
handler: handler,
static: static,
}
h.subscribe()
h.subscribe(ctx)
return h
}
func (m *Styling) subscribe() {
func (m *Styling) subscribe(ctx context.Context) {
m.subscription = m.es.Subscribe(m.AggregateTypes()...)
go func() {
for event := range m.subscription.Events {
query.ReduceEvent(m, event)
query.ReduceEvent(ctx, m, event)
}
}()
}
@@ -73,15 +73,15 @@ func (m *Styling) CurrentSequence(instanceID string) (uint64, error) {
return sequence.CurrentSequence, nil
}
func (m *Styling) EventQuery(instanceIDs ...string) (*models.SearchQuery, error) {
sequences, err := m.view.GetLatestStylingSequences(instanceIDs...)
func (m *Styling) EventQuery(instanceIDs []string) (*models.SearchQuery, error) {
sequences, err := m.view.GetLatestStylingSequences(instanceIDs)
if err != nil {
return nil, err
}
searchQuery := models.NewSearchQuery()
for _, sequence := range sequences {
for _, instanceID := range instanceIDs {
var seq uint64
for _, instanceID := range instanceIDs {
for _, sequence := range sequences {
if sequence.InstanceID == instanceID {
seq = sequence.CurrentSequence
break
@@ -90,7 +90,7 @@ func (m *Styling) EventQuery(instanceIDs ...string) (*models.SearchQuery, error)
searchQuery.AddQuery().
AggregateTypeFilter(m.AggregateTypes()...).
LatestSequenceFilter(seq).
InstanceIDFilter(sequence.InstanceID)
InstanceIDFilter(instanceID)
}
return searchQuery, nil
}
@@ -166,12 +166,12 @@ func (m *Styling) processLabelPolicy(event *models.Event) (err error) {
}
func (m *Styling) OnError(event *models.Event, err error) error {
logging.LogWithFields("SPOOL-2m9fs", "id", event.AggregateID).WithError(err).Warn("something went wrong in label policy handler")
logging.WithFields("id", event.AggregateID).WithError(err).Warn("something went wrong in label policy handler")
return spooler.HandleError(event, err, m.view.GetLatestStylingFailedEvent, m.view.ProcessedStylingFailedEvent, m.view.ProcessedStylingSequence, m.errorCountUntilSkip)
}
func (m *Styling) OnSuccess() error {
return spooler.HandleSuccess(m.view.UpdateStylingSpoolerRunTimestamp)
func (m *Styling) OnSuccess(instanceIDs []string) error {
return spooler.HandleSuccess(m.view.UpdateStylingSpoolerRunTimestamp, instanceIDs)
}
func (m *Styling) generateStylingFile(policy *iam_model.LabelPolicyView) error {

5
internal/admin/repository/eventsourcing/repository.go
View File

@@ -7,6 +7,7 @@ import (
"github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/eventstore"
"github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/spooler"
admin_view "github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/view"
eventstore2 "github.com/zitadel/zitadel/internal/eventstore"
v1 "github.com/zitadel/zitadel/internal/eventstore/v1"
es_spol "github.com/zitadel/zitadel/internal/eventstore/v1/spooler"
"github.com/zitadel/zitadel/internal/static"
@@ -22,7 +23,7 @@ type EsRepository struct {
eventstore.AdministratorRepo
}
func Start(conf Config, static static.Storage, dbClient *sql.DB) (*EsRepository, error) {
func Start(ctx context.Context, conf Config, static static.Storage, dbClient *sql.DB, esV2 *eventstore2.Eventstore) (*EsRepository, error) {
es, err := v1.Start(dbClient)
if err != nil {
return nil, err
@@ -32,7 +33,7 @@ func Start(conf Config, static static.Storage, dbClient *sql.DB) (*EsRepository,
return nil, err
}
spool := spooler.StartSpooler(conf.Spooler, es, view, dbClient, static)
spool := spooler.StartSpooler(ctx, conf.Spooler, es, esV2, view, dbClient, static)
return &EsRepository{
spooler: spool,

12
internal/admin/repository/eventsourcing/spooler/spooler.go
View File

@@ -1,14 +1,15 @@
package spooler
import (
"context"
"database/sql"
v1 "github.com/zitadel/zitadel/internal/eventstore/v1"
"github.com/zitadel/zitadel/internal/static"
"github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/handler"
"github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/view"
"github.com/zitadel/zitadel/internal/eventstore"
v1 "github.com/zitadel/zitadel/internal/eventstore/v1"
"github.com/zitadel/zitadel/internal/eventstore/v1/spooler"
"github.com/zitadel/zitadel/internal/static"
)
type SpoolerConfig struct {
@@ -19,13 +20,14 @@ type SpoolerConfig struct {
Handlers handler.Configs
}
func StartSpooler(c SpoolerConfig, es v1.Eventstore, view *view.View, sql *sql.DB, static static.Storage) *spooler.Spooler {
func StartSpooler(ctx context.Context, c SpoolerConfig, es v1.Eventstore, esV2 *eventstore.Eventstore, view *view.View, sql *sql.DB, static static.Storage) *spooler.Spooler {
spoolerConfig := spooler.Config{
Eventstore: es,
EventstoreV2: esV2,
Locker: &locker{dbClient: sql},
ConcurrentWorkers: c.ConcurrentWorkers,
ConcurrentInstances: c.ConcurrentInstances,
ViewHandlers: handler.Register(c.Handlers, c.BulkLimit, c.FailureCountUntilSkip, view, es, static),
ViewHandlers: handler.Register(ctx, c.Handlers, c.BulkLimit, c.FailureCountUntilSkip, view, es, static),
}
spool := spoolerConfig.New()
spool.Start()

14
internal/admin/repository/eventsourcing/view/sequence.go
View File

@@ -19,16 +19,16 @@ func (v *View) latestSequence(viewName, instanceID string) (*repository.CurrentS
return repository.LatestSequence(v.Db, sequencesTable, viewName, instanceID)
}
func (v *View) latestSequences(viewName string, instanceIDs ...string) ([]*repository.CurrentSequence, error) {
return repository.LatestSequences(v.Db, sequencesTable, viewName, instanceIDs...)
func (v *View) latestSequences(viewName string, instanceIDs []string) ([]*repository.CurrentSequence, error) {
return repository.LatestSequences(v.Db, sequencesTable, viewName, instanceIDs)
}
func (v *View) AllCurrentSequences(db, instanceID string) ([]*repository.CurrentSequence, error) {
return repository.AllCurrentSequences(v.Db, db+".current_sequences", instanceID)
}
func (v *View) updateSpoolerRunSequence(viewName string) error {
currentSequences, err := repository.LatestSequences(v.Db, sequencesTable, viewName)
func (v *View) updateSpoolerRunSequence(viewName string, instanceIDs []string) error {
currentSequences, err := repository.LatestSequences(v.Db, sequencesTable, viewName, instanceIDs)
if err != nil {
return err
}
@@ -41,12 +41,6 @@ func (v *View) updateSpoolerRunSequence(viewName string) error {
return repository.UpdateCurrentSequences(v.Db, sequencesTable, currentSequences)
}
func (v *View) GetCurrentSequence(db, viewName string) ([]*repository.CurrentSequence, error) {
sequenceTable := db + ".current_sequences"
fullView := db + "." + viewName
return repository.LatestSequences(v.Db, sequenceTable, fullView)
}
func (v *View) ClearView(db, viewName string) error {
truncateView := db + "." + viewName
sequenceTable := db + ".current_sequences"

8
internal/admin/repository/eventsourcing/view/styling.go
View File

@@ -35,16 +35,16 @@ func (v *View) GetLatestStylingSequence(instanceID string) (*global_view.Current
return v.latestSequence(stylingTyble, instanceID)
}
func (v *View) GetLatestStylingSequences(instanceIDs ...string) ([]*global_view.CurrentSequence, error) {
return v.latestSequences(stylingTyble, instanceIDs...)
func (v *View) GetLatestStylingSequences(instanceIDs []string) ([]*global_view.CurrentSequence, error) {
return v.latestSequences(stylingTyble, instanceIDs)
}
func (v *View) ProcessedStylingSequence(event *models.Event) error {
return v.saveCurrentSequence(stylingTyble, event)
}
func (v *View) UpdateStylingSpoolerRunTimestamp() error {
return v.updateSpoolerRunSequence(stylingTyble)
func (v *View) UpdateStylingSpoolerRunTimestamp(instanceIDs []string) error {
return v.updateSpoolerRunSequence(stylingTyble, instanceIDs)
}
func (v *View) GetLatestStylingFailedEvent(sequence uint64, instanceID string) (*global_view.FailedEvent, error) {