TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-12 04:07:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: scheduling (#3978)

Browse Source 
* fix: improve scheduling

* build pre-release

* fix: locker

* fix: user handler and print stack in case of panic in reducer

* chore: remove sentry

* fix: improve handler projection and implement tests

* more tests

* fix: race condition in tests

* Update internal/eventstore/repository/sql/query.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* fix: implemented suggested changes

* fix: lock statement

Co-authored-by: Silvan <silvan.reusser@gmail.com>
This commit is contained in:
Livio Spring
2022-07-22 12:08:39 +02:00
committed by GitHub
parent 0cc548e3f8
commit aed7010508
83 changed files with 1494 additions and 1544 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
internal/eventstore/v1/eventstore.go
View File

@@ -13,6 +13,7 @@ type Eventstore interface {
Health(ctx context.Context) error
FilterEvents(ctx context.Context, searchQuery *models.SearchQuery) (events []*models.Event, err error)
Subscribe(aggregates ...models.AggregateType) *Subscription
InstanceIDs(ctx context.Context, searchQuery *models.SearchQuery) ([]string, error)
}
var _ Eventstore = (*eventstore)(nil)
@@ -37,3 +38,10 @@ func (es *eventstore) FilterEvents(ctx context.Context, searchQuery *models.Sear
func (es *eventstore) Health(ctx context.Context) error {
return es.repo.Health(ctx)
}
func (es *eventstore) InstanceIDs(ctx context.Context, searchQuery *models.SearchQuery) ([]string, error) {
if err := searchQuery.Validate(); err != nil {
return nil, err
}
return es.repo.InstanceIDs(ctx, models.FactoryFromSearchQuery(searchQuery))
}

4
internal/eventstore/v1/internal/repository/repository.go
View File

@@ -11,6 +11,8 @@ type Repository interface {
// Filter returns all events matching the given search query
Filter(ctx context.Context, searchQuery *models.SearchQueryFactory) (events []*models.Event, err error)
//LatestSequence returns the latests sequence found by the the search query
//LatestSequence returns the latest sequence found by the search query
LatestSequence(ctx context.Context, queryFactory *models.SearchQueryFactory) (uint64, error)
//InstanceIDs returns the instance ids found by the search query
InstanceIDs(ctx context.Context, queryFactory *models.SearchQueryFactory) ([]string, error)
}

29
internal/eventstore/v1/internal/repository/sql/filter.go
View File

@@ -5,6 +5,7 @@ import (
"database/sql"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/errors"
es_models "github.com/zitadel/zitadel/internal/eventstore/v1/models"
"github.com/zitadel/zitadel/internal/telemetry/tracing"
@@ -60,3 +61,31 @@ func (db *SQL) LatestSequence(ctx context.Context, queryFactory *es_models.Searc
}
return uint64(*sequence), nil
}
func (db *SQL) InstanceIDs(ctx context.Context, queryFactory *es_models.SearchQueryFactory) ([]string, error) {
query, _, values, rowScanner := buildQuery(queryFactory)
if query == "" {
return nil, errors.ThrowInvalidArgument(nil, "SQL-Sfwg2", "invalid query factory")
}
rows, err := db.client.Query(query, values...)
if err != nil {
logging.New().WithError(err).Info("query failed")
return nil, errors.ThrowInternal(err, "SQL-Sfg3r", "unable to filter instance ids")
}
defer rows.Close()
ids := make([]string, 0)
for rows.Next() {
var id string
err := rowScanner(rows.Scan, &id)
if err != nil {
return nil, err
}
ids = append(ids, id)
}
return ids, nil
}

15
internal/eventstore/v1/internal/repository/sql/query.go
View File

@@ -44,7 +44,7 @@ func buildQuery(queryFactory *es_models.SearchQueryFactory) (query string, limit
}
query += where
if searchQuery.Columns != es_models.Columns_Max_Sequence {
if searchQuery.Columns == es_models.Columns_Event {
query += " ORDER BY event_sequence"
if searchQuery.Desc {
query += " DESC"
@@ -104,6 +104,19 @@ func prepareColumns(columns es_models.Columns) (string, func(s scan, dest interf
}
return z_errors.ThrowInternal(err, "SQL-bN5xg", "something went wrong")
}
case es_models.Columns_InstanceIDs:
return "SELECT DISTINCT instance_id FROM eventstore.events", func(row scan, dest interface{}) (err error) {
instanceID, ok := dest.(*string)
if !ok {
return z_errors.ThrowInvalidArgument(nil, "SQL-Fef5h", "type must be *string]")
}
err = row(instanceID)
if err != nil {
logging.New().WithError(err).Warn("unable to scan row")
return z_errors.ThrowInternal(err, "SQL-SFef3", "unable to scan row")
}
return nil
}
case es_models.Columns_Event:
return selectStmt, func(row scan, dest interface{}) (err error) {
event, ok := dest.(*es_models.Event)

6
internal/eventstore/v1/models/search_query.go
View File

@@ -41,6 +41,7 @@ type Columns int32
const (
Columns_Event = iota
Columns_Max_Sequence
Columns_InstanceIDs
//insert new columns-types before this columnsCount because count is needed for validation
columnsCount
)
@@ -48,7 +49,7 @@ const (
//FactoryFromSearchQuery is deprecated because it's for migration purposes. use NewSearchQueryFactory
func FactoryFromSearchQuery(q *SearchQuery) *SearchQueryFactory {
factory := &SearchQueryFactory{
columns: Columns_Event,
columns: q.Columns,
desc: q.Desc,
limit: q.Limit,
queries: make([]*query, len(q.Queries)),
@@ -232,6 +233,9 @@ func (q *query) eventTypeFilter() *Filter {
}
func (q *query) aggregateTypeFilter() *Filter {
if len(q.aggregateTypes) < 1 {
return nil
}
if len(q.aggregateTypes) == 1 {
return NewFilter(Field_AggregateType, q.aggregateTypes[0], Operation_Equals)
}

6
internal/eventstore/v1/models/search_query_old.go
View File

@@ -8,6 +8,7 @@ import (
//SearchQuery is deprecated. Use SearchQueryFactory
type SearchQuery struct {
Columns Columns
Limit uint64
Desc bool
Filters []*Filter
@@ -27,6 +28,11 @@ func NewSearchQuery() *SearchQuery {
}
}
func (q *SearchQuery) SetColumn(columns Columns) *SearchQuery {
q.Columns = columns
return q
}
func (q *SearchQuery) AddQuery() *Query {
query := &Query{
searchQuery: q,

19
internal/eventstore/v1/query/handler.go
View File

@@ -2,9 +2,9 @@ package query
import (
"context"
"runtime/debug"
"time"
"github.com/getsentry/sentry-go"
"github.com/zitadel/logging"
v1 "github.com/zitadel/zitadel/internal/eventstore/v1"
@@ -17,7 +17,7 @@ const (
type Handler interface {
ViewModel() string
EventQuery() (*models.SearchQuery, error)
EventQuery(instanceIDs ...string) (*models.SearchQuery, error)
Reduce(*models.Event) error
OnError(event *models.Event, err error) error
OnSuccess() error
@@ -37,14 +37,13 @@ func ReduceEvent(handler Handler, event *models.Event) {
err := recover()
if err != nil {
sentry.CurrentHub().Recover(err)
handler.Subscription().Unsubscribe()
logging.WithFields("HANDL-SAFe1").Errorf("reduce panicked: %v", err)
logging.WithFields("cause", err, "stack", string(debug.Stack())).Error("reduce panicked")
}
}()
currentSequence, err := handler.CurrentSequence(event.InstanceID)
if err != nil {
logging.New().WithError(err).Warn("unable to get current sequence")
logging.WithError(err).Warn("unable to get current sequence")
return
}
@@ -58,14 +57,14 @@ func ReduceEvent(handler Handler, event *models.Event) {
unprocessedEvents, err := handler.Eventstore().FilterEvents(context.Background(), searchQuery)
if err != nil {
logging.WithFields("HANDL-L6YH1", "sequence", event.Sequence).Warn("filter failed")
logging.WithFields("sequence", event.Sequence).Warn("filter failed")
return
}
for _, unprocessedEvent := range unprocessedEvents {
currentSequence, err := handler.CurrentSequence(unprocessedEvent.InstanceID)
if err != nil {
logging.Log("HANDL-BmpkC").WithError(err).Warn("unable to get current sequence")
logging.WithError(err).Warn("unable to get current sequence")
return
}
if unprocessedEvent.Sequence < currentSequence {
@@ -78,12 +77,12 @@ func ReduceEvent(handler Handler, event *models.Event) {
}
err = handler.Reduce(unprocessedEvent)
logging.WithFields("HANDL-V42TI", "sequence", unprocessedEvent.Sequence).OnError(err).Warn("reduce failed")
logging.WithFields("sequence", unprocessedEvent.Sequence).OnError(err).Warn("reduce failed")
}
if len(unprocessedEvents) == eventLimit {
logging.WithFields("QUERY-BSqe9", "sequence", event.Sequence).Warn("didnt process event")
logging.WithFields("sequence", event.Sequence).Warn("didnt process event")
return
}
err = handler.Reduce(event)
logging.WithFields("HANDL-wQDL2", "sequence", event.Sequence).OnError(err).Warn("reduce failed")
logging.WithFields("sequence", event.Sequence).OnError(err).Warn("reduce failed")
}

22
internal/eventstore/v1/spooler/config.go
View File

@@ -11,10 +11,11 @@ import (
)
type Config struct {
Eventstore v1.Eventstore
Locker Locker
ViewHandlers []query.Handler
ConcurrentWorkers int
Eventstore v1.Eventstore
Locker Locker
ViewHandlers []query.Handler
ConcurrentWorkers int
ConcurrentInstances int
}
func (c *Config) New() *Spooler {
@@ -27,11 +28,12 @@ func (c *Config) New() *Spooler {
})
return &Spooler{
handlers: c.ViewHandlers,
lockID: lockID,
eventstore: c.Eventstore,
locker: c.Locker,
queue: make(chan *spooledHandler, len(c.ViewHandlers)),
workers: c.ConcurrentWorkers,
handlers: c.ViewHandlers,
lockID: lockID,
eventstore: c.Eventstore,
locker: c.Locker,
queue: make(chan *spooledHandler, len(c.ViewHandlers)),
workers: c.ConcurrentWorkers,
concurrentInstances: c.ConcurrentInstances,
}
}

75
internal/eventstore/v1/spooler/spooler.go
View File

@@ -2,11 +2,11 @@ package spooler
import (
"context"
"runtime/debug"
"strconv"
"sync"
"time"
"github.com/getsentry/sentry-go"
"github.com/zitadel/logging"
v1 "github.com/zitadel/zitadel/internal/eventstore/v1"
@@ -19,12 +19,13 @@ import (
const systemID = "system"
type Spooler struct {
handlers []query.Handler
locker Locker
lockID string
eventstore v1.Eventstore
workers int
queue chan *spooledHandler
handlers []query.Handler
locker Locker
lockID string
eventstore v1.Eventstore
workers int
queue chan *spooledHandler
concurrentInstances int
}
type Locker interface {
@@ -33,9 +34,10 @@ type Locker interface {
type spooledHandler struct {
query.Handler
locker Locker
queuedAt time.Time
eventstore v1.Eventstore
locker Locker
queuedAt time.Time
eventstore v1.Eventstore
concurrentInstances int
}
func (s *Spooler) Start() {
@@ -55,7 +57,7 @@ func (s *Spooler) Start() {
}
go func() {
for _, handler := range s.handlers {
s.queue <- &spooledHandler{Handler: handler, locker: s.locker, queuedAt: time.Now(), eventstore: s.eventstore}
s.queue <- &spooledHandler{Handler: handler, locker: s.locker, queuedAt: time.Now(), eventstore: s.eventstore, concurrentInstances: s.concurrentInstances}
}
}()
}
@@ -73,7 +75,7 @@ func (s *spooledHandler) load(workerID string) {
err := recover()
if err != nil {
sentry.CurrentHub().Recover(err)
logging.WithFields("cause", err, "stack", string(debug.Stack())).Error("reduce panicked")
}
}()
ctx, cancel := context.WithCancel(context.Background())
@@ -82,29 +84,50 @@ func (s *spooledHandler) load(workerID string) {
if <-hasLocked {
for {
events, err := s.query(ctx)
ids, err := s.eventstore.InstanceIDs(ctx, models.NewSearchQuery().SetColumn(models.Columns_InstanceIDs).AddQuery().ExcludedInstanceIDsFilter("").SearchQuery())
if err != nil {
errs <- err
break
}
err = s.process(ctx, events, workerID)
if err != nil {
errs <- err
break
}
if uint64(len(events)) < s.QueryLimit() {
// no more events to process
// stop chan
if ctx.Err() == nil {
errs <- nil
for i := 0; i < len(ids); i = i + s.concurrentInstances {
max := i + s.concurrentInstances
if max > len(ids) {
max = len(ids)
}
err = s.processInstances(ctx, workerID, ids[i:max]...)
if err != nil {
errs <- err
}
break
}
if ctx.Err() == nil {
errs <- nil
}
break
}
}
<-ctx.Done()
}
func (s *spooledHandler) processInstances(ctx context.Context, workerID string, ids ...string) error {
for {
events, err := s.query(ctx, ids...)
if err != nil {
return err
}
if len(events) == 0 {
return nil
}
err = s.process(ctx, events, workerID)
if err != nil {
return err
}
if uint64(len(events)) < s.QueryLimit() {
// no more events to process
return nil
}
}
}
func (s *spooledHandler) awaitError(cancel func(), errs chan error, workerID string) {
select {
case err := <-errs:
@@ -135,8 +158,8 @@ func (s *spooledHandler) process(ctx context.Context, events []*models.Event, wo
return err
}
func (s *spooledHandler) query(ctx context.Context) ([]*models.Event, error) {
query, err := s.EventQuery()
func (s *spooledHandler) query(ctx context.Context, instanceIDs ...string) ([]*models.Event, error) {
query, err := s.EventQuery(instanceIDs...)
if err != nil {
return nil, err
}

5
internal/eventstore/v1/spooler/spooler_test.go
View File

@@ -47,7 +47,7 @@ func (h *testHandler) Subscription() *v1.Subscription {
return nil
}
func (h *testHandler) EventQuery() (*models.SearchQuery, error) {
func (h *testHandler) EventQuery(instanceIDs ...string) (*models.SearchQuery, error) {
if h.queryError != nil {
return nil, h.queryError
}
@@ -111,6 +111,9 @@ func (es *eventstoreStub) PushAggregates(ctx context.Context, in ...*models.Aggr
func (es *eventstoreStub) LatestSequence(ctx context.Context, in *models.SearchQueryFactory) (uint64, error) {
return 0, nil
}
func (es *eventstoreStub) InstanceIDs(ctx context.Context, in *models.SearchQuery) ([]string, error) {
return nil, nil
}
func (es *eventstoreStub) V2() *eventstore.Eventstore {
return nil
}