zitadel/internal/eventstore/v1/spooler/spooler.go

229 lines
5.8 KiB
Go
Raw Normal View History

package spooler
import (
"context"
"runtime/debug"
"strconv"
"sync"
"time"
"github.com/zitadel/logging"
v1 "github.com/zitadel/zitadel/internal/eventstore/v1"
"github.com/zitadel/zitadel/internal/eventstore/v1/models"
"github.com/zitadel/zitadel/internal/eventstore/v1/query"
"github.com/zitadel/zitadel/internal/telemetry/tracing"
"github.com/zitadel/zitadel/internal/view/repository"
)
const systemID = "system"
type Spooler struct {
handlers []query.Handler
locker Locker
lockID string
eventstore v1.Eventstore
workers int
queue chan *spooledHandler
concurrentInstances int
}
type Locker interface {
Renew(lockerID, viewModel, instanceID string, waitTime time.Duration) error
}
type spooledHandler struct {
query.Handler
locker Locker
queuedAt time.Time
eventstore v1.Eventstore
concurrentInstances int
}
func (s *Spooler) Start() {
defer logging.WithFields("lockerID", s.lockID, "workers", s.workers).Info("spooler started")
if s.workers < 1 {
return
}
for i := 0; i < s.workers; i++ {
go func(workerIdx int) {
workerID := s.lockID + "--" + strconv.Itoa(workerIdx)
for task := range s.queue {
go requeueTask(task, s.queue)
task.load(workerID)
}
}(i)
}
2020-11-27 14:32:26 +00:00
go func() {
for _, handler := range s.handlers {
s.queue <- &spooledHandler{Handler: handler, locker: s.locker, queuedAt: time.Now(), eventstore: s.eventstore, concurrentInstances: s.concurrentInstances}
2020-11-27 14:32:26 +00:00
}
}()
}
func requeueTask(task *spooledHandler, queue chan<- *spooledHandler) {
time.Sleep(task.MinimumCycleDuration() - time.Since(task.queuedAt))
task.queuedAt = time.Now()
queue <- task
}
func (s *spooledHandler) load(workerID string) {
errs := make(chan error)
defer func() {
close(errs)
err := recover()
if err != nil {
logging.WithFields("cause", err, "stack", string(debug.Stack())).Error("reduce panicked")
}
}()
ctx, cancel := context.WithCancel(context.Background())
go s.awaitError(cancel, errs, workerID)
hasLocked := s.lock(ctx, errs, workerID)
if <-hasLocked {
for {
ids, err := s.eventstore.InstanceIDs(ctx, models.NewSearchQuery().SetColumn(models.Columns_InstanceIDs).AddQuery().ExcludedInstanceIDsFilter("").SearchQuery())
if err != nil {
errs <- err
break
}
for i := 0; i < len(ids); i = i + s.concurrentInstances {
max := i + s.concurrentInstances
if max > len(ids) {
max = len(ids)
}
err = s.processInstances(ctx, workerID, ids[i:max]...)
if err != nil {
errs <- err
}
}
if ctx.Err() == nil {
errs <- nil
}
break
}
}
<-ctx.Done()
}
func (s *spooledHandler) processInstances(ctx context.Context, workerID string, ids ...string) error {
for {
events, err := s.query(ctx, ids...)
if err != nil {
return err
}
if len(events) == 0 {
return nil
}
err = s.process(ctx, events, workerID)
if err != nil {
return err
}
if uint64(len(events)) < s.QueryLimit() {
// no more events to process
return nil
}
}
}
func (s *spooledHandler) awaitError(cancel func(), errs chan error, workerID string) {
select {
case err := <-errs:
cancel()
feat: split users into human and machine (#470) * feat(management): service accounts * chore: current go version * init * refactor: apis * feat(internal): start impl of service account * chore: start impl of machine/human users * code compiles * fix: tests * fix: tests * fix: add new event types to switches * chore: add cases to event types * fix(management): definitive proto messages * fix: machine/human * fix: add missing tables as todos * fix: remove unused permissions * fix: refactoring * fix: refactor * fix: human registered * fix: user id * fix: logid * fix: proto remove //equal * chore(management): remove no comment * fix: human mfas * fix: user subobjects * chore: rename existing to better name * fix: username in user (#634) * fix: username in user * fix: username * fix remove unused code * fix add validations * fix: use new user in all apis * fix: regexp for username in api * fix: fill user data for human and machine (#638) * fix: fill Display name grant/member handlers fix: add description to grant/member objects in api fix: check if user is human in login * fix: remove description from member and grant * chore: remove todos * feat: machine keys * fix: implement missing parts * feat: machine key management view * fix: remove keys from machine view * fix: set default expiration date * fix: get key by ids * feat: add machine keys in proto * feat: machine keys * fix: add migration * fix: mig * fix: correct method name * feat: user search * feat: user search * fix: log ids * fix partial authconfig prompt, domain c perm * membership read check * contributor refresh trigger, observe org write * fix: migrations * fix(console): machine build (#660) * frontend 1 * fix html bindings * trailing comma * user permissions, project deactivate * fix(console): human view (#661) * fix search user view, user detail form * rm log * feat(console): user services list and create (#663) * fix search user view, user detail form * rm log * machine list * generic table component * create user service * proove table for undefined values * tmp disable user link if machine * lint * lint styles * user table lint * Update console/src/assets/i18n/de.json Co-authored-by: Florian Forster <florian@caos.ch> * feat(console): service user detail view, keys cr_d, fix search user autocomplete (#664) * service users for sidenav, routing * i18n * back routes * machine detail form * update machine detail, fix svc user grants * keys table * add key dialog, timestamp creation * check permission on create, delete, fix selection * lint ts, scss * Update console/src/assets/i18n/de.json * Apply suggestions from code review Co-authored-by: Florian Forster <florian@caos.ch> * allow user grants for project.write * management service * fix mgmt service * feat: Machine keys (#655) * fix: memberships (#633) * feat: add iam members to memberships * fix: search project grants * fix: rename * feat: idp and login policy configurations (#619) * feat: oidc config * fix: oidc configurations * feat: oidc idp config * feat: add oidc config test * fix: tests * fix: tests * feat: translate new events * feat: idp eventstore * feat: idp eventstore * fix: tests * feat: command side idp * feat: query side idp * feat: idp config on org * fix: tests * feat: authz idp on org * feat: org idps * feat: login policy * feat: login policy * feat: login policy * feat: add idp func on login policy * feat: add validation to loginpolicy and idp provider * feat: add default login policy * feat: login policy on org * feat: login policy on org * fix: id config handlers * fix: id config handlers * fix: create idp on org * fix: create idp on org * fix: not existing idp config * fix: default login policy * fix: add login policy on org * fix: idp provider search on org * fix: test * fix: remove idp on org * fix: test * fix: test * fix: remove admin idp * fix: logo src as byte * fix: migration * fix: tests * Update internal/iam/repository/eventsourcing/iam.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * Update internal/iam/repository/eventsourcing/iam_test.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * Update internal/iam/repository/eventsourcing/iam_test.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * Update internal/iam/repository/eventsourcing/model/login_policy.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * Update internal/iam/repository/eventsourcing/model/login_policy.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * Update internal/org/repository/eventsourcing/org_test.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * Update internal/iam/repository/eventsourcing/model/login_policy_test.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * Update internal/iam/repository/eventsourcing/model/login_policy_test.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * fix: pr comments * fix: tests * Update types.go * fix: merge request changes * fix: reduce optimization Co-authored-by: Silvan <silvan.reusser@gmail.com> Co-authored-by: Livio Amstutz <livio.a@gmail.com> * fix: reread user mfas, preferred loginname as otp account name (#636) * fix: reread user mfas * fix: use preferred login name as otp account name * fix: tests * fix: reduce (#635) * fix: management reduce optimization * fix: reduce optimization * fix: reduce optimization * fix: merge master * chore(deps): bump github.com/gorilla/schema from 1.1.0 to 1.2.0 (#627) Bumps [github.com/gorilla/schema](https://github.com/gorilla/schema) from 1.1.0 to 1.2.0. - [Release notes](https://github.com/gorilla/schema/releases) - [Commits](https://github.com/gorilla/schema/compare/v1.1.0...v1.2.0) Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/gorilla/mux from 1.7.4 to 1.8.0 (#624) Bumps [github.com/gorilla/mux](https://github.com/gorilla/mux) from 1.7.4 to 1.8.0. - [Release notes](https://github.com/gorilla/mux/releases) - [Commits](https://github.com/gorilla/mux/compare/v1.7.4...v1.8.0) Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/DATA-DOG/go-sqlmock from 1.4.1 to 1.5.0 (#591) Bumps [github.com/DATA-DOG/go-sqlmock](https://github.com/DATA-DOG/go-sqlmock) from 1.4.1 to 1.5.0. - [Release notes](https://github.com/DATA-DOG/go-sqlmock/releases) - [Commits](https://github.com/DATA-DOG/go-sqlmock/compare/v1.4.1...v1.5.0) Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore: auto assign issues and PR to ZTIADEL project board (#643) * Create main.yml * Update main.yml Co-authored-by: Livio Amstutz <livio.a@gmail.com> * fix(console): project grant members, update deps (#645) * fix: searchprojectgrantmembers * chore(deps-dev): bump @angular/cli from 10.0.6 to 10.0.7 in /console (#622) Bumps [@angular/cli](https://github.com/angular/angular-cli) from 10.0.6 to 10.0.7. - [Release notes](https://github.com/angular/angular-cli/releases) - [Commits](https://github.com/angular/angular-cli/compare/v10.0.6...v10.0.7) Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps-dev): bump @angular-devkit/build-angular in /console (#626) Bumps [@angular-devkit/build-angular](https://github.com/angular/angular-cli) from 0.1000.6 to 0.1000.7. - [Release notes](https://github.com/angular/angular-cli/releases) - [Commits](https://github.com/angular/angular-cli/commits) Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Peintner <max@caos.ch> * chore(deps-dev): bump @types/jasmine from 3.5.12 to 3.5.13 in /console (#623) Bumps [@types/jasmine](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/jasmine) from 3.5.12 to 3.5.13. - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/jasmine) Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps-dev): bump ts-node from 8.10.2 to 9.0.0 in /console (#629) Bumps [ts-node](https://github.com/TypeStrong/ts-node) from 8.10.2 to 9.0.0. - [Release notes](https://github.com/TypeStrong/ts-node/releases) - [Commits](https://github.com/TypeStrong/ts-node/compare/v8.10.2...v9.0.0) Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * update packlock Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore: delete main.yml (#648) * fix: usergrant (#650) * fix(console): mfa refresh after verification, member eventemitter (#651) * refresh mfa * fix: detail link from contributors * lint * feat: add domain verification notification (#649) * fix: dont (re)generate client secret with auth type none * fix(cors): allow Origin from request * feat: add origin allow list and fix some core issues * rename migration * fix UserIDsByDomain * feat: send email to users after domain claim * username * check origin on userinfo * update oidc pkg * fix: add migration 1.6 * change username * change username * remove unique email aggregate * change username in mgmt * search global user by login name * fix test * change user search in angular * fix tests * merge * userview in angular * fix merge * Update pkg/grpc/management/proto/management.proto Co-authored-by: Fabi <38692350+fgerschwiler@users.noreply.github.com> * Update internal/notification/static/i18n/de.yaml Co-authored-by: Fabi <38692350+fgerschwiler@users.noreply.github.com> * fix Co-authored-by: Fabi <38692350+fgerschwiler@users.noreply.github.com> * fix: translation (#647) * fix: translation * fix: translation * fix: translation * fix: remove unused code * fix: log err * fix: migration numbers (#652) * chore: issue / feature templates (#642) * feat: machine keys * fix: implement missing parts * feat: machine key management view * fix: remove keys from machine view * feat: global org read (#657) * fix: set default expiration date * fix: get key by ids * feat: add machine keys in proto * feat: machine keys * fix: add migration * fix: mig * fix: correct method name * feat: user search * feat: user search * fix: log ids * fix: migrations * fix(console): machine build (#660) * frontend 1 * fix html bindings * trailing comma * fix(console): human view (#661) * fix search user view, user detail form * rm log * feat(console): user services list and create (#663) * fix search user view, user detail form * rm log * machine list * generic table component * create user service * proove table for undefined values * tmp disable user link if machine * lint * lint styles * user table lint * Update console/src/assets/i18n/de.json Co-authored-by: Florian Forster <florian@caos.ch> * feat(console): service user detail view, keys cr_d, fix search user autocomplete (#664) * service users for sidenav, routing * i18n * back routes * machine detail form * update machine detail, fix svc user grants * keys table * add key dialog, timestamp creation * check permission on create, delete, fix selection * lint ts, scss * Update console/src/assets/i18n/de.json * Apply suggestions from code review Co-authored-by: Florian Forster <florian@caos.ch> * refactor: protos * fix(management): key expiration date * fix: check if user is human * fix: marshal key details * fix: correct generate login names * fix: logid Co-authored-by: Fabi <38692350+fgerschwiler@users.noreply.github.com> Co-authored-by: Livio Amstutz <livio.a@gmail.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Max Peintner <max@caos.ch> Co-authored-by: Fabiennne <fabienne.gerschwiler@gmail.com> Co-authored-by: Florian Forster <florian@caos.ch> * fix: naming * refactor: findings * fix: username * fix: mfa upper case * fix: tests * fix: add translations * reactivatemyorg req typeö * fix: projectType for console * fix: user changes * fix: translate events * fix: event type translation * fix: remove unused types Co-authored-by: Fabiennne <fabienne.gerschwiler@gmail.com> Co-authored-by: Max Peintner <max@caos.ch> Co-authored-by: Florian Forster <florian@caos.ch> Co-authored-by: Fabi <38692350+fgerschwiler@users.noreply.github.com> Co-authored-by: Livio Amstutz <livio.a@gmail.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2020-08-31 15:48:01 +00:00
logging.Log("SPOOL-OT8di").OnError(err).WithField("view", s.ViewModel()).WithField("worker", workerID).Debug("load canceled")
}
}
func (s *spooledHandler) process(ctx context.Context, events []*models.Event, workerID string) error {
for i, event := range events {
select {
case <-ctx.Done():
logging.WithFields("view", s.ViewModel(), "worker", workerID, "traceID", tracing.TraceIDFromCtx(ctx)).Debug("context canceled")
return nil
default:
if err := s.Reduce(event); err != nil {
err = s.OnError(event, err)
if err == nil {
continue
}
time.Sleep(100 * time.Millisecond)
return s.process(ctx, events[i:], workerID)
}
}
}
err := s.OnSuccess()
logging.WithFields("view", s.ViewModel(), "worker", workerID, "traceID", tracing.TraceIDFromCtx(ctx)).OnError(err).Warn("could not process on success func")
return err
}
func (s *spooledHandler) query(ctx context.Context, instanceIDs ...string) ([]*models.Event, error) {
query, err := s.EventQuery(instanceIDs...)
if err != nil {
return nil, err
}
query.Limit = s.QueryLimit()
return s.eventstore.FilterEvents(ctx, query)
}
//lock ensures the lock on the database.
// the returned channel will be closed if ctx is done or an error occured durring lock
func (s *spooledHandler) lock(ctx context.Context, errs chan<- error, workerID string) chan bool {
renewTimer := time.After(0)
locked := make(chan bool)
go func(locked chan bool) {
var firstLock sync.Once
defer close(locked)
for {
select {
case <-ctx.Done():
return
case <-renewTimer:
err := s.locker.Renew(workerID, s.ViewModel(), systemID, s.LockDuration())
firstLock.Do(func() {
locked <- err == nil
})
if err == nil {
renewTimer = time.After(s.LockDuration())
continue
}
if ctx.Err() == nil {
errs <- err
}
return
}
}
}(locked)
return locked
}
func HandleError(event *models.Event, failedErr error,
latestFailedEvent func(sequence uint64, instanceID string) (*repository.FailedEvent, error),
processFailedEvent func(*repository.FailedEvent) error,
processSequence func(*models.Event) error,
errorCountUntilSkip uint64) error {
failedEvent, err := latestFailedEvent(event.Sequence, event.InstanceID)
if err != nil {
return err
}
failedEvent.FailureCount++
failedEvent.ErrMsg = failedErr.Error()
failedEvent.InstanceID = event.InstanceID
err = processFailedEvent(failedEvent)
if err != nil {
return err
}
if errorCountUntilSkip <= failedEvent.FailureCount {
return processSequence(event)
}
return failedErr
}
func HandleSuccess(updateSpoolerRunTimestamp func() error) error {
return updateSpoolerRunTimestamp()
}