2020-04-29 10:11:13 +00:00
package spooler
import (
"context"
2022-07-22 10:08:39 +00:00
"runtime/debug"
2020-07-28 07:42:21 +00:00
"strconv"
2020-09-18 11:39:28 +00:00
"sync"
2020-12-21 17:42:34 +00:00
"time"
2020-07-02 06:08:55 +00:00
2022-04-26 23:01:45 +00:00
"github.com/zitadel/logging"
2022-03-15 06:19:02 +00:00
2022-04-26 23:01:45 +00:00
v1 "github.com/zitadel/zitadel/internal/eventstore/v1"
"github.com/zitadel/zitadel/internal/eventstore/v1/models"
"github.com/zitadel/zitadel/internal/eventstore/v1/query"
"github.com/zitadel/zitadel/internal/telemetry/tracing"
"github.com/zitadel/zitadel/internal/view/repository"
2020-04-29 10:11:13 +00:00
)
2022-04-19 06:26:12 +00:00
const systemID = "system"
2020-04-29 10:11:13 +00:00
type Spooler struct {
2022-07-22 10:08:39 +00:00
handlers [ ] query . Handler
locker Locker
lockID string
eventstore v1 . Eventstore
workers int
queue chan * spooledHandler
concurrentInstances int
2020-04-29 10:11:13 +00:00
}
type Locker interface {
2022-04-19 06:26:12 +00:00
Renew ( lockerID , viewModel , instanceID string , waitTime time . Duration ) error
2020-04-29 10:11:13 +00:00
}
type spooledHandler struct {
2020-07-28 07:42:21 +00:00
query . Handler
2022-07-22 10:08:39 +00:00
locker Locker
queuedAt time . Time
eventstore v1 . Eventstore
concurrentInstances int
2020-04-29 10:11:13 +00:00
}
func ( s * Spooler ) Start ( ) {
2022-03-15 06:19:02 +00:00
defer logging . WithFields ( "lockerID" , s . lockID , "workers" , s . workers ) . Info ( "spooler started" )
2020-07-28 07:42:21 +00:00
if s . workers < 1 {
2020-04-29 10:11:13 +00:00
return
}
2020-07-28 07:42:21 +00:00
for i := 0 ; i < s . workers ; i ++ {
go func ( workerIdx int ) {
workerID := s . lockID + "--" + strconv . Itoa ( workerIdx )
for task := range s . queue {
2020-09-18 11:39:28 +00:00
go requeueTask ( task , s . queue )
2020-07-28 07:42:21 +00:00
task . load ( workerID )
2020-04-29 10:11:13 +00:00
}
2020-07-28 07:42:21 +00:00
} ( i )
2020-04-29 10:11:13 +00:00
}
2020-11-27 14:32:26 +00:00
go func ( ) {
for _ , handler := range s . handlers {
2022-07-22 10:08:39 +00:00
s . queue <- & spooledHandler { Handler : handler , locker : s . locker , queuedAt : time . Now ( ) , eventstore : s . eventstore , concurrentInstances : s . concurrentInstances }
2020-11-27 14:32:26 +00:00
}
} ( )
2020-04-29 10:11:13 +00:00
}
2020-09-18 11:39:28 +00:00
func requeueTask ( task * spooledHandler , queue chan <- * spooledHandler ) {
time . Sleep ( task . MinimumCycleDuration ( ) - time . Since ( task . queuedAt ) )
task . queuedAt = time . Now ( )
queue <- task
}
2020-07-28 07:42:21 +00:00
func ( s * spooledHandler ) load ( workerID string ) {
2020-04-29 10:11:13 +00:00
errs := make ( chan error )
2021-07-06 11:36:35 +00:00
defer func ( ) {
close ( errs )
err := recover ( )
if err != nil {
2022-07-22 10:08:39 +00:00
logging . WithFields ( "cause" , err , "stack" , string ( debug . Stack ( ) ) ) . Error ( "reduce panicked" )
2021-07-06 11:36:35 +00:00
}
} ( )
2020-07-28 07:42:21 +00:00
ctx , cancel := context . WithCancel ( context . Background ( ) )
go s . awaitError ( cancel , errs , workerID )
hasLocked := s . lock ( ctx , errs , workerID )
2020-04-29 10:11:13 +00:00
if <- hasLocked {
2020-12-21 17:42:34 +00:00
for {
2022-07-22 10:08:39 +00:00
ids , err := s . eventstore . InstanceIDs ( ctx , models . NewSearchQuery ( ) . SetColumn ( models . Columns_InstanceIDs ) . AddQuery ( ) . ExcludedInstanceIDsFilter ( "" ) . SearchQuery ( ) )
2020-12-21 17:42:34 +00:00
if err != nil {
errs <- err
break
}
2022-07-22 10:08:39 +00:00
for i := 0 ; i < len ( ids ) ; i = i + s . concurrentInstances {
max := i + s . concurrentInstances
if max > len ( ids ) {
max = len ( ids )
2020-12-21 17:42:34 +00:00
}
2022-07-22 10:08:39 +00:00
err = s . processInstances ( ctx , workerID , ids [ i : max ] ... )
if err != nil {
errs <- err
}
}
if ctx . Err ( ) == nil {
errs <- nil
2020-12-21 17:42:34 +00:00
}
2022-07-22 10:08:39 +00:00
break
2020-04-29 10:11:13 +00:00
}
}
<- ctx . Done ( )
}
2022-07-22 10:08:39 +00:00
func ( s * spooledHandler ) processInstances ( ctx context . Context , workerID string , ids ... string ) error {
for {
events , err := s . query ( ctx , ids ... )
if err != nil {
return err
}
if len ( events ) == 0 {
return nil
}
err = s . process ( ctx , events , workerID )
if err != nil {
return err
}
if uint64 ( len ( events ) ) < s . QueryLimit ( ) {
// no more events to process
return nil
}
}
}
2020-07-28 07:42:21 +00:00
func ( s * spooledHandler ) awaitError ( cancel func ( ) , errs chan error , workerID string ) {
2020-04-29 10:11:13 +00:00
select {
case err := <- errs :
cancel ( )
feat: split users into human and machine (#470)
* feat(management): service accounts
* chore: current go version
* init
* refactor: apis
* feat(internal): start impl of service account
* chore: start impl of machine/human users
* code compiles
* fix: tests
* fix: tests
* fix: add new event types to switches
* chore: add cases to event types
* fix(management): definitive proto messages
* fix: machine/human
* fix: add missing tables as todos
* fix: remove unused permissions
* fix: refactoring
* fix: refactor
* fix: human registered
* fix: user id
* fix: logid
* fix: proto remove //equal
* chore(management): remove no comment
* fix: human mfas
* fix: user subobjects
* chore: rename existing to better name
* fix: username in user (#634)
* fix: username in user
* fix: username
* fix remove unused code
* fix add validations
* fix: use new user in all apis
* fix: regexp for username in api
* fix: fill user data for human and machine (#638)
* fix: fill Display name grant/member handlers
fix: add description to grant/member objects in api
fix: check if user is human in login
* fix: remove description from member and grant
* chore: remove todos
* feat: machine keys
* fix: implement missing parts
* feat: machine key management view
* fix: remove keys from machine view
* fix: set default expiration date
* fix: get key by ids
* feat: add machine keys in proto
* feat: machine keys
* fix: add migration
* fix: mig
* fix: correct method name
* feat: user search
* feat: user search
* fix: log ids
* fix partial authconfig prompt, domain c perm
* membership read check
* contributor refresh trigger, observe org write
* fix: migrations
* fix(console): machine build (#660)
* frontend 1
* fix html bindings
* trailing comma
* user permissions, project deactivate
* fix(console): human view (#661)
* fix search user view, user detail form
* rm log
* feat(console): user services list and create (#663)
* fix search user view, user detail form
* rm log
* machine list
* generic table component
* create user service
* proove table for undefined values
* tmp disable user link if machine
* lint
* lint styles
* user table lint
* Update console/src/assets/i18n/de.json
Co-authored-by: Florian Forster <florian@caos.ch>
* feat(console): service user detail view, keys cr_d, fix search user autocomplete (#664)
* service users for sidenav, routing
* i18n
* back routes
* machine detail form
* update machine detail, fix svc user grants
* keys table
* add key dialog, timestamp creation
* check permission on create, delete, fix selection
* lint ts, scss
* Update console/src/assets/i18n/de.json
* Apply suggestions from code review
Co-authored-by: Florian Forster <florian@caos.ch>
* allow user grants for project.write
* management service
* fix mgmt service
* feat: Machine keys (#655)
* fix: memberships (#633)
* feat: add iam members to memberships
* fix: search project grants
* fix: rename
* feat: idp and login policy configurations (#619)
* feat: oidc config
* fix: oidc configurations
* feat: oidc idp config
* feat: add oidc config test
* fix: tests
* fix: tests
* feat: translate new events
* feat: idp eventstore
* feat: idp eventstore
* fix: tests
* feat: command side idp
* feat: query side idp
* feat: idp config on org
* fix: tests
* feat: authz idp on org
* feat: org idps
* feat: login policy
* feat: login policy
* feat: login policy
* feat: add idp func on login policy
* feat: add validation to loginpolicy and idp provider
* feat: add default login policy
* feat: login policy on org
* feat: login policy on org
* fix: id config handlers
* fix: id config handlers
* fix: create idp on org
* fix: create idp on org
* fix: not existing idp config
* fix: default login policy
* fix: add login policy on org
* fix: idp provider search on org
* fix: test
* fix: remove idp on org
* fix: test
* fix: test
* fix: remove admin idp
* fix: logo src as byte
* fix: migration
* fix: tests
* Update internal/iam/repository/eventsourcing/iam.go
Co-authored-by: Silvan <silvan.reusser@gmail.com>
* Update internal/iam/repository/eventsourcing/iam_test.go
Co-authored-by: Silvan <silvan.reusser@gmail.com>
* Update internal/iam/repository/eventsourcing/iam_test.go
Co-authored-by: Silvan <silvan.reusser@gmail.com>
* Update internal/iam/repository/eventsourcing/model/login_policy.go
Co-authored-by: Silvan <silvan.reusser@gmail.com>
* Update internal/iam/repository/eventsourcing/model/login_policy.go
Co-authored-by: Silvan <silvan.reusser@gmail.com>
* Update internal/org/repository/eventsourcing/org_test.go
Co-authored-by: Silvan <silvan.reusser@gmail.com>
* Update internal/iam/repository/eventsourcing/model/login_policy_test.go
Co-authored-by: Silvan <silvan.reusser@gmail.com>
* Update internal/iam/repository/eventsourcing/model/login_policy_test.go
Co-authored-by: Silvan <silvan.reusser@gmail.com>
* fix: pr comments
* fix: tests
* Update types.go
* fix: merge request changes
* fix: reduce optimization
Co-authored-by: Silvan <silvan.reusser@gmail.com>
Co-authored-by: Livio Amstutz <livio.a@gmail.com>
* fix: reread user mfas, preferred loginname as otp account name (#636)
* fix: reread user mfas
* fix: use preferred login name as otp account name
* fix: tests
* fix: reduce (#635)
* fix: management reduce optimization
* fix: reduce optimization
* fix: reduce optimization
* fix: merge master
* chore(deps): bump github.com/gorilla/schema from 1.1.0 to 1.2.0 (#627)
Bumps [github.com/gorilla/schema](https://github.com/gorilla/schema) from 1.1.0 to 1.2.0.
- [Release notes](https://github.com/gorilla/schema/releases)
- [Commits](https://github.com/gorilla/schema/compare/v1.1.0...v1.2.0)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* chore(deps): bump github.com/gorilla/mux from 1.7.4 to 1.8.0 (#624)
Bumps [github.com/gorilla/mux](https://github.com/gorilla/mux) from 1.7.4 to 1.8.0.
- [Release notes](https://github.com/gorilla/mux/releases)
- [Commits](https://github.com/gorilla/mux/compare/v1.7.4...v1.8.0)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* chore(deps): bump github.com/DATA-DOG/go-sqlmock from 1.4.1 to 1.5.0 (#591)
Bumps [github.com/DATA-DOG/go-sqlmock](https://github.com/DATA-DOG/go-sqlmock) from 1.4.1 to 1.5.0.
- [Release notes](https://github.com/DATA-DOG/go-sqlmock/releases)
- [Commits](https://github.com/DATA-DOG/go-sqlmock/compare/v1.4.1...v1.5.0)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* chore: auto assign issues and PR to ZTIADEL project board (#643)
* Create main.yml
* Update main.yml
Co-authored-by: Livio Amstutz <livio.a@gmail.com>
* fix(console): project grant members, update deps (#645)
* fix: searchprojectgrantmembers
* chore(deps-dev): bump @angular/cli from 10.0.6 to 10.0.7 in /console (#622)
Bumps [@angular/cli](https://github.com/angular/angular-cli) from 10.0.6 to 10.0.7.
- [Release notes](https://github.com/angular/angular-cli/releases)
- [Commits](https://github.com/angular/angular-cli/compare/v10.0.6...v10.0.7)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* chore(deps-dev): bump @angular-devkit/build-angular in /console (#626)
Bumps [@angular-devkit/build-angular](https://github.com/angular/angular-cli) from 0.1000.6 to 0.1000.7.
- [Release notes](https://github.com/angular/angular-cli/releases)
- [Commits](https://github.com/angular/angular-cli/commits)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Max Peintner <max@caos.ch>
* chore(deps-dev): bump @types/jasmine from 3.5.12 to 3.5.13 in /console (#623)
Bumps [@types/jasmine](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/jasmine) from 3.5.12 to 3.5.13.
- [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases)
- [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/jasmine)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* chore(deps-dev): bump ts-node from 8.10.2 to 9.0.0 in /console (#629)
Bumps [ts-node](https://github.com/TypeStrong/ts-node) from 8.10.2 to 9.0.0.
- [Release notes](https://github.com/TypeStrong/ts-node/releases)
- [Commits](https://github.com/TypeStrong/ts-node/compare/v8.10.2...v9.0.0)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* update packlock
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* chore: delete main.yml (#648)
* fix: usergrant (#650)
* fix(console): mfa refresh after verification, member eventemitter (#651)
* refresh mfa
* fix: detail link from contributors
* lint
* feat: add domain verification notification (#649)
* fix: dont (re)generate client secret with auth type none
* fix(cors): allow Origin from request
* feat: add origin allow list and fix some core issues
* rename migration
* fix UserIDsByDomain
* feat: send email to users after domain claim
* username
* check origin on userinfo
* update oidc pkg
* fix: add migration 1.6
* change username
* change username
* remove unique email aggregate
* change username in mgmt
* search global user by login name
* fix test
* change user search in angular
* fix tests
* merge
* userview in angular
* fix merge
* Update pkg/grpc/management/proto/management.proto
Co-authored-by: Fabi <38692350+fgerschwiler@users.noreply.github.com>
* Update internal/notification/static/i18n/de.yaml
Co-authored-by: Fabi <38692350+fgerschwiler@users.noreply.github.com>
* fix
Co-authored-by: Fabi <38692350+fgerschwiler@users.noreply.github.com>
* fix: translation (#647)
* fix: translation
* fix: translation
* fix: translation
* fix: remove unused code
* fix: log err
* fix: migration numbers (#652)
* chore: issue / feature templates (#642)
* feat: machine keys
* fix: implement missing parts
* feat: machine key management view
* fix: remove keys from machine view
* feat: global org read (#657)
* fix: set default expiration date
* fix: get key by ids
* feat: add machine keys in proto
* feat: machine keys
* fix: add migration
* fix: mig
* fix: correct method name
* feat: user search
* feat: user search
* fix: log ids
* fix: migrations
* fix(console): machine build (#660)
* frontend 1
* fix html bindings
* trailing comma
* fix(console): human view (#661)
* fix search user view, user detail form
* rm log
* feat(console): user services list and create (#663)
* fix search user view, user detail form
* rm log
* machine list
* generic table component
* create user service
* proove table for undefined values
* tmp disable user link if machine
* lint
* lint styles
* user table lint
* Update console/src/assets/i18n/de.json
Co-authored-by: Florian Forster <florian@caos.ch>
* feat(console): service user detail view, keys cr_d, fix search user autocomplete (#664)
* service users for sidenav, routing
* i18n
* back routes
* machine detail form
* update machine detail, fix svc user grants
* keys table
* add key dialog, timestamp creation
* check permission on create, delete, fix selection
* lint ts, scss
* Update console/src/assets/i18n/de.json
* Apply suggestions from code review
Co-authored-by: Florian Forster <florian@caos.ch>
* refactor: protos
* fix(management): key expiration date
* fix: check if user is human
* fix: marshal key details
* fix: correct generate login names
* fix: logid
Co-authored-by: Fabi <38692350+fgerschwiler@users.noreply.github.com>
Co-authored-by: Livio Amstutz <livio.a@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Max Peintner <max@caos.ch>
Co-authored-by: Fabiennne <fabienne.gerschwiler@gmail.com>
Co-authored-by: Florian Forster <florian@caos.ch>
* fix: naming
* refactor: findings
* fix: username
* fix: mfa upper case
* fix: tests
* fix: add translations
* reactivatemyorg req typeö
* fix: projectType for console
* fix: user changes
* fix: translate events
* fix: event type translation
* fix: remove unused types
Co-authored-by: Fabiennne <fabienne.gerschwiler@gmail.com>
Co-authored-by: Max Peintner <max@caos.ch>
Co-authored-by: Florian Forster <florian@caos.ch>
Co-authored-by: Fabi <38692350+fgerschwiler@users.noreply.github.com>
Co-authored-by: Livio Amstutz <livio.a@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2020-08-31 15:48:01 +00:00
logging . Log ( "SPOOL-OT8di" ) . OnError ( err ) . WithField ( "view" , s . ViewModel ( ) ) . WithField ( "worker" , workerID ) . Debug ( "load canceled" )
2020-04-29 10:11:13 +00:00
}
}
2020-07-28 07:42:21 +00:00
func ( s * spooledHandler ) process ( ctx context . Context , events [ ] * models . Event , workerID string ) error {
2020-12-21 17:42:34 +00:00
for i , event := range events {
2020-04-29 10:11:13 +00:00
select {
case <- ctx . Done ( ) :
2022-03-15 06:19:02 +00:00
logging . WithFields ( "view" , s . ViewModel ( ) , "worker" , workerID , "traceID" , tracing . TraceIDFromCtx ( ctx ) ) . Debug ( "context canceled" )
2020-04-29 10:11:13 +00:00
return nil
default :
2020-06-23 12:47:47 +00:00
if err := s . Reduce ( event ) ; err != nil {
2020-12-21 17:42:34 +00:00
err = s . OnError ( event , err )
if err == nil {
continue
}
time . Sleep ( 100 * time . Millisecond )
return s . process ( ctx , events [ i : ] , workerID )
2020-04-29 10:11:13 +00:00
}
}
}
2020-12-02 07:50:59 +00:00
err := s . OnSuccess ( )
2022-03-15 06:19:02 +00:00
logging . WithFields ( "view" , s . ViewModel ( ) , "worker" , workerID , "traceID" , tracing . TraceIDFromCtx ( ctx ) ) . OnError ( err ) . Warn ( "could not process on success func" )
2020-12-02 07:50:59 +00:00
return err
2020-04-29 10:11:13 +00:00
}
2022-07-22 10:08:39 +00:00
func ( s * spooledHandler ) query ( ctx context . Context , instanceIDs ... string ) ( [ ] * models . Event , error ) {
query , err := s . EventQuery ( instanceIDs ... )
2020-04-29 10:11:13 +00:00
if err != nil {
return nil , err
}
2020-07-28 07:42:21 +00:00
query . Limit = s . QueryLimit ( )
2020-04-29 10:11:13 +00:00
return s . eventstore . FilterEvents ( ctx , query )
}
2020-09-18 11:39:28 +00:00
//lock ensures the lock on the database.
// the returned channel will be closed if ctx is done or an error occured durring lock
2020-07-28 07:42:21 +00:00
func ( s * spooledHandler ) lock ( ctx context . Context , errs chan <- error , workerID string ) chan bool {
2020-04-29 10:11:13 +00:00
renewTimer := time . After ( 0 )
2020-07-28 07:42:21 +00:00
locked := make ( chan bool )
2020-04-29 10:11:13 +00:00
go func ( locked chan bool ) {
2020-09-18 11:39:28 +00:00
var firstLock sync . Once
defer close ( locked )
2020-04-29 10:11:13 +00:00
for {
select {
case <- ctx . Done ( ) :
return
case <- renewTimer :
2022-04-19 06:26:12 +00:00
err := s . locker . Renew ( workerID , s . ViewModel ( ) , systemID , s . LockDuration ( ) )
2020-09-18 11:39:28 +00:00
firstLock . Do ( func ( ) {
locked <- err == nil
} )
2020-04-29 10:11:13 +00:00
if err == nil {
2020-12-22 11:27:55 +00:00
renewTimer = time . After ( s . LockDuration ( ) )
2020-04-29 10:11:13 +00:00
continue
}
if ctx . Err ( ) == nil {
errs <- err
}
return
}
}
} ( locked )
return locked
}
2020-09-18 11:39:28 +00:00
func HandleError ( event * models . Event , failedErr error ,
2022-04-19 06:26:12 +00:00
latestFailedEvent func ( sequence uint64 , instanceID string ) ( * repository . FailedEvent , error ) ,
2020-09-18 11:39:28 +00:00
processFailedEvent func ( * repository . FailedEvent ) error ,
2020-12-21 17:42:34 +00:00
processSequence func ( * models . Event ) error ,
errorCountUntilSkip uint64 ) error {
2022-04-19 06:26:12 +00:00
failedEvent , err := latestFailedEvent ( event . Sequence , event . InstanceID )
2020-09-18 11:39:28 +00:00
if err != nil {
return err
}
failedEvent . FailureCount ++
failedEvent . ErrMsg = failedErr . Error ( )
2022-04-19 06:26:12 +00:00
failedEvent . InstanceID = event . InstanceID
2020-09-18 11:39:28 +00:00
err = processFailedEvent ( failedEvent )
if err != nil {
return err
}
if errorCountUntilSkip <= failedEvent . FailureCount {
2020-12-18 15:47:45 +00:00
return processSequence ( event )
2020-09-18 11:39:28 +00:00
}
2020-12-21 17:42:34 +00:00
return failedErr
2020-09-18 11:39:28 +00:00
}
2020-12-02 07:50:59 +00:00
func HandleSuccess ( updateSpoolerRunTimestamp func ( ) error ) error {
return updateSpoolerRunTimestamp ( )
}