TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-11 19:07:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(spooler): correct workers (#508)

Browse Source 
* one concurrent task

* disable spooler

* fix: improve concurrency in spooler

* fix: dont block lock

* fix: break if lock failed

* fix: check if handler is working

* fix: worker id

* fix: test

* fix: use limit for spoolers configured in startup.yaml

* fix test

* fix: factory

* fix(key): only reduce if not expired

* fix(searchQueryFactory): check for string-slice in aggregateID

* fix(migrations): combine migrations

* fix: allow saving multiple objects in one request

* fix(eventstore): logging

* fix(eventstore): rethink insert i locks table

* fix: ignore failed tests for the moment

* fix: tuubel

* fix: for tests in io

* fix: ignore tests for io

* fix: rename concurrent tasks to workers

* fix: incomment tests and remove some tests

* fix: refert changes for io

* refactor(eventstore): combine types of sql in one file

* refactor(eventstore): logs, TODO's, tests

* fix(eventstore): sql package

* test(eventstore): add tests for search query factory

* chore: logs

* fix(spooler): optimize lock query
chore(migrations): rename locks.object_type to view_name
chore(migrations): refactor migrations

* test: incomment tests

* fix: rename PrepareSaves to PrepareBulkSave

* chore: go dependencies

* fix(migrations): add id in events table

* refactor(lock): less magic numbers

Co-authored-by: Livio Amstutz <livio.a@gmail.com>
This commit is contained in:
Silvan
2020-07-28 09:42:21 +02:00
committed by GitHub
parent 531060ab67
commit 41e1a7cc7b
90 changed files with 2210 additions and 1471 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
internal/notification/repository/eventsourcing/handler/handler.go
View File

@@ -6,7 +6,7 @@ import (
"github.com/caos/zitadel/internal/config/types"
"github.com/caos/zitadel/internal/crypto"
"github.com/caos/zitadel/internal/eventstore"
"github.com/caos/zitadel/internal/eventstore/spooler"
"github.com/caos/zitadel/internal/eventstore/query"
"github.com/caos/zitadel/internal/i18n"
"github.com/caos/zitadel/internal/notification/repository/eventsourcing/view"
org_event "github.com/caos/zitadel/internal/org/repository/eventsourcing"
@@ -33,12 +33,12 @@ type EventstoreRepos struct {
OrgEvents *org_event.OrgEventstore
}
func Register(configs Configs, bulkLimit, errorCount uint64, view *view.View, eventstore eventstore.Eventstore, repos EventstoreRepos, systemDefaults sd.SystemDefaults, i18n *i18n.Translator, dir http.FileSystem) []spooler.Handler {
func Register(configs Configs, bulkLimit, errorCount uint64, view *view.View, eventstore eventstore.Eventstore, repos EventstoreRepos, systemDefaults sd.SystemDefaults, i18n *i18n.Translator, dir http.FileSystem) []query.Handler {
aesCrypto, err := crypto.NewAESCrypto(systemDefaults.UserVerificationKey)
if err != nil {
logging.Log("HANDL-s90ew").WithError(err).Debug("error create new aes crypto")
}
return []spooler.Handler{
return []query.Handler{
&NotifyUser{
handler: handler{view, bulkLimit, configs.cycleDuration("User"), errorCount},
orgEvents: repos.OrgEvents,
@@ -62,3 +62,11 @@ func (configs Configs) cycleDuration(viewModel string) time.Duration {
}
return c.MinimumCycleDuration.Duration
}
func (h *handler) MinimumCycleDuration() time.Duration {
return h.cycleDuration
}
func (h *handler) QueryLimit() uint64 {
return h.bulkLimit
}

3
internal/notification/repository/eventsourcing/handler/notification.go
View File

@@ -3,7 +3,6 @@ package handler
import (
"context"
"net/http"
"time"
"github.com/caos/logging"
@@ -35,8 +34,6 @@ const (
NotifyUserID = "NOTIFICATION"
)
func (n *Notification) MinimumCycleDuration() time.Duration { return n.cycleDuration }
func (n *Notification) ViewModel() string {
return notificationTable
}

13
internal/notification/repository/eventsourcing/handler/notify_user.go
View File

@@ -2,18 +2,17 @@ package handler
import (
"context"
es_models "github.com/caos/zitadel/internal/eventstore/models"
org_model "github.com/caos/zitadel/internal/org/model"
org_events "github.com/caos/zitadel/internal/org/repository/eventsourcing"
org_es_model "github.com/caos/zitadel/internal/org/repository/eventsourcing/model"
es_model "github.com/caos/zitadel/internal/user/repository/eventsourcing/model"
"time"
"github.com/caos/logging"
"github.com/caos/zitadel/internal/eventstore"
"github.com/caos/zitadel/internal/eventstore/models"
es_models "github.com/caos/zitadel/internal/eventstore/models"
"github.com/caos/zitadel/internal/eventstore/spooler"
org_model "github.com/caos/zitadel/internal/org/model"
org_events "github.com/caos/zitadel/internal/org/repository/eventsourcing"
org_es_model "github.com/caos/zitadel/internal/org/repository/eventsourcing/model"
es_model "github.com/caos/zitadel/internal/user/repository/eventsourcing/model"
view_model "github.com/caos/zitadel/internal/user/repository/view/model"
)
@@ -27,8 +26,6 @@ const (
userTable = "notification.notify_users"
)
func (p *NotifyUser) MinimumCycleDuration() time.Duration { return p.cycleDuration }
func (p *NotifyUser) ViewModel() string {
return userTable
}

127
internal/notification/repository/eventsourcing/spooler/lock_test.go
View File

@@ -1,127 +0,0 @@
package spooler
import (
"database/sql"
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
)
type dbMock struct {
db *sql.DB
mock sqlmock.Sqlmock
}
func mockDB(t *testing.T) *dbMock {
mockDB := dbMock{}
var err error
mockDB.db, mockDB.mock, err = sqlmock.New()
if err != nil {
t.Fatalf("error occured while creating stub db %v", err)
}
mockDB.mock.MatchExpectationsInOrder(true)
return &mockDB
}
func (db *dbMock) expectCommit() *dbMock {
db.mock.ExpectCommit()
return db
}
func (db *dbMock) expectRollback() *dbMock {
db.mock.ExpectRollback()
return db
}
func (db *dbMock) expectBegin() *dbMock {
db.mock.ExpectBegin()
return db
}
func (db *dbMock) expectSavepoint() *dbMock {
db.mock.ExpectExec("SAVEPOINT").WillReturnResult(sqlmock.NewResult(1, 1))
return db
}
func (db *dbMock) expectReleaseSavepoint() *dbMock {
db.mock.ExpectExec("RELEASE SAVEPOINT").WillReturnResult(sqlmock.NewResult(1, 1))
return db
}
func (db *dbMock) expectRenew(lockerID, view string, affectedRows int64) *dbMock {
query := db.mock.
ExpectExec(`INSERT INTO notification\.locks \(object_type, locker_id, locked_until\) VALUES \(\$1, \$2, now\(\)\+\$3\) ON CONFLICT \(object_type\) DO UPDATE SET locked_until = now\(\)\+\$4, locker_id = \$5 WHERE \(locks\.locked_until < now\(\) OR locks\.locker_id = \$6\) AND locks\.object_type = \$7`).
WithArgs(view, lockerID, sqlmock.AnyArg(), sqlmock.AnyArg(), lockerID, lockerID, view).
WillReturnResult(sqlmock.NewResult(1, 1))
if affectedRows == 0 {
query.WillReturnResult(sqlmock.NewResult(0, 0))
} else {
query.WillReturnResult(sqlmock.NewResult(1, affectedRows))
}
return db
}
func Test_locker_Renew(t *testing.T) {
type fields struct {
db *dbMock
}
type args struct {
lockerID string
viewModel string
waitTime time.Duration
}
tests := []struct {
name string
fields fields
args args
wantErr bool
}{
{
name: "renew succeeded",
fields: fields{
db: mockDB(t).
expectBegin().
expectSavepoint().
expectRenew("locker", "view", 1).
expectReleaseSavepoint().
expectCommit(),
},
args: args{lockerID: "locker", viewModel: "view", waitTime: 1 * time.Second},
wantErr: false,
},
{
name: "renew now rows updated",
fields: fields{
db: mockDB(t).
expectBegin().
expectSavepoint().
expectRenew("locker", "view", 0).
expectRollback(),
},
args: args{lockerID: "locker", viewModel: "view", waitTime: 1 * time.Second},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
l := &locker{
dbClient: tt.fields.db.db,
}
if err := l.Renew(tt.args.lockerID, tt.args.viewModel, tt.args.waitTime); (err != nil) != tt.wantErr {
t.Errorf("locker.Renew() error = %v, wantErr %v", err, tt.wantErr)
}
if err := tt.fields.db.mock.ExpectationsWereMet(); err != nil {
t.Errorf("not all database expectations met: %v", err)
}
})
}
}

13
internal/notification/repository/eventsourcing/spooler/spooler.go
View File

@@ -2,28 +2,29 @@ package spooler
import (
"database/sql"
"net/http"
sd "github.com/caos/zitadel/internal/config/systemdefaults"
"github.com/caos/zitadel/internal/eventstore"
"github.com/caos/zitadel/internal/eventstore/spooler"
"github.com/caos/zitadel/internal/i18n"
"github.com/caos/zitadel/internal/notification/repository/eventsourcing/handler"
"github.com/caos/zitadel/internal/notification/repository/eventsourcing/view"
"net/http"
)
type SpoolerConfig struct {
BulkLimit uint64
FailureCountUntilSkip uint64
ConcurrentTasks int
ConcurrentWorkers int
Handlers handler.Configs
}
func StartSpooler(c SpoolerConfig, es eventstore.Eventstore, view *view.View, sql *sql.DB, eventstoreRepos handler.EventstoreRepos, systemDefaults sd.SystemDefaults, i18n *i18n.Translator, dir http.FileSystem) *spooler.Spooler {
spoolerConfig := spooler.Config{
Eventstore: es,
Locker: &locker{dbClient: sql},
ConcurrentTasks: c.ConcurrentTasks,
ViewHandlers: handler.Register(c.Handlers, c.BulkLimit, c.FailureCountUntilSkip, view, es, eventstoreRepos, systemDefaults, i18n, dir),
Eventstore: es,
Locker: &locker{dbClient: sql},
ConcurrentWorkers: c.ConcurrentWorkers,
ViewHandlers: handler.Register(c.Handlers, c.BulkLimit, c.FailureCountUntilSkip, view, es, eventstoreRepos, systemDefaults, i18n, dir),
}
spool := spoolerConfig.New()
spool.Start()