mirror of
https://github.com/zitadel/zitadel.git
synced 2025-08-11 19:07:30 +00:00
feat(eventstore): increase parallel write capabilities (#5940)
This implementation increases parallel write capabilities of the eventstore. Please have a look at the technical advisories: [05](https://zitadel.com/docs/support/advisory/a10005) and [06](https://zitadel.com/docs/support/advisory/a10006). The implementation of eventstore.push is rewritten and stored events are migrated to a new table `eventstore.events2`. If you are using cockroach: make sure that the database user of ZITADEL has `VIEWACTIVITY` grant. This is used to query events.
This commit is contained in:
Silvan
@@ -1,13 +0,0 @@
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
type FailedEvent struct {
|
||||
Database string
|
||||
ViewName string
|
||||
FailedSequence uint64
|
||||
FailureCount uint64
|
||||
ErrMsg string
|
||||
InstanceID string
|
||||
LastFailed time.Time
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"github.com/zitadel/zitadel/internal/domain"
|
||||
)
|
||||
|
||||
type GeneralSearchRequest struct {
|
||||
Offset uint64
|
||||
Limit uint64
|
||||
SortingColumn GeneralSearchKey
|
||||
Asc bool
|
||||
Queries []*GeneralSearchQuery
|
||||
}
|
||||
|
||||
type GeneralSearchKey int32
|
||||
|
||||
const (
|
||||
GeneralSearchKeyUnspecified GeneralSearchKey = iota
|
||||
)
|
||||
|
||||
type GeneralSearchQuery struct {
|
||||
Key GeneralSearchKey
|
||||
Method domain.SearchMethod
|
||||
Value interface{}
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type View struct {
|
||||
Database string
|
||||
ViewName string
|
||||
CurrentSequence uint64
|
||||
EventTimestamp time.Time
|
||||
LastSuccessfulSpoolerRun time.Time
|
||||
}
|
||||
@@ -1,180 +0,0 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jinzhu/gorm"
|
||||
|
||||
"github.com/zitadel/zitadel/internal/domain"
|
||||
"github.com/zitadel/zitadel/internal/errors"
|
||||
view_model "github.com/zitadel/zitadel/internal/view/model"
|
||||
)
|
||||
|
||||
type FailedEvent struct {
|
||||
ViewName string `gorm:"column:view_name;primary_key"`
|
||||
FailedSequence uint64 `gorm:"column:failed_sequence;primary_key"`
|
||||
FailureCount uint64 `gorm:"column:failure_count"`
|
||||
ErrMsg string `gorm:"column:err_msg"`
|
||||
InstanceID string `gorm:"column:instance_id"`
|
||||
LastFailed time.Time `gorm:"column:last_failed"`
|
||||
}
|
||||
|
||||
type failedEventSearchRequest struct {
|
||||
Offset uint64
|
||||
Limit uint64
|
||||
SortingColumn failedEventSearchKey
|
||||
Asc bool
|
||||
Queries []*FailedEventSearchQuery
|
||||
}
|
||||
|
||||
func (f failedEventSearchRequest) GetLimit() uint64 {
|
||||
return f.Limit
|
||||
}
|
||||
|
||||
func (f failedEventSearchRequest) GetOffset() uint64 {
|
||||
return f.Offset
|
||||
}
|
||||
|
||||
func (f failedEventSearchRequest) GetSortingColumn() ColumnKey {
|
||||
if f.SortingColumn == failedEventSearchKey(FailedEventKeyUndefined) {
|
||||
return nil
|
||||
}
|
||||
return f.SortingColumn
|
||||
}
|
||||
|
||||
func (f failedEventSearchRequest) GetAsc() bool {
|
||||
return f.Asc
|
||||
}
|
||||
|
||||
func (f failedEventSearchRequest) GetQueries() []SearchQuery {
|
||||
result := make([]SearchQuery, len(f.Queries))
|
||||
for i, q := range f.Queries {
|
||||
result[i] = q
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
type FailedEventSearchQuery struct {
|
||||
Key FailedEventSearchKey
|
||||
Method domain.SearchMethod
|
||||
Value interface{}
|
||||
}
|
||||
|
||||
func (req FailedEventSearchQuery) GetKey() ColumnKey {
|
||||
return failedEventSearchKey(req.Key)
|
||||
}
|
||||
|
||||
func (req FailedEventSearchQuery) GetMethod() domain.SearchMethod {
|
||||
return req.Method
|
||||
}
|
||||
|
||||
func (req FailedEventSearchQuery) GetValue() interface{} {
|
||||
return req.Value
|
||||
}
|
||||
|
||||
type FailedEventSearchKey int32
|
||||
|
||||
const (
|
||||
FailedEventKeyUndefined FailedEventSearchKey = iota
|
||||
FailedEventKeyViewName
|
||||
FailedEventKeyFailedSequence
|
||||
FailedEventKeyInstanceID
|
||||
FailedEventKeyLastFailed
|
||||
)
|
||||
|
||||
type failedEventSearchKey FailedEventSearchKey
|
||||
|
||||
func (key failedEventSearchKey) ToColumnName() string {
|
||||
switch FailedEventSearchKey(key) {
|
||||
case FailedEventKeyViewName:
|
||||
return "view_name"
|
||||
case FailedEventKeyFailedSequence:
|
||||
return "failed_sequence"
|
||||
case FailedEventKeyInstanceID:
|
||||
return "instance_id"
|
||||
case FailedEventKeyLastFailed:
|
||||
return "last_failed"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func FailedEventFromModel(failedEvent *view_model.FailedEvent) *FailedEvent {
|
||||
return &FailedEvent{
|
||||
ViewName: failedEvent.Database + "." + failedEvent.ViewName,
|
||||
FailureCount: failedEvent.FailureCount,
|
||||
FailedSequence: failedEvent.FailedSequence,
|
||||
InstanceID: failedEvent.InstanceID,
|
||||
ErrMsg: failedEvent.ErrMsg,
|
||||
}
|
||||
}
|
||||
func FailedEventToModel(failedEvent *FailedEvent) *view_model.FailedEvent {
|
||||
dbView := strings.Split(failedEvent.ViewName, ".")
|
||||
return &view_model.FailedEvent{
|
||||
Database: dbView[0],
|
||||
ViewName: dbView[1],
|
||||
FailureCount: failedEvent.FailureCount,
|
||||
FailedSequence: failedEvent.FailedSequence,
|
||||
ErrMsg: failedEvent.ErrMsg,
|
||||
LastFailed: failedEvent.LastFailed,
|
||||
}
|
||||
}
|
||||
|
||||
func SaveFailedEvent(db *gorm.DB, table string, failedEvent *FailedEvent) error {
|
||||
save := PrepareSave(table)
|
||||
err := save(db, failedEvent)
|
||||
|
||||
if err != nil {
|
||||
return errors.ThrowInternal(err, "VIEW-4F8us", "unable to updated failed events")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func RemoveFailedEvent(db *gorm.DB, table string, failedEvent *FailedEvent) error {
|
||||
delete := PrepareDeleteByKeys(table,
|
||||
Key{Key: failedEventSearchKey(FailedEventKeyViewName), Value: failedEvent.ViewName},
|
||||
Key{Key: failedEventSearchKey(FailedEventKeyFailedSequence), Value: failedEvent.FailedSequence},
|
||||
Key{Key: failedEventSearchKey(FailedEventKeyInstanceID), Value: failedEvent.InstanceID},
|
||||
)
|
||||
return delete(db)
|
||||
}
|
||||
|
||||
func LatestFailedEvent(db *gorm.DB, table, viewName, instanceID string, sequence uint64) (*FailedEvent, error) {
|
||||
failedEvent := new(FailedEvent)
|
||||
queries := []SearchQuery{
|
||||
FailedEventSearchQuery{Key: FailedEventKeyViewName, Method: domain.SearchMethodEqualsIgnoreCase, Value: viewName},
|
||||
FailedEventSearchQuery{Key: FailedEventKeyFailedSequence, Method: domain.SearchMethodEquals, Value: sequence},
|
||||
FailedEventSearchQuery{Key: FailedEventKeyInstanceID, Method: domain.SearchMethodEquals, Value: instanceID},
|
||||
}
|
||||
query := PrepareGetByQuery(table, queries...)
|
||||
err := query(db, failedEvent)
|
||||
|
||||
if err == nil && failedEvent.ViewName != "" {
|
||||
return failedEvent, nil
|
||||
}
|
||||
|
||||
if errors.IsNotFound(err) {
|
||||
return &FailedEvent{
|
||||
ViewName: viewName,
|
||||
FailedSequence: sequence,
|
||||
FailureCount: 0,
|
||||
}, nil
|
||||
}
|
||||
return nil, errors.ThrowInternalf(err, "VIEW-9LyCB", "unable to get failed events of %s", viewName)
|
||||
|
||||
}
|
||||
|
||||
func AllFailedEvents(db *gorm.DB, table, instanceID string) ([]*FailedEvent, error) {
|
||||
queries := make([]*FailedEventSearchQuery, 0, 1)
|
||||
if instanceID != "" {
|
||||
queries = append(queries, &FailedEventSearchQuery{Key: FailedEventKeyInstanceID, Method: domain.SearchMethodEquals, Value: instanceID})
|
||||
}
|
||||
failedEvents := make([]*FailedEvent, 0)
|
||||
query := PrepareSearchQuery(table, &failedEventSearchRequest{SortingColumn: failedEventSearchKey(FailedEventKeyLastFailed), Queries: queries})
|
||||
_, err := query(db, &failedEvents)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return failedEvents, nil
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"github.com/zitadel/zitadel/internal/domain"
|
||||
"github.com/zitadel/zitadel/internal/view/model"
|
||||
)
|
||||
|
||||
type GeneralSearchRequest model.GeneralSearchRequest
|
||||
type GeneralSearchQuery model.GeneralSearchQuery
|
||||
type GeneralSearchKey model.GeneralSearchKey
|
||||
|
||||
func (req GeneralSearchRequest) GetLimit() uint64 {
|
||||
return req.Limit
|
||||
}
|
||||
|
||||
func (req GeneralSearchRequest) GetOffset() uint64 {
|
||||
return req.Offset
|
||||
}
|
||||
|
||||
func (req GeneralSearchRequest) GetSortingColumn() ColumnKey {
|
||||
if req.SortingColumn == model.GeneralSearchKeyUnspecified {
|
||||
return nil
|
||||
}
|
||||
return GeneralSearchKey(req.SortingColumn)
|
||||
}
|
||||
|
||||
func (req GeneralSearchRequest) GetAsc() bool {
|
||||
return req.Asc
|
||||
}
|
||||
|
||||
func (req GeneralSearchRequest) GetQueries() []SearchQuery {
|
||||
result := make([]SearchQuery, len(req.Queries))
|
||||
for i, q := range req.Queries {
|
||||
result[i] = GeneralSearchQuery{Key: q.Key, Value: q.Value, Method: q.Method}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (req GeneralSearchQuery) GetKey() ColumnKey {
|
||||
return GeneralSearchKey(req.Key)
|
||||
}
|
||||
|
||||
func (req GeneralSearchQuery) GetMethod() domain.SearchMethod {
|
||||
return req.Method
|
||||
}
|
||||
|
||||
func (req GeneralSearchQuery) GetValue() interface{} {
|
||||
return req.Value
|
||||
}
|
||||
|
||||
func (key GeneralSearchKey) ToColumnName() string {
|
||||
return ""
|
||||
}
|
||||
@@ -139,7 +139,7 @@ func SetQuery(query *gorm.DB, key ColumnKey, value interface{}, method domain.Se
|
||||
if !ok {
|
||||
return nil, caos_errs.ThrowInvalidArgument(nil, "VIEW-Psois", "list contains only possible for strings")
|
||||
}
|
||||
query = query.Where("? <@ "+column, database.StringArray{valueText})
|
||||
query = query.Where("? <@ "+column, database.TextArray[string]{valueText})
|
||||
default:
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
@@ -13,30 +13,6 @@ import (
|
||||
caos_errs "github.com/zitadel/zitadel/internal/errors"
|
||||
)
|
||||
|
||||
func PrepareGetByKey(table string, key ColumnKey, id string) func(db *gorm.DB, res interface{}) error {
|
||||
return func(db *gorm.DB, res interface{}) error {
|
||||
tx := db.BeginTx(context.Background(), &sql.TxOptions{ReadOnly: true})
|
||||
defer func() {
|
||||
if err := tx.Commit().Error; err != nil {
|
||||
logging.OnError(err).Info("commit failed")
|
||||
}
|
||||
}()
|
||||
|
||||
err := tx.Table(table).
|
||||
Where(fmt.Sprintf("%s = ?", key.ToColumnName()), id).
|
||||
Take(res).
|
||||
Error
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return caos_errs.ThrowNotFound(err, "VIEW-XRI9c", "object not found")
|
||||
}
|
||||
logging.LogWithFields("VIEW-xVShS", "AggregateID", id).WithError(err).Warn("get from view error")
|
||||
return caos_errs.ThrowInternal(err, "VIEW-J92Td", "Errors.Internal")
|
||||
}
|
||||
}
|
||||
|
||||
func PrepareGetByQuery(table string, queries ...SearchQuery) func(db *gorm.DB, res interface{}) error {
|
||||
return func(db *gorm.DB, res interface{}) error {
|
||||
query := db.Table(table)
|
||||
@@ -161,27 +137,3 @@ func PrepareDeleteByKeys(table string, keys ...Key) func(db *gorm.DB) error {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func PrepareDeleteByObject(table string, object interface{}) func(db *gorm.DB) error {
|
||||
return func(db *gorm.DB) error {
|
||||
err := db.Table(table).
|
||||
Delete(object).
|
||||
Error
|
||||
if err != nil {
|
||||
return caos_errs.ThrowInternal(err, "VIEW-lso9w", "could not delete object")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func PrepareTruncate(table string) func(db *gorm.DB) error {
|
||||
return func(db *gorm.DB) error {
|
||||
err := db.
|
||||
Exec("TRUNCATE " + table).
|
||||
Error
|
||||
if err != nil {
|
||||
return caos_errs.ThrowInternal(err, "VIEW-lso9w", "could not truncate table")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,90 +9,6 @@ import (
|
||||
caos_errs "github.com/zitadel/zitadel/internal/errors"
|
||||
)
|
||||
|
||||
func TestPrepareGetByKey(t *testing.T) {
|
||||
type args struct {
|
||||
table string
|
||||
key ColumnKey
|
||||
value string
|
||||
}
|
||||
type res struct {
|
||||
result Test
|
||||
wantErr bool
|
||||
errFunc func(err error) bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
db *dbMock
|
||||
args args
|
||||
res res
|
||||
}{
|
||||
{
|
||||
"ok",
|
||||
mockDB(t).
|
||||
expectGetByID("TESTTABLE", "test", "VALUE"),
|
||||
args{
|
||||
table: "TESTTABLE",
|
||||
key: TestSearchKey_TEST,
|
||||
value: "VALUE",
|
||||
},
|
||||
res{
|
||||
result: Test{ID: "VALUE"},
|
||||
wantErr: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
"not found",
|
||||
mockDB(t).
|
||||
expectGetByIDErr("TESTTABLE", "test", "VALUE", gorm.ErrRecordNotFound),
|
||||
args{
|
||||
table: "TESTTABLE",
|
||||
key: TestSearchKey_TEST,
|
||||
value: "VALUE",
|
||||
},
|
||||
res{
|
||||
result: Test{ID: "VALUE"},
|
||||
wantErr: true,
|
||||
errFunc: caos_errs.IsNotFound,
|
||||
},
|
||||
},
|
||||
{
|
||||
"db err",
|
||||
mockDB(t).
|
||||
expectGetByIDErr("TESTTABLE", "test", "VALUE", gorm.ErrUnaddressable),
|
||||
args{
|
||||
table: "TESTTABLE",
|
||||
key: TestSearchKey_TEST,
|
||||
value: "VALUE",
|
||||
},
|
||||
res{
|
||||
result: Test{ID: "VALUE"},
|
||||
wantErr: true,
|
||||
errFunc: caos_errs.IsInternal,
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
res := &Test{}
|
||||
getByID := PrepareGetByKey(tt.args.table, tt.args.key, tt.args.value)
|
||||
err := getByID(tt.db.db, res)
|
||||
|
||||
if !tt.res.wantErr && err != nil {
|
||||
t.Errorf("got wrong err should be nil: %v ", err)
|
||||
}
|
||||
|
||||
if tt.res.wantErr && !tt.res.errFunc(err) {
|
||||
t.Errorf("got wrong err: %v ", err)
|
||||
}
|
||||
if err := tt.db.mock.ExpectationsWereMet(); err != nil {
|
||||
t.Errorf("there were unfulfilled expectations: %s", err)
|
||||
}
|
||||
|
||||
tt.db.close()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareGetByQuery(t *testing.T) {
|
||||
type args struct {
|
||||
table string
|
||||
@@ -483,86 +399,3 @@ func TestPrepareDeleteByKeys(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareDeleteByObject(t *testing.T) {
|
||||
type args struct {
|
||||
table string
|
||||
object interface{}
|
||||
}
|
||||
type res struct {
|
||||
result Test
|
||||
wantErr bool
|
||||
errFunc func(err error) bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
db *dbMock
|
||||
args args
|
||||
res res
|
||||
}{
|
||||
{
|
||||
"delete",
|
||||
mockDB(t).
|
||||
expectBegin(nil).
|
||||
expectRemoveByObject("TESTTABLE", Test{ID: "VALUE", Test: "TEST"}).
|
||||
expectCommit(nil),
|
||||
args{
|
||||
table: "TESTTABLE",
|
||||
object: &Test{ID: "VALUE", Test: "TEST"},
|
||||
},
|
||||
res{
|
||||
result: Test{ID: "VALUE"},
|
||||
wantErr: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
"delete multiple PK",
|
||||
mockDB(t).
|
||||
expectBegin(nil).
|
||||
expectRemoveByObjectMultiplePKs("TESTTABLE", TestMultiplePK{TestID: "TESTID", HodorID: "HODORID", Test: "TEST"}).
|
||||
expectCommit(nil),
|
||||
args{
|
||||
table: "TESTTABLE",
|
||||
object: &TestMultiplePK{TestID: "TESTID", HodorID: "HODORID", Test: "TEST"},
|
||||
},
|
||||
res{
|
||||
wantErr: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
"db error",
|
||||
mockDB(t).
|
||||
expectBegin(nil).
|
||||
expectRemoveErr("TESTTABLE", "id", "VALUE", gorm.ErrUnaddressable).
|
||||
expectCommit(nil),
|
||||
args{
|
||||
table: "TESTTABLE",
|
||||
object: &Test{ID: "VALUE", Test: "TEST"},
|
||||
},
|
||||
res{
|
||||
result: Test{ID: "VALUE"},
|
||||
wantErr: true,
|
||||
errFunc: caos_errs.IsInternal,
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
getDelete := PrepareDeleteByObject(tt.args.table, tt.args.object)
|
||||
err := getDelete(tt.db.db)
|
||||
|
||||
if !tt.res.wantErr && err != nil {
|
||||
t.Errorf("got wrong err should be nil: %v ", err)
|
||||
}
|
||||
|
||||
if tt.res.wantErr && !tt.res.errFunc(err) {
|
||||
t.Errorf("got wrong err: %v ", err)
|
||||
}
|
||||
if err := tt.db.mock.ExpectationsWereMet(); !tt.res.wantErr && err != nil {
|
||||
t.Errorf("there were unfulfilled expectations: %s", err)
|
||||
}
|
||||
|
||||
tt.db.close()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,216 +0,0 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jinzhu/gorm"
|
||||
|
||||
"github.com/zitadel/zitadel/internal/domain"
|
||||
caos_errs "github.com/zitadel/zitadel/internal/errors"
|
||||
"github.com/zitadel/zitadel/internal/view/model"
|
||||
)
|
||||
|
||||
type CurrentSequence struct {
|
||||
ViewName string `gorm:"column:view_name;primary_key"`
|
||||
CurrentSequence uint64 `gorm:"column:current_sequence"`
|
||||
EventTimestamp time.Time `gorm:"column:event_timestamp"`
|
||||
LastSuccessfulSpoolerRun time.Time `gorm:"column:last_successful_spooler_run"`
|
||||
InstanceID string `gorm:"column:instance_id;primary_key"`
|
||||
}
|
||||
|
||||
type currentSequenceViewWithSequence struct {
|
||||
ViewName string `gorm:"column:view_name;primary_key"`
|
||||
CurrentSequence uint64 `gorm:"column:current_sequence"`
|
||||
LastSuccessfulSpoolerRun time.Time `gorm:"column:last_successful_spooler_run"`
|
||||
}
|
||||
|
||||
type currentSequenceView struct {
|
||||
ViewName string `gorm:"column:view_name;primary_key"`
|
||||
LastSuccessfulSpoolerRun time.Time `gorm:"column:last_successful_spooler_run"`
|
||||
}
|
||||
|
||||
type SequenceSearchKey int32
|
||||
|
||||
const (
|
||||
SequenceSearchKeyUndefined SequenceSearchKey = iota
|
||||
SequenceSearchKeyViewName
|
||||
SequenceSearchKeyAggregateType
|
||||
SequenceSearchKeyInstanceID
|
||||
)
|
||||
|
||||
type sequenceSearchKey SequenceSearchKey
|
||||
|
||||
func (key sequenceSearchKey) ToColumnName() string {
|
||||
switch SequenceSearchKey(key) {
|
||||
case SequenceSearchKeyViewName:
|
||||
return "view_name"
|
||||
case SequenceSearchKeyAggregateType:
|
||||
return "aggregate_type"
|
||||
case SequenceSearchKeyInstanceID:
|
||||
return "instance_id"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
type sequenceSearchQuery struct {
|
||||
key sequenceSearchKey
|
||||
method domain.SearchMethod
|
||||
value interface{}
|
||||
}
|
||||
|
||||
func (q *sequenceSearchQuery) GetKey() ColumnKey {
|
||||
return q.key
|
||||
}
|
||||
|
||||
func (q *sequenceSearchQuery) GetMethod() domain.SearchMethod {
|
||||
return q.method
|
||||
}
|
||||
|
||||
func (q *sequenceSearchQuery) GetValue() interface{} {
|
||||
return q.value
|
||||
}
|
||||
|
||||
type sequenceSearchRequest struct {
|
||||
queries []sequenceSearchQuery
|
||||
}
|
||||
|
||||
func (s *sequenceSearchRequest) GetLimit() uint64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (s *sequenceSearchRequest) GetOffset() uint64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (s *sequenceSearchRequest) GetSortingColumn() ColumnKey {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *sequenceSearchRequest) GetAsc() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *sequenceSearchRequest) GetQueries() []SearchQuery {
|
||||
result := make([]SearchQuery, len(s.queries))
|
||||
for i, q := range s.queries {
|
||||
result[i] = &sequenceSearchQuery{key: q.key, value: q.value, method: q.method}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func CurrentSequenceToModel(sequence *CurrentSequence) *model.View {
|
||||
dbView := strings.Split(sequence.ViewName, ".")
|
||||
return &model.View{
|
||||
Database: dbView[0],
|
||||
ViewName: dbView[1],
|
||||
CurrentSequence: sequence.CurrentSequence,
|
||||
EventTimestamp: sequence.EventTimestamp,
|
||||
LastSuccessfulSpoolerRun: sequence.LastSuccessfulSpoolerRun,
|
||||
}
|
||||
}
|
||||
|
||||
func SaveCurrentSequence(db *gorm.DB, table, viewName, instanceID string, sequence uint64, eventTimestamp time.Time) error {
|
||||
return UpdateCurrentSequence(db, table, &CurrentSequence{viewName, sequence, eventTimestamp, time.Now(), instanceID})
|
||||
}
|
||||
|
||||
func SaveCurrentSequences(db *gorm.DB, table, viewName string, sequence uint64, eventTimestamp time.Time) error {
|
||||
err := db.Table(table).Where("view_name = ?", viewName).
|
||||
Updates(map[string]interface{}{"current_sequence": sequence, "event_timestamp": eventTimestamp, "last_successful_spooler_run": time.Now()}).Error
|
||||
if err != nil {
|
||||
return caos_errs.ThrowInternal(err, "VIEW-Sfdqs", "unable to updated processed sequence")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func UpdateCurrentSequence(db *gorm.DB, table string, currentSequence *CurrentSequence) (err error) {
|
||||
save := PrepareSave(table)
|
||||
err = save(db, currentSequence)
|
||||
if err != nil {
|
||||
return caos_errs.ThrowInternal(err, "VIEW-5kOhP", "unable to updated processed sequence")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func UpdateCurrentSequences(db *gorm.DB, table string, currentSequences []*CurrentSequence) (err error) {
|
||||
save := PrepareBulkSave(table)
|
||||
s := make([]interface{}, len(currentSequences))
|
||||
for i, currentSequence := range currentSequences {
|
||||
s[i] = currentSequence
|
||||
}
|
||||
err = save(db, s...)
|
||||
if err != nil {
|
||||
return caos_errs.ThrowInternal(err, "VIEW-5kOhP", "unable to updated processed sequence")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func LatestSequence(db *gorm.DB, table, viewName, instanceID string) (*CurrentSequence, error) {
|
||||
searchQueries := []SearchQuery{
|
||||
&sequenceSearchQuery{key: sequenceSearchKey(SequenceSearchKeyViewName), value: viewName, method: domain.SearchMethodEquals},
|
||||
&sequenceSearchQuery{key: sequenceSearchKey(SequenceSearchKeyInstanceID), value: instanceID, method: domain.SearchMethodIsOneOf},
|
||||
}
|
||||
|
||||
// ensure highest sequence of view
|
||||
db = db.Order("current_sequence DESC")
|
||||
|
||||
query := PrepareGetByQuery(table, searchQueries...)
|
||||
sequence := new(CurrentSequence)
|
||||
err := query(db, sequence)
|
||||
|
||||
if err == nil {
|
||||
return sequence, nil
|
||||
}
|
||||
|
||||
if caos_errs.IsNotFound(err) {
|
||||
return sequence, nil
|
||||
}
|
||||
return nil, caos_errs.ThrowInternalf(err, "VIEW-9LyCB", "unable to get latest sequence of %s", viewName)
|
||||
}
|
||||
|
||||
func LatestSequences(db *gorm.DB, table, viewName string, instanceIDs []string) ([]*CurrentSequence, error) {
|
||||
searchQueries := []sequenceSearchQuery{
|
||||
{key: sequenceSearchKey(SequenceSearchKeyViewName), value: viewName, method: domain.SearchMethodEquals},
|
||||
}
|
||||
if len(instanceIDs) > 0 {
|
||||
searchQueries = append(searchQueries, sequenceSearchQuery{key: sequenceSearchKey(SequenceSearchKeyInstanceID), value: instanceIDs, method: domain.SearchMethodIsOneOf})
|
||||
}
|
||||
searchRequest := &sequenceSearchRequest{
|
||||
queries: searchQueries,
|
||||
}
|
||||
|
||||
// ensure highest sequence of view
|
||||
db = db.Order("current_sequence DESC")
|
||||
|
||||
sequences := make([]*CurrentSequence, 0)
|
||||
query := PrepareSearchQuery(table, searchRequest)
|
||||
_, err := query(db, &sequences)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return sequences, nil
|
||||
}
|
||||
|
||||
func AllCurrentSequences(db *gorm.DB, table, instanceID string) ([]*CurrentSequence, error) {
|
||||
queries := make([]sequenceSearchQuery, 0, 1)
|
||||
if instanceID != "" {
|
||||
queries = append(queries, sequenceSearchQuery{key: sequenceSearchKey(SequenceSearchKeyInstanceID), value: instanceID})
|
||||
}
|
||||
sequences := make([]*CurrentSequence, 0)
|
||||
query := PrepareSearchQuery(table, &sequenceSearchRequest{queries: queries})
|
||||
_, err := query(db, &sequences)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return sequences, nil
|
||||
}
|
||||
|
||||
func ClearView(db *gorm.DB, truncateView, sequenceTable string) error {
|
||||
truncate := PrepareTruncate(truncateView)
|
||||
err := truncate(db)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return SaveCurrentSequences(db, sequenceTable, truncateView, 0, time.Now())
|
||||
}
|
||||
Reference in New Issue
Block a user