mirror of
https://github.com/zitadel/zitadel.git
synced 2025-08-12 00:27:31 +00:00
feat(eventstore): increase parallel write capabilities (#5940)
This implementation increases parallel write capabilities of the eventstore. Please have a look at the technical advisories: [05](https://zitadel.com/docs/support/advisory/a10005) and [06](https://zitadel.com/docs/support/advisory/a10006). The implementation of eventstore.push is rewritten and stored events are migrated to a new table `eventstore.events2`. If you are using cockroach: make sure that the database user of ZITADEL has `VIEWACTIVITY` grant. This is used to query events.
This commit is contained in:
Silvan
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/zitadel/logging"
|
||||
@@ -14,19 +15,20 @@ import (
|
||||
"github.com/zitadel/zitadel/internal/database"
|
||||
"github.com/zitadel/zitadel/internal/database/dialect"
|
||||
z_errors "github.com/zitadel/zitadel/internal/errors"
|
||||
"github.com/zitadel/zitadel/internal/eventstore"
|
||||
"github.com/zitadel/zitadel/internal/eventstore/repository"
|
||||
)
|
||||
|
||||
type querier interface {
|
||||
columnName(repository.Field) string
|
||||
columnName(field repository.Field, useV1 bool) string
|
||||
operation(repository.Operation) string
|
||||
conditionFormat(repository.Operation) string
|
||||
placeholder(query string) string
|
||||
eventQuery() string
|
||||
maxSequenceQuery() string
|
||||
instanceIDsQuery() string
|
||||
eventQuery(useV1 bool) string
|
||||
maxSequenceQuery(useV1 bool) string
|
||||
instanceIDsQuery(useV1 bool) string
|
||||
db() *database.DB
|
||||
orderByEventSequence(desc bool) string
|
||||
orderByEventSequence(desc, useV1 bool) string
|
||||
dialect.Database
|
||||
}
|
||||
|
||||
@@ -52,25 +54,38 @@ func (t *tx) QueryContext(ctx context.Context, scan func(rows *sql.Rows) error,
|
||||
return rows.Err()
|
||||
}
|
||||
|
||||
func query(ctx context.Context, criteria querier, searchQuery *repository.SearchQuery, dest interface{}) error {
|
||||
query, rowScanner := prepareColumns(criteria, searchQuery.Columns)
|
||||
where, values := prepareCondition(criteria, searchQuery.Filters)
|
||||
func query(ctx context.Context, criteria querier, searchQuery *eventstore.SearchQueryBuilder, dest interface{}, useV1 bool) error {
|
||||
q, err := repository.QueryFromBuilder(searchQuery)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
query, rowScanner := prepareColumns(criteria, q.Columns, useV1)
|
||||
where, values := prepareConditions(criteria, q, useV1)
|
||||
if where == "" || query == "" {
|
||||
return z_errors.ThrowInvalidArgument(nil, "SQL-rWeBw", "invalid query factory")
|
||||
}
|
||||
if searchQuery.Tx == nil {
|
||||
if travel := prepareTimeTravel(ctx, criteria, searchQuery.AllowTimeTravel); travel != "" {
|
||||
if q.Tx == nil {
|
||||
if travel := prepareTimeTravel(ctx, criteria, q.AllowTimeTravel); travel != "" {
|
||||
query += travel
|
||||
}
|
||||
}
|
||||
query += where
|
||||
|
||||
if searchQuery.Columns == repository.ColumnsEvent {
|
||||
query += criteria.orderByEventSequence(searchQuery.Desc)
|
||||
// instead of using the max function of the database (which doesn't work for postgres)
|
||||
// we select the most recent row
|
||||
if q.Columns == eventstore.ColumnsMaxSequence {
|
||||
q.Limit = 1
|
||||
q.Desc = true
|
||||
}
|
||||
|
||||
if searchQuery.Limit > 0 {
|
||||
values = append(values, searchQuery.Limit)
|
||||
switch q.Columns {
|
||||
case eventstore.ColumnsEvent,
|
||||
eventstore.ColumnsMaxSequence:
|
||||
query += criteria.orderByEventSequence(q.Desc, useV1)
|
||||
}
|
||||
|
||||
if q.Limit > 0 {
|
||||
values = append(values, q.Limit)
|
||||
query += " LIMIT ?"
|
||||
}
|
||||
|
||||
@@ -80,11 +95,11 @@ func query(ctx context.Context, criteria querier, searchQuery *repository.Search
|
||||
QueryContext(context.Context, func(rows *sql.Rows) error, string, ...interface{}) error
|
||||
}
|
||||
contextQuerier = criteria.db()
|
||||
if searchQuery.Tx != nil {
|
||||
contextQuerier = &tx{Tx: searchQuery.Tx}
|
||||
if q.Tx != nil {
|
||||
contextQuerier = &tx{Tx: q.Tx}
|
||||
}
|
||||
|
||||
err := contextQuerier.QueryContext(ctx,
|
||||
err = contextQuerier.QueryContext(ctx,
|
||||
func(rows *sql.Rows) error {
|
||||
for rows.Next() {
|
||||
err := rowScanner(rows.Scan, dest)
|
||||
@@ -102,14 +117,14 @@ func query(ctx context.Context, criteria querier, searchQuery *repository.Search
|
||||
return nil
|
||||
}
|
||||
|
||||
func prepareColumns(criteria querier, columns repository.Columns) (string, func(s scan, dest interface{}) error) {
|
||||
func prepareColumns(criteria querier, columns eventstore.Columns, useV1 bool) (string, func(s scan, dest interface{}) error) {
|
||||
switch columns {
|
||||
case repository.ColumnsMaxSequence:
|
||||
return criteria.maxSequenceQuery(), maxSequenceScanner
|
||||
case repository.ColumnsInstanceIDs:
|
||||
return criteria.instanceIDsQuery(), instanceIDsScanner
|
||||
case repository.ColumnsEvent:
|
||||
return criteria.eventQuery(), eventsScanner
|
||||
case eventstore.ColumnsMaxSequence:
|
||||
return criteria.maxSequenceQuery(useV1), maxSequenceScanner
|
||||
case eventstore.ColumnsInstanceIDs:
|
||||
return criteria.instanceIDsQuery(useV1), instanceIDsScanner
|
||||
case eventstore.ColumnsEvent:
|
||||
return criteria.eventQuery(useV1), eventsScanner(useV1)
|
||||
default:
|
||||
return "", nil
|
||||
}
|
||||
@@ -124,11 +139,11 @@ func prepareTimeTravel(ctx context.Context, criteria querier, allow bool) string
|
||||
}
|
||||
|
||||
func maxSequenceScanner(row scan, dest interface{}) (err error) {
|
||||
sequence, ok := dest.(*Sequence)
|
||||
position, ok := dest.(*sql.NullFloat64)
|
||||
if !ok {
|
||||
return z_errors.ThrowInvalidArgument(nil, "SQL-NBjA9", "type must be sequence")
|
||||
return z_errors.ThrowInvalidArgumentf(nil, "SQL-NBjA9", "type must be sql.NullInt64 got: %T", dest)
|
||||
}
|
||||
err = row(sequence)
|
||||
err = row(position)
|
||||
if err == nil || errors.Is(err, sql.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
@@ -151,84 +166,139 @@ func instanceIDsScanner(scanner scan, dest interface{}) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
func eventsScanner(scanner scan, dest interface{}) (err error) {
|
||||
events, ok := dest.(*[]*repository.Event)
|
||||
if !ok {
|
||||
return z_errors.ThrowInvalidArgument(nil, "SQL-4GP6F", "type must be event")
|
||||
}
|
||||
var (
|
||||
previousAggregateSequence Sequence
|
||||
previousAggregateTypeSequence Sequence
|
||||
)
|
||||
data := make(Data, 0)
|
||||
event := new(repository.Event)
|
||||
|
||||
err = scanner(
|
||||
&event.CreationDate,
|
||||
&event.Type,
|
||||
&event.Sequence,
|
||||
&previousAggregateSequence,
|
||||
&previousAggregateTypeSequence,
|
||||
&data,
|
||||
&event.EditorService,
|
||||
&event.EditorUser,
|
||||
&event.ResourceOwner,
|
||||
&event.InstanceID,
|
||||
&event.AggregateType,
|
||||
&event.AggregateID,
|
||||
&event.Version,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
logging.New().WithError(err).Warn("unable to scan row")
|
||||
return z_errors.ThrowInternal(err, "SQL-M0dsf", "unable to scan row")
|
||||
}
|
||||
|
||||
event.PreviousAggregateSequence = uint64(previousAggregateSequence)
|
||||
event.PreviousAggregateTypeSequence = uint64(previousAggregateTypeSequence)
|
||||
event.Data = make([]byte, len(data))
|
||||
copy(event.Data, data)
|
||||
|
||||
*events = append(*events, event)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func prepareCondition(criteria querier, filters [][]*repository.Filter) (clause string, values []interface{}) {
|
||||
values = make([]interface{}, 0, len(filters))
|
||||
|
||||
if len(filters) == 0 {
|
||||
return clause, values
|
||||
}
|
||||
|
||||
clauses := make([]string, len(filters))
|
||||
for idx, filter := range filters {
|
||||
subClauses := make([]string, 0, len(filter))
|
||||
for _, f := range filter {
|
||||
value := f.Value
|
||||
switch value.(type) {
|
||||
case map[string]interface{}:
|
||||
var err error
|
||||
value, err = json.Marshal(value)
|
||||
if err != nil {
|
||||
logging.WithError(err).Warn("unable to marshal search value")
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
subClauses = append(subClauses, getCondition(criteria, f))
|
||||
if subClauses[len(subClauses)-1] == "" {
|
||||
return "", nil
|
||||
}
|
||||
values = append(values, value)
|
||||
func eventsScanner(useV1 bool) func(scanner scan, dest interface{}) (err error) {
|
||||
return func(scanner scan, dest interface{}) (err error) {
|
||||
events, ok := dest.(*[]eventstore.Event)
|
||||
if !ok {
|
||||
return z_errors.ThrowInvalidArgument(nil, "SQL-4GP6F", "type must be event")
|
||||
}
|
||||
clauses[idx] = "( " + strings.Join(subClauses, " AND ") + " )"
|
||||
event := new(repository.Event)
|
||||
data := sql.RawBytes{}
|
||||
position := new(sql.NullFloat64)
|
||||
|
||||
if useV1 {
|
||||
err = scanner(
|
||||
&event.CreationDate,
|
||||
&event.Typ,
|
||||
&event.Seq,
|
||||
&data,
|
||||
&event.EditorUser,
|
||||
&event.ResourceOwner,
|
||||
&event.InstanceID,
|
||||
&event.AggregateType,
|
||||
&event.AggregateID,
|
||||
&event.Version,
|
||||
)
|
||||
} else {
|
||||
var revision uint8
|
||||
err = scanner(
|
||||
&event.CreationDate,
|
||||
&event.Typ,
|
||||
&event.Seq,
|
||||
position,
|
||||
&data,
|
||||
&event.EditorUser,
|
||||
&event.ResourceOwner,
|
||||
&event.InstanceID,
|
||||
&event.AggregateType,
|
||||
&event.AggregateID,
|
||||
&revision,
|
||||
)
|
||||
event.Version = eventstore.Version("v" + strconv.Itoa(int(revision)))
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
logging.New().WithError(err).Warn("unable to scan row")
|
||||
return z_errors.ThrowInternal(err, "SQL-M0dsf", "unable to scan row")
|
||||
}
|
||||
|
||||
event.Data = make([]byte, len(data))
|
||||
copy(event.Data, data)
|
||||
event.Pos = position.Float64
|
||||
|
||||
*events = append(*events, event)
|
||||
|
||||
return nil
|
||||
}
|
||||
return " WHERE " + strings.Join(clauses, " OR "), values
|
||||
}
|
||||
|
||||
func getCondition(cond querier, filter *repository.Filter) (condition string) {
|
||||
field := cond.columnName(filter.Field)
|
||||
func prepareConditions(criteria querier, query *repository.SearchQuery, useV1 bool) (string, []any) {
|
||||
clauses, args := prepareQuery(criteria, useV1, query.InstanceID, query.ExcludedInstances)
|
||||
if clauses != "" && len(query.SubQueries) > 0 {
|
||||
clauses += " AND "
|
||||
}
|
||||
subClauses := make([]string, len(query.SubQueries))
|
||||
for i, filters := range query.SubQueries {
|
||||
var subArgs []any
|
||||
subClauses[i], subArgs = prepareQuery(criteria, useV1, filters...)
|
||||
// an error is thrown in [query]
|
||||
if subClauses[i] == "" {
|
||||
return "", nil
|
||||
}
|
||||
if len(query.SubQueries) > 1 && len(subArgs) > 1 {
|
||||
subClauses[i] = "(" + subClauses[i] + ")"
|
||||
}
|
||||
args = append(args, subArgs...)
|
||||
}
|
||||
if len(subClauses) == 1 {
|
||||
clauses += subClauses[0]
|
||||
} else if len(subClauses) > 1 {
|
||||
clauses += "(" + strings.Join(subClauses, " OR ") + ")"
|
||||
}
|
||||
|
||||
additionalClauses, additionalArgs := prepareQuery(criteria, useV1, query.Position, query.Owner, query.Sequence, query.CreatedAt, query.Creator)
|
||||
if additionalClauses != "" {
|
||||
if clauses != "" {
|
||||
clauses += " AND "
|
||||
}
|
||||
clauses += additionalClauses
|
||||
args = append(args, additionalArgs...)
|
||||
}
|
||||
|
||||
if query.AwaitOpenTransactions {
|
||||
clauses += awaitOpenTransactions(useV1)
|
||||
}
|
||||
|
||||
if clauses == "" {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
return " WHERE " + clauses, args
|
||||
}
|
||||
|
||||
func prepareQuery(criteria querier, useV1 bool, filters ...*repository.Filter) (_ string, args []any) {
|
||||
clauses := make([]string, 0, len(filters))
|
||||
args = make([]any, 0, len(filters))
|
||||
for _, filter := range filters {
|
||||
if filter == nil {
|
||||
continue
|
||||
}
|
||||
arg := filter.Value
|
||||
|
||||
// marshal if payload filter
|
||||
if filter.Field == repository.FieldEventData {
|
||||
var err error
|
||||
arg, err = json.Marshal(arg)
|
||||
if err != nil {
|
||||
logging.WithError(err).Warn("unable to marshal search value")
|
||||
continue
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
clauses = append(clauses, getCondition(criteria, filter, useV1))
|
||||
// if mapping failed an error is thrown in [query]
|
||||
if clauses[len(clauses)-1] == "" {
|
||||
return "", nil
|
||||
}
|
||||
args = append(args, arg)
|
||||
}
|
||||
|
||||
return strings.Join(clauses, " AND "), args
|
||||
}
|
||||
|
||||
func getCondition(cond querier, filter *repository.Filter, useV1 bool) (condition string) {
|
||||
field := cond.columnName(filter.Field, useV1)
|
||||
operation := cond.operation(filter.Operation)
|
||||
if field == "" || operation == "" {
|
||||
return ""
|
||||
|
||||
Reference in New Issue
Block a user