2020-10-05 17:09:26 +00:00
package sql
2020-10-02 14:21:51 +00:00
import (
2020-10-21 17:00:41 +00:00
"context"
2020-10-02 14:21:51 +00:00
"database/sql"
2020-10-21 17:00:41 +00:00
"database/sql/driver"
2020-10-02 14:21:51 +00:00
"reflect"
feat(eventstore): exclude aggregate IDs when event_type occurred (#8940)
# Which Problems Are Solved
For truly event-based notification handler, we need to be able to filter
out events of aggregates which are already handled. For example when an
event like `notify.success` or `notify.failed` was created on an
aggregate, we no longer require events from that aggregate ID.
# How the Problems Are Solved
Extend the query builder to use a `NOT IN` clause which excludes
aggregate IDs when they have certain events for a certain aggregate
type. For optimization and proper index usages, certain filters are
inherited from the parent query, such as:
- Instance ID
- Instance IDs
- Position offset
This is a prettified query as used by the unit tests:
```sql
SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision
FROM eventstore.events2
WHERE instance_id = $1
AND aggregate_type = $2
AND event_type = $3
AND "position" > $4
AND aggregate_id NOT IN (
SELECT aggregate_id
FROM eventstore.events2
WHERE aggregate_type = $5
AND event_type = ANY($6)
AND instance_id = $7
AND "position" > $8
)
ORDER BY "position" DESC, in_tx_order DESC
LIMIT $9
```
I used this query to run it against the `oidc_session` aggregate looking
for added events, excluding aggregates where a token was revoked,
against a recent position. It fully used index scans:
<details>
```json
[
{
"Plan": {
"Node Type": "Index Scan",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2",
"Actual Rows": 2,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.added'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0,
"Filter": "(NOT (hashed SubPlan 1))",
"Rows Removed by Filter": 1,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "SubPlan",
"Subplan Name": "SubPlan 1",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2_1",
"Actual Rows": 1,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.access_token.revoked'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0
}
]
},
"Triggers": [
]
}
]
```
</details>
# Additional Changes
- None
# Additional Context
- Related to https://github.com/zitadel/zitadel/issues/8931
---------
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2024-11-25 15:25:11 +00:00
"regexp"
2023-10-19 10:19:10 +00:00
"strconv"
2020-10-02 14:21:51 +00:00
"testing"
"time"
2020-10-21 17:00:41 +00:00
"github.com/DATA-DOG/go-sqlmock"
2023-10-19 15:21:31 +00:00
"github.com/stretchr/testify/assert"
2022-01-06 07:29:58 +00:00
2023-02-27 21:36:43 +00:00
"github.com/zitadel/zitadel/internal/database"
2023-10-19 10:19:10 +00:00
"github.com/zitadel/zitadel/internal/database/cockroach"
2024-03-27 13:48:22 +00:00
db_mock "github.com/zitadel/zitadel/internal/database/mock"
2023-10-19 10:19:10 +00:00
"github.com/zitadel/zitadel/internal/eventstore"
2022-04-26 23:01:45 +00:00
"github.com/zitadel/zitadel/internal/eventstore/repository"
2023-12-08 14:30:55 +00:00
"github.com/zitadel/zitadel/internal/zerrors"
2020-10-02 14:21:51 +00:00
)
func Test_getCondition ( t * testing . T ) {
type args struct {
2020-10-05 17:09:26 +00:00
filter * repository . Filter
2020-10-02 14:21:51 +00:00
}
tests := [ ] struct {
name string
args args
want string
} {
{
name : "equals" ,
2020-10-06 19:28:09 +00:00
args : args { filter : repository . NewFilter ( repository . FieldAggregateID , "" , repository . OperationEquals ) } ,
2020-10-02 14:21:51 +00:00
want : "aggregate_id = ?" ,
} ,
{
name : "greater" ,
2020-10-06 19:28:09 +00:00
args : args { filter : repository . NewFilter ( repository . FieldSequence , 0 , repository . OperationGreater ) } ,
2023-10-19 10:19:10 +00:00
want : ` "sequence" > ? ` ,
2020-10-02 14:21:51 +00:00
} ,
{
name : "less" ,
2020-10-06 19:28:09 +00:00
args : args { filter : repository . NewFilter ( repository . FieldSequence , 5000 , repository . OperationLess ) } ,
2023-10-19 10:19:10 +00:00
want : ` "sequence" < ? ` ,
2020-10-02 14:21:51 +00:00
} ,
{
name : "in list" ,
2023-10-19 10:19:10 +00:00
args : args { filter : repository . NewFilter ( repository . FieldAggregateType , [ ] eventstore . AggregateType { "movies" , "actors" } , repository . OperationIn ) } ,
2020-10-02 14:21:51 +00:00
want : "aggregate_type = ANY(?)" ,
} ,
{
name : "invalid operation" ,
2023-10-19 10:19:10 +00:00
args : args { filter : repository . NewFilter ( repository . FieldAggregateType , [ ] eventstore . AggregateType { "movies" , "actors" } , repository . Operation ( - 1 ) ) } ,
2020-10-02 14:21:51 +00:00
want : "" ,
} ,
{
name : "invalid field" ,
2023-10-19 10:19:10 +00:00
args : args { filter : repository . NewFilter ( repository . Field ( - 1 ) , [ ] eventstore . AggregateType { "movies" , "actors" } , repository . OperationEquals ) } ,
2020-10-02 14:21:51 +00:00
want : "" ,
} ,
{
name : "invalid field and operation" ,
2023-10-19 10:19:10 +00:00
args : args { filter : repository . NewFilter ( repository . Field ( - 1 ) , [ ] eventstore . AggregateType { "movies" , "actors" } , repository . Operation ( - 1 ) ) } ,
2020-10-02 14:21:51 +00:00
want : "" ,
} ,
}
for _ , tt := range tests {
t . Run ( tt . name , func ( t * testing . T ) {
2020-10-05 18:39:36 +00:00
db := & CRDB { }
2023-10-19 10:19:10 +00:00
if got := getCondition ( db , tt . args . filter , false ) ; got != tt . want {
2020-10-02 14:21:51 +00:00
t . Errorf ( "getCondition() = %v, want %v" , got , tt . want )
}
} )
}
}
func Test_prepareColumns ( t * testing . T ) {
2023-10-19 15:21:31 +00:00
var reducedEvents [ ] eventstore . Event
2020-10-05 18:39:36 +00:00
type fields struct {
dbRow [ ] interface { }
}
2020-10-02 14:21:51 +00:00
type args struct {
2023-10-19 10:19:10 +00:00
columns eventstore . Columns
2020-10-02 14:21:51 +00:00
dest interface { }
dbErr error
2023-10-19 10:19:10 +00:00
useV1 bool
2020-10-02 14:21:51 +00:00
}
type res struct {
query string
expected interface { }
dbErr func ( error ) bool
}
tests := [ ] struct {
2020-10-05 18:39:36 +00:00
name string
args args
res res
fields fields
2020-10-02 14:21:51 +00:00
} {
{
name : "invalid columns" ,
2023-10-19 10:19:10 +00:00
args : args { columns : eventstore . Columns ( - 1 ) } ,
2020-10-02 14:21:51 +00:00
res : res {
query : "" ,
dbErr : func ( err error ) bool { return err == nil } ,
} ,
} ,
{
name : "max column" ,
args : args {
2024-09-24 16:43:29 +00:00
columns : eventstore . ColumnsMaxSequence ,
dest : new ( sql . NullFloat64 ) ,
2023-10-19 10:19:10 +00:00
useV1 : true ,
2020-10-02 14:21:51 +00:00
} ,
res : res {
2023-10-19 10:19:10 +00:00
query : ` SELECT event_sequence FROM eventstore.events ` ,
2024-09-24 16:43:29 +00:00
expected : sql . NullFloat64 { Float64 : 43 , Valid : true } ,
2020-10-02 14:21:51 +00:00
} ,
2020-10-05 18:39:36 +00:00
fields : fields {
2024-09-24 16:43:29 +00:00
dbRow : [ ] interface { } { sql . NullFloat64 { Float64 : 43 , Valid : true } } ,
2023-10-19 10:19:10 +00:00
} ,
} ,
{
name : "max column v2" ,
args : args {
2024-09-24 16:43:29 +00:00
columns : eventstore . ColumnsMaxSequence ,
dest : new ( sql . NullFloat64 ) ,
2023-10-19 10:19:10 +00:00
} ,
res : res {
query : ` SELECT "position" FROM eventstore.events2 ` ,
2024-09-24 16:43:29 +00:00
expected : sql . NullFloat64 { Float64 : 43 , Valid : true } ,
2023-10-19 10:19:10 +00:00
} ,
fields : fields {
2024-09-24 16:43:29 +00:00
dbRow : [ ] interface { } { sql . NullFloat64 { Float64 : 43 , Valid : true } } ,
2020-10-05 18:39:36 +00:00
} ,
2020-10-02 14:21:51 +00:00
} ,
{
name : "max sequence wrong dest type" ,
args : args {
2024-09-24 16:43:29 +00:00
columns : eventstore . ColumnsMaxSequence ,
2020-10-02 14:21:51 +00:00
dest : new ( uint64 ) ,
} ,
res : res {
2023-10-19 10:19:10 +00:00
query : ` SELECT "position" FROM eventstore.events2 ` ,
2023-12-08 14:30:55 +00:00
dbErr : zerrors . IsErrorInvalidArgument ,
2020-10-02 14:21:51 +00:00
} ,
} ,
{
2020-10-05 18:39:36 +00:00
name : "events" ,
2020-10-02 14:21:51 +00:00
args : args {
2023-10-19 10:19:10 +00:00
columns : eventstore . ColumnsEvent ,
2023-10-19 15:21:31 +00:00
dest : eventstore . Reducer ( func ( event eventstore . Event ) error {
reducedEvents = append ( reducedEvents , event )
return nil
} ) ,
useV1 : true ,
2023-10-19 10:19:10 +00:00
} ,
res : res {
query : ` SELECT creation_date, event_type, event_sequence, event_data, editor_user, resource_owner, instance_id, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events ` ,
expected : [ ] eventstore . Event {
2023-10-19 15:21:31 +00:00
& repository . Event { AggregateID : "hodor" , AggregateType : "user" , Seq : 5 , Data : nil } ,
2023-10-19 10:19:10 +00:00
} ,
} ,
fields : fields {
dbRow : [ ] interface { } { time . Time { } , eventstore . EventType ( "" ) , uint64 ( 5 ) , sql . RawBytes ( nil ) , "" , sql . NullString { } , "" , eventstore . AggregateType ( "user" ) , "hodor" , eventstore . Version ( "" ) } ,
} ,
} ,
{
name : "events v2" ,
args : args {
columns : eventstore . ColumnsEvent ,
2023-10-19 15:21:31 +00:00
dest : eventstore . Reducer ( func ( event eventstore . Event ) error {
reducedEvents = append ( reducedEvents , event )
return nil
} ) ,
2023-10-19 10:19:10 +00:00
} ,
res : res {
query : ` SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision FROM eventstore.events2 ` ,
expected : [ ] eventstore . Event {
2024-09-24 16:43:29 +00:00
& repository . Event { AggregateID : "hodor" , AggregateType : "user" , Seq : 5 , Pos : 42 , Data : nil , Version : "v1" } ,
2023-10-19 10:19:10 +00:00
} ,
} ,
fields : fields {
2024-09-24 16:43:29 +00:00
dbRow : [ ] interface { } { time . Time { } , eventstore . EventType ( "" ) , uint64 ( 5 ) , sql . NullFloat64 { Float64 : 42 , Valid : true } , sql . RawBytes ( nil ) , "" , sql . NullString { } , "" , eventstore . AggregateType ( "user" ) , "hodor" , uint8 ( 1 ) } ,
2023-10-19 10:19:10 +00:00
} ,
} ,
{
name : "event null position" ,
args : args {
columns : eventstore . ColumnsEvent ,
2023-10-19 15:21:31 +00:00
dest : eventstore . Reducer ( func ( event eventstore . Event ) error {
reducedEvents = append ( reducedEvents , event )
return nil
} ) ,
2020-10-02 14:21:51 +00:00
} ,
res : res {
2023-10-19 10:19:10 +00:00
query : ` SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision FROM eventstore.events2 ` ,
expected : [ ] eventstore . Event {
2024-09-24 16:43:29 +00:00
& repository . Event { AggregateID : "hodor" , AggregateType : "user" , Seq : 5 , Pos : 0 , Data : nil , Version : "v1" } ,
2020-10-05 18:39:36 +00:00
} ,
} ,
fields : fields {
2024-09-24 16:43:29 +00:00
dbRow : [ ] interface { } { time . Time { } , eventstore . EventType ( "" ) , uint64 ( 5 ) , sql . NullFloat64 { Float64 : 0 , Valid : false } , sql . RawBytes ( nil ) , "" , sql . NullString { } , "" , eventstore . AggregateType ( "user" ) , "hodor" , uint8 ( 1 ) } ,
2020-10-02 14:21:51 +00:00
} ,
} ,
{
2020-10-05 18:39:36 +00:00
name : "events wrong dest type" ,
2020-10-02 14:21:51 +00:00
args : args {
2023-10-19 10:19:10 +00:00
columns : eventstore . ColumnsEvent ,
2020-10-05 18:39:36 +00:00
dest : [ ] * repository . Event { } ,
2023-10-19 10:19:10 +00:00
useV1 : true ,
2020-10-02 14:21:51 +00:00
} ,
res : res {
2023-10-19 10:19:10 +00:00
query : ` SELECT creation_date, event_type, event_sequence, event_data, editor_user, resource_owner, instance_id, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events ` ,
2023-12-08 14:30:55 +00:00
dbErr : zerrors . IsErrorInvalidArgument ,
2020-10-02 14:21:51 +00:00
} ,
} ,
{
name : "event query error" ,
args : args {
2023-10-19 10:19:10 +00:00
columns : eventstore . ColumnsEvent ,
2023-10-19 15:21:31 +00:00
dest : eventstore . Reducer ( func ( event eventstore . Event ) error {
reducedEvents = append ( reducedEvents , event )
return nil
} ) ,
dbErr : sql . ErrConnDone ,
useV1 : true ,
2020-10-02 14:21:51 +00:00
} ,
res : res {
2023-10-19 10:19:10 +00:00
query : ` SELECT creation_date, event_type, event_sequence, event_data, editor_user, resource_owner, instance_id, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events ` ,
2023-12-08 14:30:55 +00:00
dbErr : zerrors . IsInternal ,
2020-10-02 14:21:51 +00:00
} ,
} ,
}
for _ , tt := range tests {
t . Run ( tt . name , func ( t * testing . T ) {
2020-10-05 18:39:36 +00:00
crdb := & CRDB { }
2023-10-19 10:19:10 +00:00
query , rowScanner := prepareColumns ( crdb , tt . args . columns , tt . args . useV1 )
2020-10-02 14:21:51 +00:00
if query != tt . res . query {
2020-10-05 18:39:36 +00:00
t . Errorf ( "prepareColumns() got = %s, want %s" , query , tt . res . query )
2020-10-02 14:21:51 +00:00
}
if tt . res . query == "" && rowScanner != nil {
t . Errorf ( "row scanner should be nil" )
}
if rowScanner == nil {
return
}
2020-10-05 18:39:36 +00:00
err := rowScanner ( prepareTestScan ( tt . args . dbErr , tt . fields . dbRow ) , tt . args . dest )
if err != nil && tt . res . dbErr == nil || err != nil && ! tt . res . dbErr ( err ) || err == nil && tt . res . dbErr != nil {
t . Errorf ( "wrong error type in rowScanner got: %v" , err )
return
}
if tt . res . dbErr != nil && tt . res . dbErr ( err ) {
return
}
2023-10-19 10:19:10 +00:00
if equalizer , ok := tt . res . expected . ( interface { Equal ( time . Time ) bool } ) ; ok {
equalizer . Equal ( tt . args . dest . ( * sql . NullTime ) . Time )
return
}
2023-10-19 15:21:31 +00:00
if _ , ok := tt . args . dest . ( eventstore . Reducer ) ; ok {
assert . Equal ( t , tt . res . expected , reducedEvents )
reducedEvents = nil
return
}
2023-10-19 10:19:10 +00:00
got := reflect . Indirect ( reflect . ValueOf ( tt . args . dest ) ) . Interface ( )
if ! reflect . DeepEqual ( got , tt . res . expected ) {
t . Errorf ( "unexpected result from rowScanner \nwant: %+v \ngot: %+v" , tt . res . expected , got )
2020-10-02 14:21:51 +00:00
}
} )
}
}
2020-10-05 17:09:26 +00:00
func prepareTestScan ( err error , res [ ] interface { } ) scan {
2020-10-02 14:21:51 +00:00
return func ( dests ... interface { } ) error {
if err != nil {
return err
}
if len ( dests ) != len ( res ) {
2023-12-08 14:30:55 +00:00
return zerrors . ThrowInvalidArgumentf ( nil , "SQL-NML1q" , "expected len %d got %d" , len ( res ) , len ( dests ) )
2020-10-02 14:21:51 +00:00
}
for i , r := range res {
2023-10-19 10:19:10 +00:00
_ , ok := dests [ i ] . ( * eventstore . Version )
if ok {
val , ok := r . ( uint8 )
if ok {
r = eventstore . Version ( "" + strconv . Itoa ( int ( val ) ) )
}
}
2020-10-02 14:21:51 +00:00
reflect . ValueOf ( dests [ i ] ) . Elem ( ) . Set ( reflect . ValueOf ( r ) )
}
return nil
}
}
func Test_prepareCondition ( t * testing . T ) {
type args struct {
2023-10-19 10:19:10 +00:00
query * repository . SearchQuery
useV1 bool
2020-10-02 14:21:51 +00:00
}
type res struct {
clause string
values [ ] interface { }
}
tests := [ ] struct {
name string
args args
res res
} {
{
name : "nil filters" ,
args : args {
2023-10-19 10:19:10 +00:00
query : & repository . SearchQuery { } ,
useV1 : true ,
} ,
res : res {
clause : "" ,
values : nil ,
} ,
} ,
{
name : "nil filters v2" ,
args : args {
query : & repository . SearchQuery { } ,
2020-10-02 14:21:51 +00:00
} ,
res : res {
clause : "" ,
values : nil ,
} ,
} ,
{
name : "empty filters" ,
args : args {
2023-10-19 10:19:10 +00:00
query : & repository . SearchQuery {
SubQueries : [ ] [ ] * repository . Filter { } ,
} ,
useV1 : true ,
} ,
res : res {
clause : "" ,
values : nil ,
} ,
} ,
{
name : "empty filters v2" ,
args : args {
query : & repository . SearchQuery {
SubQueries : [ ] [ ] * repository . Filter { } ,
} ,
2020-10-02 14:21:51 +00:00
} ,
res : res {
clause : "" ,
values : nil ,
} ,
} ,
{
name : "invalid condition" ,
args : args {
2023-10-19 10:19:10 +00:00
query : & repository . SearchQuery {
SubQueries : [ ] [ ] * repository . Filter {
{
repository . NewFilter ( repository . FieldAggregateID , "wrong" , repository . Operation ( - 1 ) ) ,
} ,
} ,
} ,
useV1 : true ,
} ,
res : res {
clause : "" ,
values : nil ,
} ,
} ,
{
name : "invalid condition v2" ,
args : args {
query : & repository . SearchQuery {
SubQueries : [ ] [ ] * repository . Filter {
{
repository . NewFilter ( repository . FieldAggregateID , "wrong" , repository . Operation ( - 1 ) ) ,
} ,
2021-07-06 11:55:57 +00:00
} ,
2020-10-02 14:21:51 +00:00
} ,
} ,
res : res {
clause : "" ,
values : nil ,
} ,
} ,
{
name : "array as condition value" ,
args : args {
2023-10-19 10:19:10 +00:00
query : & repository . SearchQuery {
AwaitOpenTransactions : true ,
SubQueries : [ ] [ ] * repository . Filter {
{
repository . NewFilter ( repository . FieldAggregateType , [ ] eventstore . AggregateType { "user" , "org" } , repository . OperationIn ) ,
} ,
2021-07-06 11:55:57 +00:00
} ,
2020-10-02 14:21:51 +00:00
} ,
2023-10-19 10:19:10 +00:00
useV1 : true ,
2020-10-02 14:21:51 +00:00
} ,
res : res {
2024-11-18 15:30:12 +00:00
clause : " WHERE aggregate_type = ANY(?) AND creation_date::TIMESTAMP < (SELECT COALESCE(MIN(start), NOW())::TIMESTAMP FROM crdb_internal.cluster_transactions where application_name = ANY(?))" ,
values : [ ] interface { } { [ ] eventstore . AggregateType { "user" , "org" } , database . TextArray [ string ] { } } ,
2023-10-19 10:19:10 +00:00
} ,
} ,
{
name : "array as condition value v2" ,
args : args {
query : & repository . SearchQuery {
AwaitOpenTransactions : true ,
SubQueries : [ ] [ ] * repository . Filter {
{
repository . NewFilter ( repository . FieldAggregateType , [ ] eventstore . AggregateType { "user" , "org" } , repository . OperationIn ) ,
} ,
} ,
} ,
} ,
res : res {
2024-11-18 15:30:12 +00:00
clause : ` WHERE aggregate_type = ANY(?) AND hlc_to_timestamp("position") < (SELECT COALESCE(MIN(start), NOW())::TIMESTAMP FROM crdb_internal.cluster_transactions where application_name = ANY(?)) ` ,
values : [ ] interface { } { [ ] eventstore . AggregateType { "user" , "org" } , database . TextArray [ string ] { } } ,
2020-10-02 14:21:51 +00:00
} ,
} ,
{
name : "multiple filters" ,
args : args {
2023-10-19 10:19:10 +00:00
query : & repository . SearchQuery {
AwaitOpenTransactions : true ,
SubQueries : [ ] [ ] * repository . Filter {
{
repository . NewFilter ( repository . FieldAggregateType , [ ] eventstore . AggregateType { "user" , "org" } , repository . OperationIn ) ,
repository . NewFilter ( repository . FieldAggregateID , "1234" , repository . OperationEquals ) ,
repository . NewFilter ( repository . FieldEventType , [ ] eventstore . EventType { "user.created" , "org.created" } , repository . OperationIn ) ,
} ,
2021-07-06 11:55:57 +00:00
} ,
2020-10-02 14:21:51 +00:00
} ,
2023-10-19 10:19:10 +00:00
useV1 : true ,
2020-10-02 14:21:51 +00:00
} ,
res : res {
2024-11-18 15:30:12 +00:00
clause : " WHERE aggregate_type = ANY(?) AND aggregate_id = ? AND event_type = ANY(?) AND creation_date::TIMESTAMP < (SELECT COALESCE(MIN(start), NOW())::TIMESTAMP FROM crdb_internal.cluster_transactions where application_name = ANY(?))" ,
values : [ ] interface { } { [ ] eventstore . AggregateType { "user" , "org" } , "1234" , [ ] eventstore . EventType { "user.created" , "org.created" } , database . TextArray [ string ] { } } ,
2023-10-19 10:19:10 +00:00
} ,
} ,
{
name : "multiple filters v2" ,
args : args {
query : & repository . SearchQuery {
AwaitOpenTransactions : true ,
SubQueries : [ ] [ ] * repository . Filter {
{
repository . NewFilter ( repository . FieldAggregateType , [ ] eventstore . AggregateType { "user" , "org" } , repository . OperationIn ) ,
repository . NewFilter ( repository . FieldAggregateID , "1234" , repository . OperationEquals ) ,
repository . NewFilter ( repository . FieldEventType , [ ] eventstore . EventType { "user.created" , "org.created" } , repository . OperationIn ) ,
} ,
} ,
} ,
} ,
res : res {
2024-11-18 15:30:12 +00:00
clause : ` WHERE aggregate_type = ANY(?) AND aggregate_id = ? AND event_type = ANY(?) AND hlc_to_timestamp("position") < (SELECT COALESCE(MIN(start), NOW())::TIMESTAMP FROM crdb_internal.cluster_transactions where application_name = ANY(?)) ` ,
values : [ ] interface { } { [ ] eventstore . AggregateType { "user" , "org" } , "1234" , [ ] eventstore . EventType { "user.created" , "org.created" } , database . TextArray [ string ] { } } ,
2020-10-02 14:21:51 +00:00
} ,
} ,
}
2023-10-19 10:19:10 +00:00
crdb := NewCRDB ( & database . DB { Database : new ( cockroach . Config ) } )
2020-10-02 14:21:51 +00:00
for _ , tt := range tests {
t . Run ( tt . name , func ( t * testing . T ) {
2023-10-19 10:19:10 +00:00
gotClause , gotValues := prepareConditions ( crdb , tt . args . query , tt . args . useV1 )
2020-10-02 14:21:51 +00:00
if gotClause != tt . res . clause {
t . Errorf ( "prepareCondition() gotClause = %v, want %v" , gotClause , tt . res . clause )
}
if len ( gotValues ) != len ( tt . res . values ) {
t . Errorf ( "wrong length of gotten values got = %d, want %d" , len ( gotValues ) , len ( tt . res . values ) )
return
}
for i , value := range gotValues {
if ! reflect . DeepEqual ( value , tt . res . values [ i ] ) {
t . Errorf ( "prepareCondition() gotValues = %v, want %v" , gotValues , tt . res . values )
}
}
} )
}
}
2020-10-21 17:00:41 +00:00
func Test_query_events_with_crdb ( t * testing . T ) {
type args struct {
2023-10-19 10:19:10 +00:00
searchQuery * eventstore . SearchQueryBuilder
2020-10-21 17:00:41 +00:00
}
type fields struct {
2023-10-19 10:19:10 +00:00
existingEvents [ ] eventstore . Command
2020-10-21 17:00:41 +00:00
client * sql . DB
}
type res struct {
eventCount int
}
tests := [ ] struct {
name string
fields fields
args args
res res
wantErr bool
} {
{
name : "aggregate type filter no events" ,
args : args {
2023-10-19 10:19:10 +00:00
searchQuery : eventstore . NewSearchQueryBuilder ( eventstore . ColumnsEvent ) .
AddQuery ( ) .
AggregateTypes ( "not found" ) .
Builder ( ) ,
2020-10-21 17:00:41 +00:00
} ,
fields : fields {
client : testCRDBClient ,
2023-10-19 10:19:10 +00:00
existingEvents : [ ] eventstore . Command {
2021-01-15 08:32:59 +00:00
generateEvent ( t , "300" ) ,
generateEvent ( t , "300" ) ,
generateEvent ( t , "300" ) ,
2020-10-21 17:00:41 +00:00
} ,
} ,
res : res {
eventCount : 0 ,
} ,
wantErr : false ,
} ,
{
name : "aggregate type filter events found" ,
args : args {
2023-10-19 10:19:10 +00:00
searchQuery : eventstore . NewSearchQueryBuilder ( eventstore . ColumnsEvent ) .
AddQuery ( ) .
AggregateTypes ( eventstore . AggregateType ( t . Name ( ) ) ) .
Builder ( ) ,
2020-10-21 17:00:41 +00:00
} ,
fields : fields {
client : testCRDBClient ,
2023-10-19 10:19:10 +00:00
existingEvents : [ ] eventstore . Command {
2021-01-15 08:32:59 +00:00
generateEvent ( t , "301" ) ,
generateEvent ( t , "302" ) ,
generateEvent ( t , "302" ) ,
generateEvent ( t , "303" , func ( e * repository . Event ) { e . AggregateType = "not in list" } ) ,
2020-10-21 17:00:41 +00:00
} ,
} ,
res : res {
eventCount : 3 ,
} ,
wantErr : false ,
} ,
{
name : "aggregate type and id filter events found" ,
args : args {
2023-10-19 10:19:10 +00:00
searchQuery : eventstore . NewSearchQueryBuilder ( eventstore . ColumnsEvent ) .
AddQuery ( ) .
AggregateTypes ( eventstore . AggregateType ( t . Name ( ) ) ) .
AggregateIDs ( "303" ) .
Builder ( ) ,
2020-10-21 17:00:41 +00:00
} ,
fields : fields {
client : testCRDBClient ,
2023-10-19 10:19:10 +00:00
existingEvents : [ ] eventstore . Command {
2021-01-15 08:32:59 +00:00
generateEvent ( t , "303" ) ,
generateEvent ( t , "303" ) ,
generateEvent ( t , "303" ) ,
generateEvent ( t , "304" , func ( e * repository . Event ) { e . AggregateType = "not in list" } ) ,
generateEvent ( t , "305" ) ,
2020-10-21 17:00:41 +00:00
} ,
} ,
res : res {
eventCount : 3 ,
} ,
wantErr : false ,
} ,
{
name : "resource owner filter events found" ,
args : args {
2023-10-19 10:19:10 +00:00
searchQuery : eventstore . NewSearchQueryBuilder ( eventstore . ColumnsEvent ) .
ResourceOwner ( "caos" ) ,
2020-10-21 17:00:41 +00:00
} ,
fields : fields {
client : testCRDBClient ,
2023-10-19 10:19:10 +00:00
existingEvents : [ ] eventstore . Command {
2022-01-06 07:29:58 +00:00
generateEvent ( t , "306" , func ( e * repository . Event ) { e . ResourceOwner = sql . NullString { String : "caos" , Valid : true } } ) ,
generateEvent ( t , "307" , func ( e * repository . Event ) { e . ResourceOwner = sql . NullString { String : "caos" , Valid : true } } ) ,
generateEvent ( t , "308" , func ( e * repository . Event ) { e . ResourceOwner = sql . NullString { String : "caos" , Valid : true } } ) ,
generateEvent ( t , "309" , func ( e * repository . Event ) { e . ResourceOwner = sql . NullString { String : "orgID" , Valid : true } } ) ,
generateEvent ( t , "309" , func ( e * repository . Event ) { e . ResourceOwner = sql . NullString { String : "orgID" , Valid : true } } ) ,
2020-10-21 17:00:41 +00:00
} ,
} ,
res : res {
eventCount : 3 ,
} ,
wantErr : false ,
} ,
{
name : "event type filter events found" ,
args : args {
2023-10-19 10:19:10 +00:00
searchQuery : eventstore . NewSearchQueryBuilder ( eventstore . ColumnsEvent ) .
AddQuery ( ) .
EventTypes ( "user.created" , "user.updated" ) .
Builder ( ) ,
2020-10-21 17:00:41 +00:00
} ,
fields : fields {
client : testCRDBClient ,
2023-10-19 10:19:10 +00:00
existingEvents : [ ] eventstore . Command {
generateEvent ( t , "311" , func ( e * repository . Event ) { e . Typ = "user.created" } ) ,
generateEvent ( t , "311" , func ( e * repository . Event ) { e . Typ = "user.updated" } ) ,
generateEvent ( t , "311" , func ( e * repository . Event ) { e . Typ = "user.deactivated" } ) ,
generateEvent ( t , "311" , func ( e * repository . Event ) { e . Typ = "user.locked" } ) ,
generateEvent ( t , "312" , func ( e * repository . Event ) { e . Typ = "user.created" } ) ,
generateEvent ( t , "312" , func ( e * repository . Event ) { e . Typ = "user.updated" } ) ,
generateEvent ( t , "312" , func ( e * repository . Event ) { e . Typ = "user.deactivated" } ) ,
generateEvent ( t , "312" , func ( e * repository . Event ) { e . Typ = "user.reactivated" } ) ,
generateEvent ( t , "313" , func ( e * repository . Event ) { e . Typ = "user.locked" } ) ,
2020-10-21 17:00:41 +00:00
} ,
} ,
res : res {
eventCount : 7 ,
} ,
wantErr : false ,
} ,
{
name : "fail because no filter" ,
args : args {
2023-10-19 10:19:10 +00:00
searchQuery : eventstore . NewSearchQueryBuilder ( eventstore . Columns ( - 1 ) ) ,
2020-10-21 17:00:41 +00:00
} ,
fields : fields {
client : testCRDBClient ,
2023-10-19 10:19:10 +00:00
existingEvents : [ ] eventstore . Command { } ,
2020-10-21 17:00:41 +00:00
} ,
res : res {
eventCount : 0 ,
} ,
wantErr : true ,
} ,
}
for _ , tt := range tests {
t . Run ( tt . name , func ( t * testing . T ) {
db := & CRDB {
2023-02-27 21:36:43 +00:00
DB : & database . DB {
DB : tt . fields . client ,
Database : new ( testDB ) ,
} ,
2020-10-21 17:00:41 +00:00
}
// setup initial data for query
2023-10-19 10:19:10 +00:00
if _ , err := db . Push ( context . Background ( ) , tt . fields . existingEvents ... ) ; err != nil {
2020-10-21 17:00:41 +00:00
t . Errorf ( "error in setup = %v" , err )
return
}
2023-10-19 10:19:10 +00:00
events := [ ] eventstore . Event { }
2023-10-19 15:21:31 +00:00
if err := query ( context . Background ( ) , db , tt . args . searchQuery , eventstore . Reducer ( func ( event eventstore . Event ) error {
events = append ( events , event )
return nil
} ) , true ) ; ( err != nil ) != tt . wantErr {
2020-10-21 17:00:41 +00:00
t . Errorf ( "CRDB.query() error = %v, wantErr %v" , err , tt . wantErr )
}
} )
}
}
2024-11-21 14:46:30 +00:00
/ * Cockroach test DB doesn ' t seem to lock
func Test_query_events_with_crdb_locking ( t * testing . T ) {
type args struct {
searchQuery * eventstore . SearchQueryBuilder
}
type fields struct {
existingEvents [ ] eventstore . Command
client * sql . DB
}
tests := [ ] struct {
name string
fields fields
args args
lockOption eventstore . LockOption
wantErr bool
} {
{
name : "skip locked" ,
fields : fields {
client : testCRDBClient ,
existingEvents : [ ] eventstore . Command {
generateEvent ( t , "306" , func ( e * repository . Event ) { e . ResourceOwner = sql . NullString { String : "caos" , Valid : true } } ) ,
generateEvent ( t , "307" , func ( e * repository . Event ) { e . ResourceOwner = sql . NullString { String : "caos" , Valid : true } } ) ,
generateEvent ( t , "308" , func ( e * repository . Event ) { e . ResourceOwner = sql . NullString { String : "caos" , Valid : true } } ) ,
} ,
} ,
args : args {
searchQuery : eventstore . NewSearchQueryBuilder ( eventstore . ColumnsEvent ) .
ResourceOwner ( "caos" ) ,
} ,
lockOption : eventstore . LockOptionNoWait ,
} ,
}
for _ , tt := range tests {
t . Run ( tt . name , func ( t * testing . T ) {
db := & CRDB {
DB : & database . DB {
DB : tt . fields . client ,
Database : new ( testDB ) ,
} ,
}
// setup initial data for query
if _ , err := db . Push ( context . Background ( ) , tt . fields . existingEvents ... ) ; err != nil {
t . Errorf ( "error in setup = %v" , err )
return
}
// first TX should lock and return all events
tx1 , err := db . DB . Begin ( )
require . NoError ( t , err )
defer func ( ) {
require . NoError ( t , tx1 . Rollback ( ) )
} ( )
searchQuery1 := tt . args . searchQuery . LockRowsDuringTx ( tx1 , tt . lockOption )
gotEvents1 := [ ] eventstore . Event { }
err = query ( context . Background ( ) , db , searchQuery1 , eventstore . Reducer ( func ( event eventstore . Event ) error {
gotEvents1 = append ( gotEvents1 , event )
return nil
} ) , true )
require . NoError ( t , err )
assert . Len ( t , gotEvents1 , len ( tt . fields . existingEvents ) )
// second TX should not return the events, and might return an error
tx2 , err := db . DB . Begin ( )
require . NoError ( t , err )
defer func ( ) {
require . NoError ( t , tx2 . Rollback ( ) )
} ( )
searchQuery2 := tt . args . searchQuery . LockRowsDuringTx ( tx1 , tt . lockOption )
gotEvents2 := [ ] eventstore . Event { }
err = query ( context . Background ( ) , db , searchQuery2 , eventstore . Reducer ( func ( event eventstore . Event ) error {
gotEvents2 = append ( gotEvents2 , event )
return nil
} ) , true )
if tt . wantErr {
require . Error ( t , err )
}
require . NoError ( t , err )
assert . Len ( t , gotEvents2 , 0 )
} )
}
}
* /
2020-10-21 17:00:41 +00:00
func Test_query_events_mocked ( t * testing . T ) {
2020-10-02 14:21:51 +00:00
type args struct {
2023-10-19 10:19:10 +00:00
query * eventstore . SearchQueryBuilder
2020-10-21 17:00:41 +00:00
dest interface { }
feat(eventstore): exclude aggregate IDs when event_type occurred (#8940)
# Which Problems Are Solved
For truly event-based notification handler, we need to be able to filter
out events of aggregates which are already handled. For example when an
event like `notify.success` or `notify.failed` was created on an
aggregate, we no longer require events from that aggregate ID.
# How the Problems Are Solved
Extend the query builder to use a `NOT IN` clause which excludes
aggregate IDs when they have certain events for a certain aggregate
type. For optimization and proper index usages, certain filters are
inherited from the parent query, such as:
- Instance ID
- Instance IDs
- Position offset
This is a prettified query as used by the unit tests:
```sql
SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision
FROM eventstore.events2
WHERE instance_id = $1
AND aggregate_type = $2
AND event_type = $3
AND "position" > $4
AND aggregate_id NOT IN (
SELECT aggregate_id
FROM eventstore.events2
WHERE aggregate_type = $5
AND event_type = ANY($6)
AND instance_id = $7
AND "position" > $8
)
ORDER BY "position" DESC, in_tx_order DESC
LIMIT $9
```
I used this query to run it against the `oidc_session` aggregate looking
for added events, excluding aggregates where a token was revoked,
against a recent position. It fully used index scans:
<details>
```json
[
{
"Plan": {
"Node Type": "Index Scan",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2",
"Actual Rows": 2,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.added'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0,
"Filter": "(NOT (hashed SubPlan 1))",
"Rows Removed by Filter": 1,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "SubPlan",
"Subplan Name": "SubPlan 1",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2_1",
"Actual Rows": 1,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.access_token.revoked'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0
}
]
},
"Triggers": [
]
}
]
```
</details>
# Additional Changes
- None
# Additional Context
- Related to https://github.com/zitadel/zitadel/issues/8931
---------
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2024-11-25 15:25:11 +00:00
useV1 bool
2020-10-02 14:21:51 +00:00
}
type res struct {
2020-10-21 17:00:41 +00:00
wantErr bool
}
type fields struct {
mock * dbMock
2020-10-02 14:21:51 +00:00
}
tests := [ ] struct {
2020-10-21 17:00:41 +00:00
name string
args args
fields fields
res res
2020-10-02 14:21:51 +00:00
} {
{
name : "with order by desc" ,
args : args {
2020-10-21 17:00:41 +00:00
dest : & [ ] * repository . Event { } ,
2023-10-19 10:19:10 +00:00
query : eventstore . NewSearchQueryBuilder ( eventstore . ColumnsEvent ) .
OrderDesc ( ) .
AwaitOpenTransactions ( ) .
AddQuery ( ) .
AggregateTypes ( "user" ) .
Builder ( ) ,
feat(eventstore): exclude aggregate IDs when event_type occurred (#8940)
# Which Problems Are Solved
For truly event-based notification handler, we need to be able to filter
out events of aggregates which are already handled. For example when an
event like `notify.success` or `notify.failed` was created on an
aggregate, we no longer require events from that aggregate ID.
# How the Problems Are Solved
Extend the query builder to use a `NOT IN` clause which excludes
aggregate IDs when they have certain events for a certain aggregate
type. For optimization and proper index usages, certain filters are
inherited from the parent query, such as:
- Instance ID
- Instance IDs
- Position offset
This is a prettified query as used by the unit tests:
```sql
SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision
FROM eventstore.events2
WHERE instance_id = $1
AND aggregate_type = $2
AND event_type = $3
AND "position" > $4
AND aggregate_id NOT IN (
SELECT aggregate_id
FROM eventstore.events2
WHERE aggregate_type = $5
AND event_type = ANY($6)
AND instance_id = $7
AND "position" > $8
)
ORDER BY "position" DESC, in_tx_order DESC
LIMIT $9
```
I used this query to run it against the `oidc_session` aggregate looking
for added events, excluding aggregates where a token was revoked,
against a recent position. It fully used index scans:
<details>
```json
[
{
"Plan": {
"Node Type": "Index Scan",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2",
"Actual Rows": 2,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.added'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0,
"Filter": "(NOT (hashed SubPlan 1))",
"Rows Removed by Filter": 1,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "SubPlan",
"Subplan Name": "SubPlan 1",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2_1",
"Actual Rows": 1,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.access_token.revoked'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0
}
]
},
"Triggers": [
]
}
]
```
</details>
# Additional Changes
- None
# Additional Context
- Related to https://github.com/zitadel/zitadel/issues/8931
---------
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2024-11-25 15:25:11 +00:00
useV1 : true ,
2020-10-02 14:21:51 +00:00
} ,
2020-10-21 17:00:41 +00:00
fields : fields {
mock : newMockClient ( t ) . expectQuery ( t ,
2024-11-18 15:30:12 +00:00
` SELECT creation_date, event_type, event_sequence, event_data, editor_user, resource_owner, instance_id, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE aggregate_type = \$1 AND creation_date::TIMESTAMP < \(SELECT COALESCE\(MIN\(start\), NOW\(\)\)::TIMESTAMP FROM crdb_internal\.cluster_transactions where application_name = ANY\(\$2\)\) ORDER BY event_sequence DESC ` ,
[ ] driver . Value { eventstore . AggregateType ( "user" ) , database . TextArray [ string ] { } } ,
2020-10-21 17:00:41 +00:00
) ,
} ,
2020-10-02 14:21:51 +00:00
res : res {
2020-10-21 17:00:41 +00:00
wantErr : false ,
2020-10-02 14:21:51 +00:00
} ,
} ,
{
name : "with limit" ,
args : args {
2020-10-21 17:00:41 +00:00
dest : & [ ] * repository . Event { } ,
2023-10-19 10:19:10 +00:00
query : eventstore . NewSearchQueryBuilder ( eventstore . ColumnsEvent ) .
OrderAsc ( ) .
AwaitOpenTransactions ( ) .
Limit ( 5 ) .
AddQuery ( ) .
AggregateTypes ( "user" ) .
Builder ( ) ,
feat(eventstore): exclude aggregate IDs when event_type occurred (#8940)
# Which Problems Are Solved
For truly event-based notification handler, we need to be able to filter
out events of aggregates which are already handled. For example when an
event like `notify.success` or `notify.failed` was created on an
aggregate, we no longer require events from that aggregate ID.
# How the Problems Are Solved
Extend the query builder to use a `NOT IN` clause which excludes
aggregate IDs when they have certain events for a certain aggregate
type. For optimization and proper index usages, certain filters are
inherited from the parent query, such as:
- Instance ID
- Instance IDs
- Position offset
This is a prettified query as used by the unit tests:
```sql
SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision
FROM eventstore.events2
WHERE instance_id = $1
AND aggregate_type = $2
AND event_type = $3
AND "position" > $4
AND aggregate_id NOT IN (
SELECT aggregate_id
FROM eventstore.events2
WHERE aggregate_type = $5
AND event_type = ANY($6)
AND instance_id = $7
AND "position" > $8
)
ORDER BY "position" DESC, in_tx_order DESC
LIMIT $9
```
I used this query to run it against the `oidc_session` aggregate looking
for added events, excluding aggregates where a token was revoked,
against a recent position. It fully used index scans:
<details>
```json
[
{
"Plan": {
"Node Type": "Index Scan",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2",
"Actual Rows": 2,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.added'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0,
"Filter": "(NOT (hashed SubPlan 1))",
"Rows Removed by Filter": 1,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "SubPlan",
"Subplan Name": "SubPlan 1",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2_1",
"Actual Rows": 1,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.access_token.revoked'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0
}
]
},
"Triggers": [
]
}
]
```
</details>
# Additional Changes
- None
# Additional Context
- Related to https://github.com/zitadel/zitadel/issues/8931
---------
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2024-11-25 15:25:11 +00:00
useV1 : true ,
2020-10-02 14:21:51 +00:00
} ,
2020-10-21 17:00:41 +00:00
fields : fields {
mock : newMockClient ( t ) . expectQuery ( t ,
2024-11-18 15:30:12 +00:00
` SELECT creation_date, event_type, event_sequence, event_data, editor_user, resource_owner, instance_id, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE aggregate_type = \$1 AND creation_date::TIMESTAMP < \(SELECT COALESCE\(MIN\(start\), NOW\(\)\)::TIMESTAMP FROM crdb_internal\.cluster_transactions where application_name = ANY\(\$2\)\) ORDER BY event_sequence LIMIT \$3 ` ,
[ ] driver . Value { eventstore . AggregateType ( "user" ) , database . TextArray [ string ] { } , uint64 ( 5 ) } ,
2020-10-21 17:00:41 +00:00
) ,
} ,
2020-10-02 14:21:51 +00:00
res : res {
2020-10-21 17:00:41 +00:00
wantErr : false ,
2020-10-02 14:21:51 +00:00
} ,
} ,
{
name : "with limit and order by desc" ,
args : args {
2020-10-21 17:00:41 +00:00
dest : & [ ] * repository . Event { } ,
2023-10-19 10:19:10 +00:00
query : eventstore . NewSearchQueryBuilder ( eventstore . ColumnsEvent ) .
OrderDesc ( ) .
AwaitOpenTransactions ( ) .
Limit ( 5 ) .
AddQuery ( ) .
AggregateTypes ( "user" ) .
Builder ( ) ,
feat(eventstore): exclude aggregate IDs when event_type occurred (#8940)
# Which Problems Are Solved
For truly event-based notification handler, we need to be able to filter
out events of aggregates which are already handled. For example when an
event like `notify.success` or `notify.failed` was created on an
aggregate, we no longer require events from that aggregate ID.
# How the Problems Are Solved
Extend the query builder to use a `NOT IN` clause which excludes
aggregate IDs when they have certain events for a certain aggregate
type. For optimization and proper index usages, certain filters are
inherited from the parent query, such as:
- Instance ID
- Instance IDs
- Position offset
This is a prettified query as used by the unit tests:
```sql
SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision
FROM eventstore.events2
WHERE instance_id = $1
AND aggregate_type = $2
AND event_type = $3
AND "position" > $4
AND aggregate_id NOT IN (
SELECT aggregate_id
FROM eventstore.events2
WHERE aggregate_type = $5
AND event_type = ANY($6)
AND instance_id = $7
AND "position" > $8
)
ORDER BY "position" DESC, in_tx_order DESC
LIMIT $9
```
I used this query to run it against the `oidc_session` aggregate looking
for added events, excluding aggregates where a token was revoked,
against a recent position. It fully used index scans:
<details>
```json
[
{
"Plan": {
"Node Type": "Index Scan",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2",
"Actual Rows": 2,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.added'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0,
"Filter": "(NOT (hashed SubPlan 1))",
"Rows Removed by Filter": 1,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "SubPlan",
"Subplan Name": "SubPlan 1",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2_1",
"Actual Rows": 1,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.access_token.revoked'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0
}
]
},
"Triggers": [
]
}
]
```
</details>
# Additional Changes
- None
# Additional Context
- Related to https://github.com/zitadel/zitadel/issues/8931
---------
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2024-11-25 15:25:11 +00:00
useV1 : true ,
2020-10-02 14:21:51 +00:00
} ,
2020-10-21 17:00:41 +00:00
fields : fields {
mock : newMockClient ( t ) . expectQuery ( t ,
2024-11-18 15:30:12 +00:00
` SELECT creation_date, event_type, event_sequence, event_data, editor_user, resource_owner, instance_id, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE aggregate_type = \$1 AND creation_date::TIMESTAMP < \(SELECT COALESCE\(MIN\(start\), NOW\(\)\)::TIMESTAMP FROM crdb_internal\.cluster_transactions where application_name = ANY\(\$2\)\) ORDER BY event_sequence DESC LIMIT \$3 ` ,
[ ] driver . Value { eventstore . AggregateType ( "user" ) , database . TextArray [ string ] { } , uint64 ( 5 ) } ,
2020-10-21 17:00:41 +00:00
) ,
} ,
res : res {
wantErr : false ,
} ,
} ,
2023-02-27 21:36:43 +00:00
{
name : "with limit and order by desc as of system time" ,
args : args {
dest : & [ ] * repository . Event { } ,
2023-10-19 10:19:10 +00:00
query : eventstore . NewSearchQueryBuilder ( eventstore . ColumnsEvent ) .
OrderDesc ( ) .
AwaitOpenTransactions ( ) .
Limit ( 5 ) .
AllowTimeTravel ( ) .
AddQuery ( ) .
AggregateTypes ( "user" ) .
Builder ( ) ,
feat(eventstore): exclude aggregate IDs when event_type occurred (#8940)
# Which Problems Are Solved
For truly event-based notification handler, we need to be able to filter
out events of aggregates which are already handled. For example when an
event like `notify.success` or `notify.failed` was created on an
aggregate, we no longer require events from that aggregate ID.
# How the Problems Are Solved
Extend the query builder to use a `NOT IN` clause which excludes
aggregate IDs when they have certain events for a certain aggregate
type. For optimization and proper index usages, certain filters are
inherited from the parent query, such as:
- Instance ID
- Instance IDs
- Position offset
This is a prettified query as used by the unit tests:
```sql
SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision
FROM eventstore.events2
WHERE instance_id = $1
AND aggregate_type = $2
AND event_type = $3
AND "position" > $4
AND aggregate_id NOT IN (
SELECT aggregate_id
FROM eventstore.events2
WHERE aggregate_type = $5
AND event_type = ANY($6)
AND instance_id = $7
AND "position" > $8
)
ORDER BY "position" DESC, in_tx_order DESC
LIMIT $9
```
I used this query to run it against the `oidc_session` aggregate looking
for added events, excluding aggregates where a token was revoked,
against a recent position. It fully used index scans:
<details>
```json
[
{
"Plan": {
"Node Type": "Index Scan",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2",
"Actual Rows": 2,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.added'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0,
"Filter": "(NOT (hashed SubPlan 1))",
"Rows Removed by Filter": 1,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "SubPlan",
"Subplan Name": "SubPlan 1",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2_1",
"Actual Rows": 1,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.access_token.revoked'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0
}
]
},
"Triggers": [
]
}
]
```
</details>
# Additional Changes
- None
# Additional Context
- Related to https://github.com/zitadel/zitadel/issues/8931
---------
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2024-11-25 15:25:11 +00:00
useV1 : true ,
2023-02-27 21:36:43 +00:00
} ,
fields : fields {
mock : newMockClient ( t ) . expectQuery ( t ,
2024-11-18 15:30:12 +00:00
` SELECT creation_date, event_type, event_sequence, event_data, editor_user, resource_owner, instance_id, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events AS OF SYSTEM TIME '-1 ms' WHERE aggregate_type = \$1 AND creation_date::TIMESTAMP < \(SELECT COALESCE\(MIN\(start\), NOW\(\)\)::TIMESTAMP FROM crdb_internal\.cluster_transactions where application_name = ANY\(\$2\)\) ORDER BY event_sequence DESC LIMIT \$3 ` ,
[ ] driver . Value { eventstore . AggregateType ( "user" ) , database . TextArray [ string ] { } , uint64 ( 5 ) } ,
2023-02-27 21:36:43 +00:00
) ,
} ,
res : res {
wantErr : false ,
} ,
} ,
2024-11-21 14:46:30 +00:00
{
name : "lock, wait" ,
args : args {
dest : & [ ] * repository . Event { } ,
query : eventstore . NewSearchQueryBuilder ( eventstore . ColumnsEvent ) .
OrderDesc ( ) .
Limit ( 5 ) .
AddQuery ( ) .
AggregateTypes ( "user" ) .
Builder ( ) . LockRowsDuringTx ( nil , eventstore . LockOptionWait ) ,
feat(eventstore): exclude aggregate IDs when event_type occurred (#8940)
# Which Problems Are Solved
For truly event-based notification handler, we need to be able to filter
out events of aggregates which are already handled. For example when an
event like `notify.success` or `notify.failed` was created on an
aggregate, we no longer require events from that aggregate ID.
# How the Problems Are Solved
Extend the query builder to use a `NOT IN` clause which excludes
aggregate IDs when they have certain events for a certain aggregate
type. For optimization and proper index usages, certain filters are
inherited from the parent query, such as:
- Instance ID
- Instance IDs
- Position offset
This is a prettified query as used by the unit tests:
```sql
SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision
FROM eventstore.events2
WHERE instance_id = $1
AND aggregate_type = $2
AND event_type = $3
AND "position" > $4
AND aggregate_id NOT IN (
SELECT aggregate_id
FROM eventstore.events2
WHERE aggregate_type = $5
AND event_type = ANY($6)
AND instance_id = $7
AND "position" > $8
)
ORDER BY "position" DESC, in_tx_order DESC
LIMIT $9
```
I used this query to run it against the `oidc_session` aggregate looking
for added events, excluding aggregates where a token was revoked,
against a recent position. It fully used index scans:
<details>
```json
[
{
"Plan": {
"Node Type": "Index Scan",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2",
"Actual Rows": 2,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.added'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0,
"Filter": "(NOT (hashed SubPlan 1))",
"Rows Removed by Filter": 1,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "SubPlan",
"Subplan Name": "SubPlan 1",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2_1",
"Actual Rows": 1,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.access_token.revoked'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0
}
]
},
"Triggers": [
]
}
]
```
</details>
# Additional Changes
- None
# Additional Context
- Related to https://github.com/zitadel/zitadel/issues/8931
---------
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2024-11-25 15:25:11 +00:00
useV1 : true ,
2024-11-21 14:46:30 +00:00
} ,
fields : fields {
mock : newMockClient ( t ) . expectQuery ( t ,
` SELECT creation_date, event_type, event_sequence, event_data, editor_user, resource_owner, instance_id, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE aggregate_type = \$1 ORDER BY event_sequence DESC LIMIT \$2 FOR UPDATE ` ,
[ ] driver . Value { eventstore . AggregateType ( "user" ) , uint64 ( 5 ) } ,
) ,
} ,
res : res {
wantErr : false ,
} ,
} ,
{
name : "lock, no wait" ,
args : args {
dest : & [ ] * repository . Event { } ,
query : eventstore . NewSearchQueryBuilder ( eventstore . ColumnsEvent ) .
OrderDesc ( ) .
Limit ( 5 ) .
AddQuery ( ) .
AggregateTypes ( "user" ) .
Builder ( ) . LockRowsDuringTx ( nil , eventstore . LockOptionNoWait ) ,
feat(eventstore): exclude aggregate IDs when event_type occurred (#8940)
# Which Problems Are Solved
For truly event-based notification handler, we need to be able to filter
out events of aggregates which are already handled. For example when an
event like `notify.success` or `notify.failed` was created on an
aggregate, we no longer require events from that aggregate ID.
# How the Problems Are Solved
Extend the query builder to use a `NOT IN` clause which excludes
aggregate IDs when they have certain events for a certain aggregate
type. For optimization and proper index usages, certain filters are
inherited from the parent query, such as:
- Instance ID
- Instance IDs
- Position offset
This is a prettified query as used by the unit tests:
```sql
SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision
FROM eventstore.events2
WHERE instance_id = $1
AND aggregate_type = $2
AND event_type = $3
AND "position" > $4
AND aggregate_id NOT IN (
SELECT aggregate_id
FROM eventstore.events2
WHERE aggregate_type = $5
AND event_type = ANY($6)
AND instance_id = $7
AND "position" > $8
)
ORDER BY "position" DESC, in_tx_order DESC
LIMIT $9
```
I used this query to run it against the `oidc_session` aggregate looking
for added events, excluding aggregates where a token was revoked,
against a recent position. It fully used index scans:
<details>
```json
[
{
"Plan": {
"Node Type": "Index Scan",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2",
"Actual Rows": 2,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.added'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0,
"Filter": "(NOT (hashed SubPlan 1))",
"Rows Removed by Filter": 1,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "SubPlan",
"Subplan Name": "SubPlan 1",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2_1",
"Actual Rows": 1,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.access_token.revoked'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0
}
]
},
"Triggers": [
]
}
]
```
</details>
# Additional Changes
- None
# Additional Context
- Related to https://github.com/zitadel/zitadel/issues/8931
---------
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2024-11-25 15:25:11 +00:00
useV1 : true ,
2024-11-21 14:46:30 +00:00
} ,
fields : fields {
mock : newMockClient ( t ) . expectQuery ( t ,
` SELECT creation_date, event_type, event_sequence, event_data, editor_user, resource_owner, instance_id, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE aggregate_type = \$1 ORDER BY event_sequence DESC LIMIT \$2 FOR UPDATE NOWAIT ` ,
[ ] driver . Value { eventstore . AggregateType ( "user" ) , uint64 ( 5 ) } ,
) ,
} ,
res : res {
wantErr : false ,
} ,
} ,
{
name : "lock, skip locked" ,
args : args {
dest : & [ ] * repository . Event { } ,
query : eventstore . NewSearchQueryBuilder ( eventstore . ColumnsEvent ) .
OrderDesc ( ) .
Limit ( 5 ) .
AddQuery ( ) .
AggregateTypes ( "user" ) .
Builder ( ) . LockRowsDuringTx ( nil , eventstore . LockOptionSkipLocked ) ,
feat(eventstore): exclude aggregate IDs when event_type occurred (#8940)
# Which Problems Are Solved
For truly event-based notification handler, we need to be able to filter
out events of aggregates which are already handled. For example when an
event like `notify.success` or `notify.failed` was created on an
aggregate, we no longer require events from that aggregate ID.
# How the Problems Are Solved
Extend the query builder to use a `NOT IN` clause which excludes
aggregate IDs when they have certain events for a certain aggregate
type. For optimization and proper index usages, certain filters are
inherited from the parent query, such as:
- Instance ID
- Instance IDs
- Position offset
This is a prettified query as used by the unit tests:
```sql
SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision
FROM eventstore.events2
WHERE instance_id = $1
AND aggregate_type = $2
AND event_type = $3
AND "position" > $4
AND aggregate_id NOT IN (
SELECT aggregate_id
FROM eventstore.events2
WHERE aggregate_type = $5
AND event_type = ANY($6)
AND instance_id = $7
AND "position" > $8
)
ORDER BY "position" DESC, in_tx_order DESC
LIMIT $9
```
I used this query to run it against the `oidc_session` aggregate looking
for added events, excluding aggregates where a token was revoked,
against a recent position. It fully used index scans:
<details>
```json
[
{
"Plan": {
"Node Type": "Index Scan",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2",
"Actual Rows": 2,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.added'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0,
"Filter": "(NOT (hashed SubPlan 1))",
"Rows Removed by Filter": 1,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "SubPlan",
"Subplan Name": "SubPlan 1",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2_1",
"Actual Rows": 1,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.access_token.revoked'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0
}
]
},
"Triggers": [
]
}
]
```
</details>
# Additional Changes
- None
# Additional Context
- Related to https://github.com/zitadel/zitadel/issues/8931
---------
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2024-11-25 15:25:11 +00:00
useV1 : true ,
2024-11-21 14:46:30 +00:00
} ,
fields : fields {
mock : newMockClient ( t ) . expectQuery ( t ,
` SELECT creation_date, event_type, event_sequence, event_data, editor_user, resource_owner, instance_id, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE aggregate_type = \$1 ORDER BY event_sequence DESC LIMIT \$2 FOR UPDATE SKIP LOCKED ` ,
[ ] driver . Value { eventstore . AggregateType ( "user" ) , uint64 ( 5 ) } ,
) ,
} ,
res : res {
wantErr : false ,
} ,
} ,
2020-10-21 17:00:41 +00:00
{
name : "error sql conn closed" ,
args : args {
dest : & [ ] * repository . Event { } ,
2023-10-19 10:19:10 +00:00
query : eventstore . NewSearchQueryBuilder ( eventstore . ColumnsEvent ) .
OrderDesc ( ) .
AwaitOpenTransactions ( ) .
Limit ( 0 ) .
AddQuery ( ) .
AggregateTypes ( "user" ) .
Builder ( ) ,
feat(eventstore): exclude aggregate IDs when event_type occurred (#8940)
# Which Problems Are Solved
For truly event-based notification handler, we need to be able to filter
out events of aggregates which are already handled. For example when an
event like `notify.success` or `notify.failed` was created on an
aggregate, we no longer require events from that aggregate ID.
# How the Problems Are Solved
Extend the query builder to use a `NOT IN` clause which excludes
aggregate IDs when they have certain events for a certain aggregate
type. For optimization and proper index usages, certain filters are
inherited from the parent query, such as:
- Instance ID
- Instance IDs
- Position offset
This is a prettified query as used by the unit tests:
```sql
SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision
FROM eventstore.events2
WHERE instance_id = $1
AND aggregate_type = $2
AND event_type = $3
AND "position" > $4
AND aggregate_id NOT IN (
SELECT aggregate_id
FROM eventstore.events2
WHERE aggregate_type = $5
AND event_type = ANY($6)
AND instance_id = $7
AND "position" > $8
)
ORDER BY "position" DESC, in_tx_order DESC
LIMIT $9
```
I used this query to run it against the `oidc_session` aggregate looking
for added events, excluding aggregates where a token was revoked,
against a recent position. It fully used index scans:
<details>
```json
[
{
"Plan": {
"Node Type": "Index Scan",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2",
"Actual Rows": 2,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.added'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0,
"Filter": "(NOT (hashed SubPlan 1))",
"Rows Removed by Filter": 1,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "SubPlan",
"Subplan Name": "SubPlan 1",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2_1",
"Actual Rows": 1,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.access_token.revoked'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0
}
]
},
"Triggers": [
]
}
]
```
</details>
# Additional Changes
- None
# Additional Context
- Related to https://github.com/zitadel/zitadel/issues/8931
---------
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2024-11-25 15:25:11 +00:00
useV1 : true ,
2020-10-21 17:00:41 +00:00
} ,
fields : fields {
mock : newMockClient ( t ) . expectQueryErr ( t ,
2024-11-18 15:30:12 +00:00
` SELECT creation_date, event_type, event_sequence, event_data, editor_user, resource_owner, instance_id, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE aggregate_type = \$1 AND creation_date::TIMESTAMP < \(SELECT COALESCE\(MIN\(start\), NOW\(\)\)::TIMESTAMP FROM crdb_internal\.cluster_transactions where application_name = ANY\(\$2\)\) ORDER BY event_sequence DESC ` ,
[ ] driver . Value { eventstore . AggregateType ( "user" ) , database . TextArray [ string ] { } } ,
2020-10-21 17:00:41 +00:00
sql . ErrConnDone ) ,
} ,
res : res {
wantErr : true ,
} ,
} ,
{
name : "error unexpected dest" ,
args : args {
dest : nil ,
2023-10-19 10:19:10 +00:00
query : eventstore . NewSearchQueryBuilder ( eventstore . ColumnsEvent ) .
OrderDesc ( ) .
AwaitOpenTransactions ( ) .
Limit ( 0 ) .
AddQuery ( ) .
AggregateTypes ( "user" ) .
Builder ( ) ,
feat(eventstore): exclude aggregate IDs when event_type occurred (#8940)
# Which Problems Are Solved
For truly event-based notification handler, we need to be able to filter
out events of aggregates which are already handled. For example when an
event like `notify.success` or `notify.failed` was created on an
aggregate, we no longer require events from that aggregate ID.
# How the Problems Are Solved
Extend the query builder to use a `NOT IN` clause which excludes
aggregate IDs when they have certain events for a certain aggregate
type. For optimization and proper index usages, certain filters are
inherited from the parent query, such as:
- Instance ID
- Instance IDs
- Position offset
This is a prettified query as used by the unit tests:
```sql
SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision
FROM eventstore.events2
WHERE instance_id = $1
AND aggregate_type = $2
AND event_type = $3
AND "position" > $4
AND aggregate_id NOT IN (
SELECT aggregate_id
FROM eventstore.events2
WHERE aggregate_type = $5
AND event_type = ANY($6)
AND instance_id = $7
AND "position" > $8
)
ORDER BY "position" DESC, in_tx_order DESC
LIMIT $9
```
I used this query to run it against the `oidc_session` aggregate looking
for added events, excluding aggregates where a token was revoked,
against a recent position. It fully used index scans:
<details>
```json
[
{
"Plan": {
"Node Type": "Index Scan",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2",
"Actual Rows": 2,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.added'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0,
"Filter": "(NOT (hashed SubPlan 1))",
"Rows Removed by Filter": 1,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "SubPlan",
"Subplan Name": "SubPlan 1",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2_1",
"Actual Rows": 1,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.access_token.revoked'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0
}
]
},
"Triggers": [
]
}
]
```
</details>
# Additional Changes
- None
# Additional Context
- Related to https://github.com/zitadel/zitadel/issues/8931
---------
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2024-11-25 15:25:11 +00:00
useV1 : true ,
2020-10-21 17:00:41 +00:00
} ,
fields : fields {
2023-08-22 12:49:02 +00:00
mock : newMockClient ( t ) . expectQueryScanErr ( t ,
2024-11-18 15:30:12 +00:00
` SELECT creation_date, event_type, event_sequence, event_data, editor_user, resource_owner, instance_id, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE aggregate_type = \$1 AND creation_date::TIMESTAMP < \(SELECT COALESCE\(MIN\(start\), NOW\(\)\)::TIMESTAMP FROM crdb_internal\.cluster_transactions where application_name = ANY\(\$2\)\) ORDER BY event_sequence DESC ` ,
[ ] driver . Value { eventstore . AggregateType ( "user" ) , database . TextArray [ string ] { } } ,
2023-10-19 10:19:10 +00:00
& repository . Event { Seq : 100 } ) ,
2020-10-21 17:00:41 +00:00
} ,
2020-10-02 14:21:51 +00:00
res : res {
2020-10-21 17:00:41 +00:00
wantErr : true ,
2020-10-02 14:21:51 +00:00
} ,
} ,
2020-10-05 20:03:21 +00:00
{
name : "error no columns" ,
args : args {
2023-10-19 10:19:10 +00:00
query : eventstore . NewSearchQueryBuilder ( eventstore . Columns ( - 1 ) ) ,
2020-10-05 20:03:21 +00:00
} ,
res : res {
2020-10-21 17:00:41 +00:00
wantErr : true ,
2020-10-05 20:03:21 +00:00
} ,
} ,
2021-07-06 11:55:57 +00:00
{
name : "with subqueries" ,
args : args {
dest : & [ ] * repository . Event { } ,
2023-10-19 10:19:10 +00:00
query : eventstore . NewSearchQueryBuilder ( eventstore . ColumnsEvent ) .
OrderDesc ( ) .
AwaitOpenTransactions ( ) .
Limit ( 5 ) .
AddQuery ( ) .
AggregateTypes ( "user" ) .
Or ( ) .
AggregateTypes ( "org" ) .
AggregateIDs ( "asdf42" ) .
Builder ( ) ,
feat(eventstore): exclude aggregate IDs when event_type occurred (#8940)
# Which Problems Are Solved
For truly event-based notification handler, we need to be able to filter
out events of aggregates which are already handled. For example when an
event like `notify.success` or `notify.failed` was created on an
aggregate, we no longer require events from that aggregate ID.
# How the Problems Are Solved
Extend the query builder to use a `NOT IN` clause which excludes
aggregate IDs when they have certain events for a certain aggregate
type. For optimization and proper index usages, certain filters are
inherited from the parent query, such as:
- Instance ID
- Instance IDs
- Position offset
This is a prettified query as used by the unit tests:
```sql
SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision
FROM eventstore.events2
WHERE instance_id = $1
AND aggregate_type = $2
AND event_type = $3
AND "position" > $4
AND aggregate_id NOT IN (
SELECT aggregate_id
FROM eventstore.events2
WHERE aggregate_type = $5
AND event_type = ANY($6)
AND instance_id = $7
AND "position" > $8
)
ORDER BY "position" DESC, in_tx_order DESC
LIMIT $9
```
I used this query to run it against the `oidc_session` aggregate looking
for added events, excluding aggregates where a token was revoked,
against a recent position. It fully used index scans:
<details>
```json
[
{
"Plan": {
"Node Type": "Index Scan",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2",
"Actual Rows": 2,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.added'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0,
"Filter": "(NOT (hashed SubPlan 1))",
"Rows Removed by Filter": 1,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "SubPlan",
"Subplan Name": "SubPlan 1",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2_1",
"Actual Rows": 1,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.access_token.revoked'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0
}
]
},
"Triggers": [
]
}
]
```
</details>
# Additional Changes
- None
# Additional Context
- Related to https://github.com/zitadel/zitadel/issues/8931
---------
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2024-11-25 15:25:11 +00:00
useV1 : true ,
2021-07-06 11:55:57 +00:00
} ,
fields : fields {
mock : newMockClient ( t ) . expectQuery ( t ,
2024-11-18 15:30:12 +00:00
` SELECT creation_date, event_type, event_sequence, event_data, editor_user, resource_owner, instance_id, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE \(aggregate_type = \$1 OR \(aggregate_type = \$2 AND aggregate_id = \$3\)\) AND creation_date::TIMESTAMP < \(SELECT COALESCE\(MIN\(start\), NOW\(\)\)::TIMESTAMP FROM crdb_internal\.cluster_transactions where application_name = ANY\(\$4\)\) ORDER BY event_sequence DESC LIMIT \$5 ` ,
[ ] driver . Value { eventstore . AggregateType ( "user" ) , eventstore . AggregateType ( "org" ) , "asdf42" , database . TextArray [ string ] { } , uint64 ( 5 ) } ,
2021-07-06 11:55:57 +00:00
) ,
} ,
res : res {
wantErr : false ,
} ,
} ,
feat(eventstore): exclude aggregate IDs when event_type occurred (#8940)
# Which Problems Are Solved
For truly event-based notification handler, we need to be able to filter
out events of aggregates which are already handled. For example when an
event like `notify.success` or `notify.failed` was created on an
aggregate, we no longer require events from that aggregate ID.
# How the Problems Are Solved
Extend the query builder to use a `NOT IN` clause which excludes
aggregate IDs when they have certain events for a certain aggregate
type. For optimization and proper index usages, certain filters are
inherited from the parent query, such as:
- Instance ID
- Instance IDs
- Position offset
This is a prettified query as used by the unit tests:
```sql
SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision
FROM eventstore.events2
WHERE instance_id = $1
AND aggregate_type = $2
AND event_type = $3
AND "position" > $4
AND aggregate_id NOT IN (
SELECT aggregate_id
FROM eventstore.events2
WHERE aggregate_type = $5
AND event_type = ANY($6)
AND instance_id = $7
AND "position" > $8
)
ORDER BY "position" DESC, in_tx_order DESC
LIMIT $9
```
I used this query to run it against the `oidc_session` aggregate looking
for added events, excluding aggregates where a token was revoked,
against a recent position. It fully used index scans:
<details>
```json
[
{
"Plan": {
"Node Type": "Index Scan",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2",
"Actual Rows": 2,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.added'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0,
"Filter": "(NOT (hashed SubPlan 1))",
"Rows Removed by Filter": 1,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "SubPlan",
"Subplan Name": "SubPlan 1",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2_1",
"Actual Rows": 1,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.access_token.revoked'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0
}
]
},
"Triggers": [
]
}
]
```
</details>
# Additional Changes
- None
# Additional Context
- Related to https://github.com/zitadel/zitadel/issues/8931
---------
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2024-11-25 15:25:11 +00:00
{
name : "aggregate / event type, position and exclusion, v1" ,
args : args {
dest : & [ ] * repository . Event { } ,
query : eventstore . NewSearchQueryBuilder ( eventstore . ColumnsEvent ) .
InstanceID ( "instanceID" ) .
OrderDesc ( ) .
Limit ( 5 ) .
PositionAfter ( 123.456 ) .
AddQuery ( ) .
AggregateTypes ( "notify" ) .
EventTypes ( "notify.foo.bar" ) .
Builder ( ) .
ExcludeAggregateIDs ( ) .
AggregateTypes ( "notify" ) .
EventTypes ( "notification.failed" , "notification.success" ) .
Builder ( ) ,
useV1 : true ,
} ,
fields : fields {
mock : newMockClient ( t ) . expectQuery ( t ,
regexp . QuoteMeta (
` SELECT creation_date, event_type, event_sequence, event_data, editor_user, resource_owner, instance_id, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE instance_id = $1 AND aggregate_type = $2 AND event_type = $3 AND "position" > $4 AND aggregate_id NOT IN (SELECT aggregate_id FROM eventstore.events WHERE aggregate_type = $5 AND event_type = ANY($6) AND instance_id = $7 AND "position" > $8) ORDER BY event_sequence DESC LIMIT $9 ` ,
) ,
[ ] driver . Value { "instanceID" , eventstore . AggregateType ( "notify" ) , eventstore . EventType ( "notify.foo.bar" ) , 123.456 , eventstore . AggregateType ( "notify" ) , [ ] eventstore . EventType { "notification.failed" , "notification.success" } , "instanceID" , 123.456 , uint64 ( 5 ) } ,
) ,
} ,
res : res {
wantErr : false ,
} ,
} ,
{
name : "aggregate / event type, position and exclusion, v2" ,
args : args {
dest : & [ ] * repository . Event { } ,
query : eventstore . NewSearchQueryBuilder ( eventstore . ColumnsEvent ) .
InstanceID ( "instanceID" ) .
OrderDesc ( ) .
Limit ( 5 ) .
PositionAfter ( 123.456 ) .
AddQuery ( ) .
AggregateTypes ( "notify" ) .
EventTypes ( "notify.foo.bar" ) .
Builder ( ) .
ExcludeAggregateIDs ( ) .
AggregateTypes ( "notify" ) .
EventTypes ( "notification.failed" , "notification.success" ) .
Builder ( ) ,
useV1 : false ,
} ,
fields : fields {
mock : newMockClient ( t ) . expectQuery ( t ,
regexp . QuoteMeta (
` SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision FROM eventstore.events2 WHERE instance_id = $1 AND aggregate_type = $2 AND event_type = $3 AND "position" > $4 AND aggregate_id NOT IN (SELECT aggregate_id FROM eventstore.events2 WHERE aggregate_type = $5 AND event_type = ANY($6) AND instance_id = $7 AND "position" > $8) ORDER BY "position" DESC, in_tx_order DESC LIMIT $9 ` ,
) ,
[ ] driver . Value { "instanceID" , eventstore . AggregateType ( "notify" ) , eventstore . EventType ( "notify.foo.bar" ) , 123.456 , eventstore . AggregateType ( "notify" ) , [ ] eventstore . EventType { "notification.failed" , "notification.success" } , "instanceID" , 123.456 , uint64 ( 5 ) } ,
) ,
} ,
res : res {
wantErr : false ,
} ,
} ,
2024-12-06 10:20:10 +00:00
{
name : "aggregate / event type, created after and exclusion, v2" ,
args : args {
dest : & [ ] * repository . Event { } ,
query : eventstore . NewSearchQueryBuilder ( eventstore . ColumnsEvent ) .
InstanceID ( "instanceID" ) .
OrderDesc ( ) .
Limit ( 5 ) .
CreationDateAfter ( time . Unix ( 123 , 456 ) ) .
AddQuery ( ) .
AggregateTypes ( "notify" ) .
EventTypes ( "notify.foo.bar" ) .
Builder ( ) .
ExcludeAggregateIDs ( ) .
AggregateTypes ( "notify" ) .
EventTypes ( "notification.failed" , "notification.success" ) .
Builder ( ) ,
useV1 : false ,
} ,
fields : fields {
mock : newMockClient ( t ) . expectQuery ( t ,
regexp . QuoteMeta (
` SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision FROM eventstore.events2 WHERE instance_id = $1 AND aggregate_type = $2 AND event_type = $3 AND created_at > $4 AND aggregate_id NOT IN (SELECT aggregate_id FROM eventstore.events2 WHERE aggregate_type = $5 AND event_type = ANY($6) AND instance_id = $7 AND created_at > $8) ORDER BY "position" DESC, in_tx_order DESC LIMIT $9 ` ,
) ,
[ ] driver . Value { "instanceID" , eventstore . AggregateType ( "notify" ) , eventstore . EventType ( "notify.foo.bar" ) , time . Unix ( 123 , 456 ) , eventstore . AggregateType ( "notify" ) , [ ] eventstore . EventType { "notification.failed" , "notification.success" } , "instanceID" , time . Unix ( 123 , 456 ) , uint64 ( 5 ) } ,
) ,
} ,
res : res {
wantErr : false ,
} ,
} ,
2020-10-02 14:21:51 +00:00
}
2023-10-19 10:19:10 +00:00
crdb := NewCRDB ( & database . DB { Database : new ( testDB ) } )
2020-10-02 14:21:51 +00:00
for _ , tt := range tests {
t . Run ( tt . name , func ( t * testing . T ) {
2020-10-21 17:00:41 +00:00
if tt . fields . mock != nil {
2023-02-27 21:36:43 +00:00
crdb . DB . DB = tt . fields . mock . client
2020-10-02 14:21:51 +00:00
}
2020-10-21 17:00:41 +00:00
feat(eventstore): exclude aggregate IDs when event_type occurred (#8940)
# Which Problems Are Solved
For truly event-based notification handler, we need to be able to filter
out events of aggregates which are already handled. For example when an
event like `notify.success` or `notify.failed` was created on an
aggregate, we no longer require events from that aggregate ID.
# How the Problems Are Solved
Extend the query builder to use a `NOT IN` clause which excludes
aggregate IDs when they have certain events for a certain aggregate
type. For optimization and proper index usages, certain filters are
inherited from the parent query, such as:
- Instance ID
- Instance IDs
- Position offset
This is a prettified query as used by the unit tests:
```sql
SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision
FROM eventstore.events2
WHERE instance_id = $1
AND aggregate_type = $2
AND event_type = $3
AND "position" > $4
AND aggregate_id NOT IN (
SELECT aggregate_id
FROM eventstore.events2
WHERE aggregate_type = $5
AND event_type = ANY($6)
AND instance_id = $7
AND "position" > $8
)
ORDER BY "position" DESC, in_tx_order DESC
LIMIT $9
```
I used this query to run it against the `oidc_session` aggregate looking
for added events, excluding aggregates where a token was revoked,
against a recent position. It fully used index scans:
<details>
```json
[
{
"Plan": {
"Node Type": "Index Scan",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2",
"Actual Rows": 2,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.added'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0,
"Filter": "(NOT (hashed SubPlan 1))",
"Rows Removed by Filter": 1,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "SubPlan",
"Subplan Name": "SubPlan 1",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "es_projection",
"Relation Name": "events2",
"Alias": "events2_1",
"Actual Rows": 1,
"Actual Loops": 1,
"Index Cond": "((instance_id = '286399006995644420'::text) AND (aggregate_type = 'oidc_session'::text) AND (event_type = 'oidc_session.access_token.revoked'::text) AND (\"position\" > 1731582100.784168))",
"Rows Removed by Index Recheck": 0
}
]
},
"Triggers": [
]
}
]
```
</details>
# Additional Changes
- None
# Additional Context
- Related to https://github.com/zitadel/zitadel/issues/8931
---------
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2024-11-25 15:25:11 +00:00
err := query ( context . Background ( ) , crdb , tt . args . query , tt . args . dest , tt . args . useV1 )
2020-10-21 17:00:41 +00:00
if ( err != nil ) != tt . res . wantErr {
t . Errorf ( "query() error = %v, wantErr %v" , err , tt . res . wantErr )
2020-10-02 14:21:51 +00:00
}
2020-10-21 17:00:41 +00:00
if tt . fields . mock == nil {
return
2020-10-02 14:21:51 +00:00
}
2020-10-21 17:00:41 +00:00
if err := tt . fields . mock . mock . ExpectationsWereMet ( ) ; err != nil {
t . Errorf ( "not all expectaions met: %v" , err )
2020-10-02 14:21:51 +00:00
}
} )
}
}
2020-10-21 17:00:41 +00:00
type dbMock struct {
mock sqlmock . Sqlmock
client * sql . DB
}
func ( m * dbMock ) expectQuery ( t * testing . T , expectedQuery string , args [ ] driver . Value , events ... * repository . Event ) * dbMock {
2023-08-22 12:49:02 +00:00
query := m . mock . ExpectQuery ( expectedQuery ) . WithArgs ( args ... )
2024-03-27 13:48:22 +00:00
rows := m . mock . NewRows ( [ ] string { "sequence" } )
2023-08-22 12:49:02 +00:00
for _ , event := range events {
2023-10-19 10:19:10 +00:00
rows = rows . AddRow ( event . Seq )
2023-08-22 12:49:02 +00:00
}
query . WillReturnRows ( rows ) . RowsWillBeClosed ( )
return m
}
func ( m * dbMock ) expectQueryScanErr ( t * testing . T , expectedQuery string , args [ ] driver . Value , events ... * repository . Event ) * dbMock {
2020-10-21 17:00:41 +00:00
query := m . mock . ExpectQuery ( expectedQuery ) . WithArgs ( args ... )
2024-03-27 13:48:22 +00:00
rows := m . mock . NewRows ( [ ] string { "sequence" } )
2020-10-21 17:00:41 +00:00
for _ , event := range events {
2023-10-19 10:19:10 +00:00
rows = rows . AddRow ( event . Seq )
2020-10-21 17:00:41 +00:00
}
query . WillReturnRows ( rows ) . RowsWillBeClosed ( )
return m
}
func ( m * dbMock ) expectQueryErr ( t * testing . T , expectedQuery string , args [ ] driver . Value , err error ) * dbMock {
m . mock . ExpectQuery ( expectedQuery ) . WithArgs ( args ... ) . WillReturnError ( err )
return m
}
func newMockClient ( t * testing . T ) * dbMock {
t . Helper ( )
2024-03-27 13:48:22 +00:00
db , mock , err := sqlmock . New ( sqlmock . ValueConverterOption ( new ( db_mock . TypeConverter ) ) )
2020-10-21 17:00:41 +00:00
if err != nil {
t . Errorf ( "unable to create mock client: %v" , err )
t . FailNow ( )
return nil
}
return & dbMock {
mock : mock ,
client : db ,
}
}