TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-11 21:37:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(eventstore): prevent allocation of filtered events (#6749)

Browse Source 
* fix(eventstore): prevent allocation of filtered events

Directly reduce each event obtained from a sql.Rows scan,
so that we do not have to allocate all events in a slice.

* reinstate the mutex as RWMutex

* scan data directly

* add todos

* fix(writemodels): add reduce of parent

* test: remove comment

* update comments

---------

Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
This commit is contained in:
Tim Möhlmann
2023-10-19 18:21:31 +03:00
committed by GitHub
parent 459761d99a
commit ab79855cf0
16 changed files with 150 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
internal/eventstore/repository/mock/repository.mock.go
View File

@@ -35,19 +35,18 @@ func (m *MockQuerier) EXPECT() *MockQuerierMockRecorder {
return m.recorder
}
// Filter mocks base method.
func (m *MockQuerier) Filter(arg0 context.Context, arg1 *eventstore.SearchQueryBuilder) ([]eventstore.Event, error) {
// FilterToReducer mocks base method.
func (m *MockQuerier) FilterToReducer(arg0 context.Context, arg1 *eventstore.SearchQueryBuilder, arg2 eventstore.Reducer) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Filter", arg0, arg1)
ret0, _ := ret[0].([]eventstore.Event)
ret1, _ := ret[1].(error)
return ret0, ret1
ret := m.ctrl.Call(m, "FilterToReducer", arg0, arg1, arg2)
ret0, _ := ret[0].(error)
return ret0
}
// Filter indicates an expected call of Filter.
func (mr *MockQuerierMockRecorder) Filter(arg0, arg1 interface{}) *gomock.Call {
// FilterToReducer indicates an expected call of FilterToReducer.
func (mr *MockQuerierMockRecorder) FilterToReducer(arg0, arg1, arg2 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Filter", reflect.TypeOf((*MockQuerier)(nil).Filter), arg0, arg1)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FilterToReducer", reflect.TypeOf((*MockQuerier)(nil).FilterToReducer), arg0, arg1, arg2)
}
// Health mocks base method.

15
internal/eventstore/repository/mock/repository.mock.impl.go
View File

@@ -30,21 +30,30 @@ func NewRepo(t *testing.T) *MockRepository {
func (m *MockRepository) ExpectFilterNoEventsNoError() *MockRepository {
m.MockQuerier.ctrl.T.Helper()
m.MockQuerier.EXPECT().Filter(gomock.Any(), gomock.Any()).Return(nil, nil)
m.MockQuerier.EXPECT().FilterToReducer(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
return m
}
func (m *MockRepository) ExpectFilterEvents(events ...eventstore.Event) *MockRepository {
m.MockQuerier.ctrl.T.Helper()
m.MockQuerier.EXPECT().Filter(gomock.Any(), gomock.Any()).Return(events, nil)
m.MockQuerier.EXPECT().FilterToReducer(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(
func(_ context.Context, _ *eventstore.SearchQueryBuilder, reduce eventstore.Reducer) error {
for _, event := range events {
if err := reduce(event); err != nil {
return err
}
}
return nil
},
)
return m
}
func (m *MockRepository) ExpectFilterEventsError(err error) *MockRepository {
m.MockQuerier.ctrl.T.Helper()
m.MockQuerier.EXPECT().Filter(gomock.Any(), gomock.Any()).Return(nil, err)
m.MockQuerier.EXPECT().FilterToReducer(gomock.Any(), gomock.Any(), gomock.Any()).Return(err)
return m
}

22
internal/eventstore/repository/sql/crdb.go
View File

@@ -247,22 +247,18 @@ func (db *CRDB) handleUniqueConstraints(ctx context.Context, tx *sql.Tx, uniqueC
return nil
}
// Filter returns all events matching the given search query
func (crdb *CRDB) Filter(ctx context.Context, searchQuery *eventstore.SearchQueryBuilder) (events []eventstore.Event, err error) {
events = make([]eventstore.Event, 0, searchQuery.GetLimit())
err = query(ctx, crdb, searchQuery, &events, false)
// FilterToReducer finds all events matching the given search query and passes them to the reduce function.
func (crdb *CRDB) FilterToReducer(ctx context.Context, searchQuery *eventstore.SearchQueryBuilder, reduce eventstore.Reducer) error {
err := query(ctx, crdb, searchQuery, reduce, false)
if err == nil {
return nil
}
pgErr := new(pgconn.PgError)
// check events2 not exists
if err != nil && errors.As(err, &pgErr) {
if pgErr.Code == "42P01" {
err = query(ctx, crdb, searchQuery, &events, true)
}
if errors.As(err, &pgErr) && pgErr.Code == "42P01" {
return query(ctx, crdb, searchQuery, reduce, true)
}
if err != nil {
return nil, err
}
return events, nil
return err
}
// LatestSequence returns the latest sequence found by the search query

17
internal/eventstore/repository/sql/query.go
View File

@@ -168,12 +168,11 @@ func instanceIDsScanner(scanner scan, dest interface{}) (err error) {
func eventsScanner(useV1 bool) func(scanner scan, dest interface{}) (err error) {
return func(scanner scan, dest interface{}) (err error) {
events, ok := dest.(*[]eventstore.Event)
reduce, ok := dest.(eventstore.Reducer)
if !ok {
return z_errors.ThrowInvalidArgument(nil, "SQL-4GP6F", "type must be event")
return z_errors.ThrowInvalidArgumentf(nil, "SQL-4GP6F", "events scanner: invalid type %T", dest)
}
event := new(repository.Event)
data := sql.RawBytes{}
position := new(sql.NullFloat64)
if useV1 {
@@ -181,7 +180,7 @@ func eventsScanner(useV1 bool) func(scanner scan, dest interface{}) (err error)
&event.CreationDate,
&event.Typ,
&event.Seq,
&data,
&event.Data,
&event.EditorUser,
&event.ResourceOwner,
&event.InstanceID,
@@ -196,7 +195,7 @@ func eventsScanner(useV1 bool) func(scanner scan, dest interface{}) (err error)
&event.Typ,
&event.Seq,
position,
&data,
&event.Data,
&event.EditorUser,
&event.ResourceOwner,
&event.InstanceID,
@@ -211,14 +210,8 @@ func eventsScanner(useV1 bool) func(scanner scan, dest interface{}) (err error)
logging.New().WithError(err).Warn("unable to scan row")
return z_errors.ThrowInternal(err, "SQL-M0dsf", "unable to scan row")
}
event.Data = make([]byte, len(data))
copy(event.Data, data)
event.Pos = position.Float64
*events = append(*events, event)
return nil
return reduce(event)
}
}

46
internal/eventstore/repository/sql/query_test.go
View File

@@ -10,6 +10,7 @@ import (
"time"
"github.com/DATA-DOG/go-sqlmock"
"github.com/stretchr/testify/assert"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/database/cockroach"
@@ -74,6 +75,8 @@ func Test_getCondition(t *testing.T) {
}
func Test_prepareColumns(t *testing.T) {
var reducedEvents []eventstore.Event
type fields struct {
dbRow []interface{}
}
@@ -146,13 +149,16 @@ func Test_prepareColumns(t *testing.T) {
name: "events",
args: args{
columns: eventstore.ColumnsEvent,
dest: &[]eventstore.Event{},
useV1: true,
dest: eventstore.Reducer(func(event eventstore.Event) error {
reducedEvents = append(reducedEvents, event)
return nil
}),
useV1: true,
},
res: res{
query: `SELECT creation_date, event_type, event_sequence, event_data, editor_user, resource_owner, instance_id, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events`,
expected: []eventstore.Event{
&repository.Event{AggregateID: "hodor", AggregateType: "user", Seq: 5, Data: make(sql.RawBytes, 0)},
&repository.Event{AggregateID: "hodor", AggregateType: "user", Seq: 5, Data: nil},
},
},
fields: fields{
@@ -163,12 +169,15 @@ func Test_prepareColumns(t *testing.T) {
name: "events v2",
args: args{
columns: eventstore.ColumnsEvent,
dest: &[]eventstore.Event{},
dest: eventstore.Reducer(func(event eventstore.Event) error {
reducedEvents = append(reducedEvents, event)
return nil
}),
},
res: res{
query: `SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision FROM eventstore.events2`,
expected: []eventstore.Event{
&repository.Event{AggregateID: "hodor", AggregateType: "user", Seq: 5, Pos: 42, Data: make(sql.RawBytes, 0), Version: "v1"},
&repository.Event{AggregateID: "hodor", AggregateType: "user", Seq: 5, Pos: 42, Data: nil, Version: "v1"},
},
},
fields: fields{
@@ -179,12 +188,15 @@ func Test_prepareColumns(t *testing.T) {
name: "event null position",
args: args{
columns: eventstore.ColumnsEvent,
dest: &[]eventstore.Event{},
dest: eventstore.Reducer(func(event eventstore.Event) error {
reducedEvents = append(reducedEvents, event)
return nil
}),
},
res: res{
query: `SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision FROM eventstore.events2`,
expected: []eventstore.Event{
&repository.Event{AggregateID: "hodor", AggregateType: "user", Seq: 5, Pos: 0, Data: make(sql.RawBytes, 0), Version: "v1"},
&repository.Event{AggregateID: "hodor", AggregateType: "user", Seq: 5, Pos: 0, Data: nil, Version: "v1"},
},
},
fields: fields{
@@ -207,9 +219,12 @@ func Test_prepareColumns(t *testing.T) {
name: "event query error",
args: args{
columns: eventstore.ColumnsEvent,
dest: &[]eventstore.Event{},
dbErr: sql.ErrConnDone,
useV1: true,
dest: eventstore.Reducer(func(event eventstore.Event) error {
reducedEvents = append(reducedEvents, event)
return nil
}),
dbErr: sql.ErrConnDone,
useV1: true,
},
res: res{
query: `SELECT creation_date, event_type, event_sequence, event_data, editor_user, resource_owner, instance_id, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events`,
@@ -242,6 +257,12 @@ func Test_prepareColumns(t *testing.T) {
equalizer.Equal(tt.args.dest.(*sql.NullTime).Time)
return
}
if _, ok := tt.args.dest.(eventstore.Reducer); ok {
assert.Equal(t, tt.res.expected, reducedEvents)
reducedEvents = nil
return
}
got := reflect.Indirect(reflect.ValueOf(tt.args.dest)).Interface()
if !reflect.DeepEqual(got, tt.res.expected) {
t.Errorf("unexpected result from rowScanner \nwant: %+v \ngot: %+v", tt.res.expected, got)
@@ -625,7 +646,10 @@ func Test_query_events_with_crdb(t *testing.T) {
}
events := []eventstore.Event{}
if err := query(context.Background(), db, tt.args.searchQuery, &events, true); (err != nil) != tt.wantErr {
if err := query(context.Background(), db, tt.args.searchQuery, eventstore.Reducer(func(event eventstore.Event) error {
events = append(events, event)
return nil
}), true); (err != nil) != tt.wantErr {
t.Errorf("CRDB.query() error = %v, wantErr %v", err, tt.wantErr)
}
})