fix(eventstore): sub queries (#1805)

* sub queries

* fix: tests

* add builder to tests

* new search query

* rename searchquerybuilder to builder

* remove comment from code

* test with multiple queries

* add filters test

* fix(contibute): listing

* add validate module

* fix: search queries

* remove unused event type in query

* ignore query if error in marshal

* go mod tidy

* update privacy policy query

* update queries

Co-authored-by: Livio Amstutz <livio.a@gmail.com>
This commit is contained in:
Silvan
2021-07-06 13:55:57 +02:00
committed by GitHub
parent 0e472a347f
commit 5349d96ce4
85 changed files with 1258 additions and 686 deletions

View File

@@ -13,9 +13,6 @@ import (
"github.com/caos/zitadel/internal/eventstore/repository"
"github.com/cockroachdb/cockroach-go/v2/crdb"
"github.com/lib/pq"
//sql import for cockroach
_ "github.com/lib/pq"
)
const (
@@ -163,7 +160,7 @@ func (db *CRDB) Push(ctx context.Context, events []*repository.Event, uniqueCons
// handleUniqueConstraints adds or removes unique constraints
func (db *CRDB) handleUniqueConstraints(ctx context.Context, tx *sql.Tx, uniqueConstraints ...*repository.UniqueConstraint) (err error) {
if uniqueConstraints == nil || len(uniqueConstraints) == 0 || (len(uniqueConstraints) == 1 && uniqueConstraints[0] == nil) {
if len(uniqueConstraints) == 0 || (len(uniqueConstraints) == 1 && uniqueConstraints[0] == nil) {
return nil
}

View File

@@ -5,10 +5,8 @@ import (
"sync"
"testing"
"github.com/lib/pq"
_ "github.com/lib/pq"
"github.com/caos/zitadel/internal/eventstore/repository"
"github.com/lib/pq"
)
func TestCRDB_placeholder(t *testing.T) {
@@ -749,8 +747,10 @@ func TestCRDB_Filter(t *testing.T) {
args: args{
searchQuery: &repository.SearchQuery{
Columns: repository.ColumnsEvent,
Filters: []*repository.Filter{
repository.NewFilter(repository.FieldAggregateType, "not found", repository.OperationEquals),
Filters: [][]*repository.Filter{
{
repository.NewFilter(repository.FieldAggregateType, "not found", repository.OperationEquals),
},
},
},
},
@@ -771,9 +771,11 @@ func TestCRDB_Filter(t *testing.T) {
args: args{
searchQuery: &repository.SearchQuery{
Columns: repository.ColumnsEvent,
Filters: []*repository.Filter{
repository.NewFilter(repository.FieldAggregateType, t.Name(), repository.OperationEquals),
repository.NewFilter(repository.FieldAggregateID, "303", repository.OperationEquals),
Filters: [][]*repository.Filter{
{
repository.NewFilter(repository.FieldAggregateType, t.Name(), repository.OperationEquals),
repository.NewFilter(repository.FieldAggregateID, "303", repository.OperationEquals),
},
},
},
},
@@ -837,8 +839,10 @@ func TestCRDB_LatestSequence(t *testing.T) {
args: args{
searchQuery: &repository.SearchQuery{
Columns: repository.ColumnsMaxSequence,
Filters: []*repository.Filter{
repository.NewFilter(repository.FieldAggregateType, "not found", repository.OperationEquals),
Filters: [][]*repository.Filter{
{
repository.NewFilter(repository.FieldAggregateType, "not found", repository.OperationEquals),
},
},
},
},
@@ -859,8 +863,10 @@ func TestCRDB_LatestSequence(t *testing.T) {
args: args{
searchQuery: &repository.SearchQuery{
Columns: repository.ColumnsMaxSequence,
Filters: []*repository.Filter{
repository.NewFilter(repository.FieldAggregateType, t.Name(), repository.OperationEquals),
Filters: [][]*repository.Filter{
{
repository.NewFilter(repository.FieldAggregateType, t.Name(), repository.OperationEquals),
},
},
},
},
@@ -1093,20 +1099,6 @@ func generateEvent(t *testing.T, aggregateID string, opts ...func(*repository.Ev
return e
}
func generateEventWithData(t *testing.T, aggregateID string, data []byte) *repository.Event {
t.Helper()
return &repository.Event{
AggregateID: aggregateID,
AggregateType: repository.AggregateType(t.Name()),
EditorService: "svc",
EditorUser: "user",
ResourceOwner: "ro",
Type: "test.created",
Version: "v1",
Data: data,
}
}
func generateAddUniqueConstraint(t *testing.T, table, uniqueField string) *repository.UniqueConstraint {
t.Helper()
e := &repository.UniqueConstraint{
@@ -1128,22 +1120,3 @@ func generateRemoveUniqueConstraint(t *testing.T, table, uniqueField string) *re
return e
}
func generateAddAsset(t *testing.T, id string, asset []byte) *repository.Asset {
t.Helper()
e := &repository.Asset{
ID: id,
Asset: asset,
Action: repository.AssetAdded,
}
return e
}
func generateRemoveAsset(t *testing.T, id string) *repository.Asset {
t.Helper()
e := &repository.Asset{
ID: id,
Action: repository.AssetRemoved,
}
return e
}

View File

@@ -100,11 +100,6 @@ func fillUniqueData(unique_type, field string) error {
return err
}
func fillAssets(id string, asset []byte) error {
_, err := testCRDBClient.Exec("INSERT INTO eventstore.assets (id, asset) VALUES ($1, $2)", id, asset)
return err
}
type migrationPaths []string
type version struct {

View File

@@ -9,7 +9,6 @@ import (
"strings"
"github.com/caos/logging"
caos_errs "github.com/caos/zitadel/internal/errors"
z_errors "github.com/caos/zitadel/internal/errors"
"github.com/caos/zitadel/internal/eventstore/repository"
"github.com/lib/pq"
@@ -26,14 +25,13 @@ type querier interface {
orderByEventSequence(desc bool) string
}
type rowScan func(scan, interface{}) error
type scan func(dest ...interface{}) error
func query(ctx context.Context, criteria querier, searchQuery *repository.SearchQuery, dest interface{}) error {
query, rowScanner := prepareColumns(criteria, searchQuery.Columns)
where, values := prepareCondition(criteria, searchQuery.Filters)
if where == "" || query == "" {
return caos_errs.ThrowInvalidArgument(nil, "SQL-rWeBw", "invalid query factory")
return z_errors.ThrowInvalidArgument(nil, "SQL-rWeBw", "invalid query factory")
}
query += where
@@ -51,7 +49,7 @@ func query(ctx context.Context, criteria querier, searchQuery *repository.Search
rows, err := criteria.db().QueryContext(ctx, query, values...)
if err != nil {
logging.Log("SQL-HP3Uk").WithError(err).Info("query failed")
return caos_errs.ThrowInternal(err, "SQL-IJuyR", "unable to filter events")
return z_errors.ThrowInternal(err, "SQL-IJuyR", "unable to filter events")
}
defer rows.Close()
@@ -125,31 +123,37 @@ func eventsScanner(scanner scan, dest interface{}) (err error) {
return nil
}
func prepareCondition(criteria querier, filters []*repository.Filter) (clause string, values []interface{}) {
values = make([]interface{}, len(filters))
clauses := make([]string, len(filters))
func prepareCondition(criteria querier, filters [][]*repository.Filter) (clause string, values []interface{}) {
values = make([]interface{}, 0, len(filters))
if len(filters) == 0 {
return clause, values
}
for i, filter := range filters {
value := filter.Value
switch value.(type) {
case []bool, []float64, []int64, []string, []repository.AggregateType, []repository.EventType, *[]bool, *[]float64, *[]int64, *[]string, *[]repository.AggregateType, *[]repository.EventType:
value = pq.Array(value)
case map[string]interface{}:
var err error
value, err = json.Marshal(value)
logging.Log("SQL-BSsNy").OnError(err).Warn("unable to marshal search value")
}
clauses[i] = getCondition(criteria, filter)
if clauses[i] == "" {
return "", nil
clauses := make([]string, len(filters))
for idx, filter := range filters {
subClauses := make([]string, 0, len(filter))
for _, f := range filter {
value := f.Value
switch value.(type) {
case []bool, []float64, []int64, []string, []repository.AggregateType, []repository.EventType, *[]bool, *[]float64, *[]int64, *[]string, *[]repository.AggregateType, *[]repository.EventType:
value = pq.Array(value)
case map[string]interface{}:
var err error
value, err = json.Marshal(value)
logging.Log("SQL-BSsNy").OnError(err).Warn("unable to marshal search value")
continue
}
subClauses = append(subClauses, getCondition(criteria, f))
if subClauses[len(subClauses)-1] == "" {
return "", nil
}
values = append(values, value)
}
values[i] = value
clauses[idx] = "( " + strings.Join(subClauses, " AND ") + " )"
}
return " WHERE " + strings.Join(clauses, " AND "), values
return " WHERE " + strings.Join(clauses, " OR "), values
}
func getCondition(cond querier, filter *repository.Filter) (condition string) {

View File

@@ -208,7 +208,7 @@ func prepareTestScan(err error, res []interface{}) scan {
func Test_prepareCondition(t *testing.T) {
type args struct {
filters []*repository.Filter
filters [][]*repository.Filter
}
type res struct {
clause string
@@ -232,7 +232,7 @@ func Test_prepareCondition(t *testing.T) {
{
name: "empty filters",
args: args{
filters: []*repository.Filter{},
filters: [][]*repository.Filter{},
},
res: res{
clause: "",
@@ -242,8 +242,10 @@ func Test_prepareCondition(t *testing.T) {
{
name: "invalid condition",
args: args{
filters: []*repository.Filter{
repository.NewFilter(repository.FieldAggregateID, "wrong", repository.Operation(-1)),
filters: [][]*repository.Filter{
{
repository.NewFilter(repository.FieldAggregateID, "wrong", repository.Operation(-1)),
},
},
},
res: res{
@@ -254,26 +256,30 @@ func Test_prepareCondition(t *testing.T) {
{
name: "array as condition value",
args: args{
filters: []*repository.Filter{
repository.NewFilter(repository.FieldAggregateType, []repository.AggregateType{"user", "org"}, repository.OperationIn),
filters: [][]*repository.Filter{
{
repository.NewFilter(repository.FieldAggregateType, []repository.AggregateType{"user", "org"}, repository.OperationIn),
},
},
},
res: res{
clause: " WHERE aggregate_type = ANY(?)",
clause: " WHERE ( aggregate_type = ANY(?) )",
values: []interface{}{pq.Array([]repository.AggregateType{"user", "org"})},
},
},
{
name: "multiple filters",
args: args{
filters: []*repository.Filter{
repository.NewFilter(repository.FieldAggregateType, []repository.AggregateType{"user", "org"}, repository.OperationIn),
repository.NewFilter(repository.FieldAggregateID, "1234", repository.OperationEquals),
repository.NewFilter(repository.FieldEventType, []repository.EventType{"user.created", "org.created"}, repository.OperationIn),
filters: [][]*repository.Filter{
{
repository.NewFilter(repository.FieldAggregateType, []repository.AggregateType{"user", "org"}, repository.OperationIn),
repository.NewFilter(repository.FieldAggregateID, "1234", repository.OperationEquals),
repository.NewFilter(repository.FieldEventType, []repository.EventType{"user.created", "org.created"}, repository.OperationIn),
},
},
},
res: res{
clause: " WHERE aggregate_type = ANY(?) AND aggregate_id = ? AND event_type = ANY(?)",
clause: " WHERE ( aggregate_type = ANY(?) AND aggregate_id = ? AND event_type = ANY(?) )",
values: []interface{}{pq.Array([]repository.AggregateType{"user", "org"}), "1234", pq.Array([]repository.EventType{"user.created", "org.created"})},
},
},
@@ -304,7 +310,6 @@ func Test_query_events_with_crdb(t *testing.T) {
}
type fields struct {
existingEvents []*repository.Event
existingAssets []*repository.Asset
client *sql.DB
}
type res struct {
@@ -322,8 +327,10 @@ func Test_query_events_with_crdb(t *testing.T) {
args: args{
searchQuery: &repository.SearchQuery{
Columns: repository.ColumnsEvent,
Filters: []*repository.Filter{
repository.NewFilter(repository.FieldAggregateType, "not found", repository.OperationEquals),
Filters: [][]*repository.Filter{
{
repository.NewFilter(repository.FieldAggregateType, "not found", repository.OperationEquals),
},
},
},
},
@@ -345,8 +352,10 @@ func Test_query_events_with_crdb(t *testing.T) {
args: args{
searchQuery: &repository.SearchQuery{
Columns: repository.ColumnsEvent,
Filters: []*repository.Filter{
repository.NewFilter(repository.FieldAggregateType, t.Name(), repository.OperationEquals),
Filters: [][]*repository.Filter{
{
repository.NewFilter(repository.FieldAggregateType, t.Name(), repository.OperationEquals),
},
},
},
},
@@ -369,9 +378,11 @@ func Test_query_events_with_crdb(t *testing.T) {
args: args{
searchQuery: &repository.SearchQuery{
Columns: repository.ColumnsEvent,
Filters: []*repository.Filter{
repository.NewFilter(repository.FieldAggregateType, t.Name(), repository.OperationEquals),
repository.NewFilter(repository.FieldAggregateID, "303", repository.OperationEquals),
Filters: [][]*repository.Filter{
{
repository.NewFilter(repository.FieldAggregateType, t.Name(), repository.OperationEquals),
repository.NewFilter(repository.FieldAggregateID, "303", repository.OperationEquals),
},
},
},
},
@@ -395,8 +406,10 @@ func Test_query_events_with_crdb(t *testing.T) {
args: args{
searchQuery: &repository.SearchQuery{
Columns: repository.ColumnsEvent,
Filters: []*repository.Filter{
repository.NewFilter(repository.FieldResourceOwner, "caos", repository.OperationEquals),
Filters: [][]*repository.Filter{
{
repository.NewFilter(repository.FieldResourceOwner, "caos", repository.OperationEquals),
},
},
},
},
@@ -420,9 +433,11 @@ func Test_query_events_with_crdb(t *testing.T) {
args: args{
searchQuery: &repository.SearchQuery{
Columns: repository.ColumnsEvent,
Filters: []*repository.Filter{
repository.NewFilter(repository.FieldEditorService, "MANAGEMENT-API", repository.OperationEquals),
repository.NewFilter(repository.FieldEditorService, "ADMIN-API", repository.OperationEquals),
Filters: [][]*repository.Filter{
{
repository.NewFilter(repository.FieldEditorService, "MANAGEMENT-API", repository.OperationEquals),
repository.NewFilter(repository.FieldEditorService, "ADMIN-API", repository.OperationEquals),
},
},
},
},
@@ -446,10 +461,12 @@ func Test_query_events_with_crdb(t *testing.T) {
args: args{
searchQuery: &repository.SearchQuery{
Columns: repository.ColumnsEvent,
Filters: []*repository.Filter{
repository.NewFilter(repository.FieldEditorUser, "adlerhurst", repository.OperationEquals),
repository.NewFilter(repository.FieldEditorUser, "nobody", repository.OperationEquals),
repository.NewFilter(repository.FieldEditorUser, "", repository.OperationEquals),
Filters: [][]*repository.Filter{
{
repository.NewFilter(repository.FieldEditorUser, "adlerhurst", repository.OperationEquals),
repository.NewFilter(repository.FieldEditorUser, "nobody", repository.OperationEquals),
repository.NewFilter(repository.FieldEditorUser, "", repository.OperationEquals),
},
},
},
},
@@ -475,9 +492,11 @@ func Test_query_events_with_crdb(t *testing.T) {
args: args{
searchQuery: &repository.SearchQuery{
Columns: repository.ColumnsEvent,
Filters: []*repository.Filter{
repository.NewFilter(repository.FieldEventType, repository.EventType("user.created"), repository.OperationEquals),
repository.NewFilter(repository.FieldEventType, repository.EventType("user.updated"), repository.OperationEquals),
Filters: [][]*repository.Filter{
{
repository.NewFilter(repository.FieldEventType, repository.EventType("user.created"), repository.OperationEquals),
repository.NewFilter(repository.FieldEventType, repository.EventType("user.updated"), repository.OperationEquals),
},
},
},
},
@@ -559,18 +578,20 @@ func Test_query_events_mocked(t *testing.T) {
query: &repository.SearchQuery{
Columns: repository.ColumnsEvent,
Desc: true,
Filters: []*repository.Filter{
Filters: [][]*repository.Filter{
{
Field: repository.FieldAggregateType,
Value: repository.AggregateType("user"),
Operation: repository.OperationEquals,
{
Field: repository.FieldAggregateType,
Value: repository.AggregateType("user"),
Operation: repository.OperationEquals,
},
},
},
},
},
fields: fields{
mock: newMockClient(t).expectQuery(t,
`SELECT creation_date, event_type, event_sequence, previous_sequence, event_data, editor_service, editor_user, resource_owner, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE aggregate_type = \$1 ORDER BY event_sequence DESC`,
`SELECT creation_date, event_type, event_sequence, previous_sequence, event_data, editor_service, editor_user, resource_owner, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE \( aggregate_type = \$1 \) ORDER BY event_sequence DESC`,
[]driver.Value{repository.AggregateType("user")},
),
},
@@ -586,18 +607,20 @@ func Test_query_events_mocked(t *testing.T) {
Columns: repository.ColumnsEvent,
Desc: false,
Limit: 5,
Filters: []*repository.Filter{
Filters: [][]*repository.Filter{
{
Field: repository.FieldAggregateType,
Value: repository.AggregateType("user"),
Operation: repository.OperationEquals,
{
Field: repository.FieldAggregateType,
Value: repository.AggregateType("user"),
Operation: repository.OperationEquals,
},
},
},
},
},
fields: fields{
mock: newMockClient(t).expectQuery(t,
`SELECT creation_date, event_type, event_sequence, previous_sequence, event_data, editor_service, editor_user, resource_owner, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE aggregate_type = \$1 ORDER BY event_sequence LIMIT \$2`,
`SELECT creation_date, event_type, event_sequence, previous_sequence, event_data, editor_service, editor_user, resource_owner, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE \( aggregate_type = \$1 \) ORDER BY event_sequence LIMIT \$2`,
[]driver.Value{repository.AggregateType("user"), uint64(5)},
),
},
@@ -613,18 +636,20 @@ func Test_query_events_mocked(t *testing.T) {
Columns: repository.ColumnsEvent,
Desc: true,
Limit: 5,
Filters: []*repository.Filter{
Filters: [][]*repository.Filter{
{
Field: repository.FieldAggregateType,
Value: repository.AggregateType("user"),
Operation: repository.OperationEquals,
{
Field: repository.FieldAggregateType,
Value: repository.AggregateType("user"),
Operation: repository.OperationEquals,
},
},
},
},
},
fields: fields{
mock: newMockClient(t).expectQuery(t,
`SELECT creation_date, event_type, event_sequence, previous_sequence, event_data, editor_service, editor_user, resource_owner, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE aggregate_type = \$1 ORDER BY event_sequence DESC LIMIT \$2`,
`SELECT creation_date, event_type, event_sequence, previous_sequence, event_data, editor_service, editor_user, resource_owner, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE \( aggregate_type = \$1 \) ORDER BY event_sequence DESC LIMIT \$2`,
[]driver.Value{repository.AggregateType("user"), uint64(5)},
),
},
@@ -640,18 +665,20 @@ func Test_query_events_mocked(t *testing.T) {
Columns: repository.ColumnsEvent,
Desc: true,
Limit: 0,
Filters: []*repository.Filter{
Filters: [][]*repository.Filter{
{
Field: repository.FieldAggregateType,
Value: repository.AggregateType("user"),
Operation: repository.OperationEquals,
{
Field: repository.FieldAggregateType,
Value: repository.AggregateType("user"),
Operation: repository.OperationEquals,
},
},
},
},
},
fields: fields{
mock: newMockClient(t).expectQueryErr(t,
`SELECT creation_date, event_type, event_sequence, previous_sequence, event_data, editor_service, editor_user, resource_owner, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE aggregate_type = \$1 ORDER BY event_sequence DESC`,
`SELECT creation_date, event_type, event_sequence, previous_sequence, event_data, editor_service, editor_user, resource_owner, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE \( aggregate_type = \$1 \) ORDER BY event_sequence DESC`,
[]driver.Value{repository.AggregateType("user")},
sql.ErrConnDone),
},
@@ -667,18 +694,20 @@ func Test_query_events_mocked(t *testing.T) {
Columns: repository.ColumnsEvent,
Desc: true,
Limit: 0,
Filters: []*repository.Filter{
Filters: [][]*repository.Filter{
{
Field: repository.FieldAggregateType,
Value: repository.AggregateType("user"),
Operation: repository.OperationEquals,
{
Field: repository.FieldAggregateType,
Value: repository.AggregateType("user"),
Operation: repository.OperationEquals,
},
},
},
},
},
fields: fields{
mock: newMockClient(t).expectQuery(t,
`SELECT creation_date, event_type, event_sequence, previous_sequence, event_data, editor_service, editor_user, resource_owner, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE aggregate_type = \$1 ORDER BY event_sequence DESC`,
`SELECT creation_date, event_type, event_sequence, previous_sequence, event_data, editor_service, editor_user, resource_owner, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE \( aggregate_type = \$1 \) ORDER BY event_sequence DESC`,
[]driver.Value{repository.AggregateType("user")},
&repository.Event{Sequence: 100}),
},
@@ -702,8 +731,10 @@ func Test_query_events_mocked(t *testing.T) {
args: args{
query: &repository.SearchQuery{
Columns: repository.ColumnsEvent,
Filters: []*repository.Filter{
{},
Filters: [][]*repository.Filter{
{
{},
},
},
},
},
@@ -711,6 +742,47 @@ func Test_query_events_mocked(t *testing.T) {
wantErr: true,
},
},
{
name: "with subqueries",
args: args{
dest: &[]*repository.Event{},
query: &repository.SearchQuery{
Columns: repository.ColumnsEvent,
Desc: true,
Limit: 5,
Filters: [][]*repository.Filter{
{
{
Field: repository.FieldAggregateType,
Value: repository.AggregateType("user"),
Operation: repository.OperationEquals,
},
},
{
{
Field: repository.FieldAggregateType,
Value: repository.AggregateType("org"),
Operation: repository.OperationEquals,
},
{
Field: repository.FieldAggregateID,
Value: "asdf42",
Operation: repository.OperationEquals,
},
},
},
},
},
fields: fields{
mock: newMockClient(t).expectQuery(t,
`SELECT creation_date, event_type, event_sequence, previous_sequence, event_data, editor_service, editor_user, resource_owner, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE \( aggregate_type = \$1 \) OR \( aggregate_type = \$2 AND aggregate_id = \$3 \) ORDER BY event_sequence DESC LIMIT \$4`,
[]driver.Value{repository.AggregateType("user"), repository.AggregateType("org"), "asdf42", uint64(5)},
),
},
res: res{
wantErr: false,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {