feat(cli): setup (#3267)

* commander

* commander

* selber!

* move to packages

* fix(errors): implement Is interface

* test: command

* test: commands

* add init steps

* setup tenant

* add default step yaml

* possibility to set password

* merge v2 into v2-commander

* fix: rename iam command side to instance

* fix: rename iam command side to instance

* fix: rename iam command side to instance

* fix: rename iam command side to instance

* fix: search query builder can filter events in memory

* fix: filters for add member

* fix(setup): add `ExternalSecure` to config

* chore: name iam to instance

* fix: matching

* remove unsued func

* base url

* base url

* test(command): filter funcs

* test: commands

* fix: rename orgiampolicy to domain policy

* start from init

* commands

* config

* fix indexes and add constraints

* fixes

* fix: merge conflicts

* fix: protos

* fix: md files

* setup

* add deprecated org iam policy again

* typo

* fix search query

* fix filter

* Apply suggestions from code review

* remove custom org from org setup

* add todos for verification

* change apps creation

* simplify package structure

* fix error

* move preparation helper for tests

* fix unique constraints

* fix config mapping in setup

* fix error handling in encryption_keys.go

* fix projection config

* fix query from old views to projection

* fix setup of mgmt api

* set iam project and fix instance projection

* imports

Co-authored-by: Livio Amstutz <livio.a@gmail.com>
Co-authored-by: fabi <fabienne.gerschwiler@gmail.com>

internal/eventstore/aggregate.go

@@ -68,3 +68,21 @@ type Aggregate struct {
 	//Version is the semver this aggregate represents
 	Version Version `json:"-"`
 }
+
+func isAggreagteTypes(a Aggregate, types ...AggregateType) bool {
+	for _, typ := range types {
+		if a.Type == typ {
+			return true
+		}
+	}
+	return false
+}
+
+func isAggregateIDs(a Aggregate, ids ...string) bool {
+	for _, id := range ids {
+		if a.ID == id {
+			return true
+		}
+	}
+	return false
+}

internal/eventstore/event.go

@@ -45,3 +45,12 @@ type Event interface {
 	//DataAsBytes returns the payload of the event. It represent the changed fields by the event
 	DataAsBytes() []byte
 }
+
+func isEventTypes(event Event, types ...EventType) bool {
+	for _, typ := range types {
+		if event.Type() == typ {
+			return true
+		}
+	}
+	return false
+}

internal/eventstore/eventstore.go

@@ -90,19 +90,20 @@ func commandsToRepository(instanceID string, cmds []Command) (events []*reposito
 			Data:          data,
 		}
 		if len(cmd.UniqueConstraints()) > 0 {
-			constraints = append(constraints, uniqueConstraintsToRepository(cmd.UniqueConstraints())...)
+			constraints = append(constraints, uniqueConstraintsToRepository(instanceID, cmd.UniqueConstraints())...)
 		}
 	}
 
 	return events, constraints, nil
 }
 
-func uniqueConstraintsToRepository(constraints []*EventUniqueConstraint) (uniqueConstraints []*repository.UniqueConstraint) {
+func uniqueConstraintsToRepository(instanceID string, constraints []*EventUniqueConstraint) (uniqueConstraints []*repository.UniqueConstraint) {
 	uniqueConstraints = make([]*repository.UniqueConstraint, len(constraints))
 	for i, constraint := range constraints {
 		uniqueConstraints[i] = &repository.UniqueConstraint{
 			UniqueType:   constraint.UniqueType,
 			UniqueField:  constraint.UniqueField,
+			InstanceID:   instanceID,
 			Action:       uniqueConstraintActionToRepository(constraint.Action),
 			ErrorMessage: constraint.ErrorMessage,
 		}

internal/eventstore/handler/crdb/init.go

@@ -15,17 +15,21 @@ import (
 )
 
 type Table struct {
-	columns    []*Column
-	primaryKey PrimaryKey
-	indices    []*Index
+	columns     []*Column
+	primaryKey  PrimaryKey
+	indices     []*Index
+	constraints []*Constraint
 }
 
-func NewTable(columns []*Column, key PrimaryKey, indices ...*Index) *Table {
-	return &Table{
+func NewTable(columns []*Column, key PrimaryKey, opts ...TableOption) *Table {
+	t := &Table{
 		columns:    columns,
 		primaryKey: key,
-		indices:    indices,
 	}
+	for _, opt := range opts {
+		opt(t)
+	}
+	return t
 }
 
 type SuffixedTable struct {
@@ -33,17 +37,27 @@ type SuffixedTable struct {
 	suffix string
 }
 
-func NewSuffixedTable(columns []*Column, key PrimaryKey, suffix string, indices ...*Index) *SuffixedTable {
+func NewSuffixedTable(columns []*Column, key PrimaryKey, suffix string, opts ...TableOption) *SuffixedTable {
 	return &SuffixedTable{
-		Table: Table{
-			columns:    columns,
-			primaryKey: key,
-			indices:    indices,
-		},
+		Table:  *NewTable(columns, key, opts...),
 		suffix: suffix,
 	}
 }
 
+type TableOption func(*Table)
+
+func WithIndex(index *Index) TableOption {
+	return func(table *Table) {
+		table.indices = append(table.indices, index)
+	}
+}
+
+func WithConstraint(constraint *Constraint) TableOption {
+	return func(table *Table) {
+		table.constraints = append(table.constraints, constraint)
+	}
+}
+
 type Column struct {
 	Name          string
 	Type          ColumnType
@@ -131,6 +145,19 @@ func Hash(bucketsCount uint16) indexOpts {
 	}
 }
 
+func NewConstraint(name string, columns []string) *Constraint {
+	i := &Constraint{
+		Name:    name,
+		Columns: columns,
+	}
+	return i
+}
+
+type Constraint struct {
+	Name    string
+	Columns []string
+}
+
 //Init implements handler.Init
 func (h *StatementHandler) Init(ctx context.Context, checks ...*handler.Check) error {
 	for _, check := range checks {
@@ -241,6 +268,9 @@ func createTableStatement(table *Table, tableName string, suffix string) string
 	for _, index := range table.indices {
 		stmt += fmt.Sprintf(", INDEX %s (%s)", index.Name, strings.Join(index.Columns, ","))
 	}
+	for _, constraint := range table.constraints {
+		stmt += fmt.Sprintf(", CONSTRAINT %s UNIQUE (%s)", constraint.Name, strings.Join(constraint.Columns, ","))
+	}
 	return stmt + ");"
 }
 
@@ -276,7 +306,7 @@ func createColumnsStatement(cols []*Column, tableName string) string {
 		if col.defaultValue != nil {
 			column += " DEFAULT " + defaultValue(col.defaultValue)
 		}
-		if col.deleteCascade != "" {
+		if len(col.deleteCascade) != 0 {
 			column += fmt.Sprintf(" REFERENCES %s (%s) ON DELETE CASCADE", tableName, col.deleteCascade)
 		}
 		columns[i] = column

internal/eventstore/repository/sql/crdb.go

@@ -80,15 +80,17 @@ const (
 	uniqueInsert = `INSERT INTO eventstore.unique_constraints
 					(
 						unique_type,
-						unique_field
+						unique_field,
+						instance_id
 					) 
 					VALUES (  
 						$1,
-						$2
+						$2,
+						$3
 					)`
 
 	uniqueDelete = `DELETE FROM eventstore.unique_constraints
-					WHERE unique_type = $1 and unique_field = $2`
+					WHERE unique_type = $1 and unique_field = $2 and instance_id = $3`
 )
 
 type CRDB struct {
@@ -159,8 +161,9 @@ func (db *CRDB) handleUniqueConstraints(ctx context.Context, tx *sql.Tx, uniqueC
 
 	for _, uniqueConstraint := range uniqueConstraints {
 		uniqueConstraint.UniqueField = strings.ToLower(uniqueConstraint.UniqueField)
-		if uniqueConstraint.Action == repository.UniqueConstraintAdd {
-			_, err := tx.ExecContext(ctx, uniqueInsert, uniqueConstraint.UniqueType, uniqueConstraint.UniqueField)
+		switch uniqueConstraint.Action {
+		case repository.UniqueConstraintAdd:
+			_, err := tx.ExecContext(ctx, uniqueInsert, uniqueConstraint.UniqueType, uniqueConstraint.UniqueField, uniqueConstraint.InstanceID)
 			if err != nil {
 				logging.WithFields(
 					"unique_type", uniqueConstraint.UniqueType,
@@ -172,8 +175,8 @@ func (db *CRDB) handleUniqueConstraints(ctx context.Context, tx *sql.Tx, uniqueC
 
 				return caos_errs.ThrowInternal(err, "SQL-dM9ds", "unable to create unique constraint ")
 			}
-		} else if uniqueConstraint.Action == repository.UniqueConstraintRemoved {
-			_, err := tx.ExecContext(ctx, uniqueDelete, uniqueConstraint.UniqueType, uniqueConstraint.UniqueField)
+		case repository.UniqueConstraintRemoved:
+			_, err := tx.ExecContext(ctx, uniqueDelete, uniqueConstraint.UniqueType, uniqueConstraint.UniqueField, uniqueConstraint.InstanceID)
 			if err != nil {
 				logging.WithFields(
 					"unique_type", uniqueConstraint.UniqueType,

internal/eventstore/repository/sql/crdb_test.go

@@ -267,11 +267,12 @@ func TestCRDB_columnName(t *testing.T) {
 
 func TestCRDB_Push_OneAggregate(t *testing.T) {
 	type args struct {
-		ctx               context.Context
-		events            []*repository.Event
-		uniqueConstraints *repository.UniqueConstraint
-		uniqueDataType    string
-		uniqueDataField   string
+		ctx                  context.Context
+		events               []*repository.Event
+		uniqueConstraints    *repository.UniqueConstraint
+		uniqueDataType       string
+		uniqueDataField      string
+		uniqueDataInstanceID string
 	}
 	type eventsRes struct {
 		pushedEventsCount int
@@ -419,7 +420,7 @@ func TestCRDB_Push_OneAggregate(t *testing.T) {
 				client: testCRDBClient,
 			}
 			if tt.args.uniqueDataType != "" && tt.args.uniqueDataField != "" {
-				err := fillUniqueData(tt.args.uniqueDataType, tt.args.uniqueDataField)
+				err := fillUniqueData(tt.args.uniqueDataType, tt.args.uniqueDataField, tt.args.uniqueDataInstanceID)
 				if err != nil {
 					t.Error("unable to prefill insert unique data: ", err)
 					return
@@ -440,7 +441,7 @@ func TestCRDB_Push_OneAggregate(t *testing.T) {
 				t.Errorf("expected push count %d got %d", tt.res.eventsRes.pushedEventsCount, eventCount)
 			}
 			if tt.args.uniqueConstraints != nil {
-				countUniqueRow := testCRDBClient.QueryRow("SELECT COUNT(*) FROM eventstore.unique_constraints where unique_type = $1 AND unique_field = $2", tt.args.uniqueConstraints.UniqueType, tt.args.uniqueConstraints.UniqueField)
+				countUniqueRow := testCRDBClient.QueryRow("SELECT COUNT(*) FROM eventstore.unique_constraints where unique_type = $1 AND unique_field = $2 AND instance_id = $3", tt.args.uniqueConstraints.UniqueType, tt.args.uniqueConstraints.UniqueField, tt.args.uniqueConstraints.InstanceID)
 				var uniqueCount int
 				err := countUniqueRow.Scan(&uniqueCount)
 				if err != nil {
@@ -1117,6 +1118,7 @@ func generateRemoveUniqueConstraint(t *testing.T, table, uniqueField string) *re
 	e := &repository.UniqueConstraint{
 		UniqueType:  table,
 		UniqueField: uniqueField,
+		InstanceID:  "",
 		Action:      repository.UniqueConstraintRemoved,
 	}
 

internal/eventstore/repository/sql/local_crdb_test.go

@@ -53,7 +53,7 @@ func initDB(db *sql.DB) error {
 	return initialise.VerifyZitadel(db)
 }
 
-func fillUniqueData(unique_type, field string) error {
-	_, err := testCRDBClient.Exec("INSERT INTO eventstore.unique_constraints (unique_type, unique_field) VALUES ($1, $2)", unique_type, field)
+func fillUniqueData(unique_type, field, instanceID string) error {
+	_, err := testCRDBClient.Exec("INSERT INTO eventstore.unique_constraints (unique_type, unique_field, instance_id) VALUES ($1, $2, $3)", unique_type, field, instanceID)
 	return err
 }

internal/eventstore/repository/unique_constraint.go

@@ -8,6 +8,9 @@ type UniqueConstraint struct {
 	//UniqueType is the type of the unique field
 	UniqueType string
 
+	//InstanceID represents the instance
+	InstanceID string
+
 	//Action defines if unique constraint should be added or removed
 	Action UniqueConstraintAction
 

internal/eventstore/search_query.go

@@ -50,22 +50,44 @@ func NewSearchQueryBuilder(columns Columns) *SearchQueryBuilder {
 	}
 }
 
+func (builder *SearchQueryBuilder) Matches(event Event, existingLen int) (matches bool) {
+	if builder.limit > 0 && uint64(existingLen) >= builder.limit {
+		return false
+	}
+	if builder.resourceOwner != "" && event.Aggregate().ResourceOwner != builder.resourceOwner {
+		return false
+	}
+	if event.Aggregate().InstanceID != "" && builder.instanceID != "" && event.Aggregate().InstanceID != builder.instanceID {
+		return false
+	}
+
+	if len(builder.queries) == 0 {
+		return true
+	}
+	for _, query := range builder.queries {
+		if query.matches(event) {
+			return true
+		}
+	}
+	return false
+}
+
 //Columns defines which fields are set
-func (factory *SearchQueryBuilder) Columns(columns Columns) *SearchQueryBuilder {
-	factory.columns = repository.Columns(columns)
-	return factory
+func (builder *SearchQueryBuilder) Columns(columns Columns) *SearchQueryBuilder {
+	builder.columns = repository.Columns(columns)
+	return builder
 }
 
 //Limit defines how many events are returned maximally.
-func (factory *SearchQueryBuilder) Limit(limit uint64) *SearchQueryBuilder {
-	factory.limit = limit
-	return factory
+func (builder *SearchQueryBuilder) Limit(limit uint64) *SearchQueryBuilder {
+	builder.limit = limit
+	return builder
 }
 
 //ResourceOwner defines the resource owner (org) of the events
-func (factory *SearchQueryBuilder) ResourceOwner(resourceOwner string) *SearchQueryBuilder {
-	factory.resourceOwner = resourceOwner
-	return factory
+func (builder *SearchQueryBuilder) ResourceOwner(resourceOwner string) *SearchQueryBuilder {
+	builder.resourceOwner = resourceOwner
+	return builder
 }
 
 //InstanceID defines the instanceID (system) of the events
@@ -75,25 +97,25 @@ func (factory *SearchQueryBuilder) InstanceID(instanceID string) *SearchQueryBui
 }
 
 //OrderDesc changes the sorting order of the returned events to descending
-func (factory *SearchQueryBuilder) OrderDesc() *SearchQueryBuilder {
-	factory.desc = true
-	return factory
+func (builder *SearchQueryBuilder) OrderDesc() *SearchQueryBuilder {
+	builder.desc = true
+	return builder
 }
 
 //OrderAsc changes the sorting order of the returned events to ascending
-func (factory *SearchQueryBuilder) OrderAsc() *SearchQueryBuilder {
-	factory.desc = false
-	return factory
+func (builder *SearchQueryBuilder) OrderAsc() *SearchQueryBuilder {
+	builder.desc = false
+	return builder
 }
 
 //AddQuery creates a new sub query.
 //All fields in the sub query are AND-connected in the storage request.
 //Multiple sub queries are OR-connected in the storage request.
-func (factory *SearchQueryBuilder) AddQuery() *SearchQuery {
+func (builder *SearchQueryBuilder) AddQuery() *SearchQuery {
 	query := &SearchQuery{
-		builder: factory,
+		builder: builder,
 	}
-	factory.queries = append(factory.queries, query)
+	builder.queries = append(builder.queries, query)
 
 	return query
 }
@@ -145,6 +167,25 @@ func (query *SearchQuery) Builder() *SearchQueryBuilder {
 	return query.builder
 }
 
+func (query *SearchQuery) matches(event Event) bool {
+	if query.eventSequenceLess > 0 && event.Sequence() >= query.eventSequenceLess {
+		return false
+	}
+	if query.eventSequenceGreater > 0 && event.Sequence() <= query.eventSequenceGreater {
+		return false
+	}
+	if ok := isAggreagteTypes(event.Aggregate(), query.aggregateTypes...); len(query.aggregateTypes) > 0 && !ok {
+		return false
+	}
+	if ok := isAggregateIDs(event.Aggregate(), query.aggregateIDs...); len(query.aggregateIDs) > 0 && !ok {
+		return false
+	}
+	if ok := isEventTypes(event, query.eventTypes...); len(query.eventTypes) > 0 && !ok {
+		return false
+	}
+	return true
+}
+
 func (builder *SearchQueryBuilder) build(instanceID string) (*repository.SearchQuery, error) {
 	if builder == nil ||
 		len(builder.queries) < 1 ||

internal/eventstore/search_query_test.go

@@ -774,3 +774,236 @@ func assertFilters(t *testing.T, i int, want, got *repository.Filter) {
 		t.Errorf("wrong value in filter %d : got: %v want: %v", i, got.Value, want.Value)
 	}
 }
+
+func TestSearchQuery_matches(t *testing.T) {
+	type args struct {
+		event Event
+	}
+	tests := []struct {
+		name  string
+		query *SearchQuery
+		event Event
+		want  bool
+	}{
+		{
+			name:  "sequence too low",
+			query: NewSearchQueryBuilder(ColumnsEvent).AddQuery().SequenceLess(10),
+			event: &BaseEvent{
+				sequence: 10,
+			},
+			want: false,
+		},
+		{
+			name:  "sequence too high",
+			query: NewSearchQueryBuilder(ColumnsEvent).AddQuery().SequenceGreater(60),
+			event: &BaseEvent{
+				sequence: 60,
+			},
+			want: false,
+		},
+		{
+			name:  "wrong aggregate type",
+			query: NewSearchQueryBuilder(ColumnsEvent).AddQuery().AggregateTypes("searched"),
+			event: &BaseEvent{
+				aggregate: Aggregate{
+					Type: "found",
+				},
+			},
+			want: false,
+		},
+		{
+			name:  "wrong aggregate id",
+			query: NewSearchQueryBuilder(ColumnsEvent).AddQuery().AggregateIDs("1", "10", "100"),
+			event: &BaseEvent{
+				aggregate: Aggregate{
+					ID: "2",
+				},
+			},
+			want: false,
+		},
+		{
+			name:  "wrong event type",
+			query: NewSearchQueryBuilder(ColumnsEvent).AddQuery().EventTypes("event.searched.type"),
+			event: &BaseEvent{
+				EventType: "event.actual.type",
+			},
+			want: false,
+		},
+		{
+			name: "matching",
+			query: NewSearchQueryBuilder(ColumnsEvent).AddQuery().
+				SequenceLess(100).SequenceGreater(50).AggregateIDs("2").AggregateTypes("actual").EventTypes("event.actual.type"),
+			event: &BaseEvent{
+				sequence: 55,
+				aggregate: Aggregate{
+					ID:   "2",
+					Type: "actual",
+				},
+				EventType: "event.actual.type",
+			},
+			want: true,
+		},
+		{
+			name:  "matching empty query",
+			query: NewSearchQueryBuilder(ColumnsEvent).AddQuery(),
+			event: &BaseEvent{
+				sequence: 55,
+				aggregate: Aggregate{
+					ID:   "2",
+					Type: "actual",
+				},
+				EventType: "event.actual.type",
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			query := &SearchQuery{
+				aggregateTypes:       tt.query.aggregateTypes,
+				aggregateIDs:         tt.query.aggregateIDs,
+				eventSequenceGreater: tt.query.eventSequenceGreater,
+				eventSequenceLess:    tt.query.eventSequenceLess,
+				eventTypes:           tt.query.eventTypes,
+				eventData:            tt.query.eventData,
+			}
+			if got := query.matches(tt.event); got != tt.want {
+				t.Errorf("SearchQuery.matches() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestSearchQueryBuilder_Matches(t *testing.T) {
+	type args struct {
+		event       Event
+		existingLen int
+	}
+	tests := []struct {
+		name    string
+		builder *SearchQueryBuilder
+		args    args
+		want    bool
+	}{
+		{
+			name:    "limit exeeded",
+			builder: NewSearchQueryBuilder(ColumnsEvent).Limit(100),
+			args: args{
+				event:       &BaseEvent{},
+				existingLen: 100,
+			},
+			want: false,
+		},
+		{
+			name:    "wrong resource owner",
+			builder: NewSearchQueryBuilder(ColumnsEvent).ResourceOwner("query"),
+			args: args{
+				event: &BaseEvent{
+					aggregate: Aggregate{
+						ResourceOwner: "ro",
+					},
+				},
+				existingLen: 0,
+			},
+			want: false,
+		},
+		{
+			name:    "wrong instance",
+			builder: NewSearchQueryBuilder(ColumnsEvent).InstanceID("instance"),
+			args: args{
+				event: &BaseEvent{
+					aggregate: Aggregate{
+						InstanceID: "different instance",
+					},
+				},
+				existingLen: 0,
+			},
+			want: false,
+		},
+		{
+			name: "query failed",
+			builder: NewSearchQueryBuilder(ColumnsEvent).
+				AddQuery().
+				SequenceGreater(1000).
+				Builder(),
+			args: args{
+				event: &BaseEvent{
+					sequence: 999,
+				},
+				existingLen: 0,
+			},
+			want: false,
+		},
+		{
+			name: "matching",
+			builder: NewSearchQueryBuilder(ColumnsEvent).
+				Limit(1000).
+				ResourceOwner("ro").
+				InstanceID("instance").
+				AddQuery().
+				SequenceGreater(1000).
+				Builder(),
+			args: args{
+				event: &BaseEvent{
+					aggregate: Aggregate{
+						ResourceOwner: "ro",
+						InstanceID:    "instance",
+					},
+					sequence: 1001,
+				},
+				existingLen: 999,
+			},
+			want: true,
+		},
+		{
+			name:    "matching builder resourceOwner and Instance",
+			builder: NewSearchQueryBuilder(ColumnsEvent),
+			args: args{
+				event: &BaseEvent{
+					aggregate: Aggregate{
+						ResourceOwner: "ro",
+						InstanceID:    "instance",
+					},
+					sequence: 1001,
+				},
+				existingLen: 999,
+			},
+			want: true,
+		},
+		{
+			name:    "matching builder resourceOwner only",
+			builder: NewSearchQueryBuilder(ColumnsEvent),
+			args: args{
+				event: &BaseEvent{
+					aggregate: Aggregate{
+						ResourceOwner: "ro",
+					},
+					sequence: 1001,
+				},
+				existingLen: 999,
+			},
+			want: true,
+		},
+		{
+			name:    "matching builder instanceID only",
+			builder: NewSearchQueryBuilder(ColumnsEvent),
+			args: args{
+				event: &BaseEvent{
+					aggregate: Aggregate{
+						InstanceID: "instance",
+					},
+					sequence: 1001,
+				},
+				existingLen: 999,
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.builder.Matches(tt.args.event, tt.args.existingLen); got != tt.want {
+				t.Errorf("SearchQueryBuilder.Matches() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}