TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-12 03:37:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: push telemetry (#6027)

Browse Source 
* document analytics config

* rework configuration and docs

* describe HandleActiveInstances better

* describe active instances on quotas better

* only projected events are considered

* cleanup

* describe changes at runtime

* push milestones

* stop tracking events

* calculate and push 4 in 6 milestones

* reduce milestone pushed

* remove docs

* fix scheduled pseudo event projection

* push 5 in 6 milestones

* push 6 in 6 milestones

* ignore client ids

* fix text array contains

* push human readable milestone type

* statement unit tests

* improve dev and db performance

* organize imports

* cleanup

* organize imports

* test projection

* check rows.Err()

* test search query

* pass linting

* review

* test 4 milestones

* simplify milestone by instance ids query

* use type NamespacedCondition

* cleanup

* lint

* lint

* dont overwrite original error

* no opt-in in examples

* cleanup

* prerelease

* enable request headers

* make limit configurable

* review fixes

* only requeue special handlers secondly

* include integration tests

* Revert "include integration tests"

This reverts commit 96db9504ec.

* pass reducers

* test handlers

* fix unit test

* feat: increment version

* lint

* remove prerelease

* fix integration tests
This commit is contained in:
Elio Bischof
2023-07-06 08:38:13 +02:00
committed by GitHub
parent fa93bb7e85
commit bb756482c7
53 changed files with 2214 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files

146
internal/query/milestone.go Normal file
View File

@@ -0,0 +1,146 @@
package query
import (
"context"
"database/sql"
"time"
sq "github.com/Masterminds/squirrel"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/api/call"
"github.com/zitadel/zitadel/internal/errors"
"github.com/zitadel/zitadel/internal/query/projection"
"github.com/zitadel/zitadel/internal/repository/milestone"
"github.com/zitadel/zitadel/internal/telemetry/tracing"
)
type Milestones struct {
SearchResponse
Milestones []*Milestone
}
type Milestone struct {
InstanceID string
Type milestone.Type
ReachedDate time.Time
PushedDate time.Time
PrimaryDomain string
}
type MilestonesSearchQueries struct {
SearchRequest
Queries []SearchQuery
}
func (q *MilestonesSearchQueries) toQuery(query sq.SelectBuilder) sq.SelectBuilder {
query = q.SearchRequest.toQuery(query)
for _, q := range q.Queries {
query = q.toQuery(query)
}
return query
}
var (
milestonesTable = table{
name: projection.MilestonesProjectionTable,
instanceIDCol: projection.MilestoneColumnInstanceID,
}
MilestoneInstanceIDColID = Column{
name: projection.MilestoneColumnInstanceID,
table: milestonesTable,
}
MilestoneTypeColID = Column{
name: projection.MilestoneColumnType,
table: milestonesTable,
}
MilestonePrimaryDomainColID = Column{
name: projection.MilestoneColumnPrimaryDomain,
table: milestonesTable,
}
MilestoneReachedDateColID = Column{
name: projection.MilestoneColumnReachedDate,
table: milestonesTable,
}
MilestonePushedDateColID = Column{
name: projection.MilestoneColumnPushedDate,
table: milestonesTable,
}
)
// SearchMilestones tries to defer the instanceID from the passed context if no instanceIDs are passed
func (q *Queries) SearchMilestones(ctx context.Context, instanceIDs []string, queries *MilestonesSearchQueries) (_ *Milestones, err error) {
ctx, span := tracing.NewSpan(ctx)
defer func() { span.EndWithError(err) }()
query, scan := prepareMilestonesQuery(ctx, q.client)
if len(instanceIDs) == 0 {
instanceIDs = []string{authz.GetInstance(ctx).InstanceID()}
}
stmt, args, err := queries.toQuery(query).Where(sq.Eq{MilestoneInstanceIDColID.identifier(): instanceIDs}).ToSql()
if err != nil {
return nil, errors.ThrowInternal(err, "QUERY-A9i5k", "Errors.Query.SQLStatement")
}
rows, err := q.client.QueryContext(ctx, stmt, args...)
if err != nil {
return nil, err
}
defer func() {
if closeErr := rows.Close(); closeErr != nil && err == nil {
err = errors.ThrowInternal(closeErr, "QUERY-CK9mI", "Errors.Query.CloseRows")
}
}()
milestones, err := scan(rows)
if err != nil {
return nil, err
}
if err = rows.Err(); err != nil {
return nil, errors.ThrowInternal(err, "QUERY-asLsI", "Errors.Internal")
}
milestones.LatestSequence, err = q.latestSequence(ctx, milestonesTable)
return milestones, err
}
func prepareMilestonesQuery(ctx context.Context, db prepareDatabase) (sq.SelectBuilder, func(*sql.Rows) (*Milestones, error)) {
return sq.Select(
MilestoneInstanceIDColID.identifier(),
MilestonePrimaryDomainColID.identifier(),
MilestoneReachedDateColID.identifier(),
MilestonePushedDateColID.identifier(),
MilestoneTypeColID.identifier(),
countColumn.identifier(),
).
From(milestonesTable.identifier() + db.Timetravel(call.Took(ctx))).
PlaceholderFormat(sq.Dollar),
func(rows *sql.Rows) (*Milestones, error) {
milestones := make([]*Milestone, 0)
var count uint64
for rows.Next() {
m := new(Milestone)
reachedDate := sql.NullTime{}
pushedDate := sql.NullTime{}
primaryDomain := sql.NullString{}
err := rows.Scan(
&m.InstanceID,
&primaryDomain,
&reachedDate,
&pushedDate,
&m.Type,
&count,
)
if err != nil {
return nil, err
}
m.PrimaryDomain = primaryDomain.String
m.ReachedDate = reachedDate.Time
m.PushedDate = pushedDate.Time
milestones = append(milestones, m)
}
return &Milestones{
Milestones: milestones,
SearchResponse: SearchResponse{
Count: count,
},
}, nil
}
}

189
internal/query/milestone_test.go Normal file
View File

@@ -0,0 +1,189 @@
package query
import (
"database/sql"
"database/sql/driver"
"errors"
"fmt"
"regexp"
"testing"
)
var (
expectedMilestoneQuery = regexp.QuoteMeta(`
SELECT projections.milestones.instance_id,
projections.milestones.primary_domain,
projections.milestones.reached_date,
projections.milestones.last_pushed_date,
projections.milestones.type,
COUNT(*) OVER ()
FROM projections.milestones AS OF SYSTEM TIME '-1 ms'
`)
milestoneCols = []string{
"instance_id",
"primary_domain",
"reached_date",
"last_pushed_date",
"type",
"ignore_client_ids",
}
)
func Test_MilestonesPrepare(t *testing.T) {
type want struct {
sqlExpectations sqlExpectation
err checkErr
}
tests := []struct {
name string
prepare interface{}
want want
object interface{}
}{
{
name: "prepareMilestonesQuery no result",
prepare: prepareMilestonesQuery,
want: want{
sqlExpectations: mockQueries(
expectedMilestoneQuery,
nil,
nil,
),
},
object: &Milestones{Milestones: []*Milestone{}},
},
{
name: "prepareMilestonesQuery",
prepare: prepareMilestonesQuery,
want: want{
sqlExpectations: mockQueries(
expectedMilestoneQuery,
milestoneCols,
[][]driver.Value{
{
"instance-id",
"primary.domain",
testNow,
testNow,
1,
1,
},
},
),
},
object: &Milestones{
SearchResponse: SearchResponse{
Count: 1,
},
Milestones: []*Milestone{
{
InstanceID: "instance-id",
Type: 1,
ReachedDate: testNow,
PushedDate: testNow,
PrimaryDomain: "primary.domain",
},
},
},
},
{
name: "prepareMilestonesQuery multiple result",
prepare: prepareMilestonesQuery,
want: want{
sqlExpectations: mockQueries(
expectedMilestoneQuery,
milestoneCols,
[][]driver.Value{
{
"instance-id",
"primary.domain",
testNow,
testNow,
1,
1,
},
{
"instance-id",
"primary.domain",
testNow,
testNow,
2,
2,
},
{
"instance-id",
"primary.domain",
testNow,
nil,
3,
3,
},
{
"instance-id",
"primary.domain",
nil,
nil,
4,
4,
},
},
),
},
object: &Milestones{
SearchResponse: SearchResponse{
Count: 4,
},
Milestones: []*Milestone{
{
InstanceID: "instance-id",
Type: 1,
ReachedDate: testNow,
PushedDate: testNow,
PrimaryDomain: "primary.domain",
},
{
InstanceID: "instance-id",
Type: 2,
ReachedDate: testNow,
PushedDate: testNow,
PrimaryDomain: "primary.domain",
},
{
InstanceID: "instance-id",
Type: 3,
ReachedDate: testNow,
PrimaryDomain: "primary.domain",
},
{
InstanceID: "instance-id",
Type: 4,
PrimaryDomain: "primary.domain",
},
},
},
},
{
name: "prepareMilestonesQuery sql err",
prepare: prepareMilestonesQuery,
want: want{
sqlExpectations: mockQueryErr(
expectedMilestoneQuery,
sql.ErrConnDone,
),
err: func(err error) (error, bool) {
if !errors.Is(err, sql.ErrConnDone) {
return fmt.Errorf("err should be sql.ErrConnDone got: %w", err), false
}
return nil, true
},
},
object: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assertPrepare(t, tt.prepare, tt.object, tt.want.sqlExpectations, tt.want.err, defaultPrepareArgs...)
})
}
}

14
internal/query/projection/assert.go Normal file
View File

@@ -0,0 +1,14 @@
package projection
import (
"github.com/zitadel/zitadel/internal/errors"
"github.com/zitadel/zitadel/internal/eventstore"
)
func assertEvent[T eventstore.Event](event eventstore.Event) (T, error) {
e, ok := event.(T)
if !ok {
return e, errors.ThrowInvalidArgumentf(nil, "HANDL-1m9fS", "reduce.wrong.event.type %T", event)
}
return e, nil
}

52
internal/query/projection/assert_test.go Normal file
View File

@@ -0,0 +1,52 @@
package projection
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/repository/instance"
)
func Test_assertEvent(t *testing.T) {
type args struct {
event eventstore.Event
assertFunc func(eventstore.Event) (eventstore.Event, error)
}
type testCase struct {
name string
args args
wantErr assert.ErrorAssertionFunc
}
tests := []testCase{
{
name: "correct event type",
args: args{
event: instance.NewInstanceAddedEvent(context.Background(), &instance.NewAggregate("instance-id").Aggregate, "instance-name"),
assertFunc: func(event eventstore.Event) (eventstore.Event, error) {
return assertEvent[*instance.InstanceAddedEvent](event)
},
},
wantErr: assert.NoError,
}, {
name: "wrong event type",
args: args{
event: instance.NewInstanceRemovedEvent(context.Background(), &instance.NewAggregate("instance-id").Aggregate, "instance-name", nil),
assertFunc: func(event eventstore.Event) (eventstore.Event, error) {
return assertEvent[*instance.InstanceAddedEvent](event)
},
},
wantErr: assert.Error,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := tt.args.assertFunc(tt.args.event)
if !tt.wantErr(t, err) {
return
}
})
}
}

16
internal/query/projection/event_test.go
View File

@@ -14,12 +14,26 @@ func testEvent(
eventType repository.EventType,
aggregateType repository.AggregateType,
data []byte,
) *repository.Event {
return timedTestEvent(eventType, aggregateType, data, time.Now())
}
func toSystemEvent(event *repository.Event) *repository.Event {
event.EditorService = "SYSTEM"
return event
}
func timedTestEvent(
eventType repository.EventType,
aggregateType repository.AggregateType,
data []byte,
creationDate time.Time,
) *repository.Event {
return &repository.Event{
Sequence: 15,
PreviousAggregateSequence: 10,
PreviousAggregateTypeSequence: 10,
CreationDate: time.Now(),
CreationDate: creationDate,
Type: eventType,
AggregateType: aggregateType,
Data: data,

8
internal/query/projection/label_policy.go
View File

@@ -402,10 +402,10 @@ func (p *labelPolicyProjection) reduceActivated(event eventstore.Event) (*handle
handler.NewCol(LabelPolicyDarkLogoURLCol, nil),
handler.NewCol(LabelPolicyDarkIconURLCol, nil),
},
[]handler.Condition{
handler.NewCond(LabelPolicyIDCol, event.Aggregate().ID),
handler.NewCond(LabelPolicyStateCol, domain.LabelPolicyStatePreview),
handler.NewCond(LabelPolicyInstanceIDCol, event.Aggregate().InstanceID),
[]handler.NamespacedCondition{
handler.NewNamespacedCondition(LabelPolicyIDCol, event.Aggregate().ID),
handler.NewNamespacedCondition(LabelPolicyStateCol, domain.LabelPolicyStatePreview),
handler.NewNamespacedCondition(LabelPolicyInstanceIDCol, event.Aggregate().InstanceID),
}), nil
}

4
internal/query/projection/label_policy_test.go
View File

@@ -170,7 +170,7 @@ func TestLabelPolicyProjection_reduces(t *testing.T) {
executer: &testExecuter{
executions: []execution{
{
expectedStmt: "INSERT INTO projections.label_policies2 (change_date, sequence, state, creation_date, resource_owner, instance_id, id, is_default, hide_login_name_suffix, font_url, watermark_disabled, should_error_popup, light_primary_color, light_warn_color, light_background_color, light_font_color, light_logo_url, light_icon_url, dark_primary_color, dark_warn_color, dark_background_color, dark_font_color, dark_logo_url, dark_icon_url) SELECT $1, $2, $3, creation_date, resource_owner, instance_id, id, is_default, hide_login_name_suffix, font_url, watermark_disabled, should_error_popup, light_primary_color, light_warn_color, light_background_color, light_font_color, light_logo_url, light_icon_url, dark_primary_color, dark_warn_color, dark_background_color, dark_font_color, dark_logo_url, dark_icon_url FROM projections.label_policies2 AS copy_table WHERE copy_table.id = $4 AND copy_table.state = $5 AND copy_table.instance_id = $6 ON CONFLICT (instance_id, id, state) DO UPDATE SET (change_date, sequence, state, creation_date, resource_owner, instance_id, id, is_default, hide_login_name_suffix, font_url, watermark_disabled, should_error_popup, light_primary_color, light_warn_color, light_background_color, light_font_color, light_logo_url, light_icon_url, dark_primary_color, dark_warn_color, dark_background_color, dark_font_color, dark_logo_url, dark_icon_url) = ($1, $2, $3, EXCLUDED.creation_date, EXCLUDED.resource_owner, EXCLUDED.instance_id, EXCLUDED.id, EXCLUDED.is_default, EXCLUDED.hide_login_name_suffix, EXCLUDED.font_url, EXCLUDED.watermark_disabled, EXCLUDED.should_error_popup, EXCLUDED.light_primary_color, EXCLUDED.light_warn_color, EXCLUDED.light_background_color, EXCLUDED.light_font_color, EXCLUDED.light_logo_url, EXCLUDED.light_icon_url, EXCLUDED.dark_primary_color, EXCLUDED.dark_warn_color, EXCLUDED.dark_background_color, EXCLUDED.dark_font_color, EXCLUDED.dark_logo_url, EXCLUDED.dark_icon_url)",
expectedStmt: "INSERT INTO projections.label_policies2 (change_date, sequence, state, creation_date, resource_owner, instance_id, id, is_default, hide_login_name_suffix, font_url, watermark_disabled, should_error_popup, light_primary_color, light_warn_color, light_background_color, light_font_color, light_logo_url, light_icon_url, dark_primary_color, dark_warn_color, dark_background_color, dark_font_color, dark_logo_url, dark_icon_url) SELECT $1, $2, $3, creation_date, resource_owner, instance_id, id, is_default, hide_login_name_suffix, font_url, watermark_disabled, should_error_popup, light_primary_color, light_warn_color, light_background_color, light_font_color, light_logo_url, light_icon_url, dark_primary_color, dark_warn_color, dark_background_color, dark_font_color, dark_logo_url, dark_icon_url FROM projections.label_policies2 AS copy_table WHERE (copy_table.id = $4) AND (copy_table.state = $5) AND (copy_table.instance_id = $6) ON CONFLICT (instance_id, id, state) DO UPDATE SET (change_date, sequence, state, creation_date, resource_owner, instance_id, id, is_default, hide_login_name_suffix, font_url, watermark_disabled, should_error_popup, light_primary_color, light_warn_color, light_background_color, light_font_color, light_logo_url, light_icon_url, dark_primary_color, dark_warn_color, dark_background_color, dark_font_color, dark_logo_url, dark_icon_url) = ($1, $2, $3, EXCLUDED.creation_date, EXCLUDED.resource_owner, EXCLUDED.instance_id, EXCLUDED.id, EXCLUDED.is_default, EXCLUDED.hide_login_name_suffix, EXCLUDED.font_url, EXCLUDED.watermark_disabled, EXCLUDED.should_error_popup, EXCLUDED.light_primary_color, EXCLUDED.light_warn_color, EXCLUDED.light_background_color, EXCLUDED.light_font_color, EXCLUDED.light_logo_url, EXCLUDED.light_icon_url, EXCLUDED.dark_primary_color, EXCLUDED.dark_warn_color, EXCLUDED.dark_background_color, EXCLUDED.dark_font_color, EXCLUDED.dark_logo_url, EXCLUDED.dark_icon_url)",
expectedArgs: []interface{}{
anyArg{},
uint64(15),
@@ -631,7 +631,7 @@ func TestLabelPolicyProjection_reduces(t *testing.T) {
executer: &testExecuter{
executions: []execution{
{
expectedStmt: "INSERT INTO projections.label_policies2 (change_date, sequence, state, creation_date, resource_owner, instance_id, id, is_default, hide_login_name_suffix, font_url, watermark_disabled, should_error_popup, light_primary_color, light_warn_color, light_background_color, light_font_color, light_logo_url, light_icon_url, dark_primary_color, dark_warn_color, dark_background_color, dark_font_color, dark_logo_url, dark_icon_url) SELECT $1, $2, $3, creation_date, resource_owner, instance_id, id, is_default, hide_login_name_suffix, font_url, watermark_disabled, should_error_popup, light_primary_color, light_warn_color, light_background_color, light_font_color, light_logo_url, light_icon_url, dark_primary_color, dark_warn_color, dark_background_color, dark_font_color, dark_logo_url, dark_icon_url FROM projections.label_policies2 AS copy_table WHERE copy_table.id = $4 AND copy_table.state = $5 AND copy_table.instance_id = $6 ON CONFLICT (instance_id, id, state) DO UPDATE SET (change_date, sequence, state, creation_date, resource_owner, instance_id, id, is_default, hide_login_name_suffix, font_url, watermark_disabled, should_error_popup, light_primary_color, light_warn_color, light_background_color, light_font_color, light_logo_url, light_icon_url, dark_primary_color, dark_warn_color, dark_background_color, dark_font_color, dark_logo_url, dark_icon_url) = ($1, $2, $3, EXCLUDED.creation_date, EXCLUDED.resource_owner, EXCLUDED.instance_id, EXCLUDED.id, EXCLUDED.is_default, EXCLUDED.hide_login_name_suffix, EXCLUDED.font_url, EXCLUDED.watermark_disabled, EXCLUDED.should_error_popup, EXCLUDED.light_primary_color, EXCLUDED.light_warn_color, EXCLUDED.light_background_color, EXCLUDED.light_font_color, EXCLUDED.light_logo_url, EXCLUDED.light_icon_url, EXCLUDED.dark_primary_color, EXCLUDED.dark_warn_color, EXCLUDED.dark_background_color, EXCLUDED.dark_font_color, EXCLUDED.dark_logo_url, EXCLUDED.dark_icon_url)",
expectedStmt: "INSERT INTO projections.label_policies2 (change_date, sequence, state, creation_date, resource_owner, instance_id, id, is_default, hide_login_name_suffix, font_url, watermark_disabled, should_error_popup, light_primary_color, light_warn_color, light_background_color, light_font_color, light_logo_url, light_icon_url, dark_primary_color, dark_warn_color, dark_background_color, dark_font_color, dark_logo_url, dark_icon_url) SELECT $1, $2, $3, creation_date, resource_owner, instance_id, id, is_default, hide_login_name_suffix, font_url, watermark_disabled, should_error_popup, light_primary_color, light_warn_color, light_background_color, light_font_color, light_logo_url, light_icon_url, dark_primary_color, dark_warn_color, dark_background_color, dark_font_color, dark_logo_url, dark_icon_url FROM projections.label_policies2 AS copy_table WHERE (copy_table.id = $4) AND (copy_table.state = $5) AND (copy_table.instance_id = $6) ON CONFLICT (instance_id, id, state) DO UPDATE SET (change_date, sequence, state, creation_date, resource_owner, instance_id, id, is_default, hide_login_name_suffix, font_url, watermark_disabled, should_error_popup, light_primary_color, light_warn_color, light_background_color, light_font_color, light_logo_url, light_icon_url, dark_primary_color, dark_warn_color, dark_background_color, dark_font_color, dark_logo_url, dark_icon_url) = ($1, $2, $3, EXCLUDED.creation_date, EXCLUDED.resource_owner, EXCLUDED.instance_id, EXCLUDED.id, EXCLUDED.is_default, EXCLUDED.hide_login_name_suffix, EXCLUDED.font_url, EXCLUDED.watermark_disabled, EXCLUDED.should_error_popup, EXCLUDED.light_primary_color, EXCLUDED.light_warn_color, EXCLUDED.light_background_color, EXCLUDED.light_font_color, EXCLUDED.light_logo_url, EXCLUDED.light_icon_url, EXCLUDED.dark_primary_color, EXCLUDED.dark_warn_color, EXCLUDED.dark_background_color, EXCLUDED.dark_font_color, EXCLUDED.dark_logo_url, EXCLUDED.dark_icon_url)",
expectedArgs: []interface{}{
anyArg{},
uint64(15),

295
internal/query/projection/milestones.go Normal file
View File

@@ -0,0 +1,295 @@
package projection
import (
"context"
"strconv"
"strings"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/eventstore/handler"
"github.com/zitadel/zitadel/internal/eventstore/handler/crdb"
"github.com/zitadel/zitadel/internal/repository/instance"
"github.com/zitadel/zitadel/internal/repository/milestone"
"github.com/zitadel/zitadel/internal/repository/project"
"github.com/zitadel/zitadel/internal/repository/user"
)
const (
MilestonesProjectionTable = "projections.milestones"
MilestoneColumnInstanceID = "instance_id"
MilestoneColumnType = "type"
MilestoneColumnPrimaryDomain = "primary_domain"
MilestoneColumnReachedDate = "reached_date"
MilestoneColumnPushedDate = "last_pushed_date"
MilestoneColumnIgnoreClientIDs = "ignore_client_ids"
)
type milestoneProjection struct {
crdb.StatementHandler
}
func newMilestoneProjection(ctx context.Context, config crdb.StatementHandlerConfig) *milestoneProjection {
p := new(milestoneProjection)
config.ProjectionName = MilestonesProjectionTable
config.Reducers = p.reducers()
config.InitCheck = crdb.NewMultiTableCheck(
crdb.NewTable([]*crdb.Column{
crdb.NewColumn(MilestoneColumnInstanceID, crdb.ColumnTypeText),
crdb.NewColumn(MilestoneColumnType, crdb.ColumnTypeEnum),
crdb.NewColumn(MilestoneColumnReachedDate, crdb.ColumnTypeTimestamp, crdb.Nullable()),
crdb.NewColumn(MilestoneColumnPushedDate, crdb.ColumnTypeTimestamp, crdb.Nullable()),
crdb.NewColumn(MilestoneColumnPrimaryDomain, crdb.ColumnTypeText, crdb.Nullable()),
crdb.NewColumn(MilestoneColumnIgnoreClientIDs, crdb.ColumnTypeTextArray, crdb.Nullable()),
},
crdb.NewPrimaryKey(MilestoneColumnInstanceID, MilestoneColumnType),
),
)
p.StatementHandler = crdb.NewStatementHandler(ctx, config)
return p
}
func (p *milestoneProjection) reducers() []handler.AggregateReducer {
return []handler.AggregateReducer{
{
Aggregate: instance.AggregateType,
EventRedusers: []handler.EventReducer{
{
Event: instance.InstanceAddedEventType,
Reduce: p.reduceInstanceAdded,
},
{
Event: instance.InstanceDomainPrimarySetEventType,
Reduce: p.reduceInstanceDomainPrimarySet,
},
{
Event: instance.InstanceRemovedEventType,
Reduce: p.reduceInstanceRemoved,
},
},
},
{
Aggregate: project.AggregateType,
EventRedusers: []handler.EventReducer{
{
Event: project.ProjectAddedType,
Reduce: p.reduceProjectAdded,
},
{
Event: project.ApplicationAddedType,
Reduce: p.reduceApplicationAdded,
},
{
Event: project.OIDCConfigAddedType,
Reduce: p.reduceOIDCConfigAdded,
},
{
Event: project.APIConfigAddedType,
Reduce: p.reduceAPIConfigAdded,
},
},
},
{
Aggregate: user.AggregateType,
EventRedusers: []handler.EventReducer{
{
// user.UserTokenAddedType is not emitted on creation of personal access tokens
// PATs have no effect on milestone.AuthenticationSucceededOnApplication or milestone.AuthenticationSucceededOnInstance
Event: user.UserTokenAddedType,
Reduce: p.reduceUserTokenAdded,
},
},
},
{
Aggregate: milestone.AggregateType,
EventRedusers: []handler.EventReducer{
{
Event: milestone.PushedEventType,
Reduce: p.reduceMilestonePushed,
},
},
},
}
}
func (p *milestoneProjection) reduceInstanceAdded(event eventstore.Event) (*handler.Statement, error) {
e, err := assertEvent[*instance.InstanceAddedEvent](event)
if err != nil {
return nil, err
}
allTypes := milestone.AllTypes()
statements := make([]func(eventstore.Event) crdb.Exec, 0, len(allTypes))
for _, msType := range allTypes {
createColumns := []handler.Column{
handler.NewCol(MilestoneColumnInstanceID, e.Aggregate().InstanceID),
handler.NewCol(MilestoneColumnType, msType),
}
if msType == milestone.InstanceCreated {
createColumns = append(createColumns, handler.NewCol(MilestoneColumnReachedDate, event.CreationDate()))
}
statements = append(statements, crdb.AddCreateStatement(createColumns))
}
return crdb.NewMultiStatement(e, statements...), nil
}
func (p *milestoneProjection) reduceInstanceDomainPrimarySet(event eventstore.Event) (*handler.Statement, error) {
e, err := assertEvent[*instance.DomainPrimarySetEvent](event)
if err != nil {
return nil, err
}
return crdb.NewUpdateStatement(
e,
[]handler.Column{
handler.NewCol(MilestoneColumnPrimaryDomain, e.Domain),
},
[]handler.Condition{
handler.NewCond(MilestoneColumnInstanceID, e.Aggregate().InstanceID),
crdb.NewIsNullCond(MilestoneColumnPushedDate),
},
), nil
}
func (p *milestoneProjection) reduceProjectAdded(event eventstore.Event) (*handler.Statement, error) {
if _, err := assertEvent[*project.ProjectAddedEvent](event); err != nil {
return nil, err
}
return p.reduceReachedIfUserEventFunc(milestone.ProjectCreated)(event)
}
func (p *milestoneProjection) reduceApplicationAdded(event eventstore.Event) (*handler.Statement, error) {
if _, err := assertEvent[*project.ApplicationAddedEvent](event); err != nil {
return nil, err
}
return p.reduceReachedIfUserEventFunc(milestone.ApplicationCreated)(event)
}
func (p *milestoneProjection) reduceOIDCConfigAdded(event eventstore.Event) (*handler.Statement, error) {
e, err := assertEvent[*project.OIDCConfigAddedEvent](event)
if err != nil {
return nil, err
}
return p.reduceAppConfigAdded(e, e.ClientID)
}
func (p *milestoneProjection) reduceAPIConfigAdded(event eventstore.Event) (*handler.Statement, error) {
e, err := assertEvent[*project.APIConfigAddedEvent](event)
if err != nil {
return nil, err
}
return p.reduceAppConfigAdded(e, e.ClientID)
}
func (p *milestoneProjection) reduceUserTokenAdded(event eventstore.Event) (*handler.Statement, error) {
e, err := assertEvent[*user.UserTokenAddedEvent](event)
if err != nil {
return nil, err
}
statements := []func(eventstore.Event) crdb.Exec{
crdb.AddUpdateStatement(
[]handler.Column{
handler.NewCol(MilestoneColumnReachedDate, event.CreationDate()),
},
[]handler.Condition{
handler.NewCond(MilestoneColumnInstanceID, event.Aggregate().InstanceID),
handler.NewCond(MilestoneColumnType, milestone.AuthenticationSucceededOnInstance),
crdb.NewIsNullCond(MilestoneColumnReachedDate),
},
),
}
// We ignore authentications without app, for example JWT profile or PAT
if e.ApplicationID != "" {
statements = append(statements, crdb.AddUpdateStatement(
[]handler.Column{
handler.NewCol(MilestoneColumnReachedDate, event.CreationDate()),
},
[]handler.Condition{
handler.NewCond(MilestoneColumnInstanceID, event.Aggregate().InstanceID),
handler.NewCond(MilestoneColumnType, milestone.AuthenticationSucceededOnApplication),
crdb.Not(crdb.NewTextArrayContainsCond(MilestoneColumnIgnoreClientIDs, e.ApplicationID)),
crdb.NewIsNullCond(MilestoneColumnReachedDate),
},
))
}
return crdb.NewMultiStatement(e, statements...), nil
}
func (p *milestoneProjection) reduceInstanceRemoved(event eventstore.Event) (*handler.Statement, error) {
if _, err := assertEvent[*instance.InstanceRemovedEvent](event); err != nil {
return nil, err
}
return p.reduceReachedFunc(milestone.InstanceDeleted)(event)
}
func (p *milestoneProjection) reduceMilestonePushed(event eventstore.Event) (*handler.Statement, error) {
e, err := assertEvent[*milestone.PushedEvent](event)
if err != nil {
return nil, err
}
if e.MilestoneType != milestone.InstanceDeleted {
return crdb.NewUpdateStatement(
event,
[]handler.Column{
handler.NewCol(MilestoneColumnPushedDate, event.CreationDate()),
},
[]handler.Condition{
handler.NewCond(MilestoneColumnInstanceID, event.Aggregate().InstanceID),
handler.NewCond(MilestoneColumnType, e.MilestoneType),
},
), nil
}
return crdb.NewDeleteStatement(
event,
[]handler.Condition{
handler.NewCond(MilestoneColumnInstanceID, event.Aggregate().InstanceID),
},
), nil
}
func (p *milestoneProjection) reduceReachedIfUserEventFunc(msType milestone.Type) func(event eventstore.Event) (*handler.Statement, error) {
return func(event eventstore.Event) (*handler.Statement, error) {
if p.isSystemEvent(event) {
return crdb.NewNoOpStatement(event), nil
}
return p.reduceReachedFunc(msType)(event)
}
}
func (p *milestoneProjection) reduceReachedFunc(msType milestone.Type) func(event eventstore.Event) (*handler.Statement, error) {
return func(event eventstore.Event) (*handler.Statement, error) {
return crdb.NewUpdateStatement(event, []handler.Column{
handler.NewCol(MilestoneColumnReachedDate, event.CreationDate()),
},
[]handler.Condition{
handler.NewCond(MilestoneColumnInstanceID, event.Aggregate().InstanceID),
handler.NewCond(MilestoneColumnType, msType),
crdb.NewIsNullCond(MilestoneColumnReachedDate),
}), nil
}
}
func (p *milestoneProjection) reduceAppConfigAdded(event eventstore.Event, clientID string) (*handler.Statement, error) {
if !p.isSystemEvent(event) {
return crdb.NewNoOpStatement(event), nil
}
return crdb.NewUpdateStatement(
event,
[]handler.Column{
crdb.NewArrayAppendCol(MilestoneColumnIgnoreClientIDs, clientID),
},
[]handler.Condition{
handler.NewCond(MilestoneColumnInstanceID, event.Aggregate().InstanceID),
handler.NewCond(MilestoneColumnType, milestone.AuthenticationSucceededOnApplication),
crdb.NewIsNullCond(MilestoneColumnReachedDate),
},
), nil
}
func (p *milestoneProjection) isSystemEvent(event eventstore.Event) bool {
if userId, err := strconv.Atoi(event.EditorUser()); err == nil && userId > 0 {
return false
}
lowerEditorService := strings.ToLower(event.EditorService())
return lowerEditorService == "" ||
lowerEditorService == "system" ||
lowerEditorService == "system-api"
}

404
internal/query/projection/milestones_test.go Normal file
View File

@@ -0,0 +1,404 @@
package projection
import (
"testing"
"time"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/errors"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/eventstore/handler"
"github.com/zitadel/zitadel/internal/eventstore/repository"
"github.com/zitadel/zitadel/internal/repository/instance"
"github.com/zitadel/zitadel/internal/repository/milestone"
"github.com/zitadel/zitadel/internal/repository/project"
"github.com/zitadel/zitadel/internal/repository/user"
)
func TestMilestonesProjection_reduces(t *testing.T) {
type args struct {
event func(t *testing.T) eventstore.Event
}
now := time.Now()
tests := []struct {
name string
args args
reduce func(event eventstore.Event) (*handler.Statement, error)
want wantReduce
}{
{
name: "reduceInstanceAdded",
args: args{
event: getEvent(timedTestEvent(
repository.EventType(instance.InstanceAddedEventType),
instance.AggregateType,
[]byte(`{}`),
now,
), instance.InstanceAddedEventMapper),
},
reduce: (&milestoneProjection{}).reduceInstanceAdded,
want: wantReduce{
aggregateType: eventstore.AggregateType("instance"),
sequence: 15,
previousSequence: 10,
executer: &testExecuter{
executions: []execution{
{
expectedStmt: "INSERT INTO projections.milestones (instance_id, type, reached_date) VALUES ($1, $2, $3)",
expectedArgs: []interface{}{
"instance-id",
milestone.InstanceCreated,
now,
},
},
{
expectedStmt: "INSERT INTO projections.milestones (instance_id, type) VALUES ($1, $2)",
expectedArgs: []interface{}{
"instance-id",
milestone.AuthenticationSucceededOnInstance,
},
},
{
expectedStmt: "INSERT INTO projections.milestones (instance_id, type) VALUES ($1, $2)",
expectedArgs: []interface{}{
"instance-id",
milestone.ProjectCreated,
},
},
{
expectedStmt: "INSERT INTO projections.milestones (instance_id, type) VALUES ($1, $2)",
expectedArgs: []interface{}{
"instance-id",
milestone.ApplicationCreated,
},
},
{
expectedStmt: "INSERT INTO projections.milestones (instance_id, type) VALUES ($1, $2)",
expectedArgs: []interface{}{
"instance-id",
milestone.AuthenticationSucceededOnApplication,
},
},
{
expectedStmt: "INSERT INTO projections.milestones (instance_id, type) VALUES ($1, $2)",
expectedArgs: []interface{}{
"instance-id",
milestone.InstanceDeleted,
},
},
},
},
},
},
{
name: "reduceInstancePrimaryDomainSet",
args: args{
event: getEvent(testEvent(
repository.EventType(instance.InstanceDomainPrimarySetEventType),
instance.AggregateType,
[]byte(`{"domain": "my.domain"}`),
), instance.DomainPrimarySetEventMapper),
},
reduce: (&milestoneProjection{}).reduceInstanceDomainPrimarySet,
want: wantReduce{
aggregateType: eventstore.AggregateType("instance"),
sequence: 15,
previousSequence: 10,
executer: &testExecuter{
executions: []execution{
{
expectedStmt: "UPDATE projections.milestones SET primary_domain = $1 WHERE (instance_id = $2) AND (last_pushed_date IS NULL)",
expectedArgs: []interface{}{
"my.domain",
"instance-id",
},
},
},
},
},
},
{
name: "reduceProjectAdded",
args: args{
event: getEvent(timedTestEvent(
repository.EventType(project.ProjectAddedType),
project.AggregateType,
[]byte(`{}`),
now,
), project.ProjectAddedEventMapper),
},
reduce: (&milestoneProjection{}).reduceProjectAdded,
want: wantReduce{
aggregateType: eventstore.AggregateType("project"),
sequence: 15,
previousSequence: 10,
executer: &testExecuter{
executions: []execution{
{
expectedStmt: "UPDATE projections.milestones SET reached_date = $1 WHERE (instance_id = $2) AND (type = $3) AND (reached_date IS NULL)",
expectedArgs: []interface{}{
now,
"instance-id",
milestone.ProjectCreated,
},
},
},
},
},
},
{
name: "reduceApplicationAdded",
args: args{
event: getEvent(timedTestEvent(
repository.EventType(project.ApplicationAddedType),
project.AggregateType,
[]byte(`{}`),
now,
), project.ApplicationAddedEventMapper),
},
reduce: (&milestoneProjection{}).reduceApplicationAdded,
want: wantReduce{
aggregateType: eventstore.AggregateType("project"),
sequence: 15,
previousSequence: 10,
executer: &testExecuter{
executions: []execution{
{
expectedStmt: "UPDATE projections.milestones SET reached_date = $1 WHERE (instance_id = $2) AND (type = $3) AND (reached_date IS NULL)",
expectedArgs: []interface{}{
now,
"instance-id",
milestone.ApplicationCreated,
},
},
},
},
},
},
{
name: "reduceOIDCConfigAdded user event",
args: args{
event: getEvent(testEvent(
repository.EventType(project.OIDCConfigAddedType),
project.AggregateType,
[]byte(`{}`),
), project.OIDCConfigAddedEventMapper),
},
reduce: (&milestoneProjection{}).reduceOIDCConfigAdded,
want: wantReduce{
aggregateType: eventstore.AggregateType("project"),
sequence: 15,
previousSequence: 10,
executer: &testExecuter{},
},
},
{
name: "reduceOIDCConfigAdded system event",
args: args{
event: getEvent(toSystemEvent(testEvent(
repository.EventType(project.OIDCConfigAddedType),
project.AggregateType,
[]byte(`{"clientId": "client-id"}`),
)), project.OIDCConfigAddedEventMapper),
},
reduce: (&milestoneProjection{}).reduceOIDCConfigAdded,
want: wantReduce{
aggregateType: eventstore.AggregateType("project"),
sequence: 15,
previousSequence: 10,
executer: &testExecuter{
executions: []execution{
{
expectedStmt: "UPDATE projections.milestones SET ignore_client_ids = array_append(ignore_client_ids, $1) WHERE (instance_id = $2) AND (type = $3) AND (reached_date IS NULL)",
expectedArgs: []interface{}{
"client-id",
"instance-id",
milestone.AuthenticationSucceededOnApplication,
},
},
},
},
},
},
{
name: "reduceAPIConfigAdded user event",
args: args{
event: getEvent(testEvent(
repository.EventType(project.APIConfigAddedType),
project.AggregateType,
[]byte(`{}`),
), project.APIConfigAddedEventMapper),
},
reduce: (&milestoneProjection{}).reduceAPIConfigAdded,
want: wantReduce{
aggregateType: eventstore.AggregateType("project"),
sequence: 15,
previousSequence: 10,
executer: &testExecuter{},
},
},
{
name: "reduceAPIConfigAdded system event",
args: args{
event: getEvent(toSystemEvent(testEvent(
repository.EventType(project.APIConfigAddedType),
project.AggregateType,
[]byte(`{"clientId": "client-id"}`),
)), project.APIConfigAddedEventMapper),
},
reduce: (&milestoneProjection{}).reduceAPIConfigAdded,
want: wantReduce{
aggregateType: eventstore.AggregateType("project"),
sequence: 15,
previousSequence: 10,
executer: &testExecuter{
executions: []execution{
{
expectedStmt: "UPDATE projections.milestones SET ignore_client_ids = array_append(ignore_client_ids, $1) WHERE (instance_id = $2) AND (type = $3) AND (reached_date IS NULL)",
expectedArgs: []interface{}{
"client-id",
"instance-id",
milestone.AuthenticationSucceededOnApplication,
},
},
},
},
},
},
{
name: "reduceUserTokenAdded",
args: args{
event: getEvent(timedTestEvent(
repository.EventType(user.UserTokenAddedType),
user.AggregateType,
[]byte(`{"applicationId": "client-id"}`),
now,
), user.UserTokenAddedEventMapper),
},
reduce: (&milestoneProjection{}).reduceUserTokenAdded,
want: wantReduce{
aggregateType: eventstore.AggregateType("user"),
sequence: 15,
previousSequence: 10,
executer: &testExecuter{
// TODO: This can be optimized to only use one statement with OR
executions: []execution{
{
expectedStmt: "UPDATE projections.milestones SET reached_date = $1 WHERE (instance_id = $2) AND (type = $3) AND (reached_date IS NULL)",
expectedArgs: []interface{}{
now,
"instance-id",
milestone.AuthenticationSucceededOnInstance,
},
},
{
expectedStmt: "UPDATE projections.milestones SET reached_date = $1 WHERE (instance_id = $2) AND (type = $3) AND (NOT (ignore_client_ids @> $4)) AND (reached_date IS NULL)",
expectedArgs: []interface{}{
now,
"instance-id",
milestone.AuthenticationSucceededOnApplication,
database.StringArray{"client-id"},
},
},
},
},
},
},
{
name: "reduceInstanceRemoved",
args: args{
event: getEvent(timedTestEvent(
repository.EventType(instance.InstanceRemovedEventType),
instance.AggregateType,
[]byte(`{}`),
now,
), instance.InstanceRemovedEventMapper),
},
reduce: (&milestoneProjection{}).reduceInstanceRemoved,
want: wantReduce{
aggregateType: eventstore.AggregateType("instance"),
sequence: 15,
previousSequence: 10,
executer: &testExecuter{
executions: []execution{
{
expectedStmt: "UPDATE projections.milestones SET reached_date = $1 WHERE (instance_id = $2) AND (type = $3) AND (reached_date IS NULL)",
expectedArgs: []interface{}{
now,
"instance-id",
milestone.InstanceDeleted,
},
},
},
},
},
},
{
name: "reduceMilestonePushed normal milestone",
args: args{
event: getEvent(timedTestEvent(
repository.EventType(milestone.PushedEventType),
milestone.AggregateType,
[]byte(`{"type": "ProjectCreated"}`),
now,
), milestone.PushedEventMapper),
},
reduce: (&milestoneProjection{}).reduceMilestonePushed,
want: wantReduce{
aggregateType: eventstore.AggregateType("milestone"),
sequence: 15,
previousSequence: 10,
executer: &testExecuter{
executions: []execution{
{
expectedStmt: "UPDATE projections.milestones SET last_pushed_date = $1 WHERE (instance_id = $2) AND (type = $3)",
expectedArgs: []interface{}{
now,
"instance-id",
milestone.ProjectCreated,
},
},
},
},
},
},
{
name: "reduceMilestonePushed instance deleted milestone",
args: args{
event: getEvent(testEvent(
repository.EventType(milestone.PushedEventType),
milestone.AggregateType,
[]byte(`{"type": "InstanceDeleted"}`),
), milestone.PushedEventMapper),
},
reduce: (&milestoneProjection{}).reduceMilestonePushed,
want: wantReduce{
aggregateType: eventstore.AggregateType("milestone"),
sequence: 15,
previousSequence: 10,
executer: &testExecuter{
executions: []execution{
{
expectedStmt: "DELETE FROM projections.milestones WHERE (instance_id = $1)",
expectedArgs: []interface{}{
"instance-id",
},
},
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
event := baseEvent(t)
got, err := tt.reduce(event)
if !errors.IsErrorInvalidArgument(err) {
t.Errorf("no wrong event mapping: %v, got: %v", err, got)
}
event = tt.args.event(t)
got, err = tt.reduce(event)
assertReduce(t, got, err, MilestonesProjectionTable, tt.want)
})
}
}

6
internal/query/projection/projection.go
View File

@@ -64,8 +64,10 @@ var (
NotificationPolicyProjection *notificationPolicyProjection
NotificationsProjection interface{}
NotificationsQuotaProjection interface{}
TelemetryPusherProjection interface{}
DeviceAuthProjection *deviceAuthProjection
SessionProjection *sessionProjection
MilestoneProjection *milestoneProjection
)
type projection interface {
@@ -143,6 +145,7 @@ func Create(ctx context.Context, sqlClient *database.DB, es *eventstore.Eventsto
NotificationPolicyProjection = newNotificationPolicyProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["notification_policies"]))
DeviceAuthProjection = newDeviceAuthProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["device_auth"]))
SessionProjection = newSessionProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["sessions"]))
MilestoneProjection = newMilestoneProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["milestones"]))
newProjectionsList()
return nil
}
@@ -191,7 +194,7 @@ func applyCustomConfig(config crdb.StatementHandlerConfig, customConfig CustomCo
// as setup and start currently create them individually, we make sure we get the right one
// will be refactored when changing to new id based projections
//
// NotificationsProjection is not added here, because it does not statement based / has no proprietary projection table
// Event handlers NotificationsProjection, NotificationsQuotaProjection and NotificationsProjection are not added here, because they do not reduce to database statements
func newProjectionsList() {
projections = []projection{
OrgProjection,
@@ -240,5 +243,6 @@ func newProjectionsList() {
NotificationPolicyProjection,
DeviceAuthProjection,
SessionProjection,
MilestoneProjection,
}
}

21
internal/query/search_query.go
View File

@@ -66,6 +66,27 @@ func (q *NotNullQuery) comp() sq.Sqlizer {
return sq.NotEq{q.Column.identifier(): nil}
}
type IsNullQuery struct {
Column Column
}
func NewIsNullQuery(col Column) (*IsNullQuery, error) {
if col.isZero() {
return nil, ErrMissingColumn
}
return &IsNullQuery{
Column: col,
}, nil
}
func (q *IsNullQuery) toQuery(query sq.SelectBuilder) sq.SelectBuilder {
return query.Where(q.comp())
}
func (q *IsNullQuery) comp() sq.Sqlizer {
return sq.Eq{q.Column.identifier(): nil}
}
type orQuery struct {
queries []SearchQuery
}