2020-09-24 06:52:10 +00:00
|
|
|
package eventstore
|
|
|
|
|
|
|
|
import (
|
2023-10-19 15:21:31 +00:00
|
|
|
"context"
|
2022-05-19 11:44:16 +00:00
|
|
|
"database/sql"
|
2022-09-02 14:05:13 +00:00
|
|
|
"time"
|
2022-05-19 11:44:16 +00:00
|
|
|
|
2023-10-19 15:21:31 +00:00
|
|
|
"github.com/zitadel/zitadel/internal/api/authz"
|
2023-12-08 14:30:55 +00:00
|
|
|
"github.com/zitadel/zitadel/internal/zerrors"
|
2020-09-24 06:52:10 +00:00
|
|
|
)
|
|
|
|
|
2022-08-31 07:52:43 +00:00
|
|
|
// SearchQueryBuilder represents the builder for your filter
|
2020-10-06 19:28:09 +00:00
|
|
|
// if invalid data are set the filter will fail
|
2020-12-01 13:44:19 +00:00
|
|
|
type SearchQueryBuilder struct {
|
2023-10-19 10:19:10 +00:00
|
|
|
columns Columns
|
|
|
|
limit uint64
|
2023-12-21 10:40:51 +00:00
|
|
|
offset uint32
|
2023-10-19 10:19:10 +00:00
|
|
|
desc bool
|
|
|
|
resourceOwner string
|
|
|
|
instanceID *string
|
2024-01-17 10:16:48 +00:00
|
|
|
instanceIDs []string
|
2023-10-19 10:19:10 +00:00
|
|
|
editorUser string
|
|
|
|
queries []*SearchQuery
|
|
|
|
tx *sql.Tx
|
|
|
|
allowTimeTravel bool
|
2024-09-24 16:43:29 +00:00
|
|
|
positionAfter float64
|
2023-10-19 10:19:10 +00:00
|
|
|
awaitOpenTransactions bool
|
|
|
|
creationDateAfter time.Time
|
2023-11-03 14:52:48 +00:00
|
|
|
creationDateBefore time.Time
|
2023-10-19 10:19:10 +00:00
|
|
|
eventSequenceGreater uint64
|
|
|
|
}
|
|
|
|
|
|
|
|
func (b *SearchQueryBuilder) GetColumns() Columns {
|
|
|
|
return b.columns
|
|
|
|
}
|
|
|
|
|
|
|
|
func (b *SearchQueryBuilder) GetLimit() uint64 {
|
|
|
|
return b.limit
|
|
|
|
}
|
|
|
|
|
2023-12-21 10:40:51 +00:00
|
|
|
func (b *SearchQueryBuilder) GetOffset() uint32 {
|
2023-12-01 12:25:41 +00:00
|
|
|
return b.offset
|
|
|
|
}
|
|
|
|
|
2023-10-19 10:19:10 +00:00
|
|
|
func (b *SearchQueryBuilder) GetDesc() bool {
|
|
|
|
return b.desc
|
|
|
|
}
|
|
|
|
|
|
|
|
func (b *SearchQueryBuilder) GetResourceOwner() string {
|
|
|
|
return b.resourceOwner
|
|
|
|
}
|
|
|
|
|
|
|
|
func (b *SearchQueryBuilder) GetInstanceID() *string {
|
|
|
|
return b.instanceID
|
|
|
|
}
|
|
|
|
|
2024-01-17 10:16:48 +00:00
|
|
|
func (b *SearchQueryBuilder) GetInstanceIDs() []string {
|
|
|
|
return b.instanceIDs
|
|
|
|
}
|
|
|
|
|
2023-10-19 10:19:10 +00:00
|
|
|
func (b *SearchQueryBuilder) GetEditorUser() string {
|
|
|
|
return b.editorUser
|
|
|
|
}
|
|
|
|
|
|
|
|
func (b *SearchQueryBuilder) GetQueries() []*SearchQuery {
|
|
|
|
return b.queries
|
|
|
|
}
|
|
|
|
|
|
|
|
func (b *SearchQueryBuilder) GetTx() *sql.Tx {
|
|
|
|
return b.tx
|
|
|
|
}
|
|
|
|
|
|
|
|
func (b *SearchQueryBuilder) GetAllowTimeTravel() bool {
|
|
|
|
return b.allowTimeTravel
|
|
|
|
}
|
|
|
|
|
2024-09-24 16:43:29 +00:00
|
|
|
func (b SearchQueryBuilder) GetPositionAfter() float64 {
|
|
|
|
return b.positionAfter
|
2023-10-19 10:19:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (b SearchQueryBuilder) GetAwaitOpenTransactions() bool {
|
|
|
|
return b.awaitOpenTransactions
|
|
|
|
}
|
|
|
|
|
|
|
|
func (q SearchQueryBuilder) GetEventSequenceGreater() uint64 {
|
|
|
|
return q.eventSequenceGreater
|
|
|
|
}
|
|
|
|
|
|
|
|
func (q SearchQueryBuilder) GetCreationDateAfter() time.Time {
|
|
|
|
return q.creationDateAfter
|
2021-07-06 11:55:57 +00:00
|
|
|
}
|
|
|
|
|
2023-11-03 14:52:48 +00:00
|
|
|
func (q SearchQueryBuilder) GetCreationDateBefore() time.Time {
|
|
|
|
return q.creationDateBefore
|
|
|
|
}
|
|
|
|
|
2023-10-19 15:21:31 +00:00
|
|
|
// ensureInstanceID makes sure that the instance id is always set
|
|
|
|
func (b *SearchQueryBuilder) ensureInstanceID(ctx context.Context) {
|
2024-01-17 10:16:48 +00:00
|
|
|
if b.instanceID == nil && len(b.instanceIDs) == 0 && authz.GetInstance(ctx).InstanceID() != "" {
|
2023-10-19 15:21:31 +00:00
|
|
|
b.InstanceID(authz.GetInstance(ctx).InstanceID())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-07-06 11:55:57 +00:00
|
|
|
type SearchQuery struct {
|
2023-10-19 10:19:10 +00:00
|
|
|
builder *SearchQueryBuilder
|
|
|
|
aggregateTypes []AggregateType
|
|
|
|
aggregateIDs []string
|
|
|
|
eventTypes []EventType
|
|
|
|
eventData map[string]interface{}
|
perf(oidc): nest position clause for session terminated query (#8738)
# Which Problems Are Solved
Optimize the query that checks for terminated sessions in the access
token verifier. The verifier is used in auth middleware, userinfo and
introspection.
# How the Problems Are Solved
The previous implementation built a query for certain events and then
appended a single `PositionAfter` clause. This caused the postgreSQL
planner to use indexes only for the instance ID, aggregate IDs,
aggregate types and event types. Followed by an expensive sequential
scan for the position. This resulting in internal over-fetching of rows
before the final filter was applied.
![Screenshot_20241007_105803](https://github.com/user-attachments/assets/f2d91976-be87-428b-b604-a211399b821c)
Furthermore, the query was searching for events which are not always
applicable. For example, there was always a session ID search and if
there was a user ID, we would also search for a browser fingerprint in
event payload (expensive). Even if those argument string would be empty.
This PR changes:
1. Nest the position query, so that a full `instance_id, aggregate_id,
aggregate_type, event_type, "position"` index can be matched.
2. Redefine the `es_wm` index to include the `position` column.
3. Only search for events for the IDs that actually have a value. Do not
search (noop) if none of session ID, user ID or fingerpint ID are set.
New query plan:
![Screenshot_20241007_110648](https://github.com/user-attachments/assets/c3234c33-1b76-4b33-a4a9-796f69f3d775)
# Additional Changes
- cleanup how we load multi-statement migrations and make that a bit
more reusable.
# Additional Context
- Related to https://github.com/zitadel/zitadel/issues/7639
2024-10-07 12:49:55 +00:00
|
|
|
positionAfter float64
|
2023-10-19 10:19:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (q SearchQuery) GetAggregateTypes() []AggregateType {
|
|
|
|
return q.aggregateTypes
|
|
|
|
}
|
|
|
|
|
|
|
|
func (q SearchQuery) GetAggregateIDs() []string {
|
|
|
|
return q.aggregateIDs
|
|
|
|
}
|
|
|
|
|
|
|
|
func (q SearchQuery) GetEventTypes() []EventType {
|
|
|
|
return q.eventTypes
|
|
|
|
}
|
|
|
|
|
|
|
|
func (q SearchQuery) GetEventData() map[string]interface{} {
|
|
|
|
return q.eventData
|
2020-09-24 06:52:10 +00:00
|
|
|
}
|
|
|
|
|
perf(oidc): nest position clause for session terminated query (#8738)
# Which Problems Are Solved
Optimize the query that checks for terminated sessions in the access
token verifier. The verifier is used in auth middleware, userinfo and
introspection.
# How the Problems Are Solved
The previous implementation built a query for certain events and then
appended a single `PositionAfter` clause. This caused the postgreSQL
planner to use indexes only for the instance ID, aggregate IDs,
aggregate types and event types. Followed by an expensive sequential
scan for the position. This resulting in internal over-fetching of rows
before the final filter was applied.
![Screenshot_20241007_105803](https://github.com/user-attachments/assets/f2d91976-be87-428b-b604-a211399b821c)
Furthermore, the query was searching for events which are not always
applicable. For example, there was always a session ID search and if
there was a user ID, we would also search for a browser fingerprint in
event payload (expensive). Even if those argument string would be empty.
This PR changes:
1. Nest the position query, so that a full `instance_id, aggregate_id,
aggregate_type, event_type, "position"` index can be matched.
2. Redefine the `es_wm` index to include the `position` column.
3. Only search for events for the IDs that actually have a value. Do not
search (noop) if none of session ID, user ID or fingerpint ID are set.
New query plan:
![Screenshot_20241007_110648](https://github.com/user-attachments/assets/c3234c33-1b76-4b33-a4a9-796f69f3d775)
# Additional Changes
- cleanup how we load multi-statement migrations and make that a bit
more reusable.
# Additional Context
- Related to https://github.com/zitadel/zitadel/issues/7639
2024-10-07 12:49:55 +00:00
|
|
|
func (q SearchQuery) GetPositionAfter() float64 {
|
|
|
|
return q.positionAfter
|
|
|
|
}
|
|
|
|
|
2020-10-06 19:28:09 +00:00
|
|
|
// Columns defines which fields of the event are needed for the query
|
2023-10-19 10:19:10 +00:00
|
|
|
type Columns int8
|
2020-09-24 06:52:10 +00:00
|
|
|
|
|
|
|
const (
|
2020-10-06 19:28:09 +00:00
|
|
|
//ColumnsEvent represents all fields of an event
|
2023-10-19 10:19:10 +00:00
|
|
|
ColumnsEvent = iota + 1
|
2024-09-24 16:43:29 +00:00
|
|
|
// ColumnsMaxSequence represents the latest sequence of the filtered events
|
|
|
|
ColumnsMaxSequence
|
2022-07-22 10:08:39 +00:00
|
|
|
// ColumnsInstanceIDs represents the instance ids of the filtered events
|
2023-10-19 10:19:10 +00:00
|
|
|
ColumnsInstanceIDs
|
2020-09-24 06:52:10 +00:00
|
|
|
|
2023-10-19 10:19:10 +00:00
|
|
|
columnsCount
|
|
|
|
)
|
2020-10-06 19:28:09 +00:00
|
|
|
|
2023-10-19 10:19:10 +00:00
|
|
|
func (c Columns) Validate() error {
|
|
|
|
if c <= 0 || c >= columnsCount {
|
2023-12-08 14:30:55 +00:00
|
|
|
return zerrors.ThrowPreconditionFailed(nil, "REPOS-x8R35", "column out of range")
|
2023-10-19 10:19:10 +00:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
2020-09-30 08:00:05 +00:00
|
|
|
|
feat: jobs for projection tables (#1730)
* job queue
* wg improvements
* start handler
* statement
* statements
* imporve handler
* improve statement
* statement in seperate file
* move handlers
* move query/old to query
* handler
* read models
* bulk works
* cleanup
* contrib
* rename readmodel to projection
* rename read_models schema to projections
* rename read_models schema to projections
* search query as func,
bulk iterates as long as new events
* add event sequence less query
* update checks for events between current sequence and sequence of first statement if it has previous sequence 0
* cleanup crdb projection
* refactor projection handler
* start with testing
* tests for handler
* remove todo
* refactor statement: remove table name,
add tests
* improve projection handler shutdown,
no savepoint if noop stmt,
tests for stmt handler
* tests
* start failed events
* seperate branch for contrib
* move statement constructors to crdb pkg
* correct import
* Subscribe for eventtypes (#1800)
* fix: is default (#1737)
* fix: use email as username on global org (#1738)
* fix: use email as username on global org
* Update user_human.go
* Update register_handler.go
* chore(deps): update docusaurus (#1739)
* chore: remove PAT and use GH Token (#1716)
* chore: remove PAT and use GH Token
* fix env
* fix env
* fix env
* md lint
* trigger ci
* change user
* fix GH bug
* replace login part
* chore: add GH Token to sem rel (#1746)
* chore: add GH Token to sem rel
* try branch
* add GH Token
* remove test branch again
* docs: changes acme to acme-caos (#1744)
* changes acme to acme-caos
* Apply suggestions from code review
Co-authored-by: Florian Forster <florian@caos.ch>
Co-authored-by: Maximilian Panne <maximilian.panne@gmail.com>
Co-authored-by: Florian Forster <florian@caos.ch>
* feat: add additional origins on applications (#1691)
* feat: add additional origins on applications
* app additional redirects
* chore(deps-dev): bump @angular/cli from 11.2.8 to 11.2.11 in /console (#1706)
* fix: show org with regex (#1688)
* fix: flag mapping (#1699)
* chore(deps-dev): bump @angular/cli from 11.2.8 to 11.2.11 in /console
Bumps [@angular/cli](https://github.com/angular/angular-cli) from 11.2.8 to 11.2.11.
- [Release notes](https://github.com/angular/angular-cli/releases)
- [Commits](https://github.com/angular/angular-cli/compare/v11.2.8...v11.2.11)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: Max Peintner <max@caos.ch>
Co-authored-by: Silvan <silvan.reusser@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* chore(deps-dev): bump stylelint from 13.10.0 to 13.13.1 in /console (#1703)
* fix: show org with regex (#1688)
* fix: flag mapping (#1699)
* chore(deps-dev): bump stylelint from 13.10.0 to 13.13.1 in /console
Bumps [stylelint](https://github.com/stylelint/stylelint) from 13.10.0 to 13.13.1.
- [Release notes](https://github.com/stylelint/stylelint/releases)
- [Changelog](https://github.com/stylelint/stylelint/blob/master/CHANGELOG.md)
- [Commits](https://github.com/stylelint/stylelint/compare/13.10.0...13.13.1)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: Max Peintner <max@caos.ch>
Co-authored-by: Silvan <silvan.reusser@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* chore(deps-dev): bump @types/node from 14.14.37 to 15.0.1 in /console (#1702)
* fix: show org with regex (#1688)
* fix: flag mapping (#1699)
* chore(deps-dev): bump @types/node from 14.14.37 to 15.0.1 in /console
Bumps [@types/node](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/node) from 14.14.37 to 15.0.1.
- [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases)
- [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/node)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: Max Peintner <max@caos.ch>
Co-authored-by: Silvan <silvan.reusser@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* chore(deps): bump ts-protoc-gen from 0.14.0 to 0.15.0 in /console (#1701)
* fix: show org with regex (#1688)
* fix: flag mapping (#1699)
* chore(deps): bump ts-protoc-gen from 0.14.0 to 0.15.0 in /console
Bumps [ts-protoc-gen](https://github.com/improbable-eng/ts-protoc-gen) from 0.14.0 to 0.15.0.
- [Release notes](https://github.com/improbable-eng/ts-protoc-gen/releases)
- [Changelog](https://github.com/improbable-eng/ts-protoc-gen/blob/master/CHANGELOG.md)
- [Commits](https://github.com/improbable-eng/ts-protoc-gen/compare/0.14.0...0.15.0)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: Max Peintner <max@caos.ch>
Co-authored-by: Silvan <silvan.reusser@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* chore(deps-dev): bump @types/jasmine from 3.6.9 to 3.6.10 in /console (#1682)
Bumps [@types/jasmine](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/jasmine) from 3.6.9 to 3.6.10.
- [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases)
- [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/jasmine)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* chore(deps): bump @types/google-protobuf in /console (#1681)
Bumps [@types/google-protobuf](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/google-protobuf) from 3.7.4 to 3.15.2.
- [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases)
- [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/google-protobuf)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* chore(deps): bump grpc from 1.24.5 to 1.24.7 in /console (#1666)
Bumps [grpc](https://github.com/grpc/grpc-node) from 1.24.5 to 1.24.7.
- [Release notes](https://github.com/grpc/grpc-node/releases)
- [Commits](https://github.com/grpc/grpc-node/compare/grpc@1.24.5...grpc@1.24.7)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* lock
* chore(deps-dev): bump @angular/language-service from 11.2.9 to 11.2.12 in /console (#1704)
* fix: show org with regex (#1688)
* fix: flag mapping (#1699)
* chore(deps-dev): bump @angular/language-service in /console
Bumps [@angular/language-service](https://github.com/angular/angular/tree/HEAD/packages/language-service) from 11.2.9 to 11.2.12.
- [Release notes](https://github.com/angular/angular/releases)
- [Changelog](https://github.com/angular/angular/blob/master/CHANGELOG.md)
- [Commits](https://github.com/angular/angular/commits/11.2.12/packages/language-service)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: Max Peintner <max@caos.ch>
Co-authored-by: Silvan <silvan.reusser@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* package lock
* downgrade grpc
* downgrade protobuf types
* revert npm packs 🥸
Co-authored-by: Max Peintner <max@caos.ch>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Silvan <silvan.reusser@gmail.com>
* docs: update run and start section texts (#1745)
* update run and start section texts
* adds showcase
Co-authored-by: Maximilian Panne <maximilian.panne@gmail.com>
* fix: additional origin list (#1753)
* fix: handle api configs in authz handler (#1755)
* fix(console): add model for api keys, fix toast, binding (#1757)
* fix: add model for api keys, fix toast, binding
* show api clientid
* fix: missing patchvalue (#1758)
* feat: refresh token (#1728)
* begin refresh tokens
* refresh tokens
* list and revoke refresh tokens
* handle remove
* tests for refresh tokens
* uniqueness and default expiration
* rename oidc token methods
* cleanup
* migration version
* Update internal/static/i18n/en.yaml
Co-authored-by: Fabi <38692350+fgerschwiler@users.noreply.github.com>
* fixes
* feat: update oidc pkg for refresh tokens
Co-authored-by: Fabi <38692350+fgerschwiler@users.noreply.github.com>
* fix: correct json name of clientId in key.json (#1760)
* fix: migration version (#1767)
* start subscription
* eventtypes
* fix(login): links (#1778)
* fix(login): href for help
* fix(login): correct link to tos
* fix: access tokens for service users and refresh token infos (#1779)
* fix: access token for service user
* handle info from refresh request
* uniqueness
* postpone access token uniqueness change
* chore(coc): recommend code of conduct (#1782)
* subscribe for events
* feat(console): refresh toggle out of granttype context (#1785)
* refresh toggle
* disable if not code flow, lint
* lint
* fix: change oidc config order
* accept refresh option within flow
Co-authored-by: Livio Amstutz <livio.a@gmail.com>
* fix: refresh token activation (#1795)
* fix: oidc grant type check
* docs: add offline_access scope
* docs: update refresh token status in supported grant types
* fix: update oidc pkg
* fix: check refresh token grant type (#1796)
* configuration structs
* org admins
* failed events
* fixes
Co-authored-by: Max Peintner <max@caos.ch>
Co-authored-by: Livio Amstutz <livio.a@gmail.com>
Co-authored-by: Florian Forster <florian@caos.ch>
Co-authored-by: mffap <mpa@caos.ch>
Co-authored-by: Maximilian Panne <maximilian.panne@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Fabi <38692350+fgerschwiler@users.noreply.github.com>
* remove comment
* aggregate reducer
* remove eventtypes
* add protoc-get-validate to mod
* fix transaltion
* upsert
* add gender on org admins,
allow to retry failed stmts after configurable time
* remove if
* sub queries
* fix: tests
* add builder to tests
* new search query
* rename searchquerybuilder to builder
* remove comment from code
* test with multiple queries
* add filters test
* current sequences
* make org and org_admins work again
* add aggregate type to current sequence
* fix(contibute): listing
* add validate module
* fix: search queries
* feat(eventstore): previous aggregate root sequence (#1810)
* feat(eventstore): previous aggregate root sequence
* fix tests
* fix: eventstore v1 test
* add col to all mocked rows
* next try
* fix mig
* rename aggregate root to aggregate type
* update comment
Co-authored-by: Livio Amstutz <livio.a@gmail.com>
Co-authored-by: Livio Amstutz <livio.a@gmail.com>
* small refactorings
* allow update multiple current sequences
* unique log id
* fix migrations
* rename org admin to org owner
* improve error handling and logging
* fix(migration): optimize prev agg root seq
* fix: projection handler test
* fix: sub queries
* small fixes
* additional event types
* correct org owner projection
* fix primary key
* feat(eventstore): jobs for projections (#2026)
* fix: template names in login (#1974)
* fix: template names in login
* fix: error.html
* fix: check for features on mgmt only (#1976)
* fix: add sentry in ui, http and projection handlers (#1977)
* fix: add sentry in ui, http and projection handlers
* fix test
* fix(eventstore): sub queries (#1805)
* sub queries
* fix: tests
* add builder to tests
* new search query
* rename searchquerybuilder to builder
* remove comment from code
* test with multiple queries
* add filters test
* fix(contibute): listing
* add validate module
* fix: search queries
* remove unused event type in query
* ignore query if error in marshal
* go mod tidy
* update privacy policy query
* update queries
Co-authored-by: Livio Amstutz <livio.a@gmail.com>
* feat: Extend oidc idp with oauth endpoints (#1980)
* feat: add oauth attributes to oidc idp configuration
* feat: return idpconfig id on create idp
* feat: tests
* feat: descriptions
* feat: docs
* feat: tests
* docs: update to beta 3 (#1984)
* fix: role assertion (#1986)
* fix: enum to display access token role assertion
* improve assertion descriptions
* fix nil pointer
* docs: eventstore (#1982)
* docs: eventstore
* Apply suggestions from code review
Co-authored-by: Florian Forster <florian@caos.ch>
Co-authored-by: Florian Forster <florian@caos.ch>
* fix(sentry): trigger sentry release (#1989)
* feat(send sentry release): send sentry release
* fix(moved step and added releasetag): moved step and added releasetag
* fix: set version for sentry release (#1990)
* feat(send sentry release): send sentry release
* fix(moved step and added releasetag): moved step and added releasetag
* fix(corrected var name): corrected var name
Co-authored-by: Livio Amstutz <livio.a@gmail.com>
* fix: log error reason on terminate session (#1973)
* fix: return default language file, if requested lang does not exist for default login texts (#1988)
* fix: return default language file, if requested lang doesnt exists
* feat: read default translation file
* feat: docs
* fix: race condition in auth request unmarshalling (#1993)
* feat: handle ui_locales in login (#1994)
* fix: handle ui_locales in login
* move supportedlanguage func into i18n package
* update oidc pkg
* fix: handle closed channels on unsubscribe (#1995)
* fix: give restore more time (#1997)
* fix: translation file read (#2009)
* feat: translation file read
* feat: readme
* fix: enable idp add button for iam users (#2010)
* fix: filter event_data (#2011)
* feat: Custom message files (#1992)
* feat: add get custom message text to admin api
* feat: read custom message texts from files
* feat: get languages in apis
* feat: get languages in apis
* feat: get languages in apis
* feat: pr feedback
* feat: docs
* feat: merge main
* fix: sms notification (#2013)
* fix: phone verifications
* feat: fix password reset as sms
* fix: phone verification
* fix: grpc status in sentry and validation interceptors (#2012)
* fix: remove oauth endpoints from oidc config proto (#2014)
* try with view
* fix(console): disable sw (#2021)
* fix: disable sw
* angular.json disable sw
* project projections
* fix typos
* customize projections
* customizable projections,
add change date to projects
Co-authored-by: Livio Amstutz <livio.a@gmail.com>
Co-authored-by: Max Peintner <max@caos.ch>
Co-authored-by: Fabi <38692350+fgerschwiler@users.noreply.github.com>
Co-authored-by: Florian Forster <florian@caos.ch>
Co-authored-by: mffap <mpa@caos.ch>
Co-authored-by: Christian Jakob <47860090+thesephirot@users.noreply.github.com>
Co-authored-by: Elio Bischof <eliobischof@gmail.com>
* env file
* typo
* correct users
* correct migration
* fix: merge fail
* fix test
* fix(tests): unordered matcher
* improve currentSequenceMatcher
* correct certs
* correct certs
* add zitadel database on database list
* refctor switch in match
* enable all handlers
* Delete io.env
* cleanup
* add handlers
* rename view to projection
* rename view to projection
* fix type typo
* remove unnecessary logs
* refactor stmts
* simplify interval calculation
* fix tests
* fix unlock test
* fix migration
* migs
* fix(operator): update cockroach and flyway versions (#2138)
* chore(deps): bump k8s.io/apiextensions-apiserver from 0.19.2 to 0.21.3
Bumps [k8s.io/apiextensions-apiserver](https://github.com/kubernetes/apiextensions-apiserver) from 0.19.2 to 0.21.3.
- [Release notes](https://github.com/kubernetes/apiextensions-apiserver/releases)
- [Commits](https://github.com/kubernetes/apiextensions-apiserver/compare/v0.19.2...v0.21.3)
---
updated-dependencies:
- dependency-name: k8s.io/apiextensions-apiserver
dependency-type: direct:production
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot] <support@github.com>
* chore(deps): bump google.golang.org/api from 0.34.0 to 0.52.0
Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.34.0 to 0.52.0.
- [Release notes](https://github.com/googleapis/google-api-go-client/releases)
- [Changelog](https://github.com/googleapis/google-api-go-client/blob/master/CHANGES.md)
- [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.34.0...v0.52.0)
---
updated-dependencies:
- dependency-name: google.golang.org/api
dependency-type: direct:production
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot] <support@github.com>
* start update dependencies
* update mods and otlp
* fix(build): update to go 1.16
* old version for k8s mods
* update k8s versions
* update orbos
* fix(operator): update cockroach and flyway version
* Update images.go
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Stefan Benz <stefan@caos.ch>
* fix import
* fix typo
* fix(migration): add org projection
* fix(projection): correct table for org events in org owners
* better insert stmt
* fix typo
* fix typo
* set max connection lifetime
* set max conns and conn lifetime in eventstore v1
* configure sql connection settings
* add mig for agg type index
* fix replace tab in yaml
* check requeue at least 500ms
* split column in column and condition
* remove useless comment
* mig versions
* fix migs
Co-authored-by: Max Peintner <max@caos.ch>
Co-authored-by: Livio Amstutz <livio.a@gmail.com>
Co-authored-by: Florian Forster <florian@caos.ch>
Co-authored-by: mffap <mpa@caos.ch>
Co-authored-by: Maximilian Panne <maximilian.panne@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Fabi <38692350+fgerschwiler@users.noreply.github.com>
Co-authored-by: Christian Jakob <47860090+thesephirot@users.noreply.github.com>
Co-authored-by: Elio Bischof <eliobischof@gmail.com>
Co-authored-by: Stefan Benz <stefan@caos.ch>
2021-08-19 06:31:56 +00:00
|
|
|
// NewSearchQueryBuilder creates a new builder for event filters
|
2020-10-06 19:28:09 +00:00
|
|
|
// aggregateTypes must contain at least one aggregate type
|
2021-07-06 11:55:57 +00:00
|
|
|
func NewSearchQueryBuilder(columns Columns) *SearchQueryBuilder {
|
2020-12-01 13:44:19 +00:00
|
|
|
return &SearchQueryBuilder{
|
2023-10-19 10:19:10 +00:00
|
|
|
columns: columns,
|
2020-09-24 06:52:10 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-12-01 12:25:41 +00:00
|
|
|
func (builder *SearchQueryBuilder) Matches(commands ...Command) []Command {
|
|
|
|
matches := make([]Command, 0, len(commands))
|
|
|
|
for i, command := range commands {
|
|
|
|
if builder.limit > 0 && builder.limit <= uint64(len(matches)) {
|
|
|
|
break
|
|
|
|
}
|
2023-12-21 10:40:51 +00:00
|
|
|
if builder.offset > 0 && uint32(i) < builder.offset {
|
2023-12-01 12:25:41 +00:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
if builder.matchCommand(command) {
|
|
|
|
matches = append(matches, command)
|
|
|
|
}
|
2022-03-28 08:05:09 +00:00
|
|
|
}
|
2023-12-01 12:25:41 +00:00
|
|
|
|
|
|
|
return matches
|
|
|
|
}
|
|
|
|
|
|
|
|
type sequencer interface {
|
|
|
|
Sequence() uint64
|
|
|
|
}
|
|
|
|
|
|
|
|
func (builder *SearchQueryBuilder) matchCommand(command Command) bool {
|
|
|
|
if builder.resourceOwner != "" && command.Aggregate().ResourceOwner != builder.resourceOwner {
|
2022-03-28 08:05:09 +00:00
|
|
|
return false
|
|
|
|
}
|
2023-12-01 12:25:41 +00:00
|
|
|
if command.Aggregate().InstanceID != "" && builder.instanceID != nil && *builder.instanceID != "" && command.Aggregate().InstanceID != *builder.instanceID {
|
2023-10-19 10:19:10 +00:00
|
|
|
return false
|
|
|
|
}
|
2023-12-01 12:25:41 +00:00
|
|
|
if seq, ok := command.(sequencer); ok {
|
|
|
|
if builder.eventSequenceGreater > 0 && seq.Sequence() <= builder.eventSequenceGreater {
|
|
|
|
return false
|
|
|
|
}
|
2022-03-28 08:05:09 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if len(builder.queries) == 0 {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
for _, query := range builder.queries {
|
2023-12-01 12:25:41 +00:00
|
|
|
if query.matches(command) {
|
2022-03-28 08:05:09 +00:00
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2022-08-31 07:52:43 +00:00
|
|
|
// Columns defines which fields are set
|
2022-03-28 08:05:09 +00:00
|
|
|
func (builder *SearchQueryBuilder) Columns(columns Columns) *SearchQueryBuilder {
|
2023-10-19 10:19:10 +00:00
|
|
|
builder.columns = columns
|
2022-03-28 08:05:09 +00:00
|
|
|
return builder
|
2020-09-24 06:52:10 +00:00
|
|
|
}
|
|
|
|
|
2022-08-31 07:52:43 +00:00
|
|
|
// Limit defines how many events are returned maximally.
|
2022-03-28 08:05:09 +00:00
|
|
|
func (builder *SearchQueryBuilder) Limit(limit uint64) *SearchQueryBuilder {
|
|
|
|
builder.limit = limit
|
|
|
|
return builder
|
2020-09-24 06:52:10 +00:00
|
|
|
}
|
|
|
|
|
2023-12-01 12:25:41 +00:00
|
|
|
// Limit defines how many events are returned maximally.
|
2023-12-21 10:40:51 +00:00
|
|
|
func (builder *SearchQueryBuilder) Offset(offset uint32) *SearchQueryBuilder {
|
2023-12-01 12:25:41 +00:00
|
|
|
builder.offset = offset
|
|
|
|
return builder
|
|
|
|
}
|
|
|
|
|
2024-01-17 10:16:48 +00:00
|
|
|
// ResourceOwner defines the resource owner (org or instance) of the events
|
2022-03-28 08:05:09 +00:00
|
|
|
func (builder *SearchQueryBuilder) ResourceOwner(resourceOwner string) *SearchQueryBuilder {
|
|
|
|
builder.resourceOwner = resourceOwner
|
|
|
|
return builder
|
2020-09-24 06:52:10 +00:00
|
|
|
}
|
|
|
|
|
2022-08-31 07:52:43 +00:00
|
|
|
// InstanceID defines the instanceID (system) of the events
|
2022-04-19 06:26:12 +00:00
|
|
|
func (builder *SearchQueryBuilder) InstanceID(instanceID string) *SearchQueryBuilder {
|
2023-10-19 10:19:10 +00:00
|
|
|
builder.instanceID = &instanceID
|
2022-04-19 06:26:12 +00:00
|
|
|
return builder
|
2022-03-15 06:19:02 +00:00
|
|
|
}
|
|
|
|
|
2024-01-17 10:16:48 +00:00
|
|
|
// InstanceIDs defines the instanceIDs (system) of the events
|
|
|
|
func (builder *SearchQueryBuilder) InstanceIDs(instanceIDs []string) *SearchQueryBuilder {
|
|
|
|
builder.instanceIDs = instanceIDs
|
|
|
|
return builder
|
|
|
|
}
|
|
|
|
|
2022-08-31 07:52:43 +00:00
|
|
|
// OrderDesc changes the sorting order of the returned events to descending
|
2022-03-28 08:05:09 +00:00
|
|
|
func (builder *SearchQueryBuilder) OrderDesc() *SearchQueryBuilder {
|
|
|
|
builder.desc = true
|
|
|
|
return builder
|
2020-09-24 06:52:10 +00:00
|
|
|
}
|
|
|
|
|
2022-08-31 07:52:43 +00:00
|
|
|
// OrderAsc changes the sorting order of the returned events to ascending
|
2022-03-28 08:05:09 +00:00
|
|
|
func (builder *SearchQueryBuilder) OrderAsc() *SearchQueryBuilder {
|
|
|
|
builder.desc = false
|
|
|
|
return builder
|
2020-09-24 06:52:10 +00:00
|
|
|
}
|
|
|
|
|
2022-08-31 07:52:43 +00:00
|
|
|
// SetTx ensures that the eventstore library uses the existing transaction
|
2022-05-19 11:44:16 +00:00
|
|
|
func (builder *SearchQueryBuilder) SetTx(tx *sql.Tx) *SearchQueryBuilder {
|
|
|
|
builder.tx = tx
|
|
|
|
return builder
|
|
|
|
}
|
|
|
|
|
2023-01-16 11:30:03 +00:00
|
|
|
func (builder *SearchQueryBuilder) EditorUser(id string) *SearchQueryBuilder {
|
|
|
|
builder.editorUser = id
|
|
|
|
return builder
|
|
|
|
}
|
|
|
|
|
2023-02-27 21:36:43 +00:00
|
|
|
// AllowTimeTravel activates the time travel feature of the database if supported
|
|
|
|
// The queries will be made based on the call time
|
|
|
|
func (builder *SearchQueryBuilder) AllowTimeTravel() *SearchQueryBuilder {
|
|
|
|
builder.allowTimeTravel = true
|
|
|
|
return builder
|
|
|
|
}
|
|
|
|
|
2023-10-19 10:19:10 +00:00
|
|
|
// PositionAfter filters for events which happened after the specified time
|
2024-09-24 16:43:29 +00:00
|
|
|
func (builder *SearchQueryBuilder) PositionAfter(position float64) *SearchQueryBuilder {
|
|
|
|
builder.positionAfter = position
|
2023-10-19 10:19:10 +00:00
|
|
|
return builder
|
|
|
|
}
|
|
|
|
|
|
|
|
// AwaitOpenTransactions filters for events which are older than the oldest transaction of the database
|
|
|
|
func (builder *SearchQueryBuilder) AwaitOpenTransactions() *SearchQueryBuilder {
|
|
|
|
builder.awaitOpenTransactions = true
|
|
|
|
return builder
|
|
|
|
}
|
|
|
|
|
|
|
|
// SequenceGreater filters for events with sequence greater the requested sequence
|
|
|
|
func (builder *SearchQueryBuilder) SequenceGreater(sequence uint64) *SearchQueryBuilder {
|
|
|
|
builder.eventSequenceGreater = sequence
|
|
|
|
return builder
|
|
|
|
}
|
|
|
|
|
|
|
|
// CreationDateAfter filters for events which happened after the specified time
|
|
|
|
func (builder *SearchQueryBuilder) CreationDateAfter(creationDate time.Time) *SearchQueryBuilder {
|
|
|
|
if creationDate.IsZero() || creationDate.Unix() == 0 {
|
|
|
|
return builder
|
|
|
|
}
|
|
|
|
builder.creationDateAfter = creationDate
|
|
|
|
return builder
|
|
|
|
}
|
|
|
|
|
2023-11-03 14:52:48 +00:00
|
|
|
// CreationDateBefore filters for events which happened before the specified time
|
|
|
|
func (builder *SearchQueryBuilder) CreationDateBefore(creationDate time.Time) *SearchQueryBuilder {
|
|
|
|
if creationDate.IsZero() || creationDate.Unix() == 0 {
|
|
|
|
return builder
|
|
|
|
}
|
|
|
|
builder.creationDateBefore = creationDate
|
|
|
|
return builder
|
|
|
|
}
|
|
|
|
|
2022-08-31 07:52:43 +00:00
|
|
|
// AddQuery creates a new sub query.
|
|
|
|
// All fields in the sub query are AND-connected in the storage request.
|
|
|
|
// Multiple sub queries are OR-connected in the storage request.
|
2022-03-28 08:05:09 +00:00
|
|
|
func (builder *SearchQueryBuilder) AddQuery() *SearchQuery {
|
2021-07-06 11:55:57 +00:00
|
|
|
query := &SearchQuery{
|
2022-03-28 08:05:09 +00:00
|
|
|
builder: builder,
|
2021-07-06 11:55:57 +00:00
|
|
|
}
|
2022-03-28 08:05:09 +00:00
|
|
|
builder.queries = append(builder.queries, query)
|
2021-07-06 11:55:57 +00:00
|
|
|
|
|
|
|
return query
|
|
|
|
}
|
|
|
|
|
2022-08-31 07:52:43 +00:00
|
|
|
// Or creates a new sub query on the search query builder
|
2021-07-06 11:55:57 +00:00
|
|
|
func (query SearchQuery) Or() *SearchQuery {
|
|
|
|
return query.builder.AddQuery()
|
|
|
|
}
|
|
|
|
|
2022-08-31 07:52:43 +00:00
|
|
|
// AggregateTypes filters for events with the given aggregate types
|
2021-07-06 11:55:57 +00:00
|
|
|
func (query *SearchQuery) AggregateTypes(types ...AggregateType) *SearchQuery {
|
|
|
|
query.aggregateTypes = types
|
|
|
|
return query
|
|
|
|
}
|
|
|
|
|
2022-08-31 07:52:43 +00:00
|
|
|
// AggregateIDs filters for events with the given aggregate id's
|
2021-07-06 11:55:57 +00:00
|
|
|
func (query *SearchQuery) AggregateIDs(ids ...string) *SearchQuery {
|
|
|
|
query.aggregateIDs = ids
|
|
|
|
return query
|
|
|
|
}
|
|
|
|
|
2022-08-31 07:52:43 +00:00
|
|
|
// EventTypes filters for events with the given event types
|
2021-07-06 11:55:57 +00:00
|
|
|
func (query *SearchQuery) EventTypes(types ...EventType) *SearchQuery {
|
|
|
|
query.eventTypes = types
|
|
|
|
return query
|
|
|
|
}
|
|
|
|
|
2022-08-31 07:52:43 +00:00
|
|
|
// EventData filters for events with the given event data.
|
|
|
|
// Use this call with care as it will be slower than the other filters.
|
2021-07-06 11:55:57 +00:00
|
|
|
func (query *SearchQuery) EventData(data map[string]interface{}) *SearchQuery {
|
|
|
|
query.eventData = data
|
|
|
|
return query
|
|
|
|
}
|
|
|
|
|
perf(oidc): nest position clause for session terminated query (#8738)
# Which Problems Are Solved
Optimize the query that checks for terminated sessions in the access
token verifier. The verifier is used in auth middleware, userinfo and
introspection.
# How the Problems Are Solved
The previous implementation built a query for certain events and then
appended a single `PositionAfter` clause. This caused the postgreSQL
planner to use indexes only for the instance ID, aggregate IDs,
aggregate types and event types. Followed by an expensive sequential
scan for the position. This resulting in internal over-fetching of rows
before the final filter was applied.
![Screenshot_20241007_105803](https://github.com/user-attachments/assets/f2d91976-be87-428b-b604-a211399b821c)
Furthermore, the query was searching for events which are not always
applicable. For example, there was always a session ID search and if
there was a user ID, we would also search for a browser fingerprint in
event payload (expensive). Even if those argument string would be empty.
This PR changes:
1. Nest the position query, so that a full `instance_id, aggregate_id,
aggregate_type, event_type, "position"` index can be matched.
2. Redefine the `es_wm` index to include the `position` column.
3. Only search for events for the IDs that actually have a value. Do not
search (noop) if none of session ID, user ID or fingerpint ID are set.
New query plan:
![Screenshot_20241007_110648](https://github.com/user-attachments/assets/c3234c33-1b76-4b33-a4a9-796f69f3d775)
# Additional Changes
- cleanup how we load multi-statement migrations and make that a bit
more reusable.
# Additional Context
- Related to https://github.com/zitadel/zitadel/issues/7639
2024-10-07 12:49:55 +00:00
|
|
|
func (query *SearchQuery) PositionAfter(position float64) *SearchQuery {
|
|
|
|
query.positionAfter = position
|
|
|
|
return query
|
|
|
|
}
|
|
|
|
|
2022-08-31 07:52:43 +00:00
|
|
|
// Builder returns the SearchQueryBuilder of the sub query
|
2021-07-06 11:55:57 +00:00
|
|
|
func (query *SearchQuery) Builder() *SearchQueryBuilder {
|
|
|
|
return query.builder
|
2020-11-23 18:31:12 +00:00
|
|
|
}
|
|
|
|
|
2023-12-01 12:25:41 +00:00
|
|
|
func (query *SearchQuery) matches(command Command) bool {
|
|
|
|
if ok := isAggregateTypes(command.Aggregate(), query.aggregateTypes...); len(query.aggregateTypes) > 0 && !ok {
|
2022-03-28 08:05:09 +00:00
|
|
|
return false
|
|
|
|
}
|
2023-12-01 12:25:41 +00:00
|
|
|
if ok := isAggregateIDs(command.Aggregate(), query.aggregateIDs...); len(query.aggregateIDs) > 0 && !ok {
|
2022-03-28 08:05:09 +00:00
|
|
|
return false
|
|
|
|
}
|
2023-12-01 12:25:41 +00:00
|
|
|
if ok := isEventTypes(command, query.eventTypes...); len(query.eventTypes) > 0 && !ok {
|
2022-03-28 08:05:09 +00:00
|
|
|
return false
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|