2022-03-23 09:02:39 +01:00
|
|
|
package setup
|
|
|
|
|
|
|
|
import (
|
2022-03-28 10:05:09 +02:00
|
|
|
"bytes"
|
2022-06-10 15:34:52 +02:00
|
|
|
"strings"
|
2022-12-09 13:04:33 +00:00
|
|
|
"time"
|
2022-03-28 10:05:09 +02:00
|
|
|
|
|
|
|
"github.com/mitchellh/mapstructure"
|
|
|
|
"github.com/spf13/viper"
|
2022-04-27 01:01:45 +02:00
|
|
|
"github.com/zitadel/logging"
|
2022-03-28 10:05:09 +02:00
|
|
|
|
2024-01-25 17:28:20 +01:00
|
|
|
"github.com/zitadel/zitadel/cmd/encryption"
|
2024-02-16 17:04:42 +01:00
|
|
|
"github.com/zitadel/zitadel/cmd/hooks"
|
2024-01-25 17:28:20 +01:00
|
|
|
"github.com/zitadel/zitadel/internal/actions"
|
2024-05-01 12:17:27 +02:00
|
|
|
internal_authz "github.com/zitadel/zitadel/internal/api/authz"
|
2024-01-25 17:28:20 +01:00
|
|
|
"github.com/zitadel/zitadel/internal/api/oidc"
|
|
|
|
"github.com/zitadel/zitadel/internal/api/ui/login"
|
2024-09-25 22:40:21 +03:00
|
|
|
"github.com/zitadel/zitadel/internal/cache"
|
2022-04-27 01:01:45 +02:00
|
|
|
"github.com/zitadel/zitadel/internal/command"
|
|
|
|
"github.com/zitadel/zitadel/internal/config/hook"
|
|
|
|
"github.com/zitadel/zitadel/internal/config/systemdefaults"
|
|
|
|
"github.com/zitadel/zitadel/internal/database"
|
2024-05-01 12:17:27 +02:00
|
|
|
"github.com/zitadel/zitadel/internal/domain"
|
2023-10-19 12:19:10 +02:00
|
|
|
"github.com/zitadel/zitadel/internal/eventstore"
|
2022-09-01 09:24:26 +02:00
|
|
|
"github.com/zitadel/zitadel/internal/id"
|
2024-01-25 17:28:20 +01:00
|
|
|
"github.com/zitadel/zitadel/internal/notification/handlers"
|
2022-11-04 10:21:58 +01:00
|
|
|
"github.com/zitadel/zitadel/internal/query/projection"
|
2024-01-25 17:28:20 +01:00
|
|
|
static_config "github.com/zitadel/zitadel/internal/static/config"
|
2022-03-23 09:02:39 +01:00
|
|
|
)
|
|
|
|
|
|
|
|
type Config struct {
|
2024-05-30 11:35:30 +02:00
|
|
|
ForMirror bool
|
2022-04-21 12:37:39 +02:00
|
|
|
Database database.Config
|
2024-09-25 22:40:21 +03:00
|
|
|
Caches *cache.CachesConfig
|
2022-04-21 12:37:39 +02:00
|
|
|
SystemDefaults systemdefaults.SystemDefaults
|
2024-05-01 12:17:27 +02:00
|
|
|
InternalAuthZ internal_authz.Config
|
2022-04-28 10:30:41 +02:00
|
|
|
ExternalDomain string
|
2022-04-21 12:37:39 +02:00
|
|
|
ExternalPort uint16
|
|
|
|
ExternalSecure bool
|
|
|
|
Log *logging.Config
|
2024-01-25 17:28:20 +01:00
|
|
|
EncryptionKeys *encryption.EncryptionKeyConfig
|
2022-04-21 12:37:39 +02:00
|
|
|
DefaultInstance command.InstanceSetup
|
2022-09-01 09:24:26 +02:00
|
|
|
Machine *id.Config
|
2022-11-04 10:21:58 +01:00
|
|
|
Projections projection.Config
|
2023-10-19 12:19:10 +02:00
|
|
|
Eventstore *eventstore.Config
|
2024-01-25 17:28:20 +01:00
|
|
|
|
|
|
|
InitProjections InitProjections
|
|
|
|
AssetStorage static_config.AssetStorageConfig
|
|
|
|
OIDC oidc.Config
|
|
|
|
Login login.Config
|
|
|
|
WebAuthNName string
|
|
|
|
Telemetry *handlers.TelemetryPusherConfig
|
2024-05-01 12:17:27 +02:00
|
|
|
SystemAPIUsers map[string]*internal_authz.SystemAPIUser
|
2024-01-25 17:28:20 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
type InitProjections struct {
|
|
|
|
Enabled bool
|
|
|
|
RetryFailedAfter time.Duration
|
|
|
|
MaxFailureCount uint8
|
|
|
|
BulkLimit uint64
|
2022-03-28 10:05:09 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
func MustNewConfig(v *viper.Viper) *Config {
|
|
|
|
config := new(Config)
|
2022-04-21 12:37:39 +02:00
|
|
|
err := v.Unmarshal(config,
|
|
|
|
viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
|
2024-05-01 12:17:27 +02:00
|
|
|
hooks.SliceTypeStringDecode[*domain.CustomMessageText],
|
|
|
|
hooks.SliceTypeStringDecode[internal_authz.RoleMapping],
|
|
|
|
hooks.MapTypeStringDecode[string, *internal_authz.SystemAPIUser],
|
|
|
|
hooks.MapHTTPHeaderStringDecode,
|
|
|
|
database.DecodeHook,
|
|
|
|
actions.HTTPConfigDecodeHook,
|
|
|
|
hook.EnumHookFunc(internal_authz.MemberTypeString),
|
2022-04-21 12:37:39 +02:00
|
|
|
hook.Base64ToBytesHookFunc(),
|
|
|
|
hook.TagToLanguageHookFunc(),
|
|
|
|
mapstructure.StringToTimeDurationHookFunc(),
|
2022-12-09 13:04:33 +00:00
|
|
|
mapstructure.StringToTimeHookFunc(time.RFC3339),
|
2022-04-21 12:37:39 +02:00
|
|
|
mapstructure.StringToSliceHookFunc(","),
|
2024-08-14 17:18:14 +03:00
|
|
|
mapstructure.TextUnmarshallerHookFunc(),
|
2022-04-21 12:37:39 +02:00
|
|
|
)),
|
|
|
|
)
|
2022-04-25 17:05:20 +02:00
|
|
|
logging.OnError(err).Fatal("unable to read default config")
|
2022-03-28 10:05:09 +02:00
|
|
|
|
|
|
|
err = config.Log.SetLogger()
|
|
|
|
logging.OnError(err).Fatal("unable to set logger")
|
|
|
|
|
2022-09-01 09:24:26 +02:00
|
|
|
id.Configure(config.Machine)
|
|
|
|
|
2022-03-28 10:05:09 +02:00
|
|
|
return config
|
2022-03-23 09:02:39 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
type Steps struct {
|
2024-10-04 16:15:41 +03:00
|
|
|
s1ProjectionTable *ProjectionTable
|
|
|
|
s2AssetsTable *AssetTable
|
|
|
|
FirstInstance *FirstInstance
|
|
|
|
s5LastFailed *LastFailed
|
|
|
|
s6OwnerRemoveColumns *OwnerRemoveColumns
|
|
|
|
s7LogstoreTables *LogstoreTables
|
|
|
|
s8AuthTokens *AuthTokenIndexes
|
|
|
|
CorrectCreationDate *CorrectCreationDate
|
|
|
|
s12AddOTPColumns *AddOTPColumns
|
|
|
|
s13FixQuotaProjection *FixQuotaConstraints
|
|
|
|
s14NewEventsTable *NewEventsTable
|
|
|
|
s15CurrentStates *CurrentProjectionState
|
|
|
|
s16UniqueConstraintsLower *UniqueConstraintToLower
|
|
|
|
s17AddOffsetToUniqueConstraints *AddOffsetToCurrentStates
|
|
|
|
s18AddLowerFieldsToLoginNames *AddLowerFieldsToLoginNames
|
|
|
|
s19AddCurrentStatesIndex *AddCurrentSequencesIndex
|
|
|
|
s20AddByUserSessionIndex *AddByUserIndexToSession
|
|
|
|
s21AddBlockFieldToLimits *AddBlockFieldToLimits
|
|
|
|
s22ActiveInstancesIndex *ActiveInstanceEvents
|
|
|
|
s23CorrectGlobalUniqueConstraints *CorrectGlobalUniqueConstraints
|
|
|
|
s24AddActorToAuthTokens *AddActorToAuthTokens
|
|
|
|
s25User11AddLowerFieldsToVerifiedEmail *User11AddLowerFieldsToVerifiedEmail
|
|
|
|
s26AuthUsers3 *AuthUsers3
|
|
|
|
s27IDPTemplate6SAMLNameIDFormat *IDPTemplate6SAMLNameIDFormat
|
|
|
|
s28AddFieldTable *AddFieldTable
|
|
|
|
s29FillFieldsForProjectGrant *FillFieldsForProjectGrant
|
|
|
|
s30FillFieldsForOrgDomainVerified *FillFieldsForOrgDomainVerified
|
|
|
|
s31AddAggregateIndexToFields *AddAggregateIndexToFields
|
|
|
|
s32AddAuthSessionID *AddAuthSessionID
|
2024-09-26 09:14:33 +02:00
|
|
|
s33SMSConfigs3TwilioAddVerifyServiceSid *SMSConfigs3TwilioAddVerifyServiceSid
|
2024-10-04 16:15:41 +03:00
|
|
|
s34AddCacheSchema *AddCacheSchema
|
perf(oidc): nest position clause for session terminated query (#8738)
# Which Problems Are Solved
Optimize the query that checks for terminated sessions in the access
token verifier. The verifier is used in auth middleware, userinfo and
introspection.
# How the Problems Are Solved
The previous implementation built a query for certain events and then
appended a single `PositionAfter` clause. This caused the postgreSQL
planner to use indexes only for the instance ID, aggregate IDs,
aggregate types and event types. Followed by an expensive sequential
scan for the position. This resulting in internal over-fetching of rows
before the final filter was applied.
![Screenshot_20241007_105803](https://github.com/user-attachments/assets/f2d91976-be87-428b-b604-a211399b821c)
Furthermore, the query was searching for events which are not always
applicable. For example, there was always a session ID search and if
there was a user ID, we would also search for a browser fingerprint in
event payload (expensive). Even if those argument string would be empty.
This PR changes:
1. Nest the position query, so that a full `instance_id, aggregate_id,
aggregate_type, event_type, "position"` index can be matched.
2. Redefine the `es_wm` index to include the `position` column.
3. Only search for events for the IDs that actually have a value. Do not
search (noop) if none of session ID, user ID or fingerpint ID are set.
New query plan:
![Screenshot_20241007_110648](https://github.com/user-attachments/assets/c3234c33-1b76-4b33-a4a9-796f69f3d775)
# Additional Changes
- cleanup how we load multi-statement migrations and make that a bit
more reusable.
# Additional Context
- Related to https://github.com/zitadel/zitadel/issues/7639
2024-10-07 15:49:55 +03:00
|
|
|
s35AddPositionToIndexEsWm *AddPositionToIndexEsWm
|
2024-10-28 09:29:34 +01:00
|
|
|
s36FillV2Milestones *FillV2Milestones
|
2022-04-12 16:20:17 +02:00
|
|
|
}
|
|
|
|
|
2022-03-28 10:05:09 +02:00
|
|
|
func MustNewSteps(v *viper.Viper) *Steps {
|
2022-06-27 12:32:34 +02:00
|
|
|
v.AutomaticEnv()
|
|
|
|
v.SetEnvPrefix("ZITADEL")
|
|
|
|
v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
|
2022-03-28 10:05:09 +02:00
|
|
|
v.SetConfigType("yaml")
|
|
|
|
err := v.ReadConfig(bytes.NewBuffer(defaultSteps))
|
|
|
|
logging.OnError(err).Fatal("unable to read setup steps")
|
|
|
|
|
2022-04-25 17:05:20 +02:00
|
|
|
for _, file := range stepFiles {
|
|
|
|
v.SetConfigFile(file)
|
|
|
|
err := v.MergeInConfig()
|
|
|
|
logging.WithFields("file", file).OnError(err).Warn("unable to read setup file")
|
|
|
|
}
|
|
|
|
|
2022-03-28 10:05:09 +02:00
|
|
|
steps := new(Steps)
|
|
|
|
err = v.Unmarshal(steps,
|
|
|
|
viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
|
|
|
|
hook.Base64ToBytesHookFunc(),
|
|
|
|
hook.TagToLanguageHookFunc(),
|
|
|
|
mapstructure.StringToTimeDurationHookFunc(),
|
2022-12-09 13:04:33 +00:00
|
|
|
mapstructure.StringToTimeHookFunc(time.RFC3339),
|
2022-03-28 10:05:09 +02:00
|
|
|
mapstructure.StringToSliceHookFunc(","),
|
2024-08-14 17:18:14 +03:00
|
|
|
mapstructure.TextUnmarshallerHookFunc(),
|
2022-03-28 10:05:09 +02:00
|
|
|
)),
|
|
|
|
)
|
|
|
|
logging.OnError(err).Fatal("unable to read steps")
|
|
|
|
return steps
|
2022-03-23 09:02:39 +01:00
|
|
|
}
|