zitadel/cmd/setup/config.go
Silvan 1d84635836
feat(eventstore): add search table (#8191)
# Which Problems Are Solved

To improve performance a new table and method is implemented on
eventstore. The goal of this table is to index searchable fields on
command side to use it on command and query side.

The table allows to store one primitive value (numeric, text) per row.

The eventstore framework is extended by the `Search`-method which allows
to search for objects.
The `Command`-interface is extended by the `SearchOperations()`-method
which does manipulate the the `search`-table.

# How the Problems Are Solved

This PR adds the capability of improving performance for command and
query side by using the `Search`-method of the eventstore instead of
using one of the `Filter`-methods.

# Open Tasks

- [x] Add feature flag
- [x] Unit tests
- [ ] ~~Benchmarks if needed~~
- [x] Ensure no behavior change
- [x] Add setup step to fill table with current data
- [x] Add projection which ensures data added between setup and start of
the new version are also added to the table

# Additional Changes

The `Search`-method is currently used by `ProjectGrant`-command side.

# Additional Context

- Closes https://github.com/zitadel/zitadel/issues/8094
2024-07-03 15:00:56 +00:00

145 lines
5.1 KiB
Go

package setup
import (
"bytes"
"strings"
"time"
"github.com/mitchellh/mapstructure"
"github.com/spf13/viper"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/cmd/encryption"
"github.com/zitadel/zitadel/cmd/hooks"
"github.com/zitadel/zitadel/internal/actions"
internal_authz "github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/api/oidc"
"github.com/zitadel/zitadel/internal/api/ui/login"
"github.com/zitadel/zitadel/internal/command"
"github.com/zitadel/zitadel/internal/config/hook"
"github.com/zitadel/zitadel/internal/config/systemdefaults"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/domain"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/id"
"github.com/zitadel/zitadel/internal/notification/handlers"
"github.com/zitadel/zitadel/internal/query/projection"
static_config "github.com/zitadel/zitadel/internal/static/config"
)
type Config struct {
ForMirror bool
Database database.Config
SystemDefaults systemdefaults.SystemDefaults
InternalAuthZ internal_authz.Config
ExternalDomain string
ExternalPort uint16
ExternalSecure bool
Log *logging.Config
EncryptionKeys *encryption.EncryptionKeyConfig
DefaultInstance command.InstanceSetup
Machine *id.Config
Projections projection.Config
Eventstore *eventstore.Config
InitProjections InitProjections
AssetStorage static_config.AssetStorageConfig
OIDC oidc.Config
Login login.Config
WebAuthNName string
Telemetry *handlers.TelemetryPusherConfig
SystemAPIUsers map[string]*internal_authz.SystemAPIUser
}
type InitProjections struct {
Enabled bool
RetryFailedAfter time.Duration
MaxFailureCount uint8
BulkLimit uint64
}
func MustNewConfig(v *viper.Viper) *Config {
config := new(Config)
err := v.Unmarshal(config,
viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
hooks.SliceTypeStringDecode[*domain.CustomMessageText],
hooks.SliceTypeStringDecode[internal_authz.RoleMapping],
hooks.MapTypeStringDecode[string, *internal_authz.SystemAPIUser],
hooks.MapHTTPHeaderStringDecode,
database.DecodeHook,
actions.HTTPConfigDecodeHook,
hook.EnumHookFunc(internal_authz.MemberTypeString),
hook.Base64ToBytesHookFunc(),
hook.TagToLanguageHookFunc(),
mapstructure.StringToTimeDurationHookFunc(),
mapstructure.StringToTimeHookFunc(time.RFC3339),
mapstructure.StringToSliceHookFunc(","),
)),
)
logging.OnError(err).Fatal("unable to read default config")
err = config.Log.SetLogger()
logging.OnError(err).Fatal("unable to set logger")
id.Configure(config.Machine)
return config
}
type Steps struct {
s1ProjectionTable *ProjectionTable
s2AssetsTable *AssetTable
FirstInstance *FirstInstance
s5LastFailed *LastFailed
s6OwnerRemoveColumns *OwnerRemoveColumns
s7LogstoreTables *LogstoreTables
s8AuthTokens *AuthTokenIndexes
CorrectCreationDate *CorrectCreationDate
s12AddOTPColumns *AddOTPColumns
s13FixQuotaProjection *FixQuotaConstraints
s14NewEventsTable *NewEventsTable
s15CurrentStates *CurrentProjectionState
s16UniqueConstraintsLower *UniqueConstraintToLower
s17AddOffsetToUniqueConstraints *AddOffsetToCurrentStates
s18AddLowerFieldsToLoginNames *AddLowerFieldsToLoginNames
s19AddCurrentStatesIndex *AddCurrentSequencesIndex
s20AddByUserSessionIndex *AddByUserIndexToSession
s21AddBlockFieldToLimits *AddBlockFieldToLimits
s22ActiveInstancesIndex *ActiveInstanceEvents
s23CorrectGlobalUniqueConstraints *CorrectGlobalUniqueConstraints
s24AddActorToAuthTokens *AddActorToAuthTokens
s25User11AddLowerFieldsToVerifiedEmail *User11AddLowerFieldsToVerifiedEmail
s26AuthUsers3 *AuthUsers3
s27IDPTemplate6SAMLNameIDFormat *IDPTemplate6SAMLNameIDFormat
s28AddFieldTable *AddFieldTable
s29FillFieldsForProjectGrant *FillFieldsForProjectGrant
}
func MustNewSteps(v *viper.Viper) *Steps {
v.AutomaticEnv()
v.SetEnvPrefix("ZITADEL")
v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
v.SetConfigType("yaml")
err := v.ReadConfig(bytes.NewBuffer(defaultSteps))
logging.OnError(err).Fatal("unable to read setup steps")
for _, file := range stepFiles {
v.SetConfigFile(file)
err := v.MergeInConfig()
logging.WithFields("file", file).OnError(err).Warn("unable to read setup file")
}
steps := new(Steps)
err = v.Unmarshal(steps,
viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
hook.Base64ToBytesHookFunc(),
hook.TagToLanguageHookFunc(),
mapstructure.StringToTimeDurationHookFunc(),
mapstructure.StringToTimeHookFunc(time.RFC3339),
mapstructure.StringToSliceHookFunc(","),
)),
)
logging.OnError(err).Fatal("unable to read steps")
return steps
}