TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2024-12-04 23:45:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf(import): optimize search for domains claimed by other organizations (#8200)

Browse Source 
# Which Problems Are Solved

Improve the performance of human imports by optimizing the query that
finds domains claimed by other organizations.

# How the Problems Are Solved

Use the fields search table introduced in
https://github.com/zitadel/zitadel/pull/8191 by storing each
organization domain as Object ID and the verified status as field value.

# Additional Changes

- Feature flag for this optimization

# Additional Context

- Performance improvements for import are evaluated and acted upon
internally at the moment

---------

Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
This commit is contained in:
Tim Möhlmann 2024-07-05 10:36:00 +03:00 committed by GitHub
parent ecfb9d0d6d
commit 7967e6f98b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 238 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
cmd/setup/30.go Normal file
View File

@ -0,0 +1,42 @@
package setup
import (
"context"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/query/projection"
"github.com/zitadel/zitadel/internal/repository/instance"
)
type FillFieldsForOrgDomainVerified struct {
eventstore *eventstore.Eventstore
}
func (mig *FillFieldsForOrgDomainVerified) Execute(ctx context.Context, _ eventstore.Event) error {
instances, err := mig.eventstore.InstanceIDs(
ctx,
0,
true,
eventstore.NewSearchQueryBuilder(eventstore.ColumnsInstanceIDs).
OrderDesc().
AddQuery().
AggregateTypes("instance").
EventTypes(instance.InstanceAddedEventType).
Builder(),
)
if err != nil {
return err
}
for _, instance := range instances {
ctx := authz.WithInstanceID(ctx, instance)
if err := projection.OrgDomainVerifiedFields.Trigger(ctx); err != nil {
return err
}
}
return nil
}
func (mig *FillFieldsForOrgDomainVerified) String() string {
return "30_fill_fields_for_org_domain_verified"
}

1
cmd/setup/config.go
View File

@ -113,6 +113,7 @@ type Steps struct {
s27IDPTemplate6SAMLNameIDFormat *IDPTemplate6SAMLNameIDFormat
s28AddFieldTable *AddFieldTable
s29FillFieldsForProjectGrant *FillFieldsForProjectGrant
s30FillFieldsForOrgDomainVerified *FillFieldsForOrgDomainVerified
}
func MustNewSteps(v *viper.Viper) *Steps {

2
cmd/setup/setup.go
View File

@ -158,6 +158,7 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string)
steps.s27IDPTemplate6SAMLNameIDFormat = &IDPTemplate6SAMLNameIDFormat{dbClient: esPusherDBClient}
steps.s28AddFieldTable = &AddFieldTable{dbClient: esPusherDBClient}
steps.s29FillFieldsForProjectGrant = &FillFieldsForProjectGrant{eventstore: eventstoreClient}
steps.s30FillFieldsForOrgDomainVerified = &FillFieldsForOrgDomainVerified{eventstore: eventstoreClient}
err = projection.Create(ctx, projectionDBClient, eventstoreClient, config.Projections, nil, nil, nil)
logging.OnError(err).Fatal("unable to start projections")
@ -198,6 +199,7 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string)
steps.s24AddActorToAuthTokens,
steps.s26AuthUsers3,
steps.s29FillFieldsForProjectGrant,
steps.s30FillFieldsForOrgDomainVerified,
} {
mustExecuteMigration(ctx, eventstoreClient, step, "migration failed")
}

4
internal/api/grpc/feature/v2/converter.go
View File

@ -115,6 +115,8 @@ func improvedPerformanceTypeToPb(typ feature.ImprovedPerformanceType) feature_pb
return feature_pb.ImprovedPerformance_IMPROVED_PERFORMANCE_PROJECT
case feature.ImprovedPerformanceTypeUserGrant:
return feature_pb.ImprovedPerformance_IMPROVED_PERFORMANCE_USER_GRANT
case feature.ImprovedPerformanceTypeOrgDomainVerified:
return feature_pb.ImprovedPerformance_IMPROVED_PERFORMANCE_ORG_DOMAIN_VERIFIED
default:
return feature_pb.ImprovedPerformance(typ)
}
@ -145,6 +147,8 @@ func improvedPerformanceToDomain(typ feature_pb.ImprovedPerformance) feature.Imp
return feature.ImprovedPerformanceTypeProject
case feature_pb.ImprovedPerformance_IMPROVED_PERFORMANCE_USER_GRANT:
return feature.ImprovedPerformanceTypeUserGrant
case feature_pb.ImprovedPerformance_IMPROVED_PERFORMANCE_ORG_DOMAIN_VERIFIED:
return feature.ImprovedPerformanceTypeOrgDomainVerified
default:
return feature.ImprovedPerformanceTypeUnknown
}

64
internal/command/org_domain.go
View File

@ -13,6 +13,8 @@ import (
"github.com/zitadel/zitadel/internal/crypto"
"github.com/zitadel/zitadel/internal/domain"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/feature"
"github.com/zitadel/zitadel/internal/query/projection"
"github.com/zitadel/zitadel/internal/repository/org"
"github.com/zitadel/zitadel/internal/telemetry/tracing"
"github.com/zitadel/zitadel/internal/zerrors"
@ -390,3 +392,65 @@ func (c *Commands) getOrgDomainWriteModel(ctx context.Context, orgID, domain str
}
return domainWriteModel, nil
}
type OrgDomainVerified struct {
OrgID string
Domain string
Verified bool
}
func (c *Commands) searchOrgDomainVerifiedByDomain(ctx context.Context, domain string) (_ *OrgDomainVerified, err error) {
if !authz.GetFeatures(ctx).ShouldUseImprovedPerformance(feature.ImprovedPerformanceTypeOrgDomainVerified) {
return c.searchOrgDomainVerifiedByDomainOld(ctx, domain)
}
ctx, span := tracing.NewSpan(ctx)
defer func() { span.EndWithError(err) }()
condition := map[eventstore.FieldType]any{
eventstore.FieldTypeAggregateType: org.AggregateType,
eventstore.FieldTypeObjectType: org.OrgDomainSearchType,
eventstore.FieldTypeObjectID: domain,
eventstore.FieldTypeObjectRevision: org.OrgDomainObjectRevision,
eventstore.FieldTypeFieldName: org.OrgDomainVerifiedSearchField,
}
results, err := c.eventstore.Search(ctx, condition)
if err != nil {
return nil, err
}
if len(results) == 0 {
_ = projection.OrgDomainVerifiedFields.Trigger(ctx)
results, err = c.eventstore.Search(ctx, condition)
if err != nil {
return nil, err
}
}
orgDomain := new(OrgDomainVerified)
for _, result := range results {
orgDomain.OrgID = result.Aggregate.ID
if err = result.Value.Unmarshal(&orgDomain.Verified); err != nil {
return nil, err
}
}
return orgDomain, nil
}
func (c *Commands) searchOrgDomainVerifiedByDomainOld(ctx context.Context, domain string) (_ *OrgDomainVerified, err error) {
ctx, span := tracing.NewSpan(ctx)
defer func() { span.EndWithError(err) }()
writeModel := NewOrgDomainVerifiedWriteModel(domain)
err = c.eventstore.FilterToQueryReducer(ctx, writeModel)
if err != nil {
return nil, err
}
return &OrgDomainVerified{
OrgID: writeModel.ResourceOwner,
Domain: writeModel.Domain,
Verified: writeModel.Verified,
}, nil
}

13
internal/command/user.go
View File

@ -40,17 +40,8 @@ func (c *Commands) ChangeUsername(ctx context.Context, orgID, userID, userName s
if err != nil {
return nil, zerrors.ThrowPreconditionFailed(err, "COMMAND-38fnu", "Errors.Org.DomainPolicy.NotExisting")
}
if !domainPolicy.UserLoginMustBeDomain {
index := strings.LastIndex(userName, "@")
if index > 1 {
domainCheck := NewOrgDomainVerifiedWriteModel(userName[index+1:])
if err := c.eventstore.FilterToQueryReducer(ctx, domainCheck); err != nil {
return nil, err
}
if domainCheck.Verified && domainCheck.ResourceOwner != orgID {
return nil, zerrors.ThrowInvalidArgument(nil, "COMMAND-Di2ei", "Errors.User.DomainNotAllowedAsUsername")
}
}
if err = c.userValidateDomain(ctx, orgID, userName, domainPolicy.UserLoginMustBeDomain); err != nil {
return nil, err
}
userAgg := UserAggregateFromWriteModel(&existingUser.WriteModel)

27
internal/command/user_human.go
View File

@ -362,7 +362,7 @@ func addHumanCommandPassword(ctx context.Context, filter preparation.FilterToQue
return nil
}
func (c *Commands) userValidateDomain(ctx context.Context, resourceOwner string, username string, mustBeDomain bool) error {
func (c *Commands) userValidateDomain(ctx context.Context, resourceOwner string, username string, mustBeDomain bool) (err error) {
if mustBeDomain {
return nil
}
@ -372,12 +372,12 @@ func (c *Commands) userValidateDomain(ctx context.Context, resourceOwner string,
return nil
}
domainCheck, err := c.orgDomainVerifiedWriteModel(ctx, username[index+1:])
domainCheck, err := c.searchOrgDomainVerifiedByDomain(ctx, username[index+1:])
if err != nil {
return err
}
if domainCheck.Verified && domainCheck.ResourceOwner != resourceOwner {
if domainCheck.Verified && domainCheck.OrgID != resourceOwner {
return zerrors.ThrowInvalidArgument(nil, "COMMAND-SFd21", "Errors.User.DomainNotAllowedAsUsername")
}
@ -479,7 +479,7 @@ func (c *Commands) importHuman(ctx context.Context, orgID string, human *domain.
if orgID == "" {
return nil, nil, nil, "", zerrors.ThrowInvalidArgument(nil, "COMMAND-00p2b", "Errors.Org.Empty")
}
if err := human.Normalize(); err != nil {
if err = human.Normalize(); err != nil {
return nil, nil, nil, "", err
}
events, humanWriteModel, err = c.createHuman(ctx, orgID, human, links, passwordless, domainPolicy, pwPolicy, initCodeGenerator, emailCodeGenerator, phoneCodeGenerator)
@ -497,24 +497,17 @@ func (c *Commands) importHuman(ctx context.Context, orgID string, human *domain.
return events, humanWriteModel, passwordlessCodeWriteModel, code, nil
}
//nolint:gocognit
func (c *Commands) createHuman(ctx context.Context, orgID string, human *domain.Human, links []*domain.UserIDPLink, passwordless bool, domainPolicy *domain.DomainPolicy, pwPolicy *domain.PasswordComplexityPolicy, initCodeGenerator, emailCodeGenerator, phoneCodeGenerator crypto.Generator) (events []eventstore.Command, addedHuman *HumanWriteModel, err error) {
if err := human.CheckDomainPolicy(domainPolicy); err != nil {
ctx, span := tracing.NewSpan(ctx)
defer func() { span.EndWithError(err) }()
if err = human.CheckDomainPolicy(domainPolicy); err != nil {
return nil, nil, err
}
human.Username = strings.TrimSpace(human.Username)
human.EmailAddress = human.EmailAddress.Normalize()
if !domainPolicy.UserLoginMustBeDomain {
index := strings.LastIndex(human.Username, "@")
if index > 1 {
domainCheck := NewOrgDomainVerifiedWriteModel(human.Username[index+1:])
if err := c.eventstore.FilterToQueryReducer(ctx, domainCheck); err != nil {
return nil, nil, err
}
if domainCheck.Verified && domainCheck.ResourceOwner != orgID {
return nil, nil, zerrors.ThrowInvalidArgument(nil, "COMMAND-SFd21", "Errors.User.DomainNotAllowedAsUsername")
}
}
if err = c.userValidateDomain(ctx, orgID, human.Username, domainPolicy.UserLoginMustBeDomain); err != nil {
return nil, nil, err
}
if human.AggregateID == "" {

12
internal/command/user_v2_human.go
View File

@ -434,15 +434,3 @@ func (c *Commands) userHumanWriteModel(ctx context.Context, userID string, profi
}
return writeModel, nil
}
func (c *Commands) orgDomainVerifiedWriteModel(ctx context.Context, domain string) (writeModel *OrgDomainVerifiedWriteModel, err error) {
ctx, span := tracing.NewSpan(ctx)
defer func() { span.EndWithError(err) }()
writeModel = NewOrgDomainVerifiedWriteModel(domain)
err = c.eventstore.FilterToQueryReducer(ctx, writeModel)
if err != nil {
return nil, err
}
return writeModel, nil
}

14
internal/command/user_v2_username.go
View File

@ -2,7 +2,6 @@ package command
import (
"context"
"strings"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/repository/user"
@ -19,17 +18,8 @@ func (c *Commands) changeUsername(ctx context.Context, cmds []eventstore.Command
if err != nil {
return cmds, zerrors.ThrowPreconditionFailed(err, "COMMAND-79pv6e1q62", "Errors.Org.DomainPolicy.NotExisting")
}
if !domainPolicy.UserLoginMustBeDomain {
index := strings.LastIndex(userName, "@")
if index > 1 {
domainCheck := NewOrgDomainVerifiedWriteModel(userName[index+1:])
if err := c.eventstore.FilterToQueryReducer(ctx, domainCheck); err != nil {
return cmds, err
}
if domainCheck.Verified && domainCheck.ResourceOwner != orgID {
return cmds, zerrors.ThrowInvalidArgument(nil, "COMMAND-Di2ei", "Errors.User.DomainNotAllowedAsUsername")
}
}
if err = c.userValidateDomain(ctx, orgID, userName, domainPolicy.UserLoginMustBeDomain); err != nil {
return cmds, err
}
return append(cmds,
user.NewUsernameChangedEvent(ctx, &wm.Aggregate().Aggregate, wm.UserName, userName, domainPolicy.UserLoginMustBeDomain),

10
internal/feature/feature.go
View File

@ -1,5 +1,7 @@
package feature
import "slices"
//go:generate enumer -type Key -transform snake -trimprefix Key
type Key int
@ -45,13 +47,9 @@ const (
ImprovedPerformanceTypeProjectGrant
ImprovedPerformanceTypeProject
ImprovedPerformanceTypeUserGrant
ImprovedPerformanceTypeOrgDomainVerified
)
func (f Features) ShouldUseImprovedPerformance(typ ImprovedPerformanceType) bool {
for _, improvedType := range f.ImprovedPerformance {
if improvedType == typ {
return true
}
}
return false
return slices.Contains(f.ImprovedPerformance, typ)
}

21
internal/query/projection/eventstore_field.go
View File

@ -7,13 +7,32 @@ import (
"github.com/zitadel/zitadel/internal/repository/project"
)
const (
fieldsProjectGrant = "project_grant_fields"
fieldsOrgDomainVerified = "org_domain_verified_fields"
)
func newFillProjectGrantFields(config handler.Config) *handler.FieldHandler {
return handler.NewFieldHandler(
&config,
"project_grant_fields",
fieldsProjectGrant,
map[eventstore.AggregateType][]eventstore.EventType{
org.AggregateType: nil,
project.AggregateType: nil,
},
)
}
func newFillOrgDomainVerifiedFields(config handler.Config) *handler.FieldHandler {
return handler.NewFieldHandler(
&config,
fieldsOrgDomainVerified,
map[eventstore.AggregateType][]eventstore.EventType{
org.AggregateType: {
org.OrgDomainAddedEventType,
org.OrgDomainVerifiedEventType,
org.OrgDomainRemovedEventType,
},
},
)
}

6
internal/query/projection/projection.go
View File

@ -78,7 +78,8 @@ var (
ExecutionProjection *handler.Handler
UserSchemaProjection *handler.Handler
ProjectGrantFields *handler.FieldHandler
ProjectGrantFields *handler.FieldHandler
OrgDomainVerifiedFields *handler.FieldHandler
)
type projection interface {
@ -161,7 +162,8 @@ func Create(ctx context.Context, sqlClient *database.DB, es handler.EventStore,
ExecutionProjection = newExecutionProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["executions"]))
UserSchemaProjection = newUserSchemaProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["user_schemas"]))
ProjectGrantFields = newFillProjectGrantFields(applyCustomConfig(projectionConfig, config.Customizations["project_grant_fields"]))
ProjectGrantFields = newFillProjectGrantFields(applyCustomConfig(projectionConfig, config.Customizations[fieldsProjectGrant]))
OrgDomainVerifiedFields = newFillOrgDomainVerifiedFields(applyCustomConfig(projectionConfig, config.Customizations[fieldsOrgDomainVerified]))
newProjectionsList()
return nil

78
internal/repository/org/domain.go
View File

@ -18,6 +18,10 @@ const (
OrgDomainVerifiedEventType = domainEventPrefix + "verified"
OrgDomainPrimarySetEventType = domainEventPrefix + "primary.set"
OrgDomainRemovedEventType = domainEventPrefix + "removed"
OrgDomainSearchType = "org_domain"
OrgDomainVerifiedSearchField = "verified"
OrgDomainObjectRevision = uint8(1)
)
func NewAddOrgDomainUniqueConstraint(orgDomain string) *eventstore.UniqueConstraint {
@ -47,6 +51,28 @@ func (e *DomainAddedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return nil
}
func (e *DomainAddedEvent) Fields() []*eventstore.FieldOperation {
return []*eventstore.FieldOperation{
eventstore.SetField(
e.Aggregate(),
domainSearchObject(e.Domain),
OrgDomainVerifiedSearchField,
&eventstore.Value{
Value: false,
ShouldIndex: false,
},
eventstore.FieldTypeInstanceID,
eventstore.FieldTypeResourceOwner,
eventstore.FieldTypeAggregateType,
eventstore.FieldTypeAggregateID,
eventstore.FieldTypeObjectType,
eventstore.FieldTypeObjectID,
eventstore.FieldTypeFieldName,
),
}
}
func NewDomainAddedEvent(ctx context.Context, aggregate *eventstore.Aggregate, domain string) *DomainAddedEvent {
return &DomainAddedEvent{
BaseEvent: *eventstore.NewBaseEventForPush(
@ -167,6 +193,28 @@ func (e *DomainVerifiedEvent) UniqueConstraints() []*eventstore.UniqueConstraint
return []*eventstore.UniqueConstraint{NewAddOrgDomainUniqueConstraint(e.Domain)}
}
func (e *DomainVerifiedEvent) Fields() []*eventstore.FieldOperation {
return []*eventstore.FieldOperation{
eventstore.SetField(
e.Aggregate(),
domainSearchObject(e.Domain),
OrgDomainVerifiedSearchField,
&eventstore.Value{
Value: true,
ShouldIndex: false,
},
eventstore.FieldTypeInstanceID,
eventstore.FieldTypeResourceOwner,
eventstore.FieldTypeAggregateType,
eventstore.FieldTypeAggregateID,
eventstore.FieldTypeObjectType,
eventstore.FieldTypeObjectID,
eventstore.FieldTypeFieldName,
),
}
}
func NewDomainVerifiedEvent(ctx context.Context, aggregate *eventstore.Aggregate, domain string) *DomainVerifiedEvent {
return &DomainVerifiedEvent{
BaseEvent: *eventstore.NewBaseEventForPush(
@ -245,6 +293,28 @@ func (e *DomainRemovedEvent) UniqueConstraints() []*eventstore.UniqueConstraint
return []*eventstore.UniqueConstraint{NewRemoveOrgDomainUniqueConstraint(e.Domain)}
}
func (e *DomainRemovedEvent) Fields() []*eventstore.FieldOperation {
return []*eventstore.FieldOperation{
eventstore.SetField(
e.Aggregate(),
domainSearchObject(e.Domain),
OrgDomainVerifiedSearchField,
&eventstore.Value{
Value: false,
ShouldIndex: false,
},
eventstore.FieldTypeInstanceID,
eventstore.FieldTypeResourceOwner,
eventstore.FieldTypeAggregateType,
eventstore.FieldTypeAggregateID,
eventstore.FieldTypeObjectType,
eventstore.FieldTypeObjectID,
eventstore.FieldTypeFieldName,
),
}
}
func NewDomainRemovedEvent(ctx context.Context, aggregate *eventstore.Aggregate, domain string, verified bool) *DomainRemovedEvent {
return &DomainRemovedEvent{
BaseEvent: *eventstore.NewBaseEventForPush(
@ -268,3 +338,11 @@ func DomainRemovedEventMapper(event eventstore.Event) (eventstore.Event, error)
return orgDomainRemoved, nil
}
func domainSearchObject(domain string) eventstore.Object {
return eventstore.Object{
Type: OrgDomainSearchType,
ID: domain,
Revision: OrgDomainObjectRevision,
}
}

5
proto/zitadel/feature/v2beta/feature.proto
View File

@ -60,4 +60,9 @@ enum ImprovedPerformance {
IMPROVED_PERFORMANCE_PROJECT_GRANT = 2;
IMPROVED_PERFORMANCE_PROJECT = 3;
IMPROVED_PERFORMANCE_USER_GRANT = 4;
// Improve performance on write side when
// users are checked against verified domains
// from other organizations.
IMPROVED_PERFORMANCE_ORG_DOMAIN_VERIFIED = 5;
}