TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-12 01:57:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(eventstore): add search table (#8191)

Browse Source 
# Which Problems Are Solved

To improve performance a new table and method is implemented on
eventstore. The goal of this table is to index searchable fields on
command side to use it on command and query side.

The table allows to store one primitive value (numeric, text) per row.

The eventstore framework is extended by the `Search`-method which allows
to search for objects.
The `Command`-interface is extended by the `SearchOperations()`-method
which does manipulate the the `search`-table.

# How the Problems Are Solved

This PR adds the capability of improving performance for command and
query side by using the `Search`-method of the eventstore instead of
using one of the `Filter`-methods.

# Open Tasks

- [x] Add feature flag
- [x] Unit tests
- [ ] ~~Benchmarks if needed~~
- [x] Ensure no behavior change
- [x] Add setup step to fill table with current data
- [x] Add projection which ensures data added between setup and start of
the new version are also added to the table

# Additional Changes

The `Search`-method is currently used by `ProjectGrant`-command side.

# Additional Context

- Closes https://github.com/zitadel/zitadel/issues/8094
This commit is contained in:
Silvan
2024-07-03 17:00:56 +02:00
committed by GitHub
parent 637f441a7d
commit 1d84635836
35 changed files with 2207 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

223
internal/repository/project/grant.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"github.com/zitadel/zitadel/internal/domain"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/zerrors"
)
@@ -17,6 +18,13 @@ var (
GrantDeactivatedType = grantEventTypePrefix + "deactivated"
GrantReactivatedType = grantEventTypePrefix + "reactivated"
GrantRemovedType = grantEventTypePrefix + "removed"
ProjectGrantSearchType = "project_grant"
ProjectGrantGrantIDSearchField = "grant_id"
ProjectGrantGrantedOrgIDSearchField = "granted_org_id"
ProjectGrantStateSearchField = "state"
ProjectGrantRoleKeySearchField = "role_key"
ProjectGrantObjectRevision = uint8(1)
)
func NewAddProjectGrantUniqueConstraint(grantedOrgID, projectID string) *eventstore.UniqueConstraint {
@@ -48,6 +56,76 @@ func (e *GrantAddedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return []*eventstore.UniqueConstraint{NewAddProjectGrantUniqueConstraint(e.GrantedOrgID, e.Aggregate().ID)}
}
func (e *GrantAddedEvent) Fields() []*eventstore.FieldOperation {
fields := make([]*eventstore.FieldOperation, 0, len(e.RoleKeys)+3)
fields = append(fields,
eventstore.SetField(
e.Aggregate(),
grantSearchObject(e.GrantID),
ProjectGrantGrantIDSearchField,
&eventstore.Value{
Value: e.GrantID,
ShouldIndex: true,
},
eventstore.FieldTypeInstanceID,
eventstore.FieldTypeResourceOwner,
eventstore.FieldTypeAggregateType,
eventstore.FieldTypeAggregateID,
eventstore.FieldTypeObjectType,
eventstore.FieldTypeObjectID,
eventstore.FieldTypeFieldName,
),
eventstore.SetField(
e.Aggregate(),
grantSearchObject(e.GrantID),
ProjectGrantGrantedOrgIDSearchField,
&eventstore.Value{
Value: e.GrantedOrgID,
ShouldIndex: true,
},
eventstore.FieldTypeInstanceID,
eventstore.FieldTypeResourceOwner,
eventstore.FieldTypeAggregateType,
eventstore.FieldTypeAggregateID,
eventstore.FieldTypeObjectType,
eventstore.FieldTypeObjectID,
eventstore.FieldTypeFieldName,
),
eventstore.SetField(
e.Aggregate(),
grantSearchObject(e.GrantID),
ProjectGrantStateSearchField,
&eventstore.Value{
Value: domain.ProjectGrantStateActive,
ShouldIndex: true,
},
eventstore.FieldTypeInstanceID,
eventstore.FieldTypeResourceOwner,
eventstore.FieldTypeAggregateType,
eventstore.FieldTypeAggregateID,
eventstore.FieldTypeObjectType,
eventstore.FieldTypeObjectID,
eventstore.FieldTypeFieldName,
),
)
for _, roleKey := range e.RoleKeys {
fields = append(fields,
eventstore.SetField(
e.Aggregate(),
grantSearchObject(e.GrantID),
ProjectGrantRoleKeySearchField,
&eventstore.Value{
Value: roleKey,
ShouldIndex: true,
},
),
)
}
return fields
}
func NewGrantAddedEvent(
ctx context.Context,
aggregate *eventstore.Aggregate,
@@ -95,6 +173,37 @@ func (e *GrantChangedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return nil
}
func (e *GrantChangedEvent) Fields() []*eventstore.FieldOperation {
fields := make([]*eventstore.FieldOperation, 0, len(e.RoleKeys)+1)
fields = append(fields,
eventstore.RemoveSearchFieldsByAggregateAndObjectAndField(
e.Aggregate(),
grantSearchObject(e.GrantID),
ProjectGrantRoleKeySearchField,
),
)
for _, roleKey := range e.RoleKeys {
fields = append(fields,
eventstore.SetField(
e.Aggregate(),
grantSearchObject(e.GrantID),
ProjectGrantRoleKeySearchField,
&eventstore.Value{
Value: roleKey,
ShouldIndex: true,
},
),
)
}
return fields
}
func NewGrantChangedEvent(
ctx context.Context,
aggregate *eventstore.Aggregate,
@@ -140,6 +249,43 @@ func (e *GrantCascadeChangedEvent) UniqueConstraints() []*eventstore.UniqueConst
return nil
}
func (e *GrantCascadeChangedEvent) Fields() []*eventstore.FieldOperation {
fields := make([]*eventstore.FieldOperation, 0, len(e.RoleKeys)+1)
fields = append(fields,
eventstore.RemoveSearchFieldsByAggregateAndObjectAndField(
e.Aggregate(),
grantSearchObject(e.GrantID),
ProjectGrantRoleKeySearchField,
),
)
for _, roleKey := range e.RoleKeys {
fields = append(fields,
eventstore.SetField(
e.Aggregate(),
grantSearchObject(e.GrantID),
ProjectGrantRoleKeySearchField,
&eventstore.Value{
Value: roleKey,
ShouldIndex: true,
},
eventstore.FieldTypeInstanceID,
eventstore.FieldTypeResourceOwner,
eventstore.FieldTypeAggregateType,
eventstore.FieldTypeAggregateID,
eventstore.FieldTypeObjectType,
eventstore.FieldTypeObjectID,
eventstore.FieldTypeFieldName,
),
)
}
return fields
}
func NewGrantCascadeChangedEvent(
ctx context.Context,
aggregate *eventstore.Aggregate,
@@ -184,6 +330,29 @@ func (e *GrantDeactivateEvent) UniqueConstraints() []*eventstore.UniqueConstrain
return nil
}
func (e *GrantDeactivateEvent) Fields() []*eventstore.FieldOperation {
return []*eventstore.FieldOperation{
eventstore.SetField(
e.Aggregate(),
grantSearchObject(e.GrantID),
ProjectGrantStateSearchField,
&eventstore.Value{
Value: domain.ProjectGrantStateInactive,
ShouldIndex: true,
},
eventstore.FieldTypeInstanceID,
eventstore.FieldTypeResourceOwner,
eventstore.FieldTypeAggregateType,
eventstore.FieldTypeAggregateID,
eventstore.FieldTypeObjectType,
eventstore.FieldTypeObjectID,
eventstore.FieldTypeFieldName,
),
}
}
func NewGrantDeactivateEvent(
ctx context.Context,
aggregate *eventstore.Aggregate,
@@ -226,6 +395,29 @@ func (e *GrantReactivatedEvent) UniqueConstraints() []*eventstore.UniqueConstrai
return nil
}
func (e *GrantReactivatedEvent) Fields() []*eventstore.FieldOperation {
return []*eventstore.FieldOperation{
eventstore.SetField(
e.Aggregate(),
grantSearchObject(e.GrantID),
ProjectGrantStateSearchField,
&eventstore.Value{
Value: domain.ProjectGrantStateActive,
ShouldIndex: true,
},
eventstore.FieldTypeInstanceID,
eventstore.FieldTypeResourceOwner,
eventstore.FieldTypeAggregateType,
eventstore.FieldTypeAggregateID,
eventstore.FieldTypeObjectType,
eventstore.FieldTypeObjectID,
eventstore.FieldTypeFieldName,
),
}
}
func NewGrantReactivatedEvent(
ctx context.Context,
aggregate *eventstore.Aggregate,
@@ -269,6 +461,29 @@ func (e *GrantRemovedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return []*eventstore.UniqueConstraint{NewRemoveProjectGrantUniqueConstraint(e.grantedOrgID, e.Aggregate().ID)}
}
func (e *GrantRemovedEvent) Fields() []*eventstore.FieldOperation {
return []*eventstore.FieldOperation{
eventstore.SetField(
e.Aggregate(),
grantSearchObject(e.GrantID),
ProjectGrantStateSearchField,
&eventstore.Value{
Value: domain.ProjectGrantStateRemoved,
ShouldIndex: true,
},
eventstore.FieldTypeInstanceID,
eventstore.FieldTypeResourceOwner,
eventstore.FieldTypeAggregateType,
eventstore.FieldTypeAggregateID,
eventstore.FieldTypeObjectType,
eventstore.FieldTypeObjectID,
eventstore.FieldTypeFieldName,
),
}
}
func NewGrantRemovedEvent(
ctx context.Context,
aggregate *eventstore.Aggregate,
@@ -298,3 +513,11 @@ func GrantRemovedEventMapper(event eventstore.Event) (eventstore.Event, error) {
return e, nil
}
func grantSearchObject(id string) eventstore.Object {
return eventstore.Object{
Type: ProjectGrantSearchType,
Revision: 1,
ID: id,
}
}

126
internal/repository/project/project.go
View File

@@ -16,6 +16,11 @@ const (
ProjectDeactivatedType = projectEventTypePrefix + "deactivated"
ProjectReactivatedType = projectEventTypePrefix + "reactivated"
ProjectRemovedType = projectEventTypePrefix + "removed"
ProjectSearchType = "project"
ProjectObjectRevision = uint8(1)
ProjectNameSearchField = "name"
ProjectStateSearchField = "state"
)
func NewAddProjectNameUniqueConstraint(projectName, resourceOwner string) *eventstore.UniqueConstraint {
@@ -49,6 +54,45 @@ func (e *ProjectAddedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return []*eventstore.UniqueConstraint{NewAddProjectNameUniqueConstraint(e.Name, e.Aggregate().ResourceOwner)}
}
func (e *ProjectAddedEvent) Fields() []*eventstore.FieldOperation {
return []*eventstore.FieldOperation{
eventstore.SetField(
e.Aggregate(),
projectSearchObject(e.Aggregate().ID),
ProjectNameSearchField,
&eventstore.Value{
Value: e.Name,
ShouldIndex: true,
},
eventstore.FieldTypeInstanceID,
eventstore.FieldTypeResourceOwner,
eventstore.FieldTypeAggregateID,
eventstore.FieldTypeAggregateType,
eventstore.FieldTypeObjectType,
eventstore.FieldTypeObjectID,
eventstore.FieldTypeObjectRevision,
eventstore.FieldTypeFieldName,
),
eventstore.SetField(
e.Aggregate(),
projectSearchObject(e.Aggregate().ID),
ProjectStateSearchField,
&eventstore.Value{
Value: domain.ProjectStateActive,
ShouldIndex: true,
},
eventstore.FieldTypeInstanceID,
eventstore.FieldTypeResourceOwner,
eventstore.FieldTypeAggregateID,
eventstore.FieldTypeAggregateType,
eventstore.FieldTypeObjectType,
eventstore.FieldTypeObjectID,
eventstore.FieldTypeObjectRevision,
eventstore.FieldTypeFieldName,
),
}
}
func NewProjectAddedEvent(
ctx context.Context,
aggregate *eventstore.Aggregate,
@@ -110,6 +154,30 @@ func (e *ProjectChangeEvent) UniqueConstraints() []*eventstore.UniqueConstraint
return nil
}
func (e *ProjectChangeEvent) Fields() []*eventstore.FieldOperation {
if e.Name == nil {
return nil
}
return []*eventstore.FieldOperation{
eventstore.SetField(
e.Aggregate(),
projectSearchObject(e.Aggregate().ID),
ProjectNameSearchField,
&eventstore.Value{
Value: *e.Name,
ShouldIndex: true,
},
eventstore.FieldTypeInstanceID,
eventstore.FieldTypeResourceOwner,
eventstore.FieldTypeAggregateType,
eventstore.FieldTypeAggregateID,
eventstore.FieldTypeObjectType,
eventstore.FieldTypeObjectID,
eventstore.FieldTypeFieldName,
),
}
}
func NewProjectChangeEvent(
ctx context.Context,
aggregate *eventstore.Aggregate,
@@ -190,6 +258,28 @@ func (e *ProjectDeactivatedEvent) UniqueConstraints() []*eventstore.UniqueConstr
return nil
}
func (e *ProjectDeactivatedEvent) Fields() []*eventstore.FieldOperation {
return []*eventstore.FieldOperation{
eventstore.SetField(
e.Aggregate(),
projectSearchObject(e.Aggregate().ID),
ProjectStateSearchField,
&eventstore.Value{
Value: domain.ProjectStateInactive,
ShouldIndex: true,
},
eventstore.FieldTypeInstanceID,
eventstore.FieldTypeResourceOwner,
eventstore.FieldTypeAggregateType,
eventstore.FieldTypeAggregateID,
eventstore.FieldTypeObjectType,
eventstore.FieldTypeObjectID,
eventstore.FieldTypeFieldName,
),
}
}
func NewProjectDeactivatedEvent(ctx context.Context, aggregate *eventstore.Aggregate) *ProjectDeactivatedEvent {
return &ProjectDeactivatedEvent{
BaseEvent: *eventstore.NewBaseEventForPush(
@@ -218,6 +308,28 @@ func (e *ProjectReactivatedEvent) UniqueConstraints() []*eventstore.UniqueConstr
return nil
}
func (e *ProjectReactivatedEvent) Fields() []*eventstore.FieldOperation {
return []*eventstore.FieldOperation{
eventstore.SetField(
e.Aggregate(),
projectSearchObject(e.Aggregate().ID),
ProjectStateSearchField,
&eventstore.Value{
Value: domain.ProjectStateRemoved,
ShouldIndex: true,
},
eventstore.FieldTypeInstanceID,
eventstore.FieldTypeResourceOwner,
eventstore.FieldTypeAggregateType,
eventstore.FieldTypeAggregateID,
eventstore.FieldTypeObjectType,
eventstore.FieldTypeObjectID,
eventstore.FieldTypeFieldName,
),
}
}
func NewProjectReactivatedEvent(ctx context.Context, aggregate *eventstore.Aggregate) *ProjectReactivatedEvent {
return &ProjectReactivatedEvent{
BaseEvent: *eventstore.NewBaseEventForPush(
@@ -255,6 +367,12 @@ func (e *ProjectRemovedEvent) UniqueConstraints() []*eventstore.UniqueConstraint
return constraints
}
func (e *ProjectRemovedEvent) Fields() []*eventstore.FieldOperation {
return []*eventstore.FieldOperation{
eventstore.RemoveSearchFieldsByAggregate(e.Aggregate()),
}
}
func NewProjectRemovedEvent(
ctx context.Context,
aggregate *eventstore.Aggregate,
@@ -277,3 +395,11 @@ func ProjectRemovedEventMapper(event eventstore.Event) (eventstore.Event, error)
BaseEvent: *eventstore.BaseEventFromRepo(event),
}, nil
}
func projectSearchObject(id string) eventstore.Object {
return eventstore.Object{
Type: ProjectSearchType,
Revision: ProjectObjectRevision,
ID: id,
}
}

120
internal/repository/project/role.go
View File

@@ -14,6 +14,12 @@ var (
RoleAddedType = roleEventTypePrefix + "added"
RoleChangedType = roleEventTypePrefix + "changed"
RoleRemovedType = roleEventTypePrefix + "removed"
ProjectRoleSearchType = "project_role"
ProjectRoleRevision = uint8(1)
ProjectRoleKeySearchField = "key"
ProjectRoleDisplayNameSearchField = "display_name"
ProjectRoleGroupSearchField = "group"
)
func NewAddProjectRoleUniqueConstraint(roleKey, projectID string) *eventstore.UniqueConstraint {
@@ -45,6 +51,59 @@ func (e *RoleAddedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return []*eventstore.UniqueConstraint{NewAddProjectRoleUniqueConstraint(e.Key, e.Aggregate().ID)}
}
func (e *RoleAddedEvent) Fields() []*eventstore.FieldOperation {
return []*eventstore.FieldOperation{
eventstore.SetField(
e.Aggregate(),
projectRoleSearchObject(e.Key),
ProjectRoleKeySearchField,
&eventstore.Value{
Value: e.Key,
ShouldIndex: true,
},
eventstore.FieldTypeInstanceID,
eventstore.FieldTypeResourceOwner,
eventstore.FieldTypeAggregateType,
eventstore.FieldTypeAggregateID,
eventstore.FieldTypeObjectType,
eventstore.FieldTypeObjectID,
eventstore.FieldTypeFieldName,
),
eventstore.SetField(
e.Aggregate(),
projectRoleSearchObject(e.Key),
ProjectRoleDisplayNameSearchField,
&eventstore.Value{
Value: e.DisplayName,
ShouldIndex: true,
},
eventstore.FieldTypeInstanceID,
eventstore.FieldTypeResourceOwner,
eventstore.FieldTypeAggregateType,
eventstore.FieldTypeAggregateID,
eventstore.FieldTypeObjectType,
eventstore.FieldTypeObjectID,
eventstore.FieldTypeFieldName,
),
eventstore.SetField(
e.Aggregate(),
projectRoleSearchObject(e.Key),
ProjectRoleGroupSearchField,
&eventstore.Value{
Value: e.Group,
ShouldIndex: true,
},
eventstore.FieldTypeInstanceID,
eventstore.FieldTypeResourceOwner,
eventstore.FieldTypeAggregateType,
eventstore.FieldTypeAggregateID,
eventstore.FieldTypeObjectType,
eventstore.FieldTypeObjectID,
eventstore.FieldTypeFieldName,
),
}
}
func NewRoleAddedEvent(
ctx context.Context,
aggregate *eventstore.Aggregate,
@@ -93,6 +152,50 @@ func (e *RoleChangedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return nil
}
func (e *RoleChangedEvent) Fields() []*eventstore.FieldOperation {
operations := make([]*eventstore.FieldOperation, 0, 2)
if e.DisplayName != nil {
operations = append(operations, eventstore.SetField(
e.Aggregate(),
projectRoleSearchObject(e.Key),
ProjectRoleDisplayNameSearchField,
&eventstore.Value{
Value: *e.DisplayName,
ShouldIndex: true,
},
eventstore.FieldTypeInstanceID,
eventstore.FieldTypeResourceOwner,
eventstore.FieldTypeAggregateType,
eventstore.FieldTypeAggregateID,
eventstore.FieldTypeObjectType,
eventstore.FieldTypeObjectID,
eventstore.FieldTypeFieldName,
))
}
if e.Group != nil {
operations = append(operations, eventstore.SetField(
e.Aggregate(),
projectRoleSearchObject(e.Key),
ProjectRoleGroupSearchField,
&eventstore.Value{
Value: *e.Group,
ShouldIndex: true,
},
eventstore.FieldTypeInstanceID,
eventstore.FieldTypeResourceOwner,
eventstore.FieldTypeAggregateType,
eventstore.FieldTypeAggregateID,
eventstore.FieldTypeObjectType,
eventstore.FieldTypeObjectID,
eventstore.FieldTypeFieldName,
))
}
return operations
}
func NewRoleChangedEvent(
ctx context.Context,
aggregate *eventstore.Aggregate,
@@ -162,6 +265,15 @@ func (e *RoleRemovedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return []*eventstore.UniqueConstraint{NewRemoveProjectRoleUniqueConstraint(e.Key, e.Aggregate().ID)}
}
func (e *RoleRemovedEvent) Fields() []*eventstore.FieldOperation {
return []*eventstore.FieldOperation{
eventstore.RemoveSearchFieldsByAggregateAndObject(
e.Aggregate(),
projectRoleSearchObject(e.Key),
),
}
}
func NewRoleRemovedEvent(
ctx context.Context,
aggregate *eventstore.Aggregate,
@@ -188,3 +300,11 @@ func RoleRemovedEventMapper(event eventstore.Event) (eventstore.Event, error) {
return e, nil
}
func projectRoleSearchObject(id string) eventstore.Object {
return eventstore.Object{
Type: ProjectRoleSearchType,
Revision: ProjectRoleRevision,
ID: id,
}
}