fix(permissions_v2): add membership fields migration (#9199)

# Which Problems Are Solved

Memberships did not have a fields table fill migration.

# How the Problems Are Solved

Add filling of membership fields to the repeatable steps.

# Additional Changes

- Use the same repeatable step for multiple fill fields handlers.
- Fix an error for PostgreSQL 15 where a subquery in a `FROM` clause
needs an alias ing the `permitted_orgs` function.

# Additional Context

- Part of https://github.com/zitadel/zitadel/issues/9188
- Introduced in https://github.com/zitadel/zitadel/pull/9152
This commit is contained in:
Tim Möhlmann 2025-01-17 16:16:26 +01:00 committed by GitHub
parent 9532c9bea5
commit 94cbf97534
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
22 changed files with 164 additions and 105 deletions

View File

@ -1,44 +0,0 @@
package setup
import (
"context"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/query/projection"
"github.com/zitadel/zitadel/internal/repository/instance"
)
type FillFieldsForInstanceDomains struct {
eventstore *eventstore.Eventstore
}
func (mig *FillFieldsForInstanceDomains) Execute(ctx context.Context, _ eventstore.Event) error {
instances, err := mig.eventstore.InstanceIDs(
ctx,
eventstore.NewSearchQueryBuilder(eventstore.ColumnsInstanceIDs).
OrderDesc().
AddQuery().
AggregateTypes("instance").
EventTypes(instance.InstanceAddedEventType).
Builder(),
)
if err != nil {
return err
}
for _, instance := range instances {
ctx := authz.WithInstanceID(ctx, instance)
if err := projection.InstanceDomainFields.Trigger(ctx); err != nil {
return err
}
}
return nil
}
func (mig *FillFieldsForInstanceDomains) String() string {
return "repeatable_fill_fields_for_instance_domains"
}
func (f *FillFieldsForInstanceDomains) Check(lastRun map[string]interface{}) bool {
return true
}

View File

@ -44,7 +44,7 @@ BEGIN
WHERE om.role = ANY(matched_roles)
AND om.instance_id = instanceID
AND om.user_id = userId
);
) AS orgs;
RETURN;
END;
$$;

51
cmd/setup/fill_fields.go Normal file
View File

@ -0,0 +1,51 @@
package setup
import (
"context"
"fmt"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/eventstore/handler/v2"
"github.com/zitadel/zitadel/internal/repository/instance"
)
type RepeatableFillFields struct {
eventstore *eventstore.Eventstore
handlers []*handler.FieldHandler
}
func (mig *RepeatableFillFields) Execute(ctx context.Context, _ eventstore.Event) error {
instances, err := mig.eventstore.InstanceIDs(
ctx,
eventstore.NewSearchQueryBuilder(eventstore.ColumnsInstanceIDs).
OrderDesc().
AddQuery().
AggregateTypes(instance.AggregateType).
EventTypes(instance.InstanceAddedEventType).
Builder(),
)
if err != nil {
return err
}
for _, instance := range instances {
ctx := authz.WithInstanceID(ctx, instance)
for _, handler := range mig.handlers {
logging.WithFields("migration", mig.String(), "instance_id", instance, "handler", handler.String()).Info("run fields trigger")
if err := handler.Trigger(ctx); err != nil {
return fmt.Errorf("%s: %s: %w", mig.String(), handler.String(), err)
}
}
}
return nil
}
func (mig *RepeatableFillFields) String() string {
return "repeatable_fill_fields"
}
func (f *RepeatableFillFields) Check(lastRun map[string]interface{}) bool {
return true
}

View File

@ -28,6 +28,7 @@ import (
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/domain"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/eventstore/handler/v2"
old_es "github.com/zitadel/zitadel/internal/eventstore/repository/sql"
new_es "github.com/zitadel/zitadel/internal/eventstore/v3"
"github.com/zitadel/zitadel/internal/i18n"
@ -189,8 +190,12 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string)
&DeleteStaleOrgFields{
eventstore: eventstoreClient,
},
&FillFieldsForInstanceDomains{
&RepeatableFillFields{
eventstore: eventstoreClient,
handlers: []*handler.FieldHandler{
projection.InstanceDomainFields,
projection.MembershipFields,
},
},
&SyncRolePermissions{
eventstore: eventstoreClient,

4
go.mod
View File

@ -28,6 +28,8 @@ require (
github.com/go-jose/go-jose/v4 v4.0.4
github.com/go-ldap/ldap/v3 v3.4.8
github.com/go-webauthn/webauthn v0.10.2
github.com/goccy/go-json v0.10.3
github.com/golang/protobuf v1.5.4
github.com/gorilla/csrf v1.7.2
github.com/gorilla/mux v1.8.1
github.com/gorilla/schema v1.4.1
@ -106,11 +108,9 @@ require (
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-sql-driver/mysql v1.7.1 // indirect
github.com/go-webauthn/x v0.1.9 // indirect
github.com/goccy/go-json v0.10.3 // indirect
github.com/golang-jwt/jwt/v4 v4.5.1 // indirect
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
github.com/golang/mock v1.6.0 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/go-tpm v0.9.0 // indirect
github.com/google/pprof v0.0.0-20240528025155-186aa0362fba // indirect
github.com/google/s2a-go v0.1.7 // indirect

View File

@ -10,12 +10,11 @@ import (
"testing"
"time"
"github.com/zitadel/logging"
"github.com/brianvoe/gofakeit/v6"
"github.com/muhlemmer/gu"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zitadel/logging"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/types/known/timestamppb"

View File

@ -4,10 +4,11 @@ package integration_test
import (
"context"
"github.com/zitadel/zitadel/internal/integration"
"os"
"testing"
"time"
"github.com/zitadel/zitadel/internal/integration"
)
var (

View File

@ -5,22 +5,24 @@ package integration_test
import (
"context"
_ "embed"
"net/http"
"path"
"testing"
"time"
"github.com/brianvoe/gofakeit/v6"
"github.com/muhlemmer/gu"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/text/language"
"google.golang.org/grpc/codes"
"github.com/zitadel/zitadel/internal/api/scim/resources"
"github.com/zitadel/zitadel/internal/api/scim/schemas"
"github.com/zitadel/zitadel/internal/integration"
"github.com/zitadel/zitadel/internal/integration/scim"
"github.com/zitadel/zitadel/pkg/grpc/management"
"github.com/zitadel/zitadel/pkg/grpc/user/v2"
"golang.org/x/text/language"
"google.golang.org/grpc/codes"
"net/http"
"path"
"testing"
"time"
)
var (

View File

@ -4,16 +4,18 @@ package integration_test
import (
"context"
"github.com/brianvoe/gofakeit/v6"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zitadel/zitadel/internal/integration"
"github.com/zitadel/zitadel/internal/integration/scim"
"github.com/zitadel/zitadel/pkg/grpc/user/v2"
"google.golang.org/grpc/codes"
"net/http"
"testing"
"time"
"github.com/brianvoe/gofakeit/v6"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/grpc/codes"
"github.com/zitadel/zitadel/internal/integration"
"github.com/zitadel/zitadel/internal/integration/scim"
"github.com/zitadel/zitadel/pkg/grpc/user/v2"
)
func TestDeleteUser_errors(t *testing.T) {

View File

@ -4,21 +4,23 @@ package integration_test
import (
"context"
"net/http"
"path"
"testing"
"time"
"github.com/brianvoe/gofakeit/v6"
"github.com/muhlemmer/gu"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/text/language"
"github.com/zitadel/zitadel/internal/api/scim/resources"
"github.com/zitadel/zitadel/internal/api/scim/schemas"
"github.com/zitadel/zitadel/internal/integration"
"github.com/zitadel/zitadel/internal/integration/scim"
"github.com/zitadel/zitadel/pkg/grpc/management"
"github.com/zitadel/zitadel/pkg/grpc/user/v2"
"golang.org/x/text/language"
"net/http"
"path"
"testing"
"time"
)
func TestGetUser(t *testing.T) {

View File

@ -5,20 +5,22 @@ package integration_test
import (
"context"
_ "embed"
"net/http"
"path"
"testing"
"time"
"github.com/muhlemmer/gu"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/text/language"
"github.com/zitadel/zitadel/internal/api/scim/resources"
"github.com/zitadel/zitadel/internal/api/scim/schemas"
"github.com/zitadel/zitadel/internal/integration"
"github.com/zitadel/zitadel/internal/integration/scim"
"github.com/zitadel/zitadel/pkg/grpc/management"
"github.com/zitadel/zitadel/pkg/grpc/user/v2"
"golang.org/x/text/language"
"net/http"
"path"
"testing"
"time"
)
var (

View File

@ -4,9 +4,9 @@ import (
"context"
"time"
"github.com/zitadel/logging"
"golang.org/x/text/language"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/api/http"
"github.com/zitadel/zitadel/internal/command/preparation"

View File

@ -4,6 +4,7 @@ import (
"context"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/command/preparation"
"github.com/zitadel/zitadel/internal/eventstore"

View File

@ -58,9 +58,9 @@ func (wm *ProjectMemberWriteModel) Query() *eventstore.SearchQueryBuilder {
AddQuery().
AggregateTypes(project.AggregateType).
AggregateIDs(wm.MemberWriteModel.AggregateID).
EventTypes(project.MemberAddedType,
project.MemberChangedType,
project.MemberRemovedType,
project.MemberCascadeRemovedType).
EventTypes(project.MemberAddedEventType,
project.MemberChangedEventType,
project.MemberRemovedEventType,
project.MemberCascadeRemovedEventType).
Builder()
}

View File

@ -32,6 +32,9 @@ func (f *fieldProjection) Reducers() []AggregateReducer {
var _ Projection = (*fieldProjection)(nil)
// NewFieldHandler returns a projection handler which backfills the `eventstore.fields` table with historic events which
// might have existed before they had and Field Operations defined.
// The events are filtered by the mapped aggregate types and each event type for that aggregate.
func NewFieldHandler(config *Config, name string, eventTypes map[eventstore.AggregateType][]eventstore.EventType) *FieldHandler {
return &FieldHandler{
Handler: Handler{
@ -51,6 +54,7 @@ func NewFieldHandler(config *Config, name string, eventTypes map[eventstore.Aggr
}
}
// Trigger executes the backfill job of events for the instance currently in the context.
func (h *FieldHandler) Trigger(ctx context.Context, opts ...TriggerOpt) (err error) {
config := new(triggerConfig)
for _, opt := range opts {

View File

@ -7,6 +7,7 @@ import (
"time"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/zerrors"

View File

@ -12,6 +12,7 @@ const (
fieldsProjectGrant = "project_grant_fields"
fieldsOrgDomainVerified = "org_domain_verified_fields"
fieldsInstanceDomain = "instance_domain_fields"
fieldsMemberships = "membership_fields"
)
func newFillProjectGrantFields(config handler.Config) *handler.FieldHandler {
@ -52,3 +53,33 @@ func newFillInstanceDomainFields(config handler.Config) *handler.FieldHandler {
},
)
}
func newFillMembershipFields(config handler.Config) *handler.FieldHandler {
return handler.NewFieldHandler(
&config,
fieldsMemberships,
map[eventstore.AggregateType][]eventstore.EventType{
instance.AggregateType: {
instance.MemberAddedEventType,
instance.MemberChangedEventType,
instance.MemberRemovedEventType,
instance.MemberCascadeRemovedEventType,
instance.InstanceRemovedEventType,
},
org.AggregateType: {
org.MemberAddedEventType,
org.MemberChangedEventType,
org.MemberRemovedEventType,
org.MemberCascadeRemovedEventType,
org.OrgRemovedEventType,
},
project.AggregateType: {
project.MemberAddedEventType,
project.MemberChangedEventType,
project.MemberRemovedEventType,
project.MemberCascadeRemovedEventType,
project.ProjectRemovedType,
},
},
)
}

View File

@ -60,19 +60,19 @@ func (p *projectMemberProjection) Reducers() []handler.AggregateReducer {
Aggregate: project.AggregateType,
EventReducers: []handler.EventReducer{
{
Event: project.MemberAddedType,
Event: project.MemberAddedEventType,
Reduce: p.reduceAdded,
},
{
Event: project.MemberChangedType,
Event: project.MemberChangedEventType,
Reduce: p.reduceChanged,
},
{
Event: project.MemberCascadeRemovedType,
Event: project.MemberCascadeRemovedEventType,
Reduce: p.reduceCascadeRemoved,
},
{
Event: project.MemberRemovedType,
Event: project.MemberRemovedEventType,
Reduce: p.reduceRemoved,
},
{
@ -114,7 +114,7 @@ func (p *projectMemberProjection) Reducers() []handler.AggregateReducer {
func (p *projectMemberProjection) reduceAdded(event eventstore.Event) (*handler.Statement, error) {
e, ok := event.(*project.MemberAddedEvent)
if !ok {
return nil, zerrors.ThrowInvalidArgumentf(nil, "HANDL-bgx5Q", "reduce.wrong.event.type %s", project.MemberAddedType)
return nil, zerrors.ThrowInvalidArgumentf(nil, "HANDL-bgx5Q", "reduce.wrong.event.type %s", project.MemberAddedEventType)
}
ctx := setMemberContext(e.Aggregate())
userOwner, err := getUserResourceOwner(ctx, p.es, e.Aggregate().InstanceID, e.UserID)
@ -131,7 +131,7 @@ func (p *projectMemberProjection) reduceAdded(event eventstore.Event) (*handler.
func (p *projectMemberProjection) reduceChanged(event eventstore.Event) (*handler.Statement, error) {
e, ok := event.(*project.MemberChangedEvent)
if !ok {
return nil, zerrors.ThrowInvalidArgumentf(nil, "HANDL-90WJ1", "reduce.wrong.event.type %s", project.MemberChangedType)
return nil, zerrors.ThrowInvalidArgumentf(nil, "HANDL-90WJ1", "reduce.wrong.event.type %s", project.MemberChangedEventType)
}
return reduceMemberChanged(
*member.NewMemberChangedEvent(&e.BaseEvent, e.UserID, e.Roles...),
@ -142,7 +142,7 @@ func (p *projectMemberProjection) reduceChanged(event eventstore.Event) (*handle
func (p *projectMemberProjection) reduceCascadeRemoved(event eventstore.Event) (*handler.Statement, error) {
e, ok := event.(*project.MemberCascadeRemovedEvent)
if !ok {
return nil, zerrors.ThrowInvalidArgumentf(nil, "HANDL-aGd43", "reduce.wrong.event.type %s", project.MemberCascadeRemovedType)
return nil, zerrors.ThrowInvalidArgumentf(nil, "HANDL-aGd43", "reduce.wrong.event.type %s", project.MemberCascadeRemovedEventType)
}
return reduceMemberCascadeRemoved(
*member.NewCascadeRemovedEvent(&e.BaseEvent, e.UserID),
@ -153,7 +153,7 @@ func (p *projectMemberProjection) reduceCascadeRemoved(event eventstore.Event) (
func (p *projectMemberProjection) reduceRemoved(event eventstore.Event) (*handler.Statement, error) {
e, ok := event.(*project.MemberRemovedEvent)
if !ok {
return nil, zerrors.ThrowInvalidArgumentf(nil, "HANDL-eJZPh", "reduce.wrong.event.type %s", project.MemberRemovedType)
return nil, zerrors.ThrowInvalidArgumentf(nil, "HANDL-eJZPh", "reduce.wrong.event.type %s", project.MemberRemovedEventType)
}
return reduceMemberRemoved(e,
withMemberCond(MemberUserIDCol, e.UserID),

View File

@ -32,7 +32,7 @@ func TestProjectMemberProjection_reduces(t *testing.T) {
args: args{
event: getEvent(
testEvent(
project.MemberAddedType,
project.MemberAddedEventType,
project.AggregateType,
[]byte(`{
"userId": "user-id",
@ -56,7 +56,7 @@ func TestProjectMemberProjection_reduces(t *testing.T) {
args: args{
event: getEvent(
testEvent(
project.MemberAddedType,
project.MemberAddedEventType,
project.AggregateType,
[]byte(`{
"userId": "user-id",
@ -110,7 +110,7 @@ func TestProjectMemberProjection_reduces(t *testing.T) {
args: args{
event: getEvent(
testEvent(
project.MemberAddedType,
project.MemberAddedEventType,
project.AggregateType,
[]byte(`{
"userId": "user-id",
@ -176,7 +176,7 @@ func TestProjectMemberProjection_reduces(t *testing.T) {
args: args{
event: getEvent(
testEvent(
project.MemberChangedType,
project.MemberChangedEventType,
project.AggregateType,
[]byte(`{
"userId": "user-id",
@ -210,7 +210,7 @@ func TestProjectMemberProjection_reduces(t *testing.T) {
args: args{
event: getEvent(
testEvent(
project.MemberCascadeRemovedType,
project.MemberCascadeRemovedEventType,
project.AggregateType,
[]byte(`{
"userId": "user-id"
@ -240,7 +240,7 @@ func TestProjectMemberProjection_reduces(t *testing.T) {
args: args{
event: getEvent(
testEvent(
project.MemberRemovedType,
project.MemberRemovedEventType,
project.AggregateType,
[]byte(`{
"userId": "user-id"

View File

@ -85,6 +85,7 @@ var (
ProjectGrantFields *handler.FieldHandler
OrgDomainVerifiedFields *handler.FieldHandler
InstanceDomainFields *handler.FieldHandler
MembershipFields *handler.FieldHandler
)
type projection interface {
@ -174,6 +175,7 @@ func Create(ctx context.Context, sqlClient *database.DB, es handler.EventStore,
ProjectGrantFields = newFillProjectGrantFields(applyCustomConfig(projectionConfig, config.Customizations[fieldsProjectGrant]))
OrgDomainVerifiedFields = newFillOrgDomainVerifiedFields(applyCustomConfig(projectionConfig, config.Customizations[fieldsOrgDomainVerified]))
InstanceDomainFields = newFillInstanceDomainFields(applyCustomConfig(projectionConfig, config.Customizations[fieldsInstanceDomain]))
MembershipFields = newFillMembershipFields(applyCustomConfig(projectionConfig, config.Customizations[fieldsMemberships]))
newProjectionsList()
return nil

View File

@ -10,10 +10,10 @@ func init() {
eventstore.RegisterFilterEventMapper(AggregateType, ProjectDeactivatedType, ProjectDeactivatedEventMapper)
eventstore.RegisterFilterEventMapper(AggregateType, ProjectReactivatedType, ProjectReactivatedEventMapper)
eventstore.RegisterFilterEventMapper(AggregateType, ProjectRemovedType, ProjectRemovedEventMapper)
eventstore.RegisterFilterEventMapper(AggregateType, MemberAddedType, MemberAddedEventMapper)
eventstore.RegisterFilterEventMapper(AggregateType, MemberChangedType, MemberChangedEventMapper)
eventstore.RegisterFilterEventMapper(AggregateType, MemberRemovedType, MemberRemovedEventMapper)
eventstore.RegisterFilterEventMapper(AggregateType, MemberCascadeRemovedType, MemberCascadeRemovedEventMapper)
eventstore.RegisterFilterEventMapper(AggregateType, MemberAddedEventType, MemberAddedEventMapper)
eventstore.RegisterFilterEventMapper(AggregateType, MemberChangedEventType, MemberChangedEventMapper)
eventstore.RegisterFilterEventMapper(AggregateType, MemberRemovedEventType, MemberRemovedEventMapper)
eventstore.RegisterFilterEventMapper(AggregateType, MemberCascadeRemovedEventType, MemberCascadeRemovedEventMapper)
eventstore.RegisterFilterEventMapper(AggregateType, RoleAddedType, RoleAddedEventMapper)
eventstore.RegisterFilterEventMapper(AggregateType, RoleChangedType, RoleChangedEventMapper)
eventstore.RegisterFilterEventMapper(AggregateType, RoleRemovedType, RoleRemovedEventMapper)

View File

@ -8,10 +8,10 @@ import (
)
var (
MemberAddedType = projectEventTypePrefix + member.AddedEventType
MemberChangedType = projectEventTypePrefix + member.ChangedEventType
MemberRemovedType = projectEventTypePrefix + member.RemovedEventType
MemberCascadeRemovedType = projectEventTypePrefix + member.CascadeRemovedEventType
MemberAddedEventType = projectEventTypePrefix + member.AddedEventType
MemberChangedEventType = projectEventTypePrefix + member.ChangedEventType
MemberRemovedEventType = projectEventTypePrefix + member.RemovedEventType
MemberCascadeRemovedEventType = projectEventTypePrefix + member.CascadeRemovedEventType
)
const (
@ -37,7 +37,7 @@ func NewProjectMemberAddedEvent(
eventstore.NewBaseEventForPush(
ctx,
aggregate,
MemberAddedType,
MemberAddedEventType,
),
userID,
roles...,
@ -74,7 +74,7 @@ func NewProjectMemberChangedEvent(
eventstore.NewBaseEventForPush(
ctx,
aggregate,
MemberChangedType,
MemberChangedEventType,
),
userID,
roles...,
@ -110,7 +110,7 @@ func NewProjectMemberRemovedEvent(
eventstore.NewBaseEventForPush(
ctx,
aggregate,
MemberRemovedType,
MemberRemovedEventType,
),
userID,
),
@ -145,7 +145,7 @@ func NewProjectMemberCascadeRemovedEvent(
eventstore.NewBaseEventForPush(
ctx,
aggregate,
MemberCascadeRemovedType,
MemberCascadeRemovedEventType,
),
userID,
),