TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-11 21:37:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: projections auto create their tables (#3324)

Browse Source 
* begin init checks for projections

* first projection checks

* debug notification providers with query fixes

* more projections and first index

* more projections

* more projections

* finish projections

* fix tests (remove db name)

* create tables in setup

* fix logging / error handling

* add tenant to views

* rename tenant to instance_id

* add instance_id to all projections

* add instance_id to all queries

* correct instance_id on projections

* add instance_id to failed_events

* use separate context for instance

* implement features projection

* implement features projection

* remove unique constraint from setup when migration failed

* add error to failed setup event

* add instance_id to primary keys

* fix IAM projection

* remove old migrations folder

* fix keysFromYAML test
This commit is contained in:
Livio Amstutz
2022-03-23 09:02:39 +01:00
committed by GitHub
parent 9e13b70a3d
commit 56b916a2b0
400 changed files with 6508 additions and 8890 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/authz/repository/eventsourcing/eventstore/token_verifier.go
View File

@@ -236,7 +236,7 @@ func (repo *TokenVerifierRepo) VerifierClientID(ctx context.Context, appName str
ctx, span := tracing.NewSpan(ctx)
defer func() { span.EndWithError(err) }()
iam, err := repo.Query.IAMByID(ctx, domain.IAMID)
iam, err := repo.Query.IAM(ctx)
if err != nil {
return "", "", err
}

11
internal/authz/repository/eventsourcing/eventstore/user_membership.go
View File

@@ -28,6 +28,7 @@ func (repo *UserMembershipRepo) SearchMyMemberships(ctx context.Context) ([]*aut
func (repo *UserMembershipRepo) searchUserMemberships(ctx context.Context) ([]*user_view_model.UserMembershipView, error) {
ctxData := authz.GetCtxData(ctx)
instance := authz.GetInstance(ctx)
orgMemberships, orgCount, err := repo.View.SearchUserMemberships(&user_model.UserMembershipSearchRequest{
Queries: []*user_model.UserMembershipSearchQuery{
{
@@ -40,6 +41,11 @@ func (repo *UserMembershipRepo) searchUserMemberships(ctx context.Context) ([]*u
Method: domain.SearchMethodEquals,
Value: ctxData.OrgID,
},
{
Key: user_model.UserMembershipSearchKeyInstanceID,
Method: domain.SearchMethodEquals,
Value: instance.ID,
},
},
})
if err != nil {
@@ -57,6 +63,11 @@ func (repo *UserMembershipRepo) searchUserMemberships(ctx context.Context) ([]*u
Method: domain.SearchMethodEquals,
Value: domain.IAMID,
},
{
Key: user_model.UserMembershipSearchKeyInstanceID,
Method: domain.SearchMethodEquals,
Value: instance.ID,
},
},
})
if err != nil {

2
internal/authz/repository/eventsourcing/handler/handler.go
View File

@@ -30,8 +30,6 @@ func (h *handler) Eventstore() v1.Eventstore {
func Register(configs Configs, bulkLimit, errorCount uint64, view *view.View, es v1.Eventstore, systemDefaults sd.SystemDefaults) []query.Handler {
return []query.Handler{
newUserGrant(
handler{view, bulkLimit, configs.cycleDuration("UserGrants"), errorCount, es}),
newUserMembership(
handler{view, bulkLimit, configs.cycleDuration("UserMemberships"), errorCount, es}),
}

313
internal/authz/repository/eventsourcing/handler/user_grant.go
View File

@@ -1,313 +0,0 @@
package handler
import (
"context"
"strings"
v1 "github.com/caos/zitadel/internal/eventstore/v1"
es_sdk "github.com/caos/zitadel/internal/eventstore/v1/sdk"
iam_model "github.com/caos/zitadel/internal/iam/model"
iam_view "github.com/caos/zitadel/internal/iam/repository/view"
"github.com/caos/logging"
"github.com/caos/zitadel/internal/domain"
"github.com/caos/zitadel/internal/errors"
caos_errs "github.com/caos/zitadel/internal/errors"
es_models "github.com/caos/zitadel/internal/eventstore/v1/models"
"github.com/caos/zitadel/internal/eventstore/v1/query"
"github.com/caos/zitadel/internal/eventstore/v1/spooler"
iam_es_model "github.com/caos/zitadel/internal/iam/repository/eventsourcing/model"
org_es_model "github.com/caos/zitadel/internal/org/repository/eventsourcing/model"
proj_es_model "github.com/caos/zitadel/internal/project/repository/eventsourcing/model"
view_model "github.com/caos/zitadel/internal/usergrant/repository/view/model"
)
const (
userGrantTable = "authz.user_grants"
)
type UserGrant struct {
handler
iamProjectID string
subscription *v1.Subscription
}
func newUserGrant(
handler handler,
) *UserGrant {
h := &UserGrant{
handler: handler,
}
h.subscribe()
return h
}
func (k *UserGrant) subscribe() {
k.subscription = k.es.Subscribe(k.AggregateTypes()...)
go func() {
for event := range k.subscription.Events {
query.ReduceEvent(k, event)
}
}()
}
func (u *UserGrant) ViewModel() string {
return userGrantTable
}
func (u *UserGrant) Subscription() *v1.Subscription {
return u.subscription
}
func (_ *UserGrant) AggregateTypes() []es_models.AggregateType {
return []es_models.AggregateType{iam_es_model.IAMAggregate, org_es_model.OrgAggregate, proj_es_model.ProjectAggregate}
}
func (u *UserGrant) CurrentSequence() (uint64, error) {
sequence, err := u.view.GetLatestUserGrantSequence()
if err != nil {
return 0, err
}
return sequence.CurrentSequence, nil
}
func (u *UserGrant) EventQuery() (*es_models.SearchQuery, error) {
if u.iamProjectID == "" {
err := u.setIamProjectID()
if err != nil {
return nil, err
}
}
sequence, err := u.view.GetLatestUserGrantSequence()
if err != nil {
return nil, err
}
return es_models.NewSearchQuery().
AggregateTypeFilter(iam_es_model.IAMAggregate, org_es_model.OrgAggregate, proj_es_model.ProjectAggregate).
LatestSequenceFilter(sequence.CurrentSequence), nil
}
func (u *UserGrant) Reduce(event *es_models.Event) (err error) {
switch event.AggregateType {
case proj_es_model.ProjectAggregate:
err = u.processProject(event)
case iam_es_model.IAMAggregate:
err = u.processIAMMember(event, "IAM", false)
case org_es_model.OrgAggregate:
return u.processOrg(event)
}
return err
}
func (u *UserGrant) processProject(event *es_models.Event) (err error) {
switch event.Type {
case proj_es_model.ProjectMemberAdded, proj_es_model.ProjectMemberChanged,
proj_es_model.ProjectMemberRemoved, proj_es_model.ProjectMemberCascadeRemoved:
member := new(proj_es_model.ProjectMember)
err := member.SetData(event)
if err != nil {
return err
}
return u.processMember(event, "PROJECT", event.AggregateID, member.UserID, member.Roles)
case proj_es_model.ProjectGrantMemberAdded, proj_es_model.ProjectGrantMemberChanged,
proj_es_model.ProjectGrantMemberRemoved,
proj_es_model.ProjectGrantMemberCascadeRemoved:
member := new(proj_es_model.ProjectGrantMember)
err := member.SetData(event)
if err != nil {
return err
}
return u.processMember(event, "PROJECT_GRANT", member.GrantID, member.UserID, member.Roles)
default:
return u.view.ProcessedUserGrantSequence(event)
}
}
func (u *UserGrant) processOrg(event *es_models.Event) (err error) {
switch event.Type {
case org_es_model.OrgMemberAdded, org_es_model.OrgMemberChanged,
org_es_model.OrgMemberRemoved, org_es_model.OrgMemberCascadeRemoved:
member := new(org_es_model.OrgMember)
err := member.SetData(event)
if err != nil {
return err
}
return u.processMember(event, "ORG", "", member.UserID, member.Roles)
default:
return u.view.ProcessedUserGrantSequence(event)
}
}
func (u *UserGrant) processIAMMember(event *es_models.Event, rolePrefix string, suffix bool) error {
member := new(iam_es_model.IAMMember)
switch event.Type {
case iam_es_model.IAMMemberAdded, iam_es_model.IAMMemberChanged:
member.SetData(event)
grant, err := u.view.UserGrantByIDs(domain.IAMID, u.iamProjectID, member.UserID)
if err != nil && !errors.IsNotFound(err) {
return err
}
if errors.IsNotFound(err) {
grant = &view_model.UserGrantView{
ID: u.iamProjectID + member.UserID,
ResourceOwner: domain.IAMID,
OrgName: domain.IAMID,
ProjectID: u.iamProjectID,
UserID: member.UserID,
RoleKeys: member.Roles,
CreationDate: event.CreationDate,
}
if suffix {
grant.RoleKeys = suffixRoles(event.AggregateID, grant.RoleKeys)
}
} else {
newRoles := member.Roles
if grant.RoleKeys != nil {
grant.RoleKeys = mergeExistingRoles(rolePrefix, "", grant.RoleKeys, newRoles)
} else {
grant.RoleKeys = newRoles
}
}
grant.Sequence = event.Sequence
grant.ChangeDate = event.CreationDate
return u.view.PutUserGrant(grant, event)
case iam_es_model.IAMMemberRemoved,
iam_es_model.IAMMemberCascadeRemoved:
member.SetData(event)
grant, err := u.view.UserGrantByIDs(domain.IAMID, u.iamProjectID, member.UserID)
if err != nil {
return err
}
return u.view.DeleteUserGrant(grant.ID, event)
default:
return u.view.ProcessedUserGrantSequence(event)
}
}
func (u *UserGrant) processMember(event *es_models.Event, rolePrefix, roleSuffix string, userID string, roleKeys []string) error {
switch event.Type {
case org_es_model.OrgMemberAdded, proj_es_model.ProjectMemberAdded, proj_es_model.ProjectGrantMemberAdded,
org_es_model.OrgMemberChanged, proj_es_model.ProjectMemberChanged, proj_es_model.ProjectGrantMemberChanged:
grant, err := u.view.UserGrantByIDs(event.ResourceOwner, u.iamProjectID, userID)
if err != nil && !errors.IsNotFound(err) {
return err
}
if roleSuffix != "" {
roleKeys = suffixRoles(roleSuffix, roleKeys)
}
if errors.IsNotFound(err) {
grant = &view_model.UserGrantView{
ID: u.iamProjectID + event.ResourceOwner + userID,
ResourceOwner: event.ResourceOwner,
ProjectID: u.iamProjectID,
UserID: userID,
RoleKeys: roleKeys,
CreationDate: event.CreationDate,
}
} else {
newRoles := roleKeys
if grant.RoleKeys != nil {
grant.RoleKeys = mergeExistingRoles(rolePrefix, roleSuffix, grant.RoleKeys, newRoles)
} else {
grant.RoleKeys = newRoles
}
}
grant.Sequence = event.Sequence
grant.ChangeDate = event.CreationDate
return u.view.PutUserGrant(grant, event)
case org_es_model.OrgMemberRemoved,
org_es_model.OrgMemberCascadeRemoved,
proj_es_model.ProjectMemberRemoved,
proj_es_model.ProjectMemberCascadeRemoved,
proj_es_model.ProjectGrantMemberRemoved,
proj_es_model.ProjectGrantMemberCascadeRemoved:
grant, err := u.view.UserGrantByIDs(event.ResourceOwner, u.iamProjectID, userID)
if err != nil && !errors.IsNotFound(err) {
return err
}
if errors.IsNotFound(err) {
return u.view.ProcessedUserGrantSequence(event)
}
if roleSuffix != "" {
roleKeys = suffixRoles(roleSuffix, roleKeys)
}
if grant.RoleKeys == nil {
return u.view.ProcessedUserGrantSequence(event)
}
grant.RoleKeys = mergeExistingRoles(rolePrefix, roleSuffix, grant.RoleKeys, nil)
return u.view.PutUserGrant(grant, event)
default:
return u.view.ProcessedUserGrantSequence(event)
}
}
func suffixRoles(suffix string, roles []string) []string {
suffixedRoles := make([]string, len(roles))
for i := 0; i < len(roles); i++ {
suffixedRoles[i] = roles[i] + ":" + suffix
}
return suffixedRoles
}
func mergeExistingRoles(rolePrefix, suffix string, existingRoles, newRoles []string) []string {
mergedRoles := make([]string, 0)
for _, existingRole := range existingRoles {
if !strings.HasPrefix(existingRole, rolePrefix) {
mergedRoles = append(mergedRoles, existingRole)
continue
}
if suffix != "" && !strings.HasSuffix(existingRole, suffix) {
mergedRoles = append(mergedRoles, existingRole)
}
}
return append(mergedRoles, newRoles...)
}
func (u *UserGrant) setIamProjectID() error {
if u.iamProjectID != "" {
return nil
}
iam, err := u.getIAMByID(context.Background())
if err != nil {
return err
}
if iam.SetUpDone < domain.StepCount-1 {
return caos_errs.ThrowPreconditionFailed(nil, "HANDL-s5DTs", "Setup not done")
}
u.iamProjectID = iam.IAMProjectID
return nil
}
func (u *UserGrant) OnError(event *es_models.Event, err error) error {
logging.LogWithFields("SPOOL-VcVoJ", "id", event.AggregateID).WithError(err).Warn("something went wrong in user grant handler")
return spooler.HandleError(event, err, u.view.GetLatestUserGrantFailedEvent, u.view.ProcessedUserGrantFailedEvent, u.view.ProcessedUserGrantSequence, u.errorCountUntilSkip)
}
func (u *UserGrant) OnSuccess() error {
return spooler.HandleSuccess(u.view.UpdateUserGrantSpoolerRunTimestamp)
}
func (u *UserGrant) getIAMByID(ctx context.Context) (*iam_model.IAM, error) {
query, err := iam_view.IAMByIDQuery(domain.IAMID, 0)
if err != nil {
return nil, err
}
iam := &iam_es_model.IAM{
ObjectRoot: es_models.ObjectRoot{
AggregateID: domain.IAMID,
},
}
err = es_sdk.Filter(ctx, u.Eventstore().FilterEvents, iam.AppendEvents, query)
if err != nil && errors.IsNotFound(err) && iam.Sequence == 0 {
return nil, err
}
return iam_es_model.IAMToModel(iam), nil
}

70
internal/authz/repository/eventsourcing/view/user_grant.go
View File

@@ -1,70 +0,0 @@
package view
import (
"github.com/caos/zitadel/internal/errors"
"github.com/caos/zitadel/internal/eventstore/v1/models"
grant_model "github.com/caos/zitadel/internal/usergrant/model"
"github.com/caos/zitadel/internal/usergrant/repository/view"
"github.com/caos/zitadel/internal/usergrant/repository/view/model"
"github.com/caos/zitadel/internal/view/repository"
)
const (
userGrantTable = "authz.user_grants"
)
func (v *View) UserGrantByID(grantID string) (*model.UserGrantView, error) {
return view.UserGrantByID(v.Db, userGrantTable, grantID)
}
func (v *View) UserGrantByIDs(resourceOwnerID, projectID, userID string) (*model.UserGrantView, error) {
return view.UserGrantByIDs(v.Db, userGrantTable, resourceOwnerID, projectID, userID)
}
func (v *View) UserGrantsByUserID(userID string) ([]*model.UserGrantView, error) {
return view.UserGrantsByUserID(v.Db, userGrantTable, userID)
}
func (v *View) UserGrantsByProjectID(projectID string) ([]*model.UserGrantView, error) {
return view.UserGrantsByProjectID(v.Db, userGrantTable, projectID)
}
func (v *View) SearchUserGrants(request *grant_model.UserGrantSearchRequest) ([]*model.UserGrantView, uint64, error) {
return view.SearchUserGrants(v.Db, userGrantTable, request)
}
func (v *View) PutUserGrant(grant *model.UserGrantView, event *models.Event) error {
err := view.PutUserGrant(v.Db, userGrantTable, grant)
if err != nil {
return err
}
return v.ProcessedUserGrantSequence(event)
}
func (v *View) DeleteUserGrant(grantID string, event *models.Event) error {
err := view.DeleteUserGrant(v.Db, userGrantTable, grantID)
if err != nil && !errors.IsNotFound(err) {
return err
}
return v.ProcessedUserGrantSequence(event)
}
func (v *View) GetLatestUserGrantSequence() (*repository.CurrentSequence, error) {
return v.latestSequence(userGrantTable)
}
func (v *View) ProcessedUserGrantSequence(event *models.Event) error {
return v.saveCurrentSequence(userGrantTable, event)
}
func (v *View) UpdateUserGrantSpoolerRunTimestamp() error {
return v.updateSpoolerRunSequence(userGrantTable)
}
func (v *View) GetLatestUserGrantFailedEvent(sequence uint64) (*repository.FailedEvent, error) {
return v.latestFailedEvent(userGrantTable, sequence)
}
func (v *View) ProcessedUserGrantFailedEvent(failedEvent *repository.FailedEvent) error {
return v.saveFailedEvent(failedEvent)
}