TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-11 21:27:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(eventstore): increase parallel write capabilities (#5940)

Browse Source 
This implementation increases parallel write capabilities of the eventstore.
Please have a look at the technical advisories: [05](https://zitadel.com/docs/support/advisory/a10005) and  [06](https://zitadel.com/docs/support/advisory/a10006).
The implementation of eventstore.push is rewritten and stored events are migrated to a new table `eventstore.events2`.
If you are using cockroach: make sure that the database user of ZITADEL has `VIEWACTIVITY` grant. This is used to query events.
This commit is contained in:
Silvan
2023-10-19 12:19:10 +02:00
committed by GitHub
parent 259faba3f0
commit b5564572bc
791 changed files with 30326 additions and 43202 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
internal/query/user_membership.go
View File

@@ -3,6 +3,7 @@ package query
import (
"context"
"database/sql"
"sync"
"time"
sq "github.com/Masterminds/squirrel"
@@ -11,6 +12,7 @@ import (
"github.com/zitadel/zitadel/internal/api/call"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/errors"
"github.com/zitadel/zitadel/internal/eventstore/handler/v2"
"github.com/zitadel/zitadel/internal/query/projection"
"github.com/zitadel/zitadel/internal/telemetry/tracing"
)
@@ -22,7 +24,7 @@ type Memberships struct {
type Membership struct {
UserID string
Roles database.StringArray
Roles database.TextArray[string]
CreationDate time.Time
ChangeDate time.Time
Sequence uint64
@@ -105,17 +107,48 @@ func (q *MembershipSearchQuery) toQuery(query sq.SelectBuilder) sq.SelectBuilder
return query
}
func (q *Queries) Memberships(ctx context.Context, queries *MembershipSearchQuery, withOwnerRemoved bool) (memberships *Memberships, err error) {
func (q *Queries) Memberships(ctx context.Context, queries *MembershipSearchQuery, withOwnerRemoved, shouldTrigger bool) (memberships *Memberships, err error) {
ctx, span := tracing.NewSpan(ctx)
defer func() { span.EndWithError(err) }()
query, queryArgs, scan := prepareMembershipsQuery(ctx, q.client, withOwnerRemoved)
if shouldTrigger {
wg := sync.WaitGroup{}
wg.Add(4)
go func() {
spanCtx, triggerSpan := tracing.NewNamedSpan(ctx, "TriggerOrgMemberProjection")
_, _ = projection.OrgMemberProjection.Trigger(spanCtx, handler.WithAwaitRunning())
triggerSpan.End()
wg.Done()
}()
go func() {
spanCtx, triggerSpan := tracing.NewNamedSpan(ctx, "TriggerInstanceMemberProjection")
_, _ = projection.InstanceMemberProjection.Trigger(spanCtx, handler.WithAwaitRunning())
triggerSpan.End()
wg.Done()
}()
go func() {
spanCtx, triggerSpan := tracing.NewNamedSpan(ctx, "TriggerProjectMemberProjection")
_, _ = projection.ProjectMemberProjection.Trigger(spanCtx, handler.WithAwaitRunning())
triggerSpan.End()
wg.Done()
}()
go func() {
spanCtx, triggerSpan := tracing.NewNamedSpan(ctx, "TriggerProjectGrantMemberProjection")
_, _ = projection.ProjectGrantMemberProjection.Trigger(spanCtx, handler.WithAwaitRunning())
triggerSpan.End()
wg.Done()
}()
wg.Wait()
}
query, queryArgs, scan := prepareMembershipsQuery(ctx, q.client, withOwnerRemoved, queries)
eq := sq.Eq{membershipInstanceID.identifier(): authz.GetInstance(ctx).InstanceID()}
stmt, args, err := queries.toQuery(query).Where(eq).ToSql()
if err != nil {
return nil, errors.ThrowInvalidArgument(err, "QUERY-T84X9", "Errors.Query.InvalidRequest")
}
latestSequence, err := q.latestSequence(ctx, orgMemberTable, instanceMemberTable, projectMemberTable, projectGrantMemberTable)
latestSequence, err := q.latestState(ctx, orgMemberTable, instanceMemberTable, projectMemberTable, projectGrantMemberTable)
if err != nil {
return nil, err
}
@@ -128,14 +161,14 @@ func (q *Queries) Memberships(ctx context.Context, queries *MembershipSearchQuer
if err != nil {
return nil, err
}
memberships.LatestSequence = latestSequence
memberships.State = latestSequence
return memberships, nil
}
var (
//membershipAlias is a hack to satisfy checks in the queries
membershipAlias = table{
name: "memberships",
name: "members",
instanceIDCol: projection.MemberInstanceID,
}
membershipUserID = Column{
@@ -201,11 +234,11 @@ var (
}
)
func getMembershipFromQuery(withOwnerRemoved bool) (string, []interface{}) {
orgMembers, orgMembersArgs := prepareOrgMember(withOwnerRemoved)
iamMembers, iamMembersArgs := prepareIAMMember(withOwnerRemoved)
projectMembers, projectMembersArgs := prepareProjectMember(withOwnerRemoved)
projectGrantMembers, projectGrantMembersArgs := prepareProjectGrantMember(withOwnerRemoved)
func getMembershipFromQuery(withOwnerRemoved bool, queries *MembershipSearchQuery) (string, []interface{}) {
orgMembers, orgMembersArgs := prepareOrgMember(withOwnerRemoved, queries)
iamMembers, iamMembersArgs := prepareIAMMember(withOwnerRemoved, queries)
projectMembers, projectMembersArgs := prepareProjectMember(withOwnerRemoved, queries)
projectGrantMembers, projectGrantMembersArgs := prepareProjectGrantMember(withOwnerRemoved, queries)
args := make([]interface{}, 0)
args = append(append(append(append(args, orgMembersArgs...), iamMembersArgs...), projectMembersArgs...), projectGrantMembersArgs...)
@@ -221,8 +254,8 @@ func getMembershipFromQuery(withOwnerRemoved bool) (string, []interface{}) {
args
}
func prepareMembershipsQuery(ctx context.Context, db prepareDatabase, withOwnerRemoved bool) (sq.SelectBuilder, []interface{}, func(*sql.Rows) (*Memberships, error)) {
query, args := getMembershipFromQuery(withOwnerRemoved)
func prepareMembershipsQuery(ctx context.Context, db prepareDatabase, withOwnerRemoved bool, queries *MembershipSearchQuery) (sq.SelectBuilder, []interface{}, func(*sql.Rows) (*Memberships, error)) {
query, args := getMembershipFromQuery(withOwnerRemoved, queries)
return sq.Select(
membershipUserID.identifier(),
membershipRoles.identifier(),
@@ -237,11 +270,13 @@ func prepareMembershipsQuery(ctx context.Context, db prepareDatabase, withOwnerR
ProjectGrantColumnGrantedOrgID.identifier(),
ProjectColumnName.identifier(),
OrgColumnName.identifier(),
InstanceColumnName.identifier(),
countColumn.identifier(),
).From(query).
LeftJoin(join(ProjectColumnID, membershipProjectID)).
LeftJoin(join(OrgColumnID, membershipOrgID)).
LeftJoin(join(ProjectGrantColumnGrantID, membershipGrantID) + db.Timetravel(call.Took(ctx))).
LeftJoin(join(ProjectGrantColumnGrantID, membershipGrantID)).
LeftJoin(join(InstanceColumnID, membershipInstanceID) + db.Timetravel(call.Took(ctx))).
PlaceholderFormat(sq.Dollar),
args,
func(rows *sql.Rows) (*Memberships, error) {
@@ -252,12 +287,13 @@ func prepareMembershipsQuery(ctx context.Context, db prepareDatabase, withOwnerR
var (
membership = new(Membership)
orgID = sql.NullString{}
iamID = sql.NullString{}
instanceID = sql.NullString{}
projectID = sql.NullString{}
grantID = sql.NullString{}
grantedOrgID = sql.NullString{}
projectName = sql.NullString{}
orgName = sql.NullString{}
instanceName = sql.NullString{}
)
err := rows.Scan(
@@ -268,12 +304,13 @@ func prepareMembershipsQuery(ctx context.Context, db prepareDatabase, withOwnerR
&membership.Sequence,
&membership.ResourceOwner,
&orgID,
&iamID,
&instanceID,
&projectID,
&grantID,
&grantedOrgID,
&projectName,
&orgName,
&instanceName,
&count,
)
@@ -286,10 +323,10 @@ func prepareMembershipsQuery(ctx context.Context, db prepareDatabase, withOwnerR
OrgID: orgID.String,
Name: orgName.String,
}
} else if iamID.Valid {
} else if instanceID.Valid {
membership.IAM = &IAMMembership{
IAMID: iamID.String,
Name: iamID.String,
IAMID: instanceID.String,
Name: instanceName.String,
}
} else if projectID.Valid && grantID.Valid && grantedOrgID.Valid {
membership.ProjectGrant = &ProjectGrantMembership{
@@ -321,7 +358,7 @@ func prepareMembershipsQuery(ctx context.Context, db prepareDatabase, withOwnerR
}
}
func prepareOrgMember(withOwnerRemoved bool) (string, []interface{}) {
func prepareOrgMember(withOwnerRemoved bool, query *MembershipSearchQuery) (string, []interface{}) {
builder := sq.Select(
OrgMemberUserID.identifier(),
OrgMemberRoles.identifier(),
@@ -335,15 +372,23 @@ func prepareOrgMember(withOwnerRemoved bool) (string, []interface{}) {
"NULL::TEXT AS "+membershipProjectID.name,
"NULL::TEXT AS "+membershipGrantID.name,
).From(orgMemberTable.identifier())
for _, q := range query.Queries {
if q.Col().table.name == membershipAlias.name {
builder = q.toQuery(builder)
}
}
if !withOwnerRemoved {
eq := sq.Eq{}
addOrgMemberWithoutOwnerRemoved(eq)
builder = builder.Where(eq)
}
return builder.MustSql()
}
func prepareIAMMember(withOwnerRemoved bool) (string, []interface{}) {
func prepareIAMMember(withOwnerRemoved bool, query *MembershipSearchQuery) (string, []interface{}) {
builder := sq.Select(
InstanceMemberUserID.identifier(),
InstanceMemberRoles.identifier(),
@@ -357,6 +402,13 @@ func prepareIAMMember(withOwnerRemoved bool) (string, []interface{}) {
"NULL::TEXT AS "+membershipProjectID.name,
"NULL::TEXT AS "+membershipGrantID.name,
).From(instanceMemberTable.identifier())
for _, q := range query.Queries {
if q.Col().table.name == membershipAlias.name {
builder = q.toQuery(builder)
}
}
if !withOwnerRemoved {
eq := sq.Eq{}
addIamMemberWithoutOwnerRemoved(eq)
@@ -365,7 +417,7 @@ func prepareIAMMember(withOwnerRemoved bool) (string, []interface{}) {
return builder.MustSql()
}
func prepareProjectMember(withOwnerRemoved bool) (string, []interface{}) {
func prepareProjectMember(withOwnerRemoved bool, query *MembershipSearchQuery) (string, []interface{}) {
builder := sq.Select(
ProjectMemberUserID.identifier(),
ProjectMemberRoles.identifier(),
@@ -379,6 +431,13 @@ func prepareProjectMember(withOwnerRemoved bool) (string, []interface{}) {
ProjectMemberProjectID.identifier(),
"NULL::TEXT AS "+membershipGrantID.name,
).From(projectMemberTable.identifier())
for _, q := range query.Queries {
if q.Col().table.name == membershipAlias.name {
builder = q.toQuery(builder)
}
}
if !withOwnerRemoved {
eq := sq.Eq{}
addProjectMemberWithoutOwnerRemoved(eq)
@@ -387,7 +446,7 @@ func prepareProjectMember(withOwnerRemoved bool) (string, []interface{}) {
return builder.MustSql()
}
func prepareProjectGrantMember(withOwnerRemoved bool) (string, []interface{}) {
func prepareProjectGrantMember(withOwnerRemoved bool, query *MembershipSearchQuery) (string, []interface{}) {
builder := sq.Select(
ProjectGrantMemberUserID.identifier(),
ProjectGrantMemberRoles.identifier(),
@@ -401,6 +460,13 @@ func prepareProjectGrantMember(withOwnerRemoved bool) (string, []interface{}) {
ProjectGrantMemberProjectID.identifier(),
ProjectGrantMemberGrantID.identifier(),
).From(projectGrantMemberTable.identifier())
for _, q := range query.Queries {
if q.Col().table.name == membershipAlias.name {
builder = q.toQuery(builder)
}
}
if !withOwnerRemoved {
eq := sq.Eq{}
addProjectGrantMemberWithoutOwnerRemoved(eq)