TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-11 21:37:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(eventstore): increase parallel write capabilities (#5940)

Browse Source 
This implementation increases parallel write capabilities of the eventstore.
Please have a look at the technical advisories: [05](https://zitadel.com/docs/support/advisory/a10005) and  [06](https://zitadel.com/docs/support/advisory/a10006).
The implementation of eventstore.push is rewritten and stored events are migrated to a new table `eventstore.events2`.
If you are using cockroach: make sure that the database user of ZITADEL has `VIEWACTIVITY` grant. This is used to query events.
This commit is contained in:
Silvan
2023-10-19 12:19:10 +02:00
committed by GitHub
parent 259faba3f0
commit b5564572bc
791 changed files with 30326 additions and 43202 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/api/grpc/management/actions.go
View File

@@ -19,7 +19,7 @@ func (s *Server) ListActions(ctx context.Context, req *mgmt_pb.ListActionsReques
return nil, err
}
return &mgmt_pb.ListActionsResponse{
Details: obj_grpc.ToListDetails(actions.Count, actions.Sequence, actions.Timestamp),
Details: obj_grpc.ToListDetails(actions.Count, actions.Sequence, actions.LastRun),
Result: action_grpc.ActionsToPb(actions.Actions),
}, nil
}

4
internal/api/grpc/management/idp.go
View File

@@ -30,7 +30,7 @@ func (s *Server) ListOrgIDPs(ctx context.Context, req *mgmt_pb.ListOrgIDPsReques
}
return &mgmt_pb.ListOrgIDPsResponse{
Result: idp_grpc.IDPViewsToPb(resp.IDPs),
Details: object_pb.ToListDetails(resp.Count, resp.Sequence, resp.Timestamp),
Details: object_pb.ToListDetails(resp.Count, resp.Sequence, resp.LastRun),
}, nil
}
@@ -167,7 +167,7 @@ func (s *Server) ListProviders(ctx context.Context, req *mgmt_pb.ListProvidersRe
}
return &mgmt_pb.ListProvidersResponse{
Result: idp_grpc.ProvidersToPb(resp.Templates),
Details: object_pb.ToListDetails(resp.Count, resp.Sequence, resp.Timestamp),
Details: object_pb.ToListDetails(resp.Count, resp.Sequence, resp.LastRun),
}, nil
}

9
internal/api/grpc/management/org.go
View File

@@ -52,9 +52,10 @@ func (s *Server) ListOrgChanges(ctx context.Context, req *mgmt_pb.ListOrgChanges
AllowTimeTravel().
Limit(limit).
OrderDesc().
AwaitOpenTransactions().
ResourceOwner(authz.GetCtxData(ctx).OrgID).
AddQuery().
SequenceGreater(sequence).
AddQuery().
AggregateTypes(org.AggregateType).
AggregateIDs(authz.GetCtxData(ctx).OrgID).
Builder()
@@ -176,7 +177,7 @@ func (s *Server) ListOrgDomains(ctx context.Context, req *mgmt_pb.ListOrgDomains
}
return &mgmt_pb.ListOrgDomainsResponse{
Result: org_grpc.DomainsToPb(domains.Domains),
Details: object.ToListDetails(domains.Count, domains.Sequence, domains.Timestamp),
Details: object.ToListDetails(domains.Count, domains.Sequence, domains.LastRun),
}, nil
}
@@ -272,7 +273,7 @@ func (s *Server) ListOrgMembers(ctx context.Context, req *mgmt_pb.ListOrgMembers
}
return &mgmt_pb.ListOrgMembersResponse{
Result: member_grpc.MembersToPb(s.assetAPIPrefix(ctx), members.Members),
Details: object.ToListDetails(members.Count, members.Sequence, members.Timestamp),
Details: object.ToListDetails(members.Count, members.Sequence, members.LastRun),
}, nil
}
@@ -350,7 +351,7 @@ func (s *Server) ListOrgMetadata(ctx context.Context, req *mgmt_pb.ListOrgMetada
}
return &mgmt_pb.ListOrgMetadataResponse{
Result: metadata.OrgMetadataListToPb(res.Metadata),
Details: obj_grpc.ToListDetails(res.Count, res.Sequence, res.Timestamp),
Details: obj_grpc.ToListDetails(res.Count, res.Sequence, res.LastRun),
}, nil
}

6
internal/api/grpc/management/policy_login.go
View File

@@ -74,7 +74,7 @@ func (s *Server) ListLoginPolicyIDPs(ctx context.Context, req *mgmt_pb.ListLogin
}
return &mgmt_pb.ListLoginPolicyIDPsResponse{
Result: idp.IDPLoginPolicyLinksToPb(res.Links),
Details: object.ToListDetails(res.Count, res.Sequence, res.Timestamp),
Details: object.ToListDetails(res.Count, res.Sequence, res.LastRun),
}, nil
}
@@ -123,7 +123,7 @@ func (s *Server) ListLoginPolicySecondFactors(ctx context.Context, req *mgmt_pb.
return nil, err
}
return &mgmt_pb.ListLoginPolicySecondFactorsResponse{
Details: object.ToListDetails(result.Count, result.Sequence, result.Timestamp),
Details: object.ToListDetails(result.Count, result.Sequence, result.LastRun),
Result: policy_grpc.ModelSecondFactorTypesToPb(result.Factors),
}, nil
}
@@ -154,7 +154,7 @@ func (s *Server) ListLoginPolicyMultiFactors(ctx context.Context, req *mgmt_pb.L
return nil, err
}
return &mgmt_pb.ListLoginPolicyMultiFactorsResponse{
Details: object.ToListDetails(res.Count, res.Sequence, res.Timestamp),
Details: object.ToListDetails(res.Count, res.Sequence, res.LastRun),
Result: policy_grpc.ModelMultiFactorTypesToPb(res.Factors),
}, nil
}

16
internal/api/grpc/management/project.go
View File

@@ -53,7 +53,7 @@ func (s *Server) ListProjects(ctx context.Context, req *mgmt_pb.ListProjectsRequ
}
return &mgmt_pb.ListProjectsResponse{
Result: project_grpc.ProjectViewsToPb(projects.Projects),
Details: object_grpc.ToListDetails(projects.Count, projects.Sequence, projects.Timestamp),
Details: object_grpc.ToListDetails(projects.Count, projects.Sequence, projects.LastRun),
}, nil
}
@@ -74,8 +74,9 @@ func (s *Server) ListProjectGrantChanges(ctx context.Context, req *mgmt_pb.ListP
Limit(limit).
OrderDesc().
ResourceOwner(authz.GetCtxData(ctx).OrgID).
AddQuery().
AwaitOpenTransactions().
SequenceGreater(sequence).
AddQuery().
AggregateTypes(project.AggregateType).
AggregateIDs(req.ProjectId).
EventData(map[string]interface{}{
@@ -115,7 +116,7 @@ func (s *Server) ListGrantedProjects(ctx context.Context, req *mgmt_pb.ListGrant
}
return &mgmt_pb.ListGrantedProjectsResponse{
Result: project_grpc.GrantedProjectViewsToPb(projects.ProjectGrants),
Details: object_grpc.ToListDetails(projects.Count, projects.Sequence, projects.Timestamp),
Details: object_grpc.ToListDetails(projects.Count, projects.Sequence, projects.LastRun),
}, nil
}
@@ -134,7 +135,7 @@ func (s *Server) ListGrantedProjectRoles(ctx context.Context, req *mgmt_pb.ListG
}
return &mgmt_pb.ListGrantedProjectRolesResponse{
Result: project_grpc.RoleViewsToPb(roles.ProjectRoles),
Details: object_grpc.ToListDetails(roles.Count, roles.Sequence, roles.Timestamp),
Details: object_grpc.ToListDetails(roles.Count, roles.Sequence, roles.LastRun),
}, nil
}
@@ -153,10 +154,11 @@ func (s *Server) ListProjectChanges(ctx context.Context, req *mgmt_pb.ListProjec
query := eventstore.NewSearchQueryBuilder(eventstore.ColumnsEvent).
AllowTimeTravel().
Limit(limit).
AwaitOpenTransactions().
OrderDesc().
ResourceOwner(authz.GetCtxData(ctx).OrgID).
AddQuery().
SequenceGreater(sequence).
AddQuery().
AggregateTypes(project.AggregateType).
AggregateIDs(req.ProjectId).
Builder()
@@ -259,7 +261,7 @@ func (s *Server) ListProjectRoles(ctx context.Context, req *mgmt_pb.ListProjectR
}
return &mgmt_pb.ListProjectRolesResponse{
Result: project_grpc.RoleViewsToPb(roles.ProjectRoles),
Details: object_grpc.ToListDetails(roles.Count, roles.Sequence, roles.Timestamp),
Details: object_grpc.ToListDetails(roles.Count, roles.Sequence, roles.LastRun),
}, nil
}
@@ -349,7 +351,7 @@ func (s *Server) ListProjectMembers(ctx context.Context, req *mgmt_pb.ListProjec
}
return &mgmt_pb.ListProjectMembersResponse{
Result: member_grpc.MembersToPb(s.assetAPIPrefix(ctx), members.Members),
Details: object_grpc.ToListDetails(members.Count, members.Sequence, members.Timestamp),
Details: object_grpc.ToListDetails(members.Count, members.Sequence, members.LastRun),
}, nil
}

7
internal/api/grpc/management/project_application.go
View File

@@ -36,7 +36,7 @@ func (s *Server) ListApps(ctx context.Context, req *mgmt_pb.ListAppsRequest) (*m
}
return &mgmt_pb.ListAppsResponse{
Result: project_grpc.AppsToPb(apps.Apps),
Details: object_grpc.ToListDetails(apps.Count, apps.Sequence, apps.Timestamp),
Details: object_grpc.ToListDetails(apps.Count, apps.Sequence, apps.LastRun),
}, nil
}
@@ -55,10 +55,11 @@ func (s *Server) ListAppChanges(ctx context.Context, req *mgmt_pb.ListAppChanges
query := eventstore.NewSearchQueryBuilder(eventstore.ColumnsEvent).
AllowTimeTravel().
Limit(limit).
AwaitOpenTransactions().
OrderDesc().
ResourceOwner(authz.GetCtxData(ctx).OrgID).
AddQuery().
SequenceGreater(sequence).
AddQuery().
AggregateTypes(project.AggregateType).
AggregateIDs(req.ProjectId).
EventData(map[string]interface{}{
@@ -278,7 +279,7 @@ func (s *Server) ListAppKeys(ctx context.Context, req *mgmt_pb.ListAppKeysReques
}
return &mgmt_pb.ListAppKeysResponse{
Result: authn_grpc.KeysToPb(keys.AuthNKeys),
Details: object_grpc.ToListDetails(keys.Count, keys.Sequence, keys.Timestamp),
Details: object_grpc.ToListDetails(keys.Count, keys.Sequence, keys.LastRun),
}, nil
}

6
internal/api/grpc/management/project_grant.go
View File

@@ -37,7 +37,7 @@ func (s *Server) ListProjectGrants(ctx context.Context, req *mgmt_pb.ListProject
}
return &mgmt_pb.ListProjectGrantsResponse{
Result: proj_grpc.GrantedProjectViewsToPb(grants.ProjectGrants),
Details: object_grpc.ToListDetails(grants.Count, grants.Sequence, grants.Timestamp),
Details: object_grpc.ToListDetails(grants.Count, grants.Sequence, grants.LastRun),
}, nil
}
@@ -60,7 +60,7 @@ func (s *Server) ListAllProjectGrants(ctx context.Context, req *mgmt_pb.ListAllP
}
return &mgmt_pb.ListAllProjectGrantsResponse{
Result: proj_grpc.GrantedProjectViewsToPb(grants.ProjectGrants),
Details: object_grpc.ToListDetails(grants.Count, grants.Sequence, grants.Timestamp),
Details: object_grpc.ToListDetails(grants.Count, grants.Sequence, grants.LastRun),
}, nil
}
@@ -170,7 +170,7 @@ func (s *Server) ListProjectGrantMembers(ctx context.Context, req *mgmt_pb.ListP
}
return &mgmt_pb.ListProjectGrantMembersResponse{
Result: member_grpc.MembersToPb(s.assetAPIPrefix(ctx), response.Members),
Details: object_grpc.ToListDetails(response.Count, response.Sequence, response.Timestamp),
Details: object_grpc.ToListDetails(response.Count, response.Sequence, response.LastRun),
}, nil
}

19
internal/api/grpc/management/user.go
View File

@@ -78,7 +78,7 @@ func (s *Server) ListUsers(ctx context.Context, req *mgmt_pb.ListUsersRequest) (
}
return &mgmt_pb.ListUsersResponse{
Result: user_grpc.UsersToPb(res.Users, s.assetAPIPrefix(ctx)),
Details: obj_grpc.ToListDetails(res.Count, res.Sequence, res.Timestamp),
Details: obj_grpc.ToListDetails(res.Count, res.Sequence, res.LastRun),
}, nil
}
@@ -97,10 +97,11 @@ func (s *Server) ListUserChanges(ctx context.Context, req *mgmt_pb.ListUserChang
query := eventstore.NewSearchQueryBuilder(eventstore.ColumnsEvent).
AllowTimeTravel().
Limit(limit).
AwaitOpenTransactions().
OrderDesc().
ResourceOwner(authz.GetCtxData(ctx).OrgID).
AddQuery().
SequenceGreater(sequence).
AddQuery().
AggregateTypes(user.AggregateType).
AggregateIDs(req.UserId).
Builder()
@@ -151,7 +152,7 @@ func (s *Server) ListUserMetadata(ctx context.Context, req *mgmt_pb.ListUserMeta
}
return &mgmt_pb.ListUserMetadataResponse{
Result: metadata.UserMetadataListToPb(res.Metadata),
Details: obj_grpc.ToListDetails(res.Count, res.Sequence, res.Timestamp),
Details: obj_grpc.ToListDetails(res.Count, res.Sequence, res.LastRun),
}, nil
}
@@ -383,7 +384,7 @@ func (s *Server) removeUserDependencies(ctx context.Context, userID string) ([]*
}
memberships, err := s.query.Memberships(ctx, &query.MembershipSearchQuery{
Queries: []query.SearchQuery{membershipsUserQuery},
}, true)
}, true, false)
if err != nil {
return nil, nil, err
}
@@ -752,7 +753,7 @@ func (s *Server) ListMachineKeys(ctx context.Context, req *mgmt_pb.ListMachineKe
}
return &mgmt_pb.ListMachineKeysResponse{
Result: authn.KeysToPb(result.AuthNKeys),
Details: obj_grpc.ToListDetails(result.Count, result.Sequence, result.Timestamp),
Details: obj_grpc.ToListDetails(result.Count, result.Sequence, result.LastRun),
}, nil
}
@@ -845,7 +846,7 @@ func (s *Server) ListPersonalAccessTokens(ctx context.Context, req *mgmt_pb.List
}
return &mgmt_pb.ListPersonalAccessTokensResponse{
Result: user_grpc.PersonalAccessTokensToPb(result.PersonalAccessTokens),
Details: obj_grpc.ToListDetails(result.Count, result.Sequence, result.Timestamp),
Details: obj_grpc.ToListDetails(result.Count, result.Sequence, result.LastRun),
}, nil
}
@@ -884,7 +885,7 @@ func (s *Server) ListHumanLinkedIDPs(ctx context.Context, req *mgmt_pb.ListHuman
}
return &mgmt_pb.ListHumanLinkedIDPsResponse{
Result: idp_grpc.IDPUserLinksToPb(res.Links),
Details: obj_grpc.ToListDetails(res.Count, res.Sequence, res.Timestamp),
Details: obj_grpc.ToListDetails(res.Count, res.Sequence, res.LastRun),
}, nil
}
func (s *Server) RemoveHumanLinkedIDP(ctx context.Context, req *mgmt_pb.RemoveHumanLinkedIDPRequest) (*mgmt_pb.RemoveHumanLinkedIDPResponse, error) {
@@ -902,13 +903,13 @@ func (s *Server) ListUserMemberships(ctx context.Context, req *mgmt_pb.ListUserM
if err != nil {
return nil, err
}
response, err := s.query.Memberships(ctx, request, false)
response, err := s.query.Memberships(ctx, request, false, false)
if err != nil {
return nil, err
}
return &mgmt_pb.ListUserMembershipsResponse{
Result: user_grpc.MembershipsToMembershipsPb(response.Memberships),
Details: obj_grpc.ToListDetails(response.Count, response.Sequence, response.Timestamp),
Details: obj_grpc.ToListDetails(response.Count, response.Sequence, response.LastRun),
}, nil
}

2
internal/api/grpc/management/user_grant.go
View File

@@ -39,7 +39,7 @@ func (s *Server) ListUserGrants(ctx context.Context, req *mgmt_pb.ListUserGrantR
}
return &mgmt_pb.ListUserGrantResponse{
Result: user.UserGrantsToPb(s.assetAPIPrefix(ctx), res.UserGrants),
Details: obj_grpc.ToListDetails(res.Count, res.Sequence, res.Timestamp),
Details: obj_grpc.ToListDetails(res.Count, res.Sequence, res.LastRun),
}, nil
}

15
internal/api/grpc/management/user_integration_test.go
View File

@@ -10,7 +10,6 @@ import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/text/language"
"google.golang.org/grpc/codes"
@@ -49,7 +48,6 @@ func TestMain(m *testing.M) {
// Get calls would return a Not Found error.
func TestImport_and_Get(t *testing.T) {
const N = 100
var misses int
for i := 0; i < N; i++ {
firstName := strconv.Itoa(i)
@@ -76,18 +74,11 @@ func TestImport_and_Get(t *testing.T) {
_, err = Client.GetUserByID(CTX, &management.GetUserByIDRequest{Id: res.GetUserId()})
if s, ok := status.FromError(err); ok {
if s == nil {
return
}
if s.Code() == codes.NotFound {
t.Log(s)
misses++
return
}
s, ok := status.FromError(err)
if ok && s != nil && s.Code() == codes.NotFound {
t.Errorf("iteration %d: user with id %q not found", i, res.GetUserId())
}
require.NoError(t, err) // catch and fail on any other error
})
}
assert.Zerof(t, misses, "Not Found errors %d out of %d", misses, N)
}