TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-11 21:47:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(storage): generic cache interface (#8628)

Browse Source 
# Which Problems Are Solved

We identified the need of caching.
Currently we have a number of places where we use different ways of
caching, like go maps or LRU.
We might also want shared chaches in the future, like Redis-based or in
special SQL tables.

# How the Problems Are Solved

Define a generic Cache interface which allows different implementations.

- A noop implementation is provided and enabled as.
- An implementation using go maps is provided
  - disabled in defaults.yaml
  - enabled in integration tests
- Authz middleware instance objects are cached using the interface.

# Additional Changes

- Enabled integration test command raceflag
- Fix a race condition in the limits integration test client
- Fix a number of flaky integration tests. (Because zitadel is super
fast now!) 🎸 🚀

# Additional Context

Related to https://github.com/zitadel/zitadel/issues/8648
This commit is contained in:
Tim Möhlmann
2024-09-25 22:40:21 +03:00
committed by GitHub
parent a6ea83168d
commit 4eaa3163b6
28 changed files with 1290 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
internal/api/grpc/admin/import.go
View File

@@ -123,7 +123,9 @@ func (s *Server) ImportData(ctx context.Context, req *admin_pb.ImportDataRequest
return nil, ctxTimeout.Err()
case result := <-ch:
logging.OnError(result.err).Errorf("error while importing: %v", result.err)
logging.Infof("Import done: %s", result.count.getProgress())
if result.count != nil {
logging.Infof("Import done: %s", result.count.getProgress())
}
return result.ret, result.err
}
} else {

74
internal/api/grpc/system/integration_test/limits_auditlogretention_test.go
View File

@@ -31,11 +31,11 @@ func TestServer_Limits_AuditLogRetention(t *testing.T) {
farPast := timestamppb.New(beforeTime.Add(-10 * time.Hour).UTC())
zeroCounts := &eventCounts{}
seededCount := requireEventually(t, iamOwnerCtx, isoInstance.Client, userID, projectID, appID, projectGrantID, func(c assert.TestingT, counts *eventCounts) {
counts.assertAll(t, c, "seeded events are > 0", assert.Greater, zeroCounts)
counts.assertAll(c, "seeded events are > 0", assert.Greater, zeroCounts)
}, "wait for seeded event assertions to pass")
produceEvents(iamOwnerCtx, t, isoInstance.Client, userID, appID, projectID, projectGrantID)
addedCount := requireEventually(t, iamOwnerCtx, isoInstance.Client, userID, projectID, appID, projectGrantID, func(c assert.TestingT, counts *eventCounts) {
counts.assertAll(t, c, "added events are > seeded events", assert.Greater, seededCount)
counts.assertAll(c, "added events are > seeded events", assert.Greater, seededCount)
}, "wait for added event assertions to pass")
_, err := integration.SystemClient().SetLimits(CTX, &system.SetLimitsRequest{
InstanceId: isoInstance.ID(),
@@ -44,8 +44,8 @@ func TestServer_Limits_AuditLogRetention(t *testing.T) {
require.NoError(t, err)
var limitedCounts *eventCounts
requireEventually(t, iamOwnerCtx, isoInstance.Client, userID, projectID, appID, projectGrantID, func(c assert.TestingT, counts *eventCounts) {
counts.assertAll(t, c, "limited events < added events", assert.Less, addedCount)
counts.assertAll(t, c, "limited events > 0", assert.Greater, zeroCounts)
counts.assertAll(c, "limited events < added events", assert.Less, addedCount)
counts.assertAll(c, "limited events > 0", assert.Greater, zeroCounts)
limitedCounts = counts
}, "wait for limited event assertions to pass")
listedEvents, err := isoInstance.Client.Admin.ListEvents(iamOwnerCtx, &admin.ListEventsRequest{CreationDateFilter: &admin.ListEventsRequest_From{
@@ -63,7 +63,7 @@ func TestServer_Limits_AuditLogRetention(t *testing.T) {
})
require.NoError(t, err)
requireEventually(t, iamOwnerCtx, isoInstance.Client, userID, projectID, appID, projectGrantID, func(c assert.TestingT, counts *eventCounts) {
counts.assertAll(t, c, "with reset limit, added events are > seeded events", assert.Greater, seededCount)
counts.assertAll(c, "with reset limit, added events are > seeded events", assert.Greater, seededCount)
}, "wait for reset event assertions to pass")
}
@@ -77,7 +77,7 @@ func requireEventually(
) (counts *eventCounts) {
countTimeout := 30 * time.Second
assertTimeout := countTimeout + time.Second
countCtx, cancel := context.WithTimeout(ctx, countTimeout)
countCtx, cancel := context.WithTimeout(ctx, time.Minute)
defer cancel()
require.EventuallyWithT(t, func(c *assert.CollectT) {
counts = countEvents(countCtx, c, cc, userID, projectID, appID, projectGrantID)
@@ -168,63 +168,77 @@ type eventCounts struct {
all, myUser, aUser, grant, project, app, org int
}
func (e *eventCounts) assertAll(t *testing.T, c assert.TestingT, name string, compare assert.ComparisonAssertionFunc, than *eventCounts) {
t.Run(name, func(t *testing.T) {
compare(c, e.all, than.all, "ListEvents")
compare(c, e.myUser, than.myUser, "ListMyUserChanges")
compare(c, e.aUser, than.aUser, "ListUserChanges")
compare(c, e.grant, than.grant, "ListProjectGrantChanges")
compare(c, e.project, than.project, "ListProjectChanges")
compare(c, e.app, than.app, "ListAppChanges")
compare(c, e.org, than.org, "ListOrgChanges")
})
func (e *eventCounts) assertAll(c assert.TestingT, name string, compare assert.ComparisonAssertionFunc, than *eventCounts) {
compare(c, e.all, than.all, name+"ListEvents")
compare(c, e.myUser, than.myUser, name+"ListMyUserChanges")
compare(c, e.aUser, than.aUser, name+"ListUserChanges")
compare(c, e.grant, than.grant, name+"ListProjectGrantChanges")
compare(c, e.project, than.project, name+"ListProjectChanges")
compare(c, e.app, than.app, name+"ListAppChanges")
compare(c, e.org, than.org, name+"ListOrgChanges")
}
func countEvents(ctx context.Context, t assert.TestingT, cc *integration.Client, userID, projectID, appID, grantID string) *eventCounts {
counts := new(eventCounts)
var wg sync.WaitGroup
wg.Add(7)
var mutex sync.Mutex
assertResultLocked := func(err error, f func(counts *eventCounts)) {
mutex.Lock()
assert.NoError(t, err)
f(counts)
mutex.Unlock()
}
go func() {
defer wg.Done()
result, err := cc.Admin.ListEvents(ctx, &admin.ListEventsRequest{})
assert.NoError(t, err)
counts.all = len(result.GetEvents())
assertResultLocked(err, func(counts *eventCounts) {
counts.all = len(result.GetEvents())
})
}()
go func() {
defer wg.Done()
result, err := cc.Auth.ListMyUserChanges(ctx, &auth.ListMyUserChangesRequest{})
assert.NoError(t, err)
counts.myUser = len(result.GetResult())
assertResultLocked(err, func(counts *eventCounts) {
counts.myUser = len(result.GetResult())
})
}()
go func() {
defer wg.Done()
result, err := cc.Mgmt.ListUserChanges(ctx, &management.ListUserChangesRequest{UserId: userID})
assert.NoError(t, err)
counts.aUser = len(result.GetResult())
assertResultLocked(err, func(counts *eventCounts) {
counts.aUser = len(result.GetResult())
})
}()
go func() {
defer wg.Done()
result, err := cc.Mgmt.ListAppChanges(ctx, &management.ListAppChangesRequest{ProjectId: projectID, AppId: appID})
assert.NoError(t, err)
counts.app = len(result.GetResult())
assertResultLocked(err, func(counts *eventCounts) {
counts.app = len(result.GetResult())
})
}()
go func() {
defer wg.Done()
result, err := cc.Mgmt.ListOrgChanges(ctx, &management.ListOrgChangesRequest{})
assert.NoError(t, err)
counts.org = len(result.GetResult())
assertResultLocked(err, func(counts *eventCounts) {
counts.org = len(result.GetResult())
})
}()
go func() {
defer wg.Done()
result, err := cc.Mgmt.ListProjectChanges(ctx, &management.ListProjectChangesRequest{ProjectId: projectID})
assert.NoError(t, err)
counts.project = len(result.GetResult())
assertResultLocked(err, func(counts *eventCounts) {
counts.project = len(result.GetResult())
})
}()
go func() {
defer wg.Done()
result, err := cc.Mgmt.ListProjectGrantChanges(ctx, &management.ListProjectGrantChangesRequest{ProjectId: projectID, GrantId: grantID})
assert.NoError(t, err)
counts.grant = len(result.GetResult())
assertResultLocked(err, func(counts *eventCounts) {
counts.grant = len(result.GetResult())
})
}()
wg.Wait()
return counts

21
internal/api/oidc/integration_test/token_jwt_profile_test.go
View File

@@ -11,6 +11,7 @@ import (
"github.com/zitadel/oidc/v3/pkg/client/profile"
"github.com/zitadel/oidc/v3/pkg/client/rp"
"github.com/zitadel/oidc/v3/pkg/oidc"
"golang.org/x/oauth2"
oidc_api "github.com/zitadel/zitadel/internal/api/oidc"
"github.com/zitadel/zitadel/internal/domain"
@@ -98,13 +99,19 @@ func TestServer_JWTProfile(t *testing.T) {
tokenSource, err := profile.NewJWTProfileTokenSourceFromKeyFileData(CTX, Instance.OIDCIssuer(), tt.keyData, tt.scope)
require.NoError(t, err)
tokens, err := tokenSource.TokenCtx(CTX)
if tt.wantErr {
require.Error(t, err)
return
}
require.NoError(t, err)
require.NotNil(t, tokens)
var tokens *oauth2.Token
require.EventuallyWithT(
t, func(collect *assert.CollectT) {
tokens, err = tokenSource.TokenCtx(CTX)
if tt.wantErr {
assert.Error(collect, err)
return
}
assert.NoError(collect, err)
assert.NotNil(collect, tokens)
},
time.Minute, time.Second,
)
provider, err := rp.NewRelyingPartyOIDC(CTX, Instance.OIDCIssuer(), "", "", redirectURI, tt.scope)
require.NoError(t, err)