mirror of
https://github.com/zitadel/zitadel.git
synced 2025-10-20 11:32:35 +00:00
perf: project quotas and usages (#6441)
* project quota added
* project quota removed
* add periods table
* make log record generic
* accumulate usage
* query usage
* count action run seconds
* fix filter in ReportQuotaUsage
* fix existing tests
* fix logstore tests
* fix typo
* fix: add quota unit tests command side
* fix: add quota unit tests command side
* fix: add quota unit tests command side
* move notifications into debouncer and improve limit querying
* cleanup
* comment
* fix: add quota unit tests command side
* fix remaining quota usage query
* implement InmemLogStorage
* cleanup and linting
* improve test
* fix: add quota unit tests command side
* fix: add quota unit tests command side
* fix: add quota unit tests command side
* fix: add quota unit tests command side
* action notifications and fixes for notifications query
* revert console prefix
* fix: add quota unit tests command side
* fix: add quota integration tests
* improve accountable requests
* improve accountable requests
* fix: add quota integration tests
* fix: add quota integration tests
* fix: add quota integration tests
* comment
* remove ability to store logs in db and other changes requested from review
* changes requested from review
* changes requested from review
* Update internal/api/http/middleware/access_interceptor.go
Co-authored-by: Silvan <silvan.reusser@gmail.com>
* tests: fix quotas integration tests
* improve incrementUsageStatement
* linting
* fix: delete e2e tests as intergation tests cover functionality
* Update internal/api/http/middleware/access_interceptor.go
Co-authored-by: Silvan <silvan.reusser@gmail.com>
* backup
* fix conflict
* create rc
* create prerelease
* remove issue release labeling
* fix tracing
---------
Co-authored-by: Livio Spring <livio.a@gmail.com>
Co-authored-by: Stefan Benz <stefan@caos.ch>
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
(cherry picked from commit 1a49b7d298)
This commit is contained in:
Elio Bischof
committed by
Livio Spring
Livio Spring
parent
b688d6f842
commit
5823fdbef9
203
internal/api/grpc/system/quota_integration_test.go
Normal file
203
internal/api/grpc/system/quota_integration_test.go
Normal file
@@ -0,0 +1,203 @@
|
||||
//go:build integration
|
||||
|
||||
package system_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/protobuf/types/known/durationpb"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/zitadel/zitadel/internal/integration"
|
||||
"github.com/zitadel/zitadel/internal/repository/quota"
|
||||
"github.com/zitadel/zitadel/pkg/grpc/admin"
|
||||
quota_pb "github.com/zitadel/zitadel/pkg/grpc/quota"
|
||||
"github.com/zitadel/zitadel/pkg/grpc/system"
|
||||
)
|
||||
|
||||
var callURL = "http://localhost:" + integration.PortQuotaServer
|
||||
|
||||
func TestServer_QuotaNotification_Limit(t *testing.T) {
|
||||
_, instanceID, iamOwnerCtx := Tester.UseIsolatedInstance(CTX, SystemCTX)
|
||||
amount := 10
|
||||
percent := 50
|
||||
percentAmount := amount * percent / 100
|
||||
|
||||
_, err := Tester.Client.System.AddQuota(SystemCTX, &system.AddQuotaRequest{
|
||||
InstanceId: instanceID,
|
||||
Unit: quota_pb.Unit_UNIT_REQUESTS_ALL_AUTHENTICATED,
|
||||
From: timestamppb.Now(),
|
||||
ResetInterval: durationpb.New(time.Minute * 5),
|
||||
Amount: uint64(amount),
|
||||
Limit: true,
|
||||
Notifications: []*quota_pb.Notification{
|
||||
{
|
||||
Percent: uint32(percent),
|
||||
Repeat: true,
|
||||
CallUrl: callURL,
|
||||
},
|
||||
{
|
||||
Percent: 100,
|
||||
Repeat: true,
|
||||
CallUrl: callURL,
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
for i := 0; i < percentAmount; i++ {
|
||||
_, err := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{})
|
||||
require.NoErrorf(t, err, "error in %d call of %d", i, percentAmount)
|
||||
}
|
||||
awaitNotification(t, Tester.QuotaNotificationChan, quota.RequestsAllAuthenticated, percent)
|
||||
|
||||
for i := 0; i < (amount - percentAmount); i++ {
|
||||
_, err := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{})
|
||||
require.NoErrorf(t, err, "error in %d call of %d", i, percentAmount)
|
||||
}
|
||||
awaitNotification(t, Tester.QuotaNotificationChan, quota.RequestsAllAuthenticated, 100)
|
||||
|
||||
_, limitErr := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{})
|
||||
require.Error(t, limitErr)
|
||||
}
|
||||
|
||||
func TestServer_QuotaNotification_NoLimit(t *testing.T) {
|
||||
_, instanceID, iamOwnerCtx := Tester.UseIsolatedInstance(CTX, SystemCTX)
|
||||
amount := 10
|
||||
percent := 50
|
||||
percentAmount := amount * percent / 100
|
||||
|
||||
_, err := Tester.Client.System.AddQuota(SystemCTX, &system.AddQuotaRequest{
|
||||
InstanceId: instanceID,
|
||||
Unit: quota_pb.Unit_UNIT_REQUESTS_ALL_AUTHENTICATED,
|
||||
From: timestamppb.Now(),
|
||||
ResetInterval: durationpb.New(time.Minute * 5),
|
||||
Amount: uint64(amount),
|
||||
Limit: false,
|
||||
Notifications: []*quota_pb.Notification{
|
||||
{
|
||||
Percent: uint32(percent),
|
||||
Repeat: false,
|
||||
CallUrl: callURL,
|
||||
},
|
||||
{
|
||||
Percent: 100,
|
||||
Repeat: true,
|
||||
CallUrl: callURL,
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
for i := 0; i < percentAmount; i++ {
|
||||
_, err := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{})
|
||||
require.NoErrorf(t, err, "error in %d call of %d", i, percentAmount)
|
||||
}
|
||||
awaitNotification(t, Tester.QuotaNotificationChan, quota.RequestsAllAuthenticated, percent)
|
||||
|
||||
for i := 0; i < (amount - percentAmount); i++ {
|
||||
_, err := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{})
|
||||
require.NoErrorf(t, err, "error in %d call of %d", i, percentAmount)
|
||||
}
|
||||
awaitNotification(t, Tester.QuotaNotificationChan, quota.RequestsAllAuthenticated, 100)
|
||||
|
||||
for i := 0; i < amount; i++ {
|
||||
_, err := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{})
|
||||
require.NoErrorf(t, err, "error in %d call of %d", i, percentAmount)
|
||||
}
|
||||
awaitNotification(t, Tester.QuotaNotificationChan, quota.RequestsAllAuthenticated, 200)
|
||||
|
||||
_, limitErr := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{})
|
||||
require.NoError(t, limitErr)
|
||||
}
|
||||
|
||||
func awaitNotification(t *testing.T, bodies chan []byte, unit quota.Unit, percent int) {
|
||||
for {
|
||||
select {
|
||||
case body := <-bodies:
|
||||
plain := new(bytes.Buffer)
|
||||
if err := json.Indent(plain, body, "", " "); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("received notificationDueEvent", plain.String())
|
||||
event := struct {
|
||||
Unit quota.Unit `json:"unit"`
|
||||
ID string `json:"id"`
|
||||
CallURL string `json:"callURL"`
|
||||
PeriodStart time.Time `json:"periodStart"`
|
||||
Threshold uint16 `json:"threshold"`
|
||||
Usage uint64 `json:"usage"`
|
||||
}{}
|
||||
if err := json.Unmarshal(body, &event); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if event.ID == "" {
|
||||
continue
|
||||
}
|
||||
if event.Unit == unit && event.Threshold == uint16(percent) {
|
||||
return
|
||||
}
|
||||
case <-time.After(60 * time.Second):
|
||||
t.Fatalf("timed out waiting for unit %s and percent %d", strconv.Itoa(int(unit)), percent)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestServer_AddAndRemoveQuota(t *testing.T) {
|
||||
_, instanceID, _ := Tester.UseIsolatedInstance(CTX, SystemCTX)
|
||||
|
||||
got, err := Tester.Client.System.AddQuota(SystemCTX, &system.AddQuotaRequest{
|
||||
InstanceId: instanceID,
|
||||
Unit: quota_pb.Unit_UNIT_REQUESTS_ALL_AUTHENTICATED,
|
||||
From: timestamppb.Now(),
|
||||
ResetInterval: durationpb.New(time.Minute),
|
||||
Amount: 10,
|
||||
Limit: true,
|
||||
Notifications: []*quota_pb.Notification{
|
||||
{
|
||||
Percent: 20,
|
||||
Repeat: true,
|
||||
CallUrl: callURL,
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, got.Details.ResourceOwner, instanceID)
|
||||
|
||||
gotAlreadyExisting, errAlreadyExisting := Tester.Client.System.AddQuota(SystemCTX, &system.AddQuotaRequest{
|
||||
InstanceId: instanceID,
|
||||
Unit: quota_pb.Unit_UNIT_REQUESTS_ALL_AUTHENTICATED,
|
||||
From: timestamppb.Now(),
|
||||
ResetInterval: durationpb.New(time.Minute),
|
||||
Amount: 10,
|
||||
Limit: true,
|
||||
Notifications: []*quota_pb.Notification{
|
||||
{
|
||||
Percent: 20,
|
||||
Repeat: true,
|
||||
CallUrl: callURL,
|
||||
},
|
||||
},
|
||||
})
|
||||
require.Error(t, errAlreadyExisting)
|
||||
require.Nil(t, gotAlreadyExisting)
|
||||
|
||||
gotRemove, errRemove := Tester.Client.System.RemoveQuota(SystemCTX, &system.RemoveQuotaRequest{
|
||||
InstanceId: instanceID,
|
||||
Unit: quota_pb.Unit_UNIT_REQUESTS_ALL_AUTHENTICATED,
|
||||
})
|
||||
require.NoError(t, errRemove)
|
||||
require.Equal(t, gotRemove.Details.ResourceOwner, instanceID)
|
||||
|
||||
gotRemoveAlready, errRemoveAlready := Tester.Client.System.RemoveQuota(SystemCTX, &system.RemoveQuotaRequest{
|
||||
InstanceId: instanceID,
|
||||
Unit: quota_pb.Unit_UNIT_REQUESTS_ALL_AUTHENTICATED,
|
||||
})
|
||||
require.Error(t, errRemoveAlready)
|
||||
require.Nil(t, gotRemoveAlready)
|
||||
}
|
||||
32
internal/api/grpc/system/server_integration_test.go
Normal file
32
internal/api/grpc/system/server_integration_test.go
Normal file
@@ -0,0 +1,32 @@
|
||||
//go:build integration
|
||||
|
||||
package system_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/zitadel/zitadel/internal/integration"
|
||||
)
|
||||
|
||||
var (
|
||||
CTX context.Context
|
||||
SystemCTX context.Context
|
||||
Tester *integration.Tester
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
os.Exit(func() int {
|
||||
ctx, _, cancel := integration.Contexts(5 * time.Minute)
|
||||
defer cancel()
|
||||
CTX = ctx
|
||||
|
||||
Tester = integration.NewTester(ctx)
|
||||
defer Tester.Done()
|
||||
|
||||
SystemCTX = Tester.WithAuthorization(ctx, integration.SystemUser)
|
||||
return m.Run()
|
||||
}())
|
||||
}
|
||||
Reference in New Issue
Block a user