perf: project quotas and usages (#6441)

* project quota added * project quota removed * add periods table * make log record generic * accumulate usage * query usage * count action run seconds * fix filter in ReportQuotaUsage * fix existing tests * fix logstore tests * fix typo * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * move notifications into debouncer and improve limit querying * cleanup * comment * fix: add quota unit tests command side * fix remaining quota usage query * implement InmemLogStorage * cleanup and linting * improve test * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * action notifications and fixes for notifications query * revert console prefix * fix: add quota unit tests command side * fix: add quota integration tests * improve accountable requests * improve accountable requests * fix: add quota integration tests * fix: add quota integration tests * fix: add quota integration tests * comment * remove ability to store logs in db and other changes requested from review * changes requested from review * changes requested from review * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * tests: fix quotas integration tests * improve incrementUsageStatement * linting * fix: delete e2e tests as intergation tests cover functionality * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * backup * fix conflict * create rc * create prerelease * remove issue release labeling * fix tracing --------- Co-authored-by: Livio Spring <livio.a@gmail.com> Co-authored-by: Stefan Benz <stefan@caos.ch> Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2025-08-11 21:17:32 +00:00 · 2023-09-15 16:58:45 +02:00
parent b4d0d2c9a7
commit 1a49b7d298
66 changed files with 3423 additions and 1413 deletions
--- a/internal/api/grpc/server/middleware/access_interceptor.go
+++ b/internal/api/grpc/server/middleware/access_interceptor.go
@@ -10,11 +10,11 @@ import (

 	"github.com/zitadel/zitadel/internal/api/authz"
 	"github.com/zitadel/zitadel/internal/logstore"
-	"github.com/zitadel/zitadel/internal/logstore/emitters/access"
+	"github.com/zitadel/zitadel/internal/logstore/record"
 	"github.com/zitadel/zitadel/internal/telemetry/tracing"
 )

-func AccessStorageInterceptor(svc *logstore.Service) grpc.UnaryServerInterceptor {
+func AccessStorageInterceptor(svc *logstore.Service[*record.AccessLog]) grpc.UnaryServerInterceptor {
 	return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (_ interface{}, err error) {
 		if !svc.Enabled() {
 			return handler(ctx, req)
@@ -36,9 +36,9 @@ func AccessStorageInterceptor(svc *logstore.Service) grpc.UnaryServerInterceptor
 		resMd, _ := metadata.FromOutgoingContext(ctx)
 		instance := authz.GetInstance(ctx)

-		record := &access.Record{
+		r := &record.AccessLog{
 			LogDate:         time.Now(),
-			Protocol:        access.GRPC,
+			Protocol:        record.GRPC,
 			RequestURL:      info.FullMethod,
 			ResponseStatus:  respStatus,
 			RequestHeaders:  reqMd,
@@ -49,7 +49,7 @@ func AccessStorageInterceptor(svc *logstore.Service) grpc.UnaryServerInterceptor
 			RequestedHost:   instance.RequestedHost(),
 		}

-		svc.Handle(interceptorCtx, record)
+		svc.Handle(interceptorCtx, r)
 		return resp, handlerErr
 	}
 }
--- a/internal/api/grpc/server/middleware/quota_interceptor.go
+++ b/internal/api/grpc/server/middleware/quota_interceptor.go
@@ -9,19 +9,16 @@ import (
 	"github.com/zitadel/zitadel/internal/api/authz"
 	"github.com/zitadel/zitadel/internal/errors"
 	"github.com/zitadel/zitadel/internal/logstore"
+	"github.com/zitadel/zitadel/internal/logstore/record"
 	"github.com/zitadel/zitadel/internal/telemetry/tracing"
 )

-func QuotaExhaustedInterceptor(svc *logstore.Service, ignoreService ...string) grpc.UnaryServerInterceptor {
-
-	prunedIgnoredServices := make([]string, len(ignoreService))
+func QuotaExhaustedInterceptor(svc *logstore.Service[*record.AccessLog], ignoreService ...string) grpc.UnaryServerInterceptor {
 	for idx, service := range ignoreService {
 		if !strings.HasPrefix(service, "/") {
-			service = "/" + service
+			ignoreService[idx] = "/" + service
 		}
-		prunedIgnoredServices[idx] = service
 	}
-
 	return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (_ interface{}, err error) {
 		if !svc.Enabled() {
 			return handler(ctx, req)
@@ -29,7 +26,13 @@ func QuotaExhaustedInterceptor(svc *logstore.Service, ignoreService ...string) g
 		interceptorCtx, span := tracing.NewServerInterceptorSpan(ctx)
 		defer func() { span.EndWithError(err) }()

-		for _, service := range prunedIgnoredServices {
+		// The auth interceptor will ensure that only authorized or public requests are allowed.
+		// So if there's no authorization context, we don't need to check for limitation
+		if authz.GetCtxData(ctx).IsZero() {
+			return handler(ctx, req)
+		}
+
+		for _, service := range ignoreService {
 			if strings.HasPrefix(info.FullMethod, service) {
 				return handler(ctx, req)
 			}
--- a/internal/api/grpc/server/server.go
+++ b/internal/api/grpc/server/server.go
@@ -12,6 +12,7 @@ import (
 	grpc_api "github.com/zitadel/zitadel/internal/api/grpc"
 	"github.com/zitadel/zitadel/internal/api/grpc/server/middleware"
 	"github.com/zitadel/zitadel/internal/logstore"
+	"github.com/zitadel/zitadel/internal/logstore/record"
 	"github.com/zitadel/zitadel/internal/query"
 	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 	system_pb "github.com/zitadel/zitadel/pkg/grpc/system"
@@ -39,7 +40,7 @@ func CreateServer(
 	queries *query.Queries,
 	hostHeaderName string,
 	tlsConfig *tls.Config,
-	accessSvc *logstore.Service,
+	accessSvc *logstore.Service[*record.AccessLog],
 ) *grpc.Server {
 	metricTypes := []metrics.MetricType{metrics.MetricTypeTotalCount, metrics.MetricTypeRequestCount, metrics.MetricTypeStatusCode}
 	serverOptions := []grpc.ServerOption{
@@ -49,14 +50,14 @@ func CreateServer(
 				middleware.DefaultTracingServer(),
 				middleware.MetricsHandler(metricTypes, grpc_api.Probes...),
 				middleware.NoCacheInterceptor(),
-				middleware.ErrorHandler(),
 				middleware.InstanceInterceptor(queries, hostHeaderName, system_pb.SystemService_ServiceDesc.ServiceName, healthpb.Health_ServiceDesc.ServiceName),
 				middleware.AccessStorageInterceptor(accessSvc),
+				middleware.ErrorHandler(),
 				middleware.AuthorizationInterceptor(verifier, authConfig),
+				middleware.QuotaExhaustedInterceptor(accessSvc, system_pb.SystemService_ServiceDesc.ServiceName),
 				middleware.TranslationHandler(),
 				middleware.ValidationHandler(),
 				middleware.ServiceHandler(),
-				middleware.QuotaExhaustedInterceptor(accessSvc, system_pb.SystemService_ServiceDesc.ServiceName),
 			),
 		),
 	}
--- a/internal/api/grpc/system/quota_integration_test.go
+++ b/internal/api/grpc/system/quota_integration_test.go
@@ -0,0 +1,203 @@
+//go:build integration
+
+package system_test
+
+import (
+	"bytes"
+	"encoding/json"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/types/known/durationpb"
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	"github.com/zitadel/zitadel/internal/integration"
+	"github.com/zitadel/zitadel/internal/repository/quota"
+	"github.com/zitadel/zitadel/pkg/grpc/admin"
+	quota_pb "github.com/zitadel/zitadel/pkg/grpc/quota"
+	"github.com/zitadel/zitadel/pkg/grpc/system"
+)
+
+var callURL = "http://localhost:" + integration.PortQuotaServer
+
+func TestServer_QuotaNotification_Limit(t *testing.T) {
+	_, instanceID, iamOwnerCtx := Tester.UseIsolatedInstance(CTX, SystemCTX)
+	amount := 10
+	percent := 50
+	percentAmount := amount * percent / 100
+
+	_, err := Tester.Client.System.AddQuota(SystemCTX, &system.AddQuotaRequest{
+		InstanceId:    instanceID,
+		Unit:          quota_pb.Unit_UNIT_REQUESTS_ALL_AUTHENTICATED,
+		From:          timestamppb.Now(),
+		ResetInterval: durationpb.New(time.Minute * 5),
+		Amount:        uint64(amount),
+		Limit:         true,
+		Notifications: []*quota_pb.Notification{
+			{
+				Percent: uint32(percent),
+				Repeat:  true,
+				CallUrl: callURL,
+			},
+			{
+				Percent: 100,
+				Repeat:  true,
+				CallUrl: callURL,
+			},
+		},
+	})
+	require.NoError(t, err)
+
+	for i := 0; i < percentAmount; i++ {
+		_, err := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{})
+		require.NoErrorf(t, err, "error in %d call of %d", i, percentAmount)
+	}
+	awaitNotification(t, Tester.QuotaNotificationChan, quota.RequestsAllAuthenticated, percent)
+
+	for i := 0; i < (amount - percentAmount); i++ {
+		_, err := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{})
+		require.NoErrorf(t, err, "error in %d call of %d", i, percentAmount)
+	}
+	awaitNotification(t, Tester.QuotaNotificationChan, quota.RequestsAllAuthenticated, 100)
+
+	_, limitErr := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{})
+	require.Error(t, limitErr)
+}
+
+func TestServer_QuotaNotification_NoLimit(t *testing.T) {
+	_, instanceID, iamOwnerCtx := Tester.UseIsolatedInstance(CTX, SystemCTX)
+	amount := 10
+	percent := 50
+	percentAmount := amount * percent / 100
+
+	_, err := Tester.Client.System.AddQuota(SystemCTX, &system.AddQuotaRequest{
+		InstanceId:    instanceID,
+		Unit:          quota_pb.Unit_UNIT_REQUESTS_ALL_AUTHENTICATED,
+		From:          timestamppb.Now(),
+		ResetInterval: durationpb.New(time.Minute * 5),
+		Amount:        uint64(amount),
+		Limit:         false,
+		Notifications: []*quota_pb.Notification{
+			{
+				Percent: uint32(percent),
+				Repeat:  false,
+				CallUrl: callURL,
+			},
+			{
+				Percent: 100,
+				Repeat:  true,
+				CallUrl: callURL,
+			},
+		},
+	})
+	require.NoError(t, err)
+
+	for i := 0; i < percentAmount; i++ {
+		_, err := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{})
+		require.NoErrorf(t, err, "error in %d call of %d", i, percentAmount)
+	}
+	awaitNotification(t, Tester.QuotaNotificationChan, quota.RequestsAllAuthenticated, percent)
+
+	for i := 0; i < (amount - percentAmount); i++ {
+		_, err := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{})
+		require.NoErrorf(t, err, "error in %d call of %d", i, percentAmount)
+	}
+	awaitNotification(t, Tester.QuotaNotificationChan, quota.RequestsAllAuthenticated, 100)
+
+	for i := 0; i < amount; i++ {
+		_, err := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{})
+		require.NoErrorf(t, err, "error in %d call of %d", i, percentAmount)
+	}
+	awaitNotification(t, Tester.QuotaNotificationChan, quota.RequestsAllAuthenticated, 200)
+
+	_, limitErr := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{})
+	require.NoError(t, limitErr)
+}
+
+func awaitNotification(t *testing.T, bodies chan []byte, unit quota.Unit, percent int) {
+	for {
+		select {
+		case body := <-bodies:
+			plain := new(bytes.Buffer)
+			if err := json.Indent(plain, body, "", "  "); err != nil {
+				t.Fatal(err)
+			}
+			t.Log("received notificationDueEvent", plain.String())
+			event := struct {
+				Unit        quota.Unit `json:"unit"`
+				ID          string     `json:"id"`
+				CallURL     string     `json:"callURL"`
+				PeriodStart time.Time  `json:"periodStart"`
+				Threshold   uint16     `json:"threshold"`
+				Usage       uint64     `json:"usage"`
+			}{}
+			if err := json.Unmarshal(body, &event); err != nil {
+				t.Error(err)
+			}
+			if event.ID == "" {
+				continue
+			}
+			if event.Unit == unit && event.Threshold == uint16(percent) {
+				return
+			}
+		case <-time.After(60 * time.Second):
+			t.Fatalf("timed out waiting for unit %s and percent %d", strconv.Itoa(int(unit)), percent)
+		}
+	}
+}
+
+func TestServer_AddAndRemoveQuota(t *testing.T) {
+	_, instanceID, _ := Tester.UseIsolatedInstance(CTX, SystemCTX)
+
+	got, err := Tester.Client.System.AddQuota(SystemCTX, &system.AddQuotaRequest{
+		InstanceId:    instanceID,
+		Unit:          quota_pb.Unit_UNIT_REQUESTS_ALL_AUTHENTICATED,
+		From:          timestamppb.Now(),
+		ResetInterval: durationpb.New(time.Minute),
+		Amount:        10,
+		Limit:         true,
+		Notifications: []*quota_pb.Notification{
+			{
+				Percent: 20,
+				Repeat:  true,
+				CallUrl: callURL,
+			},
+		},
+	})
+	require.NoError(t, err)
+	require.Equal(t, got.Details.ResourceOwner, instanceID)
+
+	gotAlreadyExisting, errAlreadyExisting := Tester.Client.System.AddQuota(SystemCTX, &system.AddQuotaRequest{
+		InstanceId:    instanceID,
+		Unit:          quota_pb.Unit_UNIT_REQUESTS_ALL_AUTHENTICATED,
+		From:          timestamppb.Now(),
+		ResetInterval: durationpb.New(time.Minute),
+		Amount:        10,
+		Limit:         true,
+		Notifications: []*quota_pb.Notification{
+			{
+				Percent: 20,
+				Repeat:  true,
+				CallUrl: callURL,
+			},
+		},
+	})
+	require.Error(t, errAlreadyExisting)
+	require.Nil(t, gotAlreadyExisting)
+
+	gotRemove, errRemove := Tester.Client.System.RemoveQuota(SystemCTX, &system.RemoveQuotaRequest{
+		InstanceId: instanceID,
+		Unit:       quota_pb.Unit_UNIT_REQUESTS_ALL_AUTHENTICATED,
+	})
+	require.NoError(t, errRemove)
+	require.Equal(t, gotRemove.Details.ResourceOwner, instanceID)
+
+	gotRemoveAlready, errRemoveAlready := Tester.Client.System.RemoveQuota(SystemCTX, &system.RemoveQuotaRequest{
+		InstanceId: instanceID,
+		Unit:       quota_pb.Unit_UNIT_REQUESTS_ALL_AUTHENTICATED,
+	})
+	require.Error(t, errRemoveAlready)
+	require.Nil(t, gotRemoveAlready)
+}
--- a/internal/api/grpc/system/server_integration_test.go
+++ b/internal/api/grpc/system/server_integration_test.go
@@ -0,0 +1,32 @@
+//go:build integration
+
+package system_test
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/zitadel/zitadel/internal/integration"
+)
+
+var (
+	CTX       context.Context
+	SystemCTX context.Context
+	Tester    *integration.Tester
+)
+
+func TestMain(m *testing.M) {
+	os.Exit(func() int {
+		ctx, _, cancel := integration.Contexts(5 * time.Minute)
+		defer cancel()
+		CTX = ctx
+
+		Tester = integration.NewTester(ctx)
+		defer Tester.Done()
+
+		SystemCTX = Tester.WithAuthorization(ctx, integration.SystemUser)
+		return m.Run()
+	}())
+}