perf: project quotas and usages (#6441)

* project quota added * project quota removed * add periods table * make log record generic * accumulate usage * query usage * count action run seconds * fix filter in ReportQuotaUsage * fix existing tests * fix logstore tests * fix typo * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * move notifications into debouncer and improve limit querying * cleanup * comment * fix: add quota unit tests command side * fix remaining quota usage query * implement InmemLogStorage * cleanup and linting * improve test * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * action notifications and fixes for notifications query * revert console prefix * fix: add quota unit tests command side * fix: add quota integration tests * improve accountable requests * improve accountable requests * fix: add quota integration tests * fix: add quota integration tests * fix: add quota integration tests * comment * remove ability to store logs in db and other changes requested from review * changes requested from review * changes requested from review * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * tests: fix quotas integration tests * improve incrementUsageStatement * linting * fix: delete e2e tests as intergation tests cover functionality * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * backup * fix conflict * create rc * create prerelease * remove issue release labeling * fix tracing --------- Co-authored-by: Livio Spring <livio.a@gmail.com> Co-authored-by: Stefan Benz <stefan@caos.ch> Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2025-08-11 20:57:31 +00:00 · 2023-09-15 16:58:45 +02:00
parent b4d0d2c9a7
commit 1a49b7d298
66 changed files with 3423 additions and 1413 deletions
--- a/internal/api/http/middleware/access_interceptor.go
+++ b/internal/api/http/middleware/access_interceptor.go
@@ -14,12 +14,12 @@ import (
 	"github.com/zitadel/zitadel/internal/api/grpc/server/middleware"
 	http_utils "github.com/zitadel/zitadel/internal/api/http"
 	"github.com/zitadel/zitadel/internal/logstore"
-	"github.com/zitadel/zitadel/internal/logstore/emitters/access"
+	"github.com/zitadel/zitadel/internal/logstore/record"
 	"github.com/zitadel/zitadel/internal/telemetry/tracing"
 )

 type AccessInterceptor struct {
-	svc           *logstore.Service
+	svc           *logstore.Service[*record.AccessLog]
 	cookieHandler *http_utils.CookieHandler
 	limitConfig   *AccessConfig
 	storeOnly     bool
@@ -33,7 +33,7 @@ type AccessConfig struct {
 // NewAccessInterceptor intercepts all requests and stores them to the logstore.
 // If storeOnly is false, it also checks if requests are exhausted.
 // If requests are exhausted, it also returns http.StatusTooManyRequests and sets a cookie
-func NewAccessInterceptor(svc *logstore.Service, cookieHandler *http_utils.CookieHandler, cookieConfig *AccessConfig) *AccessInterceptor {
+func NewAccessInterceptor(svc *logstore.Service[*record.AccessLog], cookieHandler *http_utils.CookieHandler, cookieConfig *AccessConfig) *AccessInterceptor {
 	return &AccessInterceptor{
 		svc:           svc,
 		cookieHandler: cookieHandler,
@@ -50,7 +50,7 @@ func (a *AccessInterceptor) WithoutLimiting() *AccessInterceptor {
 	}
 }

-func (a *AccessInterceptor) AccessService() *logstore.Service {
+func (a *AccessInterceptor) AccessService() *logstore.Service[*record.AccessLog] {
 	return a.svc
 }

@@ -81,46 +81,70 @@ func (a *AccessInterceptor) DeleteExhaustedCookie(writer http.ResponseWriter) {
 	a.cookieHandler.DeleteCookie(writer, a.limitConfig.ExhaustedCookieKey)
 }

+func (a *AccessInterceptor) HandleIgnorePathPrefixes(ignoredPathPrefixes []string) func(next http.Handler) http.Handler {
+	return a.handle(ignoredPathPrefixes...)
+}
+
 func (a *AccessInterceptor) Handle(next http.Handler) http.Handler {
-	if !a.svc.Enabled() {
-		return next
-	}
-	return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
-		ctx := request.Context()
-		tracingCtx, checkSpan := tracing.NewNamedSpan(ctx, "checkAccess")
-		wrappedWriter := &statusRecorder{ResponseWriter: writer, status: 0}
-		limited := a.Limit(tracingCtx)
-		checkSpan.End()
-		if limited {
-			a.SetExhaustedCookie(wrappedWriter, request)
-			http.Error(wrappedWriter, "quota for authenticated requests is exhausted", http.StatusTooManyRequests)
+	return a.handle()(next)
+}
+
+func (a *AccessInterceptor) handle(ignoredPathPrefixes ...string) func(http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		if !a.svc.Enabled() {
+			return next
 		}
-		if !limited && !a.storeOnly {
-			a.DeleteExhaustedCookie(wrappedWriter)
-		}
-		if !limited {
-			next.ServeHTTP(wrappedWriter, request)
-		}
-		tracingCtx, writeSpan := tracing.NewNamedSpan(tracingCtx, "writeAccess")
-		defer writeSpan.End()
-		requestURL := request.RequestURI
-		unescapedURL, err := url.QueryUnescape(requestURL)
-		if err != nil {
-			logging.WithError(err).WithField("url", requestURL).Warning("failed to unescape request url")
-		}
-		instance := authz.GetInstance(tracingCtx)
-		a.svc.Handle(tracingCtx, &access.Record{
-			LogDate:         time.Now(),
-			Protocol:        access.HTTP,
-			RequestURL:      unescapedURL,
-			ResponseStatus:  uint32(wrappedWriter.status),
-			RequestHeaders:  request.Header,
-			ResponseHeaders: writer.Header(),
-			InstanceID:      instance.InstanceID(),
-			ProjectID:       instance.ProjectID(),
-			RequestedDomain: instance.RequestedDomain(),
-			RequestedHost:   instance.RequestedHost(),
+		return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+			ctx := request.Context()
+			tracingCtx, checkSpan := tracing.NewNamedSpan(ctx, "checkAccessQuota")
+			wrappedWriter := &statusRecorder{ResponseWriter: writer, status: 0}
+			for _, ignoredPathPrefix := range ignoredPathPrefixes {
+				if !strings.HasPrefix(request.RequestURI, ignoredPathPrefix) {
+					continue
+				}
+				checkSpan.End()
+				next.ServeHTTP(wrappedWriter, request)
+				a.writeLog(tracingCtx, wrappedWriter, writer, request, true)
+				return
+			}
+			limited := a.Limit(tracingCtx)
+			checkSpan.End()
+			if limited {
+				a.SetExhaustedCookie(wrappedWriter, request)
+				http.Error(wrappedWriter, "quota for authenticated requests is exhausted", http.StatusTooManyRequests)
+			}
+			if !limited && !a.storeOnly {
+				a.DeleteExhaustedCookie(wrappedWriter)
+			}
+			if !limited {
+				next.ServeHTTP(wrappedWriter, request)
+			}
+			a.writeLog(tracingCtx, wrappedWriter, writer, request, a.storeOnly)
 		})
+	}
+}
+
+func (a *AccessInterceptor) writeLog(ctx context.Context, wrappedWriter *statusRecorder, writer http.ResponseWriter, request *http.Request, notCountable bool) {
+	ctx, writeSpan := tracing.NewNamedSpan(ctx, "writeAccess")
+	defer writeSpan.End()
+	requestURL := request.RequestURI
+	unescapedURL, err := url.QueryUnescape(requestURL)
+	if err != nil {
+		logging.WithError(err).WithField("url", requestURL).Warning("failed to unescape request url")
+	}
+	instance := authz.GetInstance(ctx)
+	a.svc.Handle(ctx, &record.AccessLog{
+		LogDate:         time.Now(),
+		Protocol:        record.HTTP,
+		RequestURL:      unescapedURL,
+		ResponseStatus:  uint32(wrappedWriter.status),
+		RequestHeaders:  request.Header,
+		ResponseHeaders: writer.Header(),
+		InstanceID:      instance.InstanceID(),
+		ProjectID:       instance.ProjectID(),
+		RequestedDomain: instance.RequestedDomain(),
+		RequestedHost:   instance.RequestedHost(),
+		NotCountable:    notCountable,
 	})
 }