perf: project quotas and usages (#6441)

* project quota added

* project quota removed

* add periods table

* make log record generic

* accumulate usage

* query usage

* count action run seconds

* fix filter in ReportQuotaUsage

* fix existing tests

* fix logstore tests

* fix typo

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* move notifications into debouncer and improve limit querying

* cleanup

* comment

* fix: add quota unit tests command side

* fix remaining quota usage query

* implement InmemLogStorage

* cleanup and linting

* improve test

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* action notifications and fixes for notifications query

* revert console prefix

* fix: add quota unit tests command side

* fix: add quota integration tests

* improve accountable requests

* improve accountable requests

* fix: add quota integration tests

* fix: add quota integration tests

* fix: add quota integration tests

* comment

* remove ability to store logs in db and other changes requested from review

* changes requested from review

* changes requested from review

* Update internal/api/http/middleware/access_interceptor.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* tests: fix quotas integration tests

* improve incrementUsageStatement

* linting

* fix: delete e2e tests as intergation tests cover functionality

* Update internal/api/http/middleware/access_interceptor.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* backup

* fix conflict

* create rc

* create prerelease

* remove issue release labeling

* fix tracing

---------

Co-authored-by: Livio Spring <livio.a@gmail.com>
Co-authored-by: Stefan Benz <stefan@caos.ch>
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>

internal/integration/assert.go

@@ -24,8 +24,8 @@ type DetailsMsg interface {
 //
 // The resource owner is compared with expected and is
 // therefore the only value that has to be set.
-func AssertDetails[D DetailsMsg](t testing.TB, exptected, actual D) {
-	wantDetails, gotDetails := exptected.GetDetails(), actual.GetDetails()
+func AssertDetails[D DetailsMsg](t testing.TB, expected, actual D) {
+	wantDetails, gotDetails := expected.GetDetails(), actual.GetDetails()
 	if wantDetails == nil {
 		assert.Nil(t, gotDetails)
 		return

internal/integration/config/zitadel.yaml

@@ -22,22 +22,17 @@ FirstInstance:
       PasswordChangeRequired: false
 
 LogStore:
-  Access:
-    Database:
-      Enabled: true
-      Debounce:
-        MinFrequency: 0s
-        MaxBulkSize: 0
   Execution:
-    Database:
-      Enabled: true
     Stdout:
       Enabled: true
 
 Quotas:
   Access:
+    Enabled: true
     ExhaustedCookieKey: "zitadel.quota.limiting"
     ExhaustedCookieMaxAge: "60s"
+  Execution:
+    Enabled: true
 
 Projections:
   Customizations:

internal/integration/integration.go

@@ -8,7 +8,11 @@ import (
 	_ "embed"
 	"errors"
 	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
 	"os"
+	"reflect"
 	"strings"
 	"sync"
 	"time"
@@ -30,6 +34,7 @@ import (
 	"github.com/zitadel/zitadel/internal/domain"
 	caos_errs "github.com/zitadel/zitadel/internal/errors"
 	"github.com/zitadel/zitadel/internal/eventstore/v1/models"
+	"github.com/zitadel/zitadel/internal/net"
 	"github.com/zitadel/zitadel/internal/query"
 	"github.com/zitadel/zitadel/internal/webauthn"
 	"github.com/zitadel/zitadel/pkg/grpc/admin"
@@ -71,6 +76,11 @@ const (
 	UserPassword          = "VeryS3cret!"
 )
 
+const (
+	PortMilestoneServer = "8081"
+	PortQuotaServer     = "8082"
+)
+
 // User information with a Personal Access Token.
 type User struct {
 	*query.User
@@ -101,6 +111,11 @@ type Tester struct {
 	Organisation *query.Org
 	Users        InstanceUserMap
 
+	MilestoneChan           chan []byte
+	milestoneServer         *httptest.Server
+	QuotaNotificationChan   chan []byte
+	quotaNotificationServer *httptest.Server
+
 	Client   Client
 	WebAuthN *webauthn.Client
 	wg       sync.WaitGroup // used for shutdown
@@ -271,6 +286,8 @@ func (s *Tester) Done() {
 
 	s.Shutdown <- os.Interrupt
 	s.wg.Wait()
+	s.milestoneServer.Close()
+	s.quotaNotificationServer.Close()
 }
 
 // NewTester start a new Zitadel server by passing the default commandline.
@@ -279,13 +296,13 @@ func (s *Tester) Done() {
 // INTEGRATION_DB_FLAVOR environment variable and can have the values "cockroach"
 // or "postgres". Defaults to "cockroach".
 //
-// The deault Instance and Organisation are read from the DB and system
+// The default Instance and Organisation are read from the DB and system
 // users are created as needed.
 //
 // After the server is started, a [grpc.ClientConn] will be created and
 // the server is polled for it's health status.
 //
-// Note: the database must already be setup and intialized before
+// Note: the database must already be setup and initialized before
 // using NewTester. See the CONTRIBUTING.md document for details.
 func NewTester(ctx context.Context) *Tester {
 	args := strings.Split(commandLine, " ")
@@ -311,6 +328,13 @@ func NewTester(ctx context.Context) *Tester {
 	tester := Tester{
 		Users: make(InstanceUserMap),
 	}
+	tester.MilestoneChan = make(chan []byte, 100)
+	tester.milestoneServer, err = runMilestoneServer(ctx, tester.MilestoneChan)
+	logging.OnError(err).Fatal()
+	tester.QuotaNotificationChan = make(chan []byte, 100)
+	tester.quotaNotificationServer, err = runQuotaServer(ctx, tester.QuotaNotificationChan)
+	logging.OnError(err).Fatal()
+
 	tester.wg.Add(1)
 	go func(wg *sync.WaitGroup) {
 		logging.OnError(cmd.Execute()).Fatal()
@@ -328,7 +352,6 @@ func NewTester(ctx context.Context) *Tester {
 	tester.createMachineUserOrgOwner(ctx)
 	tester.createMachineUserInstanceOwner(ctx)
 	tester.WebAuthN = webauthn.NewClient(tester.Config.WebAuthNName, tester.Config.ExternalDomain, "https://"+tester.Host())
-
 	return &tester
 }
 
@@ -338,3 +361,51 @@ func Contexts(timeout time.Duration) (ctx, errCtx context.Context, cancel contex
 	ctx, cancel = context.WithTimeout(context.Background(), timeout)
 	return ctx, errCtx, cancel
 }
+
+func runMilestoneServer(ctx context.Context, bodies chan []byte) (*httptest.Server, error) {
+	mockServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := io.ReadAll(r.Body)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		if r.Header.Get("single-value") != "single-value" {
+			http.Error(w, "single-value header not set", http.StatusInternalServerError)
+			return
+		}
+		if reflect.DeepEqual(r.Header.Get("multi-value"), "multi-value-1,multi-value-2") {
+			http.Error(w, "single-value header not set", http.StatusInternalServerError)
+			return
+		}
+		bodies <- body
+		w.WriteHeader(http.StatusOK)
+	}))
+	config := net.ListenConfig()
+	listener, err := config.Listen(ctx, "tcp", ":"+PortMilestoneServer)
+	if err != nil {
+		return nil, err
+	}
+	mockServer.Listener = listener
+	mockServer.Start()
+	return mockServer, nil
+}
+
+func runQuotaServer(ctx context.Context, bodies chan []byte) (*httptest.Server, error) {
+	mockServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := io.ReadAll(r.Body)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		bodies <- body
+		w.WriteHeader(http.StatusOK)
+	}))
+	config := net.ListenConfig()
+	listener, err := config.Listen(ctx, "tcp", ":"+PortQuotaServer)
+	if err != nil {
+		return nil, err
+	}
+	mockServer.Listener = listener
+	mockServer.Start()
+	return mockServer, nil
+}