perf: project quotas and usages (#6441)

* project quota added * project quota removed * add periods table * make log record generic * accumulate usage * query usage * count action run seconds * fix filter in ReportQuotaUsage * fix existing tests * fix logstore tests * fix typo * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * move notifications into debouncer and improve limit querying * cleanup * comment * fix: add quota unit tests command side * fix remaining quota usage query * implement InmemLogStorage * cleanup and linting * improve test * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * action notifications and fixes for notifications query * revert console prefix * fix: add quota unit tests command side * fix: add quota integration tests * improve accountable requests * improve accountable requests * fix: add quota integration tests * fix: add quota integration tests * fix: add quota integration tests * comment * remove ability to store logs in db and other changes requested from review * changes requested from review * changes requested from review * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * tests: fix quotas integration tests * improve incrementUsageStatement * linting * fix: delete e2e tests as intergation tests cover functionality * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * backup * fix conflict * create rc * create prerelease * remove issue release labeling * fix tracing --------- Co-authored-by: Livio Spring <livio.a@gmail.com> Co-authored-by: Stefan Benz <stefan@caos.ch> Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2025-08-11 21:27:42 +00:00 · 2023-09-15 16:58:45 +02:00
parent b4d0d2c9a7
commit 1a49b7d298
66 changed files with 3423 additions and 1413 deletions
--- a/internal/logstore/record/access.go
+++ b/internal/logstore/record/access.go
@@ -0,0 +1,135 @@
+package record
+
+import (
+	"net/http"
+	"strings"
+	"time"
+
+	"google.golang.org/grpc/codes"
+
+	zitadel_http "github.com/zitadel/zitadel/internal/api/http"
+)
+
+type AccessLog struct {
+	LogDate        time.Time      `json:"logDate"`
+	Protocol       AccessProtocol `json:"protocol"`
+	RequestURL     string         `json:"requestUrl"`
+	ResponseStatus uint32         `json:"responseStatus"`
+	// RequestHeaders and ResponseHeaders are plain maps so varying implementations
+	// between HTTP and gRPC don't interfere with each other
+	RequestHeaders  map[string][]string `json:"requestHeaders"`
+	ResponseHeaders map[string][]string `json:"responseHeaders"`
+	InstanceID      string              `json:"instanceId"`
+	ProjectID       string              `json:"projectId"`
+	RequestedDomain string              `json:"requestedDomain"`
+	RequestedHost   string              `json:"requestedHost"`
+	// NotCountable can be used by the logging service to explicitly stating,
+	// that the request must not increase the amount of countable (authenticated) requests
+	NotCountable bool `json:"-"`
+	normalized   bool `json:"-"`
+}
+
+type AccessProtocol uint8
+
+const (
+	GRPC AccessProtocol = iota
+	HTTP
+
+	redacted = "[REDACTED]"
+)
+
+var (
+	unaccountableEndpoints = []string{
+		"/zitadel.system.v1.SystemService/",
+		"/zitadel.admin.v1.AdminService/Healthz",
+		"/zitadel.management.v1.ManagementService/Healthz",
+		"/zitadel.management.v1.ManagementService/GetOIDCInformation",
+		"/zitadel.auth.v1.AuthService/Healthz",
+	}
+)
+
+func (a AccessLog) IsAuthenticated() bool {
+	if a.NotCountable {
+		return false
+	}
+	if !a.normalized {
+		panic("access log not normalized, Normalize() must be called before IsAuthenticated()")
+	}
+	_, hasHTTPAuthHeader := a.RequestHeaders[strings.ToLower(zitadel_http.Authorization)]
+	// ignore requests, which were unauthorized or do not require an authorization (even if one was sent)
+	// also ignore if the limit was already reached or if the server returned an internal error
+	// not that endpoints paths are only checked with the gRPC representation as HTTP (gateway) will not log them
+	return hasHTTPAuthHeader &&
+		(a.Protocol == HTTP &&
+			a.ResponseStatus != http.StatusInternalServerError &&
+			a.ResponseStatus != http.StatusTooManyRequests &&
+			a.ResponseStatus != http.StatusUnauthorized) ||
+		(a.Protocol == GRPC &&
+			a.ResponseStatus != uint32(codes.Internal) &&
+			a.ResponseStatus != uint32(codes.ResourceExhausted) &&
+			a.ResponseStatus != uint32(codes.Unauthenticated) &&
+			!a.isUnaccountableEndpoint())
+}
+
+func (a AccessLog) isUnaccountableEndpoint() bool {
+	for _, endpoint := range unaccountableEndpoints {
+		if strings.HasPrefix(a.RequestURL, endpoint) {
+			return true
+		}
+	}
+	return false
+}
+
+func (a AccessLog) Normalize() *AccessLog {
+	a.RequestedDomain = cutString(a.RequestedDomain, 200)
+	a.RequestURL = cutString(a.RequestURL, 200)
+	a.RequestHeaders = normalizeHeaders(a.RequestHeaders, strings.ToLower(zitadel_http.Authorization), "grpcgateway-authorization", "cookie", "grpcgateway-cookie")
+	a.ResponseHeaders = normalizeHeaders(a.ResponseHeaders, "set-cookie")
+	a.normalized = true
+	return &a
+}
+
+// normalizeHeaders lowers all header keys and redacts secrets
+func normalizeHeaders(header map[string][]string, redactKeysLower ...string) map[string][]string {
+	return pruneKeys(redactKeys(lowerKeys(header), redactKeysLower...))
+}
+
+func lowerKeys(header map[string][]string) map[string][]string {
+	lower := make(map[string][]string, len(header))
+	for k, v := range header {
+		lower[strings.ToLower(k)] = v
+	}
+	return lower
+}
+
+func redactKeys(header map[string][]string, redactKeysLower ...string) map[string][]string {
+	redactedKeys := make(map[string][]string, len(header))
+	for k, v := range header {
+		redactedKeys[k] = v
+	}
+	for _, redactKey := range redactKeysLower {
+		if _, ok := redactedKeys[redactKey]; ok {
+			redactedKeys[redactKey] = []string{redacted}
+		}
+	}
+	return redactedKeys
+}
+
+const maxValuesPerKey = 10
+
+func pruneKeys(header map[string][]string) map[string][]string {
+	prunedKeys := make(map[string][]string, len(header))
+	for key, value := range header {
+		valueItems := make([]string, 0, maxValuesPerKey)
+		for i, valueItem := range value {
+			// Max 10 header values per key
+			if i > maxValuesPerKey {
+				break
+			}
+			// Max 200 value length
+			valueItems = append(valueItems, cutString(valueItem, 200))
+		}
+		prunedKeys[key] = valueItems
+	}
+	return prunedKeys
+}
--- a/internal/logstore/record/access_test.go
+++ b/internal/logstore/record/access_test.go
@@ -0,0 +1,78 @@
+package record
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestRecord_Normalize(t *testing.T) {
+	tests := []struct {
+		name   string
+		record AccessLog
+		want   *AccessLog
+	}{{
+		name: "headers with certain keys should be redacted",
+		record: AccessLog{
+			RequestHeaders: map[string][]string{
+				"authorization":             {"AValue"},
+				"grpcgateway-authorization": {"AValue"},
+				"cookie":                    {"AValue"},
+				"grpcgateway-cookie":        {"AValue"},
+			}, ResponseHeaders: map[string][]string{
+				"set-cookie": {"AValue"},
+			},
+		},
+		want: &AccessLog{
+			RequestHeaders: map[string][]string{
+				"authorization":             {"[REDACTED]"},
+				"grpcgateway-authorization": {"[REDACTED]"},
+				"cookie":                    {"[REDACTED]"},
+				"grpcgateway-cookie":        {"[REDACTED]"},
+			}, ResponseHeaders: map[string][]string{
+				"set-cookie": {"[REDACTED]"},
+			},
+		},
+	}, {
+		name: "header keys should be lower cased",
+		record: AccessLog{
+			RequestHeaders:  map[string][]string{"AKey": {"AValue"}},
+			ResponseHeaders: map[string][]string{"AKey": {"AValue"}}},
+		want: &AccessLog{
+			RequestHeaders:  map[string][]string{"akey": {"AValue"}},
+			ResponseHeaders: map[string][]string{"akey": {"AValue"}}},
+	}, {
+		name: "an already prune record should stay unchanged",
+		record: AccessLog{
+			RequestURL: "https://my.zitadel.cloud/",
+			RequestHeaders: map[string][]string{
+				"authorization": {"[REDACTED]"},
+			},
+			ResponseHeaders: map[string][]string{},
+		},
+		want: &AccessLog{
+			RequestURL: "https://my.zitadel.cloud/",
+			RequestHeaders: map[string][]string{
+				"authorization": {"[REDACTED]"},
+			},
+			ResponseHeaders: map[string][]string{},
+		},
+	}, {
+		name: "empty record should stay empty",
+		record: AccessLog{
+			RequestHeaders:  map[string][]string{},
+			ResponseHeaders: map[string][]string{},
+		},
+		want: &AccessLog{
+			RequestHeaders:  map[string][]string{},
+			ResponseHeaders: map[string][]string{},
+		},
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.want.normalized = true
+			if got := tt.record.Normalize(); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Normalize() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
--- a/internal/logstore/record/execution.go
+++ b/internal/logstore/record/execution.go
@@ -0,0 +1,22 @@
+package record
+
+import (
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+type ExecutionLog struct {
+	LogDate    time.Time              `json:"logDate"`
+	Took       time.Duration          `json:"took"`
+	Message    string                 `json:"message"`
+	LogLevel   logrus.Level           `json:"logLevel"`
+	InstanceID string                 `json:"instanceId"`
+	ActionID   string                 `json:"actionId,omitempty"`
+	Metadata   map[string]interface{} `json:"metadata,omitempty"`
+}
+
+func (e ExecutionLog) Normalize() *ExecutionLog {
+	e.Message = cutString(e.Message, 2000)
+	return &e
+}
--- a/internal/logstore/record/prune.go
+++ b/internal/logstore/record/prune.go
@@ -0,0 +1,8 @@
+package record
+
+func cutString(str string, pos int) string {
+	if len(str) <= pos {
+		return str
+	}
+	return str[:pos-1]
+}