perf: project quotas and usages (#6441)

* project quota added * project quota removed * add periods table * make log record generic * accumulate usage * query usage * count action run seconds * fix filter in ReportQuotaUsage * fix existing tests * fix logstore tests * fix typo * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * move notifications into debouncer and improve limit querying * cleanup * comment * fix: add quota unit tests command side * fix remaining quota usage query * implement InmemLogStorage * cleanup and linting * improve test * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * action notifications and fixes for notifications query * revert console prefix * fix: add quota unit tests command side * fix: add quota integration tests * improve accountable requests * improve accountable requests * fix: add quota integration tests * fix: add quota integration tests * fix: add quota integration tests * comment * remove ability to store logs in db and other changes requested from review * changes requested from review * changes requested from review * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * tests: fix quotas integration tests * improve incrementUsageStatement * linting * fix: delete e2e tests as intergation tests cover functionality * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * backup * fix conflict * create rc * create prerelease * remove issue release labeling * fix tracing --------- Co-authored-by: Livio Spring <livio.a@gmail.com> Co-authored-by: Stefan Benz <stefan@caos.ch> Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2025-08-12 06:07:33 +00:00 · 2023-09-15 16:58:45 +02:00
parent b4d0d2c9a7
commit 1a49b7d298
66 changed files with 3423 additions and 1413 deletions
--- a/internal/logstore/record/access_test.go
+++ b/internal/logstore/record/access_test.go
@@ -0,0 +1,78 @@
+package record
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestRecord_Normalize(t *testing.T) {
+	tests := []struct {
+		name   string
+		record AccessLog
+		want   *AccessLog
+	}{{
+		name: "headers with certain keys should be redacted",
+		record: AccessLog{
+			RequestHeaders: map[string][]string{
+				"authorization":             {"AValue"},
+				"grpcgateway-authorization": {"AValue"},
+				"cookie":                    {"AValue"},
+				"grpcgateway-cookie":        {"AValue"},
+			}, ResponseHeaders: map[string][]string{
+				"set-cookie": {"AValue"},
+			},
+		},
+		want: &AccessLog{
+			RequestHeaders: map[string][]string{
+				"authorization":             {"[REDACTED]"},
+				"grpcgateway-authorization": {"[REDACTED]"},
+				"cookie":                    {"[REDACTED]"},
+				"grpcgateway-cookie":        {"[REDACTED]"},
+			}, ResponseHeaders: map[string][]string{
+				"set-cookie": {"[REDACTED]"},
+			},
+		},
+	}, {
+		name: "header keys should be lower cased",
+		record: AccessLog{
+			RequestHeaders:  map[string][]string{"AKey": {"AValue"}},
+			ResponseHeaders: map[string][]string{"AKey": {"AValue"}}},
+		want: &AccessLog{
+			RequestHeaders:  map[string][]string{"akey": {"AValue"}},
+			ResponseHeaders: map[string][]string{"akey": {"AValue"}}},
+	}, {
+		name: "an already prune record should stay unchanged",
+		record: AccessLog{
+			RequestURL: "https://my.zitadel.cloud/",
+			RequestHeaders: map[string][]string{
+				"authorization": {"[REDACTED]"},
+			},
+			ResponseHeaders: map[string][]string{},
+		},
+		want: &AccessLog{
+			RequestURL: "https://my.zitadel.cloud/",
+			RequestHeaders: map[string][]string{
+				"authorization": {"[REDACTED]"},
+			},
+			ResponseHeaders: map[string][]string{},
+		},
+	}, {
+		name: "empty record should stay empty",
+		record: AccessLog{
+			RequestHeaders:  map[string][]string{},
+			ResponseHeaders: map[string][]string{},
+		},
+		want: &AccessLog{
+			RequestHeaders:  map[string][]string{},
+			ResponseHeaders: map[string][]string{},
+		},
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.want.normalized = true
+			if got := tt.record.Normalize(); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Normalize() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}