perf: project quotas and usages (#6441)

* project quota added * project quota removed * add periods table * make log record generic * accumulate usage * query usage * count action run seconds * fix filter in ReportQuotaUsage * fix existing tests * fix logstore tests * fix typo * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * move notifications into debouncer and improve limit querying * cleanup * comment * fix: add quota unit tests command side * fix remaining quota usage query * implement InmemLogStorage * cleanup and linting * improve test * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * action notifications and fixes for notifications query * revert console prefix * fix: add quota unit tests command side * fix: add quota integration tests * improve accountable requests * improve accountable requests * fix: add quota integration tests * fix: add quota integration tests * fix: add quota integration tests * comment * remove ability to store logs in db and other changes requested from review * changes requested from review * changes requested from review * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * tests: fix quotas integration tests * improve incrementUsageStatement * linting * fix: delete e2e tests as intergation tests cover functionality * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * backup * fix conflict * create rc * create prerelease * remove issue release labeling * fix tracing --------- Co-authored-by: Livio Spring <livio.a@gmail.com> Co-authored-by: Stefan Benz <stefan@caos.ch> Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2025-08-11 19:17:32 +00:00 · 2023-09-15 16:58:45 +02:00
parent b4d0d2c9a7
commit 1a49b7d298
66 changed files with 3423 additions and 1413 deletions
--- a/internal/logstore/mock/inmem.go
+++ b/internal/logstore/mock/inmem.go
@@ -0,0 +1,120 @@
+package mock
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/benbjohnson/clock"
+
+	"github.com/zitadel/zitadel/internal/logstore"
+	"github.com/zitadel/zitadel/internal/query"
+	"github.com/zitadel/zitadel/internal/repository/quota"
+)
+
+var _ logstore.UsageStorer[*Record] = (*InmemLogStorage)(nil)
+var _ logstore.LogCleanupper[*Record] = (*InmemLogStorage)(nil)
+var _ logstore.Queries = (*InmemLogStorage)(nil)
+
+type InmemLogStorage struct {
+	mux     sync.Mutex
+	clock   clock.Clock
+	emitted []*Record
+	bulks   []int
+	quota   *query.Quota
+}
+
+func NewInMemoryStorage(clock clock.Clock, quota *query.Quota) *InmemLogStorage {
+	return &InmemLogStorage{
+		clock:   clock,
+		emitted: make([]*Record, 0),
+		bulks:   make([]int, 0),
+		quota:   quota,
+	}
+}
+
+func (l *InmemLogStorage) QuotaUnit() quota.Unit {
+	return quota.Unimplemented
+}
+
+func (l *InmemLogStorage) Emit(_ context.Context, bulk []*Record) error {
+	if len(bulk) == 0 {
+		return nil
+	}
+	l.mux.Lock()
+	defer l.mux.Unlock()
+	l.emitted = append(l.emitted, bulk...)
+	l.bulks = append(l.bulks, len(bulk))
+	return nil
+}
+
+func (l *InmemLogStorage) QueryUsage(_ context.Context, _ string, start time.Time) (uint64, error) {
+	l.mux.Lock()
+	defer l.mux.Unlock()
+
+	var count uint64
+	for _, r := range l.emitted {
+		if r.ts.After(start) {
+			count++
+		}
+	}
+	return count, nil
+}
+
+func (l *InmemLogStorage) Cleanup(_ context.Context, keep time.Duration) error {
+	l.mux.Lock()
+	defer l.mux.Unlock()
+
+	clean := make([]*Record, 0)
+	from := l.clock.Now().Add(-(keep + 1))
+	for _, r := range l.emitted {
+		if r.ts.After(from) {
+			clean = append(clean, r)
+		}
+	}
+	l.emitted = clean
+	return nil
+}
+
+func (l *InmemLogStorage) Bulks() []int {
+	l.mux.Lock()
+	defer l.mux.Unlock()
+
+	return l.bulks
+}
+
+func (l *InmemLogStorage) Len() int {
+	l.mux.Lock()
+	defer l.mux.Unlock()
+
+	return len(l.emitted)
+}
+
+func (l *InmemLogStorage) GetQuota(ctx context.Context, instanceID string, unit quota.Unit) (qu *query.Quota, err error) {
+	return l.quota, nil
+}
+
+func (l *InmemLogStorage) GetQuotaUsage(ctx context.Context, instanceID string, unit quota.Unit, periodStart time.Time) (usage uint64, err error) {
+	return uint64(l.Len()), nil
+}
+
+func (l *InmemLogStorage) GetRemainingQuotaUsage(ctx context.Context, instanceID string, unit quota.Unit) (remaining *uint64, err error) {
+	if !l.quota.Limit {
+		return nil, nil
+	}
+	var r uint64
+	used := uint64(l.Len())
+	if used > l.quota.Amount {
+		return &r, nil
+	}
+	r = l.quota.Amount - used
+	return &r, nil
+}
+
+func (l *InmemLogStorage) GetDueQuotaNotifications(ctx context.Context, instanceID string, unit quota.Unit, qu *query.Quota, periodStart time.Time, usedAbs uint64) (dueNotifications []*quota.NotificationDueEvent, err error) {
+	return nil, nil
+}
+
+func (l *InmemLogStorage) ReportQuotaUsage(ctx context.Context, dueNotifications []*quota.NotificationDueEvent) error {
+	return nil
+}
--- a/internal/logstore/mock/record.go
+++ b/internal/logstore/mock/record.go
@@ -0,0 +1,25 @@
+package mock
+
+import (
+	"time"
+
+	"github.com/benbjohnson/clock"
+
+	"github.com/zitadel/zitadel/internal/logstore"
+)
+
+var _ logstore.LogRecord[*Record] = (*Record)(nil)
+
+func NewRecord(clock clock.Clock) *Record {
+	return &Record{ts: clock.Now()}
+}
+
+type Record struct {
+	ts       time.Time
+	redacted bool
+}
+
+func (r Record) Normalize() *Record {
+	r.redacted = true
+	return &r
+}