TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-12 04:57:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: project quotas and usages (#6441)

Browse Source 
* project quota added

* project quota removed

* add periods table

* make log record generic

* accumulate usage

* query usage

* count action run seconds

* fix filter in ReportQuotaUsage

* fix existing tests

* fix logstore tests

* fix typo

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* move notifications into debouncer and improve limit querying

* cleanup

* comment

* fix: add quota unit tests command side

* fix remaining quota usage query

* implement InmemLogStorage

* cleanup and linting

* improve test

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* action notifications and fixes for notifications query

* revert console prefix

* fix: add quota unit tests command side

* fix: add quota integration tests

* improve accountable requests

* improve accountable requests

* fix: add quota integration tests

* fix: add quota integration tests

* fix: add quota integration tests

* comment

* remove ability to store logs in db and other changes requested from review

* changes requested from review

* changes requested from review

* Update internal/api/http/middleware/access_interceptor.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* tests: fix quotas integration tests

* improve incrementUsageStatement

* linting

* fix: delete e2e tests as intergation tests cover functionality

* Update internal/api/http/middleware/access_interceptor.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* backup

* fix conflict

* create rc

* create prerelease

* remove issue release labeling

* fix tracing

---------

Co-authored-by: Livio Spring <livio.a@gmail.com>
Co-authored-by: Stefan Benz <stefan@caos.ch>
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
(cherry picked from commit 1a49b7d298)
This commit is contained in:
Elio Bischof
2023-09-15 16:58:45 +02:00
committed by Livio Spring
parent b688d6f842
commit 5823fdbef9
66 changed files with 3423 additions and 1413 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
internal/logstore/debouncer.go
View File

@@ -9,19 +9,17 @@ import (
"github.com/zitadel/logging"
)
type bulkSink interface {
sendBulk(ctx context.Context, bulk []LogRecord) error
type bulkSink[T LogRecord[T]] interface {
SendBulk(ctx context.Context, bulk []T) error
}
var _ bulkSink = bulkSinkFunc(nil)
type bulkSinkFunc[T LogRecord[T]] func(ctx context.Context, bulk []T) error
type bulkSinkFunc func(ctx context.Context, items []LogRecord) error
func (s bulkSinkFunc) sendBulk(ctx context.Context, items []LogRecord) error {
return s(ctx, items)
func (s bulkSinkFunc[T]) SendBulk(ctx context.Context, bulk []T) error {
return s(ctx, bulk)
}
type debouncer struct {
type debouncer[T LogRecord[T]] struct {
// Storing context.Context in a struct is generally bad practice
// https://go.dev/blog/context-and-structs
// However, debouncer starts a go routine that triggers side effects itself.
@@ -33,8 +31,8 @@ type debouncer struct {
ticker *clock.Ticker
mux sync.Mutex
cfg DebouncerConfig
storage bulkSink
cache []LogRecord
storage bulkSink[T]
cache []T
cacheLen uint
}
@@ -43,8 +41,8 @@ type DebouncerConfig struct {
MaxBulkSize uint
}
func newDebouncer(binarySignaledCtx context.Context, cfg DebouncerConfig, clock clock.Clock, ship bulkSink) *debouncer {
a := &debouncer{
func newDebouncer[T LogRecord[T]](binarySignaledCtx context.Context, cfg DebouncerConfig, clock clock.Clock, ship bulkSink[T]) *debouncer[T] {
a := &debouncer[T]{
binarySignaledCtx: binarySignaledCtx,
clock: clock,
cfg: cfg,
@@ -58,7 +56,7 @@ func newDebouncer(binarySignaledCtx context.Context, cfg DebouncerConfig, clock
return a
}
func (d *debouncer) add(item LogRecord) {
func (d *debouncer[T]) add(item T) {
d.mux.Lock()
defer d.mux.Unlock()
d.cache = append(d.cache, item)
@@ -69,13 +67,13 @@ func (d *debouncer) add(item LogRecord) {
}
}
func (d *debouncer) ship() {
func (d *debouncer[T]) ship() {
if d.cacheLen == 0 {
return
}
d.mux.Lock()
defer d.mux.Unlock()
if err := d.storage.sendBulk(d.binarySignaledCtx, d.cache); err != nil {
if err := d.storage.SendBulk(d.binarySignaledCtx, d.cache); err != nil {
logging.WithError(err).WithField("size", len(d.cache)).Error("storing bulk failed")
}
d.cache = nil
@@ -85,7 +83,7 @@ func (d *debouncer) ship() {
}
}
func (d *debouncer) shipOnTicks() {
func (d *debouncer[T]) shipOnTicks() {
for range d.ticker.C {
d.ship()
}