TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-12 03:57:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: project quotas and usages (#6441)

Browse Source 
* project quota added

* project quota removed

* add periods table

* make log record generic

* accumulate usage

* query usage

* count action run seconds

* fix filter in ReportQuotaUsage

* fix existing tests

* fix logstore tests

* fix typo

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* move notifications into debouncer and improve limit querying

* cleanup

* comment

* fix: add quota unit tests command side

* fix remaining quota usage query

* implement InmemLogStorage

* cleanup and linting

* improve test

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* action notifications and fixes for notifications query

* revert console prefix

* fix: add quota unit tests command side

* fix: add quota integration tests

* improve accountable requests

* improve accountable requests

* fix: add quota integration tests

* fix: add quota integration tests

* fix: add quota integration tests

* comment

* remove ability to store logs in db and other changes requested from review

* changes requested from review

* changes requested from review

* Update internal/api/http/middleware/access_interceptor.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* tests: fix quotas integration tests

* improve incrementUsageStatement

* linting

* fix: delete e2e tests as intergation tests cover functionality

* Update internal/api/http/middleware/access_interceptor.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* backup

* fix conflict

* create rc

* create prerelease

* remove issue release labeling

* fix tracing

---------

Co-authored-by: Livio Spring <livio.a@gmail.com>
Co-authored-by: Stefan Benz <stefan@caos.ch>
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
This commit is contained in:
Elio Bischof
2023-09-15 16:58:45 +02:00
committed by GitHub
parent b4d0d2c9a7
commit 1a49b7d298
66 changed files with 3423 additions and 1413 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
internal/logstore/emitter.go
View File

@@ -2,56 +2,52 @@ package logstore
import (
"context"
"fmt"
"time"
"github.com/benbjohnson/clock"
"github.com/zitadel/logging"
)
type EmitterConfig struct {
Enabled bool
Keep time.Duration
CleanupInterval time.Duration
Debounce *DebouncerConfig
Enabled bool
Debounce *DebouncerConfig
}
type emitter struct {
type emitter[T LogRecord[T]] struct {
enabled bool
ctx context.Context
debouncer *debouncer
emitter LogEmitter
debouncer *debouncer[T]
emitter LogEmitter[T]
clock clock.Clock
}
type LogRecord interface {
Normalize() LogRecord
type LogRecord[T any] interface {
Normalize() T
}
type LogRecordFunc func() LogRecord
type LogRecordFunc[T any] func() T
func (r LogRecordFunc) Normalize() LogRecord {
func (r LogRecordFunc[T]) Normalize() T {
return r()
}
type LogEmitter interface {
Emit(ctx context.Context, bulk []LogRecord) error
type LogEmitter[T LogRecord[T]] interface {
Emit(ctx context.Context, bulk []T) error
}
type LogEmitterFunc func(ctx context.Context, bulk []LogRecord) error
type LogEmitterFunc[T LogRecord[T]] func(ctx context.Context, bulk []T) error
func (l LogEmitterFunc) Emit(ctx context.Context, bulk []LogRecord) error {
func (l LogEmitterFunc[T]) Emit(ctx context.Context, bulk []T) error {
return l(ctx, bulk)
}
type LogCleanupper interface {
LogEmitter
type LogCleanupper[T LogRecord[T]] interface {
Cleanup(ctx context.Context, keep time.Duration) error
LogEmitter[T]
}
// NewEmitter accepts Clock from github.com/benbjohnson/clock so we can control timers and tickers in the unit tests
func NewEmitter(ctx context.Context, clock clock.Clock, cfg *EmitterConfig, logger LogEmitter) (*emitter, error) {
svc := &emitter{
func NewEmitter[T LogRecord[T]](ctx context.Context, clock clock.Clock, cfg *EmitterConfig, logger LogEmitter[T]) (*emitter[T], error) {
svc := &emitter[T]{
enabled: cfg != nil && cfg.Enabled,
ctx: ctx,
emitter: logger,
@@ -63,36 +59,12 @@ func NewEmitter(ctx context.Context, clock clock.Clock, cfg *EmitterConfig, logg
}
if cfg.Debounce != nil && (cfg.Debounce.MinFrequency > 0 || cfg.Debounce.MaxBulkSize > 0) {
svc.debouncer = newDebouncer(ctx, *cfg.Debounce, clock, newStorageBulkSink(svc.emitter))
}
cleanupper, ok := logger.(LogCleanupper)
if !ok {
if cfg.Keep != 0 {
return nil, fmt.Errorf("cleaning up for this storage type is not supported, so keep duration must be 0, but is %d", cfg.Keep)
}
if cfg.CleanupInterval != 0 {
return nil, fmt.Errorf("cleaning up for this storage type is not supported, so cleanup interval duration must be 0, but is %d", cfg.Keep)
}
return svc, nil
}
if cfg.Keep != 0 && cfg.CleanupInterval != 0 {
go svc.startCleanupping(cleanupper, cfg.CleanupInterval, cfg.Keep)
svc.debouncer = newDebouncer[T](ctx, *cfg.Debounce, clock, newStorageBulkSink(svc.emitter))
}
return svc, nil
}
func (s *emitter) startCleanupping(cleanupper LogCleanupper, cleanupInterval, keep time.Duration) {
for range s.clock.Tick(cleanupInterval) {
if err := cleanupper.Cleanup(s.ctx, keep); err != nil {
logging.WithError(err).Error("cleaning up logs failed")
}
}
}
func (s *emitter) Emit(ctx context.Context, record LogRecord) (err error) {
func (s *emitter[T]) Emit(ctx context.Context, record T) (err error) {
if !s.enabled {
return nil
}
@@ -102,11 +74,11 @@ func (s *emitter) Emit(ctx context.Context, record LogRecord) (err error) {
return nil
}
return s.emitter.Emit(ctx, []LogRecord{record})
return s.emitter.Emit(ctx, []T{record})
}
func newStorageBulkSink(emitter LogEmitter) bulkSinkFunc {
return func(ctx context.Context, bulk []LogRecord) error {
func newStorageBulkSink[T LogRecord[T]](emitter LogEmitter[T]) bulkSinkFunc[T] {
return func(ctx context.Context, bulk []T) error {
return emitter.Emit(ctx, bulk)
}
}