TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-11 19:07:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: project quotas and usages (#6441)

Browse Source 
* project quota added

* project quota removed

* add periods table

* make log record generic

* accumulate usage

* query usage

* count action run seconds

* fix filter in ReportQuotaUsage

* fix existing tests

* fix logstore tests

* fix typo

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* move notifications into debouncer and improve limit querying

* cleanup

* comment

* fix: add quota unit tests command side

* fix remaining quota usage query

* implement InmemLogStorage

* cleanup and linting

* improve test

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* action notifications and fixes for notifications query

* revert console prefix

* fix: add quota unit tests command side

* fix: add quota integration tests

* improve accountable requests

* improve accountable requests

* fix: add quota integration tests

* fix: add quota integration tests

* fix: add quota integration tests

* comment

* remove ability to store logs in db and other changes requested from review

* changes requested from review

* changes requested from review

* Update internal/api/http/middleware/access_interceptor.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* tests: fix quotas integration tests

* improve incrementUsageStatement

* linting

* fix: delete e2e tests as intergation tests cover functionality

* Update internal/api/http/middleware/access_interceptor.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* backup

* fix conflict

* create rc

* create prerelease

* remove issue release labeling

* fix tracing

---------

Co-authored-by: Livio Spring <livio.a@gmail.com>
Co-authored-by: Stefan Benz <stefan@caos.ch>
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
This commit is contained in:
Elio Bischof
2023-09-15 16:58:45 +02:00
committed by GitHub
parent b4d0d2c9a7
commit 1a49b7d298
66 changed files with 3423 additions and 1413 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
internal/logstore/service.go
View File

@@ -2,124 +2,70 @@ package logstore
import (
"context"
"math"
"time"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/repository/quota"
)
const handleThresholdTimeout = time.Minute
type QuotaQuerier interface {
GetCurrentQuotaPeriod(ctx context.Context, instanceID string, unit quota.Unit) (config *quota.AddedEvent, periodStart time.Time, err error)
GetDueQuotaNotifications(ctx context.Context, config *quota.AddedEvent, periodStart time.Time, used uint64) ([]*quota.NotificationDueEvent, error)
}
type UsageQuerier interface {
LogEmitter
type UsageStorer[T LogRecord[T]] interface {
LogEmitter[T]
QuotaUnit() quota.Unit
QueryUsage(ctx context.Context, instanceId string, start time.Time) (uint64, error)
}
type UsageReporter interface {
Report(ctx context.Context, notifications []*quota.NotificationDueEvent) (err error)
}
type UsageReporterFunc func(context.Context, []*quota.NotificationDueEvent) (err error)
func (u UsageReporterFunc) Report(ctx context.Context, notifications []*quota.NotificationDueEvent) (err error) {
return u(ctx, notifications)
}
type Service struct {
usageQuerier UsageQuerier
quotaQuerier QuotaQuerier
usageReporter UsageReporter
enabledSinks []*emitter
type Service[T LogRecord[T]] struct {
queries Queries
usageStorer UsageStorer[T]
enabledSinks []*emitter[T]
sinkEnabled bool
reportingEnabled bool
}
func New(quotaQuerier QuotaQuerier, usageReporter UsageReporter, usageQuerierSink *emitter, additionalSink ...*emitter) *Service {
var usageQuerier UsageQuerier
type Queries interface {
GetRemainingQuotaUsage(ctx context.Context, instanceID string, unit quota.Unit) (remaining *uint64, err error)
}
func New[T LogRecord[T]](queries Queries, usageQuerierSink *emitter[T], additionalSink ...*emitter[T]) *Service[T] {
var usageStorer UsageStorer[T]
if usageQuerierSink != nil {
usageQuerier = usageQuerierSink.emitter.(UsageQuerier)
usageStorer = usageQuerierSink.emitter.(UsageStorer[T])
}
svc := &Service{
svc := &Service[T]{
queries: queries,
reportingEnabled: usageQuerierSink != nil && usageQuerierSink.enabled,
usageQuerier: usageQuerier,
quotaQuerier: quotaQuerier,
usageReporter: usageReporter,
usageStorer: usageStorer,
}
for _, s := range append([]*emitter{usageQuerierSink}, additionalSink...) {
for _, s := range append([]*emitter[T]{usageQuerierSink}, additionalSink...) {
if s != nil && s.enabled {
svc.enabledSinks = append(svc.enabledSinks, s)
}
}
svc.sinkEnabled = len(svc.enabledSinks) > 0
return svc
}
func (s *Service) Enabled() bool {
func (s *Service[T]) Enabled() bool {
return s.sinkEnabled
}
func (s *Service) Handle(ctx context.Context, record LogRecord) {
func (s *Service[T]) Handle(ctx context.Context, record T) {
for _, sink := range s.enabledSinks {
logging.OnError(sink.Emit(ctx, record.Normalize())).WithField("record", record).Warn("failed to emit log record")
}
}
func (s *Service) Limit(ctx context.Context, instanceID string) *uint64 {
func (s *Service[T]) Limit(ctx context.Context, instanceID string) *uint64 {
var err error
defer func() {
logging.OnError(err).Warn("failed to check is usage should be limited")
logging.OnError(err).Warn("failed to check if usage should be limited")
}()
if !s.reportingEnabled || instanceID == "" {
return nil
}
quota, periodStart, err := s.quotaQuerier.GetCurrentQuotaPeriod(ctx, instanceID, s.usageQuerier.QuotaUnit())
if err != nil || quota == nil {
return nil
}
usage, err := s.usageQuerier.QueryUsage(ctx, instanceID, periodStart)
remaining, err := s.queries.GetRemainingQuotaUsage(ctx, instanceID, s.usageStorer.QuotaUnit())
if err != nil {
// TODO: shouldn't we just limit then or return the error and decide there?
return nil
}
go s.handleThresholds(ctx, quota, periodStart, usage)
var remaining *uint64
if quota.Limit {
r := uint64(math.Max(0, float64(quota.Amount)-float64(usage)))
remaining = &r
}
return remaining
}
func (s *Service) handleThresholds(ctx context.Context, quota *quota.AddedEvent, periodStart time.Time, usage uint64) {
var err error
defer func() {
logging.OnError(err).Warn("handling quota thresholds failed")
}()
detatchedCtx, cancel := context.WithTimeout(authz.Detach(ctx), handleThresholdTimeout)
defer cancel()
notifications, err := s.quotaQuerier.GetDueQuotaNotifications(detatchedCtx, quota, periodStart, usage)
if err != nil || len(notifications) == 0 {
return
}
err = s.usageReporter.Report(detatchedCtx, notifications)
}