perf: project quotas and usages (#6441)

* project quota added

* project quota removed

* add periods table

* make log record generic

* accumulate usage

* query usage

* count action run seconds

* fix filter in ReportQuotaUsage

* fix existing tests

* fix logstore tests

* fix typo

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* move notifications into debouncer and improve limit querying

* cleanup

* comment

* fix: add quota unit tests command side

* fix remaining quota usage query

* implement InmemLogStorage

* cleanup and linting

* improve test

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* action notifications and fixes for notifications query

* revert console prefix

* fix: add quota unit tests command side

* fix: add quota integration tests

* improve accountable requests

* improve accountable requests

* fix: add quota integration tests

* fix: add quota integration tests

* fix: add quota integration tests

* comment

* remove ability to store logs in db and other changes requested from review

* changes requested from review

* changes requested from review

* Update internal/api/http/middleware/access_interceptor.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* tests: fix quotas integration tests

* improve incrementUsageStatement

* linting

* fix: delete e2e tests as intergation tests cover functionality

* Update internal/api/http/middleware/access_interceptor.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* backup

* fix conflict

* create rc

* create prerelease

* remove issue release labeling

* fix tracing

---------

Co-authored-by: Livio Spring <livio.a@gmail.com>
Co-authored-by: Stefan Benz <stefan@caos.ch>
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
This commit is contained in:
Elio Bischof
2023-09-15 16:58:45 +02:00
committed by GitHub
parent b4d0d2c9a7
commit 1a49b7d298
66 changed files with 3423 additions and 1413 deletions

View File

@@ -4,16 +4,14 @@ import (
"time"
"github.com/zitadel/zitadel/internal/logstore"
"github.com/zitadel/zitadel/internal/repository/quota"
"github.com/zitadel/zitadel/internal/query"
)
type emitterOption func(config *logstore.EmitterConfig)
func emitterConfig(options ...emitterOption) *logstore.EmitterConfig {
cfg := &logstore.EmitterConfig{
Enabled: true,
Keep: time.Hour,
CleanupInterval: time.Hour,
Enabled: true,
Debounce: &logstore.DebouncerConfig{
MinFrequency: 0,
MaxBulkSize: 0,
@@ -37,17 +35,10 @@ func withDisabled() emitterOption {
}
}
func withCleanupping(keep, interval time.Duration) emitterOption {
return func(c *logstore.EmitterConfig) {
c.Keep = keep
c.CleanupInterval = interval
}
}
type quotaOption func(config *query.Quota)
type quotaOption func(config *quota.AddedEvent)
func quotaConfig(quotaOptions ...quotaOption) quota.AddedEvent {
q := &quota.AddedEvent{
func quotaConfig(quotaOptions ...quotaOption) *query.Quota {
q := &query.Quota{
Amount: 90,
Limit: false,
ResetInterval: 90 * time.Second,
@@ -56,18 +47,18 @@ func quotaConfig(quotaOptions ...quotaOption) quota.AddedEvent {
for _, opt := range quotaOptions {
opt(q)
}
return *q
return q
}
func withAmountAndInterval(n uint64) quotaOption {
return func(c *quota.AddedEvent) {
return func(c *query.Quota) {
c.Amount = n
c.ResetInterval = time.Duration(n) * time.Second
}
}
func withLimiting() quotaOption {
return func(c *quota.AddedEvent) {
return func(c *query.Quota) {
c.Limit = true
}
}