perf: project quotas and usages (#6441)

* project quota added

* project quota removed

* add periods table

* make log record generic

* accumulate usage

* query usage

* count action run seconds

* fix filter in ReportQuotaUsage

* fix existing tests

* fix logstore tests

* fix typo

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* move notifications into debouncer and improve limit querying

* cleanup

* comment

* fix: add quota unit tests command side

* fix remaining quota usage query

* implement InmemLogStorage

* cleanup and linting

* improve test

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* action notifications and fixes for notifications query

* revert console prefix

* fix: add quota unit tests command side

* fix: add quota integration tests

* improve accountable requests

* improve accountable requests

* fix: add quota integration tests

* fix: add quota integration tests

* fix: add quota integration tests

* comment

* remove ability to store logs in db and other changes requested from review

* changes requested from review

* changes requested from review

* Update internal/api/http/middleware/access_interceptor.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* tests: fix quotas integration tests

* improve incrementUsageStatement

* linting

* fix: delete e2e tests as intergation tests cover functionality

* Update internal/api/http/middleware/access_interceptor.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* backup

* fix conflict

* create rc

* create prerelease

* remove issue release labeling

* fix tracing

---------

Co-authored-by: Livio Spring <livio.a@gmail.com>
Co-authored-by: Stefan Benz <stefan@caos.ch>
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
This commit is contained in:
Elio Bischof
2023-09-15 16:58:45 +02:00
committed by GitHub
parent b4d0d2c9a7
commit 1a49b7d298
66 changed files with 3423 additions and 1413 deletions

View File

@@ -7,6 +7,7 @@ import (
"time"
"github.com/rakyll/statik/fs"
"github.com/zitadel/oidc/v2/pkg/oidc"
"github.com/zitadel/oidc/v2/pkg/op"
"golang.org/x/text/language"
@@ -79,13 +80,32 @@ type OPStorage struct {
assetAPIPrefix func(ctx context.Context) string
}
func NewProvider(config Config, defaultLogoutRedirectURI string, externalSecure bool, command *command.Commands, query *query.Queries, repo repository.Repository, encryptionAlg crypto.EncryptionAlgorithm, cryptoKey []byte, es *eventstore.Eventstore, projections *database.DB, userAgentCookie, instanceHandler, accessHandler func(http.Handler) http.Handler) (op.OpenIDProvider, error) {
func NewProvider(
config Config,
defaultLogoutRedirectURI string,
externalSecure bool,
command *command.Commands,
query *query.Queries,
repo repository.Repository,
encryptionAlg crypto.EncryptionAlgorithm,
cryptoKey []byte,
es *eventstore.Eventstore,
projections *database.DB,
userAgentCookie, instanceHandler func(http.Handler) http.Handler,
accessHandler *middleware.AccessInterceptor,
) (op.OpenIDProvider, error) {
opConfig, err := createOPConfig(config, defaultLogoutRedirectURI, cryptoKey)
if err != nil {
return nil, caos_errs.ThrowInternal(err, "OIDC-EGrqd", "cannot create op config: %w")
}
storage := newStorage(config, command, query, repo, encryptionAlg, es, projections, externalSecure)
options, err := createOptions(config, externalSecure, userAgentCookie, instanceHandler, accessHandler)
options, err := createOptions(
config,
externalSecure,
userAgentCookie,
instanceHandler,
accessHandler.HandleIgnorePathPrefixes(ignoredQuotaLimitEndpoint(config.CustomEndpoints)),
)
if err != nil {
return nil, caos_errs.ThrowInternal(err, "OIDC-D3gq1", "cannot create options: %w")
}
@@ -101,6 +121,21 @@ func NewProvider(config Config, defaultLogoutRedirectURI string, externalSecure
return provider, nil
}
func ignoredQuotaLimitEndpoint(endpoints *EndpointConfig) []string {
authURL := op.DefaultEndpoints.Authorization.Relative()
keysURL := op.DefaultEndpoints.JwksURI.Relative()
if endpoints == nil {
return []string{oidc.DiscoveryEndpoint, authURL, keysURL}
}
if endpoints.Auth != nil && endpoints.Auth.Path != "" {
authURL = endpoints.Auth.Path
}
if endpoints.Keys != nil && endpoints.Keys.Path != "" {
keysURL = endpoints.Keys.Path
}
return []string{oidc.DiscoveryEndpoint, authURL, keysURL}
}
func createOPConfig(config Config, defaultLogoutRedirectURI string, cryptoKey []byte) (*op.Config, error) {
supportedLanguages, err := getSupportedLanguages()
if err != nil {