perf: project quotas and usages (#6441)

* project quota added

* project quota removed

* add periods table

* make log record generic

* accumulate usage

* query usage

* count action run seconds

* fix filter in ReportQuotaUsage

* fix existing tests

* fix logstore tests

* fix typo

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* move notifications into debouncer and improve limit querying

* cleanup

* comment

* fix: add quota unit tests command side

* fix remaining quota usage query

* implement InmemLogStorage

* cleanup and linting

* improve test

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* action notifications and fixes for notifications query

* revert console prefix

* fix: add quota unit tests command side

* fix: add quota integration tests

* improve accountable requests

* improve accountable requests

* fix: add quota integration tests

* fix: add quota integration tests

* fix: add quota integration tests

* comment

* remove ability to store logs in db and other changes requested from review

* changes requested from review

* changes requested from review

* Update internal/api/http/middleware/access_interceptor.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* tests: fix quotas integration tests

* improve incrementUsageStatement

* linting

* fix: delete e2e tests as intergation tests cover functionality

* Update internal/api/http/middleware/access_interceptor.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* backup

* fix conflict

* create rc

* create prerelease

* remove issue release labeling

* fix tracing

---------

Co-authored-by: Livio Spring <livio.a@gmail.com>
Co-authored-by: Stefan Benz <stefan@caos.ch>
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
This commit is contained in:
Elio Bischof
2023-09-15 16:58:45 +02:00
committed by GitHub
parent b4d0d2c9a7
commit 1a49b7d298
66 changed files with 3423 additions and 1413 deletions

View File

@@ -38,8 +38,8 @@ func NewProvider(
es *eventstore.Eventstore,
projections *database.DB,
instanceHandler,
userAgentCookie,
accessHandler func(http.Handler) http.Handler,
userAgentCookie func(http.Handler) http.Handler,
accessHandler *middleware.AccessInterceptor,
) (*provider.Provider, error) {
metricTypes := []metrics.MetricType{metrics.MetricTypeRequestCount, metrics.MetricTypeStatusCode, metrics.MetricTypeTotalCount}
@@ -63,7 +63,7 @@ func NewProvider(
middleware.NoCacheInterceptor().Handler,
instanceHandler,
userAgentCookie,
accessHandler,
accessHandler.HandleIgnorePathPrefixes(ignoredQuotaLimitEndpoint(conf.ProviderConfig)),
http_utils.CopyHeadersToContext,
),
provider.WithCustomTimeFormat("2006-01-02T15:04:05.999Z"),
@@ -100,3 +100,22 @@ func newStorage(
defaultLoginURL: fmt.Sprintf("%s%s?%s=", login.HandlerPrefix, login.EndpointLogin, login.QueryAuthRequestID),
}, nil
}
func ignoredQuotaLimitEndpoint(config *provider.Config) []string {
metadataEndpoint := HandlerPrefix + provider.DefaultMetadataEndpoint
certificateEndpoint := HandlerPrefix + provider.DefaultCertificateEndpoint
ssoEndpoint := HandlerPrefix + provider.DefaultSingleSignOnEndpoint
if config.MetadataConfig != nil && config.MetadataConfig.Path != "" {
metadataEndpoint = HandlerPrefix + config.MetadataConfig.Path
}
if config.IDPConfig == nil || config.IDPConfig.Endpoints == nil {
return []string{metadataEndpoint, certificateEndpoint, ssoEndpoint}
}
if config.IDPConfig.Endpoints.Certificate != nil && config.IDPConfig.Endpoints.Certificate.Relative() != "" {
certificateEndpoint = HandlerPrefix + config.IDPConfig.Endpoints.Certificate.Relative()
}
if config.IDPConfig.Endpoints.SingleSignOn != nil && config.IDPConfig.Endpoints.SingleSignOn.Relative() != "" {
ssoEndpoint = HandlerPrefix + config.IDPConfig.Endpoints.SingleSignOn.Relative()
}
return []string{metadataEndpoint, certificateEndpoint, ssoEndpoint}
}