TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-11 18:17:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: project quotas and usages (#6441)

Browse Source 
* project quota added

* project quota removed

* add periods table

* make log record generic

* accumulate usage

* query usage

* count action run seconds

* fix filter in ReportQuotaUsage

* fix existing tests

* fix logstore tests

* fix typo

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* move notifications into debouncer and improve limit querying

* cleanup

* comment

* fix: add quota unit tests command side

* fix remaining quota usage query

* implement InmemLogStorage

* cleanup and linting

* improve test

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* action notifications and fixes for notifications query

* revert console prefix

* fix: add quota unit tests command side

* fix: add quota integration tests

* improve accountable requests

* improve accountable requests

* fix: add quota integration tests

* fix: add quota integration tests

* fix: add quota integration tests

* comment

* remove ability to store logs in db and other changes requested from review

* changes requested from review

* changes requested from review

* Update internal/api/http/middleware/access_interceptor.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* tests: fix quotas integration tests

* improve incrementUsageStatement

* linting

* fix: delete e2e tests as intergation tests cover functionality

* Update internal/api/http/middleware/access_interceptor.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* backup

* fix conflict

* create rc

* create prerelease

* remove issue release labeling

* fix tracing

---------

Co-authored-by: Livio Spring <livio.a@gmail.com>
Co-authored-by: Stefan Benz <stefan@caos.ch>
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
(cherry picked from commit 1a49b7d298)
This commit is contained in:
Elio Bischof
2023-09-15 16:58:45 +02:00
committed by Livio Spring
parent b688d6f842
commit 5823fdbef9
66 changed files with 3423 additions and 1413 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
cmd/defaults.yaml
View File

@@ -452,54 +452,29 @@ Actions:
LogStore:
Access:
Database:
# If enabled, all access logs are stored in the database table logstore.access
Enabled: false # ZITADEL_LOGSTORE_ACCESS_DATABASE_ENABLED
# Logs that are older than the keep duration are cleaned up continuously
# 2160h are 90 days, 3 months
Keep: 2160h # ZITADEL_LOGSTORE_ACCESS_DATABASE_KEEP
# CleanupInterval defines the time between cleanup iterations
CleanupInterval: 4h # ZITADEL_LOGSTORE_ACCESS_DATABASE_CLEANUPINTERVAL
# Debouncing enables to asynchronously emit log entries, so the normal execution performance is not impaired
# Log entries are held in memory until one of the conditions MinFrequency or MaxBulkSize meets.
Debounce:
MinFrequency: 2m # ZITADEL_LOGSTORE_ACCESS_DATABASE_DEBOUNCE_MINFREQUENCY
MaxBulkSize: 100 # ZITADEL_LOGSTORE_ACCESS_DATABASE_DEBOUNCE_MAXBULKSIZE
Stdout:
# If enabled, all access logs are printed to the binary's standard output
Enabled: false # ZITADEL_LOGSTORE_ACCESS_STDOUT_ENABLED
# Debouncing enables to asynchronously emit log entries, so the normal execution performance is not impaired
# Log entries are held in memory until one of the conditions MinFrequency or MaxBulkSize meets.
Debounce:
MinFrequency: 0s # ZITADEL_LOGSTORE_ACCESS_STDOUT_DEBOUNCE_MINFREQUENCY
MaxBulkSize: 0 # ZITADEL_LOGSTORE_ACCESS_STDOUT_DEBOUNCE_MAXBULKSIZE
Execution:
Database:
# If enabled, all action execution logs are stored in the database table logstore.execution
Enabled: false # ZITADEL_LOGSTORE_EXECUTION_DATABASE_ENABLED
# Logs that are older than the keep duration are cleaned up continuously
# 2160h are 90 days, 3 months
Keep: 2160h # ZITADEL_LOGSTORE_EXECUTION_DATABASE_KEEP
# CleanupInterval defines the time between cleanup iterations
CleanupInterval: 4h # ZITADEL_LOGSTORE_EXECUTION_DATABASE_CLEANUPINTERVAL
# Debouncing enables to asynchronously emit log entries, so the normal execution performance is not impaired
# Log entries are held in memory until one of the conditions MinFrequency or MaxBulkSize meets.
Debounce:
MinFrequency: 0s # ZITADEL_LOGSTORE_EXECUTION_DATABASE_DEBOUNCE_MINFREQUENCY
MaxBulkSize: 0 # ZITADEL_LOGSTORE_EXECUTION_DATABASE_DEBOUNCE_MAXBULKSIZE
Stdout:
# If enabled, all execution logs are printed to the binary's standard output
Enabled: true # ZITADEL_LOGSTORE_EXECUTION_STDOUT_ENABLED
# Debouncing enables to asynchronously emit log entries, so the normal execution performance is not impaired
# Log entries are held in memory until one of the conditions MinFrequency or MaxBulkSize meets.
Debounce:
MinFrequency: 0s # ZITADEL_LOGSTORE_EXECUTION_STDOUT_DEBOUNCE_MINFREQUENCY
MaxBulkSize: 0 # ZITADEL_LOGSTORE_EXECUTION_STDOUT_DEBOUNCE_MAXBULKSIZE
Quotas:
Access:
# If enabled, authenticated requests are counted and potentially limited depending on the configured quota of the instance
Enabled: false # ZITADEL_QUOTAS_ACCESS_ENABLED
Debounce:
MinFrequency: 0s # ZITADEL_QUOTAS_ACCESS_DEBOUNCE_MINFREQUENCY
MaxBulkSize: 0 # ZITADEL_QUOTAS_ACCESS_DEBOUNCE_MAXBULKSIZE
ExhaustedCookieKey: "zitadel.quota.exhausted" # ZITADEL_QUOTAS_ACCESS_EXHAUSTEDCOOKIEKEY
ExhaustedCookieMaxAge: "300s" # ZITADEL_QUOTAS_ACCESS_EXHAUSTEDCOOKIEMAXAGE
Execution:
# If enabled, all action executions are counted and potentially limited depending on the configured quota of the instance
Enabled: false # ZITADEL_QUOTAS_EXECUTION_DATABASE_ENABLED
Debounce:
MinFrequency: 0s # ZITADEL_QUOTAS_EXECUTION_DEBOUNCE_MINFREQUENCY
MaxBulkSize: 0 # ZITADEL_QUOTAS_EXECUTION_DEBOUNCE_MAXBULKSIZE
Eventstore:
PushTimeout: 15s # ZITADEL_EVENTSTORE_PUSHTIMEOUT

6
cmd/start/config.go
View File

@@ -72,7 +72,11 @@ type Config struct {
}
type QuotasConfig struct {
Access *middleware.AccessConfig
Access struct {
logstore.EmitterConfig `mapstructure:",squash"`
middleware.AccessConfig `mapstructure:",squash"`
}
Execution *logstore.EmitterConfig
}
func MustNewConfig(v *viper.Viper) *Config {

31
cmd/start/start.go
View File

@@ -64,6 +64,8 @@ import (
"github.com/zitadel/zitadel/internal/logstore/emitters/access"
"github.com/zitadel/zitadel/internal/logstore/emitters/execution"
"github.com/zitadel/zitadel/internal/logstore/emitters/stdout"
"github.com/zitadel/zitadel/internal/logstore/record"
"github.com/zitadel/zitadel/internal/net"
"github.com/zitadel/zitadel/internal/notification"
"github.com/zitadel/zitadel/internal/query"
"github.com/zitadel/zitadel/internal/static"
@@ -108,7 +110,6 @@ type Server struct {
AuthzRepo authz_repo.Repository
Storage static.Storage
Commands *command.Commands
LogStore *logstore.Service
Router *mux.Router
TLSConfig *tls.Config
Shutdown chan<- os.Signal
@@ -209,17 +210,16 @@ func startZitadel(config *Config, masterKey string, server chan<- *Server) error
}
clock := clockpkg.New()
actionsExecutionStdoutEmitter, err := logstore.NewEmitter(ctx, clock, config.LogStore.Execution.Stdout, stdout.NewStdoutEmitter())
actionsExecutionStdoutEmitter, err := logstore.NewEmitter[*record.ExecutionLog](ctx, clock, &logstore.EmitterConfig{Enabled: config.LogStore.Execution.Stdout.Enabled}, stdout.NewStdoutEmitter[*record.ExecutionLog]())
if err != nil {
return err
}
actionsExecutionDBEmitter, err := logstore.NewEmitter(ctx, clock, config.LogStore.Execution.Database, execution.NewDatabaseLogStorage(dbClient))
actionsExecutionDBEmitter, err := logstore.NewEmitter[*record.ExecutionLog](ctx, clock, config.Quotas.Execution, execution.NewDatabaseLogStorage(dbClient, commands, queries))
if err != nil {
return err
}
usageReporter := logstore.UsageReporterFunc(commands.ReportQuotaUsage)
actionsLogstoreSvc := logstore.New(queries, usageReporter, actionsExecutionDBEmitter, actionsExecutionStdoutEmitter)
actionsLogstoreSvc := logstore.New(queries, actionsExecutionDBEmitter, actionsExecutionStdoutEmitter)
actions.SetLogstoreService(actionsLogstoreSvc)
notification.Start(
@@ -259,8 +259,6 @@ func startZitadel(config *Config, masterKey string, server chan<- *Server) error
storage,
authZRepo,
keys,
queries,
usageReporter,
permissionCheck,
)
if err != nil {
@@ -281,7 +279,6 @@ func startZitadel(config *Config, masterKey string, server chan<- *Server) error
AuthzRepo: authZRepo,
Storage: storage,
Commands: commands,
LogStore: actionsLogstoreSvc,
Router: router,
TLSConfig: tlsConfig,
Shutdown: shutdown,
@@ -304,8 +301,6 @@ func startAPIs(
store static.Storage,
authZRepo authz_repo.Repository,
keys *encryptionKeys,
quotaQuerier logstore.QuotaQuerier,
usageReporter logstore.UsageReporter,
permissionCheck domain.PermissionCheck,
) error {
repo := struct {
@@ -321,22 +316,22 @@ func startAPIs(
return err
}
accessStdoutEmitter, err := logstore.NewEmitter(ctx, clock, config.LogStore.Access.Stdout, stdout.NewStdoutEmitter())
accessStdoutEmitter, err := logstore.NewEmitter[*record.AccessLog](ctx, clock, &logstore.EmitterConfig{Enabled: config.LogStore.Access.Stdout.Enabled}, stdout.NewStdoutEmitter[*record.AccessLog]())
if err != nil {
return err
}
accessDBEmitter, err := logstore.NewEmitter(ctx, clock, config.LogStore.Access.Database, access.NewDatabaseLogStorage(dbClient))
accessDBEmitter, err := logstore.NewEmitter[*record.AccessLog](ctx, clock, &config.Quotas.Access.EmitterConfig, access.NewDatabaseLogStorage(dbClient, commands, queries))
if err != nil {
return err
}
accessSvc := logstore.New(quotaQuerier, usageReporter, accessDBEmitter, accessStdoutEmitter)
accessSvc := logstore.New[*record.AccessLog](queries, accessDBEmitter, accessStdoutEmitter)
exhaustedCookieHandler := http_util.NewCookieHandler(
http_util.WithUnsecure(),
http_util.WithNonHttpOnly(),
http_util.WithMaxAge(int(math.Floor(config.Quotas.Access.ExhaustedCookieMaxAge.Seconds()))),
)
limitingAccessInterceptor := middleware.NewAccessInterceptor(accessSvc, exhaustedCookieHandler, config.Quotas.Access)
limitingAccessInterceptor := middleware.NewAccessInterceptor(accessSvc, exhaustedCookieHandler, &config.Quotas.Access.AccessConfig)
apis, err := api.New(ctx, config.Port, router, queries, verifier, config.InternalAuthZ, tlsConfig, config.HTTP2HostHeader, config.HTTP1HostHeader, limitingAccessInterceptor)
if err != nil {
return fmt.Errorf("error creating api %w", err)
@@ -399,13 +394,13 @@ func startAPIs(
}
apis.RegisterHandlerOnPrefix(openapi.HandlerPrefix, openAPIHandler)
oidcProvider, err := oidc.NewProvider(config.OIDC, login.DefaultLoggedOutPath, config.ExternalSecure, commands, queries, authRepo, keys.OIDC, keys.OIDCKey, eventstore, dbClient, userAgentInterceptor, instanceInterceptor.Handler, limitingAccessInterceptor.Handle)
oidcProvider, err := oidc.NewProvider(config.OIDC, login.DefaultLoggedOutPath, config.ExternalSecure, commands, queries, authRepo, keys.OIDC, keys.OIDCKey, eventstore, dbClient, userAgentInterceptor, instanceInterceptor.Handler, limitingAccessInterceptor)
if err != nil {
return fmt.Errorf("unable to start oidc provider: %w", err)
}
apis.RegisterHandlerPrefixes(oidcProvider.HttpHandler(), "/.well-known/openid-configuration", "/oidc/v1", "/oauth/v2")
samlProvider, err := saml.NewProvider(config.SAML, config.ExternalSecure, commands, queries, authRepo, keys.OIDC, keys.SAML, eventstore, dbClient, instanceInterceptor.Handler, userAgentInterceptor, limitingAccessInterceptor.Handle)
samlProvider, err := saml.NewProvider(config.SAML, config.ExternalSecure, commands, queries, authRepo, keys.OIDC, keys.SAML, eventstore, dbClient, instanceInterceptor.Handler, userAgentInterceptor, limitingAccessInterceptor)
if err != nil {
return fmt.Errorf("unable to start saml provider: %w", err)
}
@@ -417,7 +412,7 @@ func startAPIs(
}
apis.RegisterHandlerOnPrefix(console.HandlerPrefix, c)
l, err := login.CreateLogin(config.Login, commands, queries, authRepo, store, console.HandlerPrefix+"/", op.AuthCallbackURL(oidcProvider), provider.AuthCallbackURL(samlProvider), config.ExternalSecure, userAgentInterceptor, op.NewIssuerInterceptor(oidcProvider.IssuerFromRequest).Handler, provider.NewIssuerInterceptor(samlProvider.IssuerFromRequest).Handler, instanceInterceptor.Handler, assetsCache.Handler, limitingAccessInterceptor.Handle, keys.User, keys.IDPConfig, keys.CSRFCookieKey)
l, err := login.CreateLogin(config.Login, commands, queries, authRepo, store, console.HandlerPrefix+"/", op.AuthCallbackURL(oidcProvider), provider.AuthCallbackURL(samlProvider), config.ExternalSecure, userAgentInterceptor, op.NewIssuerInterceptor(oidcProvider.IssuerFromRequest).Handler, provider.NewIssuerInterceptor(samlProvider.IssuerFromRequest).Handler, instanceInterceptor.Handler, assetsCache.Handler, limitingAccessInterceptor.WithoutLimiting().Handle, keys.User, keys.IDPConfig, keys.CSRFCookieKey)
if err != nil {
return fmt.Errorf("unable to start login: %w", err)
}
@@ -438,7 +433,7 @@ func listen(ctx context.Context, router *mux.Router, port uint16, tlsConfig *tls
http2Server := &http2.Server{}
http1Server := &http.Server{Handler: h2c.NewHandler(router, http2Server), TLSConfig: tlsConfig}
lc := listenConfig()
lc := net.ListenConfig()
lis, err := lc.Listen(ctx, "tcp", fmt.Sprintf(":%d", port))
if err != nil {
return fmt.Errorf("tcp listener on %d failed: %w", port, err)

11
cmd/start/start_port.go
View File

@@ -1,11 +0,0 @@
//go:build !integration
package start
import (
"net"
)
func listenConfig() *net.ListenConfig {
return &net.ListenConfig{}
}

25
cmd/start/start_port_integration.go
View File

@@ -1,25 +0,0 @@
//go:build integration
package start
import (
"net"
"syscall"
"golang.org/x/sys/unix"
)
func listenConfig() *net.ListenConfig {
return &net.ListenConfig{
Control: reusePort,
}
}
func reusePort(network, address string, conn syscall.RawConn) error {
return conn.Control(func(descriptor uintptr) {
err := syscall.SetsockoptInt(int(descriptor), syscall.SOL_SOCKET, unix.SO_REUSEPORT, 1)
if err != nil {
panic(err)
}
})
}