feat: add quotas (#4779)

adds possibilities to cap authenticated requests and execution seconds of actions on a defined intervall

cmd/defaults.yaml

@@ -320,6 +320,55 @@ Actions:
       - localhost
       - "127.0.0.1"
 
+LogStore:
+  Access:
+    Database:
+      # If enabled, all access logs are stored in the database table logstore.access
+      Enabled: false
+      # Logs that are older than the keep duration are cleaned up continuously
+      Keep: 2160h # 90 days
+      # CleanupInterval defines the time between cleanup iterations
+      CleanupInterval: 4h
+      # Debouncing enables to asynchronously emit log entries, so the normal execution performance is not impaired
+      # Log entries are held in-memory until one of the conditions MinFrequency or MaxBulkSize meets.
+      Debounce:
+        MinFrequency: 2m
+        MaxBulkSize: 100
+    Stdout:
+      # If enabled, all access logs are printed to the binaries standard output
+      Enabled: false
+      # Debouncing enables to asynchronously emit log entries, so the normal execution performance is not impaired
+      # Log entries are held in-memory until one of the conditions MinFrequency or MaxBulkSize meets.
+      Debounce:
+        MinFrequency: 0s
+        MaxBulkSize: 0
+  Execution:
+    Database:
+      # If enabled, all action execution logs are stored in the database table logstore.execution
+      Enabled: false
+      # Logs that are older than the keep duration are cleaned up continuously
+      Keep: 2160h # 90 days
+      # CleanupInterval defines the time between cleanup iterations
+      CleanupInterval: 4h
+      # Debouncing enables to asynchronously emit log entries, so the normal execution performance is not impaired
+      # Log entries are held in-memory until one of the conditions MinFrequency or MaxBulkSize meets.
+      Debounce:
+        MinFrequency: 0s
+        MaxBulkSize: 0
+    Stdout:
+      # If enabled, all execution logs are printed to the binaries standard output
+      Enabled: true
+      # Debouncing enables to asynchronously emit log entries, so the normal execution performance is not impaired
+      # Log entries are held in-memory until one of the conditions MinFrequency or MaxBulkSize meets.
+      Debounce:
+        MinFrequency: 0s
+        MaxBulkSize: 0
+
+Quotas:
+  Access:
+    ExhaustedCookieKey: "zitadel.quota.exhausted"
+    ExhaustedCookieMaxAge: "300s"
+
 Eventstore:
   PushTimeout: 15s
 
@@ -573,6 +622,40 @@ DefaultInstance:
       Text: The password of your user has changed. If this change was not done by you, please be advised to immediately reset your password.
       ButtonText: Login
 
+  Quotas:
+    # Items takes a slice of quota configurations, whereas for each unit type and instance, one or zero quotas may exist.
+    # The following unit types are supported
+
+    # "requests.all.authenticated"
+    # The sum of all requests to the ZITADEL API with an authorization header,
+    # excluding the following exceptions
+    # - Calls to the System API
+    # - Calls that cause internal server errors
+    # - Failed authorizations
+    # - Requests after the quota already exceeded
+
+    # "actions.all.runs.seconds"
+    # The sum of all actions run durations in seconds
+    Items:
+#      - Unit: "requests.all.authenticated"
+#        # From defines the starting time from which the current quota period is calculated from.
+#        # This is relevant for querying the current usage.
+#        From: "2023-01-01T00:00:00Z"
+#        # ResetInterval defines the quota periods duration
+#        ResetInterval: 720h # 30 days
+#        # Amount defines the number of units for this quota
+#        Amount: 25000
+#        # Limit defines whether ZITADEL should block further usage when the configured amount is used
+#        Limit: false
+#        # Notifications are emitted by ZITADEL when certain quota percentages are reached
+#        Notifications:
+#            # Percent defines the relative amount of used units, after which a notification should be emitted.
+#          - Percent: 100
+#            # Repeat defines, whether a notification should be emitted each time when a multitude of the configured Percent is used.
+#            Repeat: true
+#            # CallURL is called when a relative amount of the quota is used.
+#            CallURL: "https://httpbin.org/post"
+
 InternalAuthZ:
   RolePermissionMappings:
     - Role: "IAM_OWNER"

cmd/initialise/init.go

@@ -37,7 +37,7 @@ func New() *cobra.Command {
 		Short: "initialize ZITADEL instance",
 		Long: `Sets up the minimum requirements to start ZITADEL.
 
-Prereqesits:
+Prerequisites:
 - cockroachdb
 
 The user provided by flags needs privileges to

cmd/setup/07.go

@@ -0,0 +1,32 @@
+package setup
+
+import (
+	"context"
+	"database/sql"
+	_ "embed"
+	"strings"
+)
+
+var (
+	//go:embed 07/logstore.sql
+	createLogstoreSchema07 string
+	//go:embed 07/access.sql
+	createAccessLogsTable07 string
+	//go:embed 07/execution.sql
+	createExecutionLogsTable07 string
+)
+
+type LogstoreTables struct {
+	dbClient *sql.DB
+	username string
+}
+
+func (mig *LogstoreTables) Execute(ctx context.Context) error {
+	stmt := strings.ReplaceAll(createLogstoreSchema07, "%[1]s", mig.username) + createAccessLogsTable07 + createExecutionLogsTable07
+	_, err := mig.dbClient.ExecContext(ctx, stmt)
+	return err
+}
+
+func (mig *LogstoreTables) String() string {
+	return "07_logstore"
+}

cmd/setup/07/access.sql

@@ -0,0 +1,14 @@
+CREATE TABLE IF NOT EXISTS logstore.access (
+    log_date TIMESTAMPTZ NOT NULL
+    , protocol INT NOT NULL
+    , request_url TEXT NOT NULL
+    , response_status INT NOT NULL
+    , request_headers JSONB
+    , response_headers JSONB
+    , instance_id TEXT NOT NULL
+    , project_id TEXT NOT NULL
+    , requested_domain TEXT
+    , requested_host TEXT
+
+    , INDEX protocol_date_desc (instance_id, protocol, log_date DESC) STORING (request_url, response_status, request_headers)
+);

cmd/setup/07/execution.sql

@@ -0,0 +1,11 @@
+CREATE TABLE IF NOT EXISTS logstore.execution (
+    log_date TIMESTAMPTZ NOT NULL
+    , took INTERVAL
+    , message TEXT NOT NULL
+    , loglevel INT NOT NULL
+    , instance_id TEXT NOT NULL
+    , action_id TEXT NOT NULL
+    , metadata JSONB
+
+    , INDEX log_date_desc (instance_id, log_date DESC) STORING (took)
+);

cmd/setup/07/logstore.sql

@@ -0,0 +1,3 @@
+CREATE SCHEMA IF NOT EXISTS logstore;
+
+GRANT ALL ON ALL TABLES IN SCHEMA logstore TO %[1]s;

cmd/setup/config.go

@@ -62,6 +62,7 @@ type Steps struct {
 	s4EventstoreIndexes  *EventstoreIndexes
 	s5LastFailed         *LastFailed
 	s6OwnerRemoveColumns *OwnerRemoveColumns
+	s7LogstoreTables     *LogstoreTables
 }
 
 type encryptionKeyConfig struct {

cmd/setup/setup.go

@@ -84,6 +84,7 @@ func Setup(config *Config, steps *Steps, masterKey string) {
 	steps.s4EventstoreIndexes = &EventstoreIndexes{dbClient: dbClient, dbType: config.Database.Type()}
 	steps.s5LastFailed = &LastFailed{dbClient: dbClient}
 	steps.s6OwnerRemoveColumns = &OwnerRemoveColumns{dbClient: dbClient}
+	steps.s7LogstoreTables = &LogstoreTables{dbClient: dbClient, username: config.Database.Username()}
 
 	err = projection.Create(ctx, dbClient, eventstoreClient, config.Projections, nil, nil)
 	logging.OnError(err).Fatal("unable to start projections")
@@ -113,6 +114,8 @@ func Setup(config *Config, steps *Steps, masterKey string) {
 	logging.OnError(err).Fatal("unable to migrate step 5")
 	err = migration.Migrate(ctx, eventstoreClient, steps.s6OwnerRemoveColumns)
 	logging.OnError(err).Fatal("unable to migrate step 6")
+	err = migration.Migrate(ctx, eventstoreClient, steps.s7LogstoreTables)
+	logging.OnError(err).Fatal("unable to migrate step 7")
 
 	for _, repeatableStep := range repeatableSteps {
 		err = migration.Migrate(ctx, eventstoreClient, repeatableStep)

cmd/start/config.go

@@ -24,6 +24,7 @@ import (
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/id"
+	"github.com/zitadel/zitadel/internal/logstore"
 	"github.com/zitadel/zitadel/internal/query/projection"
 	static_config "github.com/zitadel/zitadel/internal/static/config"
 	metrics "github.com/zitadel/zitadel/internal/telemetry/metrics/config"
@@ -62,6 +63,12 @@ type Config struct {
 	Machine           *id.Config
 	Actions           *actions.Config
 	Eventstore        *eventstore.Config
+	LogStore          *logstore.Configs
+	Quotas            *QuotasConfig
+}
+
+type QuotasConfig struct {
+	Access *middleware.AccessConfig
 }
 
 func MustNewConfig(v *viper.Viper) *Config {

cmd/start/start.go

@@ -15,8 +15,7 @@ import (
 
 	"github.com/zitadel/saml/pkg/provider"
 
-	"github.com/zitadel/zitadel/internal/api/saml"
-
+	clockpkg "github.com/benbjohnson/clock"
 	"github.com/gorilla/mux"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -27,6 +26,7 @@ import (
 
 	"github.com/zitadel/zitadel/cmd/key"
 	cmd_tls "github.com/zitadel/zitadel/cmd/tls"
+	"github.com/zitadel/zitadel/internal/actions"
 	admin_es "github.com/zitadel/zitadel/internal/admin/repository/eventsourcing"
 	"github.com/zitadel/zitadel/internal/api"
 	"github.com/zitadel/zitadel/internal/api/assets"
@@ -38,6 +38,7 @@ import (
 	http_util "github.com/zitadel/zitadel/internal/api/http"
 	"github.com/zitadel/zitadel/internal/api/http/middleware"
 	"github.com/zitadel/zitadel/internal/api/oidc"
+	"github.com/zitadel/zitadel/internal/api/saml"
 	"github.com/zitadel/zitadel/internal/api/ui/console"
 	"github.com/zitadel/zitadel/internal/api/ui/login"
 	auth_es "github.com/zitadel/zitadel/internal/auth/repository/eventsourcing"
@@ -48,6 +49,10 @@ import (
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/id"
+	"github.com/zitadel/zitadel/internal/logstore"
+	"github.com/zitadel/zitadel/internal/logstore/emitters/access"
+	"github.com/zitadel/zitadel/internal/logstore/emitters/execution"
+	"github.com/zitadel/zitadel/internal/logstore/emitters/stdout"
 	"github.com/zitadel/zitadel/internal/notification"
 	"github.com/zitadel/zitadel/internal/query"
 	"github.com/zitadel/zitadel/internal/static"
@@ -146,6 +151,23 @@ func startZitadel(config *Config, masterKey string) error {
 		return fmt.Errorf("cannot start commands: %w", err)
 	}
 
+	clock := clockpkg.New()
+	actionsExecutionStdoutEmitter, err := logstore.NewEmitter(ctx, clock, config.LogStore.Execution.Stdout, stdout.NewStdoutEmitter())
+	if err != nil {
+		return err
+	}
+	actionsExecutionDBEmitter, err := logstore.NewEmitter(ctx, clock, config.LogStore.Execution.Database, execution.NewDatabaseLogStorage(dbClient))
+	if err != nil {
+		return err
+	}
+
+	usageReporter := logstore.UsageReporterFunc(commands.ReportUsage)
+	actionsLogstoreSvc := logstore.New(commands, usageReporter, actionsExecutionDBEmitter, actionsExecutionStdoutEmitter)
+	if actionsLogstoreSvc.Enabled() {
+		logging.Warn("execution logs are currently in beta")
+	}
+	actions.SetLogstoreService(actionsLogstoreSvc)
+
 	notification.Start(ctx, config.Projections.Customizations["notifications"], config.ExternalPort, config.ExternalSecure, commands, queries, eventstoreClient, assets.AssetAPIFromDomain(config.ExternalSecure, config.ExternalPort), config.SystemDefaults.Notifications.FileSystemPath, keys.User, keys.SMTP, keys.SMS)
 
 	router := mux.NewRouter()
@@ -153,14 +175,28 @@ func startZitadel(config *Config, masterKey string) error {
 	if err != nil {
 		return err
 	}
-	err = startAPIs(ctx, router, commands, queries, eventstoreClient, dbClient, config, storage, authZRepo, keys)
+	err = startAPIs(ctx, clock, router, commands, queries, eventstoreClient, dbClient, config, storage, authZRepo, keys, commands, usageReporter)
 	if err != nil {
 		return err
 	}
 	return listen(ctx, router, config.Port, tlsConfig)
 }
 
-func startAPIs(ctx context.Context, router *mux.Router, commands *command.Commands, queries *query.Queries, eventstore *eventstore.Eventstore, dbClient *sql.DB, config *Config, store static.Storage, authZRepo authz_repo.Repository, keys *encryptionKeys) error {
+func startAPIs(
+	ctx context.Context,
+	clock clockpkg.Clock,
+	router *mux.Router,
+	commands *command.Commands,
+	queries *query.Queries,
+	eventstore *eventstore.Eventstore,
+	dbClient *sql.DB,
+	config *Config,
+	store static.Storage,
+	authZRepo authz_repo.Repository,
+	keys *encryptionKeys,
+	quotaQuerier logstore.QuotaQuerier,
+	usageReporter logstore.UsageReporter,
+) error {
 	repo := struct {
 		authz_repo.Repository
 		*query.Queries
@@ -173,7 +209,22 @@ func startAPIs(ctx context.Context, router *mux.Router, commands *command.Comman
 	if err != nil {
 		return err
 	}
-	apis := api.New(config.Port, router, queries, verifier, config.InternalAuthZ, config.ExternalSecure, tlsConfig, config.HTTP2HostHeader, config.HTTP1HostHeader)
+
+	accessStdoutEmitter, err := logstore.NewEmitter(ctx, clock, config.LogStore.Access.Stdout, stdout.NewStdoutEmitter())
+	if err != nil {
+		return err
+	}
+	accessDBEmitter, err := logstore.NewEmitter(ctx, clock, config.LogStore.Access.Database, access.NewDatabaseLogStorage(dbClient))
+	if err != nil {
+		return err
+	}
+
+	accessSvc := logstore.New(quotaQuerier, usageReporter, accessDBEmitter, accessStdoutEmitter)
+	if accessSvc.Enabled() {
+		logging.Warn("access logs are currently in beta")
+	}
+	accessInterceptor := middleware.NewAccessInterceptor(accessSvc, config.Quotas.Access)
+	apis := api.New(config.Port, router, queries, verifier, config.InternalAuthZ, config.ExternalSecure, tlsConfig, config.HTTP2HostHeader, config.HTTP1HostHeader, accessSvc)
 	authRepo, err := auth_es.Start(ctx, config.Auth, config.SystemDefaults, commands, queries, dbClient, eventstore, keys.OIDC, keys.User)
 	if err != nil {
 		return fmt.Errorf("error starting auth repo: %w", err)
@@ -194,40 +245,40 @@ func startAPIs(ctx context.Context, router *mux.Router, commands *command.Comman
 	if err := apis.RegisterServer(ctx, auth.CreateServer(commands, queries, authRepo, config.SystemDefaults, keys.User, config.ExternalSecure, config.AuditLogRetention)); err != nil {
 		return err
 	}
-
 	instanceInterceptor := middleware.InstanceInterceptor(queries, config.HTTP1HostHeader, login.IgnoreInstanceEndpoints...)
 	assetsCache := middleware.AssetsCacheInterceptor(config.AssetStorage.Cache.MaxAge, config.AssetStorage.Cache.SharedMaxAge)
-	apis.RegisterHandler(assets.HandlerPrefix, assets.NewHandler(commands, verifier, config.InternalAuthZ, id.SonyFlakeGenerator(), store, queries, instanceInterceptor.Handler, assetsCache.Handler))
+	apis.RegisterHandler(assets.HandlerPrefix, assets.NewHandler(commands, verifier, config.InternalAuthZ, id.SonyFlakeGenerator(), store, queries, instanceInterceptor.Handler, assetsCache.Handler, accessInterceptor.Handle))
 
 	userAgentInterceptor, err := middleware.NewUserAgentHandler(config.UserAgentCookie, keys.UserAgentCookieKey, id.SonyFlakeGenerator(), config.ExternalSecure, login.EndpointResources)
 	if err != nil {
 		return err
 	}
 
+	// TODO: Record openapi access logs?
 	openAPIHandler, err := openapi.Start()
 	if err != nil {
 		return fmt.Errorf("unable to start openapi handler: %w", err)
 	}
 	apis.RegisterHandler(openapi.HandlerPrefix, openAPIHandler)
 
-	oidcProvider, err := oidc.NewProvider(ctx, config.OIDC, login.DefaultLoggedOutPath, config.ExternalSecure, commands, queries, authRepo, keys.OIDC, keys.OIDCKey, eventstore, dbClient, userAgentInterceptor, instanceInterceptor.Handler)
+	oidcProvider, err := oidc.NewProvider(ctx, config.OIDC, login.DefaultLoggedOutPath, config.ExternalSecure, commands, queries, authRepo, keys.OIDC, keys.OIDCKey, eventstore, dbClient, userAgentInterceptor, instanceInterceptor.Handler, accessInterceptor.Handle)
 	if err != nil {
 		return fmt.Errorf("unable to start oidc provider: %w", err)
 	}
 
-	samlProvider, err := saml.NewProvider(ctx, config.SAML, config.ExternalSecure, commands, queries, authRepo, keys.OIDC, keys.SAML, eventstore, dbClient, instanceInterceptor.Handler, userAgentInterceptor)
+	samlProvider, err := saml.NewProvider(ctx, config.SAML, config.ExternalSecure, commands, queries, authRepo, keys.OIDC, keys.SAML, eventstore, dbClient, instanceInterceptor.Handler, userAgentInterceptor, accessInterceptor.Handle)
 	if err != nil {
 		return fmt.Errorf("unable to start saml provider: %w", err)
 	}
 	apis.RegisterHandler(saml.HandlerPrefix, samlProvider.HttpHandler())
 
-	c, err := console.Start(config.Console, config.ExternalSecure, oidcProvider.IssuerFromRequest, instanceInterceptor.Handler, config.CustomerPortal)
+	c, err := console.Start(config.Console, config.ExternalSecure, oidcProvider.IssuerFromRequest, instanceInterceptor.Handler, accessInterceptor.Handle, config.CustomerPortal)
 	if err != nil {
 		return fmt.Errorf("unable to start console: %w", err)
 	}
 	apis.RegisterHandler(console.HandlerPrefix, c)
 
-	l, err := login.CreateLogin(config.Login, commands, queries, authRepo, store, console.HandlerPrefix+"/", op.AuthCallbackURL(oidcProvider), provider.AuthCallbackURL(samlProvider), config.ExternalSecure, userAgentInterceptor, op.NewIssuerInterceptor(oidcProvider.IssuerFromRequest).Handler, provider.NewIssuerInterceptor(samlProvider.IssuerFromRequest).Handler, instanceInterceptor.Handler, assetsCache.Handler, keys.User, keys.IDPConfig, keys.CSRFCookieKey)
+	l, err := login.CreateLogin(config.Login, commands, queries, authRepo, store, console.HandlerPrefix+"/", op.AuthCallbackURL(oidcProvider), provider.AuthCallbackURL(samlProvider), config.ExternalSecure, userAgentInterceptor, op.NewIssuerInterceptor(oidcProvider.IssuerFromRequest).Handler, provider.NewIssuerInterceptor(samlProvider.IssuerFromRequest).Handler, instanceInterceptor.Handler, assetsCache.Handler, accessInterceptor.Handle, keys.User, keys.IDPConfig, keys.CSRFCookieKey)
 	if err != nil {
 		return fmt.Errorf("unable to start login: %w", err)
 	}