feat(cache): redis cache (#8822)

# Which Problems Are Solved Add a cache implementation using Redis single mode. This does not add support for Redis Cluster or sentinel. # How the Problems Are Solved Added the `internal/cache/redis` package. All operations occur atomically, including setting of secondary indexes, using LUA scripts where needed. The [`miniredis`](https://github.com/alicebob/miniredis) package is used to run unit tests. # Additional Changes - Move connector code to `internal/cache/connector/...` and remove duplicate code from `query` and `command` packages. - Fix a missed invalidation on the restrictions projection # Additional Context Closes #8130
2025-08-11 18:17:35 +00:00 · 2024-11-04 11:44:51 +01:00
parent 9c3e5e467b
commit 250f2344c8
50 changed files with 1767 additions and 293 deletions
--- a/cmd/defaults.yaml
+++ b/cmd/defaults.yaml
@@ -185,34 +185,136 @@ Database:

 # Caches are EXPERIMENTAL. The following config may have breaking changes in the future.
 # If no config is provided, caching is disabled by default.
-# Caches:
+Caches:
  # Connectors are reused by caches.
-#  Connectors:
+  Connectors:
    # Memory connector works with local server memory.
    # It is the simplest (and probably fastest) cache implementation.
    # Unsuitable for deployments with multiple containers,
    # as each container's cache may hold a different state of the same object.
-#    Memory:
-#      Enabled: true
+    Memory:
+      Enabled: false
      # AutoPrune removes invalidated or expired object from the cache.
-#      AutoPrune:
-#        Interval: 15m
-#        TimeOut: 30s
+      AutoPrune:
+        Interval: 1m
+        TimeOut: 5s
+    Postgres:
+      Enabled: false
+      AutoPrune:
+        Interval: 15m
+        TimeOut: 30s
+    Redis:
+      Enabled: false
+      # The network type, either tcp or unix.
+      # Default is tcp.
+      # Network string
+      # host:port address.
+      Addr: localhost:6379
+      # ClientName will execute the `CLIENT SETNAME ClientName` command for each conn.
+      ClientName: ZITADEL_cache
+      # Use the specified Username to authenticate the current connection
+      # with one of the connections defined in the ACL list when connecting
+      # to a Redis 6.0 instance, or greater, that is using the Redis ACL system.
+      Username: zitadel
+      # Optional password. Must match the password specified in the
+      # requirepass server configuration option (if connecting to a Redis 5.0 instance, or lower),
+      # or the User Password when connecting to a Redis 6.0 instance, or greater,
+      # that is using the Redis ACL system.
+      Password: ""
+      # Each ZITADEL cache uses an incremental DB namespace.
+      # This option offsets the first DB so it doesn't conflict with other databases on the same server.
+      # Note that ZITADEL uses FLUSHDB command to truncate a cache.
+      # This can have destructive consequences when overlapping DB namespaces are used.
+      DBOffset: 10
+      # Maximum number of retries before giving up.
+      # Default is 3 retries; -1 (not 0) disables retries.
+      MaxRetries: 3
+      # Minimum backoff between each retry.
+      # Default is 8 milliseconds; -1 disables backoff.
+      MinRetryBackoff: 8ms
+      # Maximum backoff between each retry.
+      # Default is 512 milliseconds; -1 disables backoff.
+      MaxRetryBackoff: 512ms
+      # Dial timeout for establishing new connections.
+      # Default is 5 seconds.
+      DialTimeout: 1s
+      # Timeout for socket reads. If reached, commands will fail
+      # with a timeout instead of blocking. Supported values:
+      #   - `0` - default timeout (3 seconds).
+      #   - `-1` - no timeout (block indefinitely).
+      #   - `-2` - disables SetReadDeadline calls completely.
+      ReadTimeout: 100ms
+      # Timeout for socket writes. If reached, commands will fail
+      # with a timeout instead of blocking.  Supported values:
+      #   - `0` - default timeout (3 seconds).
+      #   - `-1` - no timeout (block indefinitely).
+      #   - `-2` - disables SetWriteDeadline calls completely.
+      WriteTimeout: 100ms
+      # Type of connection pool.
+      # true for FIFO pool, false for LIFO pool.
+      # Note that FIFO has slightly higher overhead compared to LIFO,
+      # but it helps closing idle connections faster reducing the pool size.
+      PoolFIFO: false
+      # Base number of socket connections.
+      # Default is 10 connections per every available CPU as reported by runtime.GOMAXPROCS.
+      # If there is not enough connections in the pool, new connections will be allocated in excess of PoolSize,
+      # you can limit it through MaxActiveConns
+      PoolSize: 20
+      # Amount of time client waits for connection if all connections
+      # are busy before returning an error.
+      # Default is ReadTimeout + 1 second.
+      PoolTimeout: 100ms
+      # Minimum number of idle connections which is useful when establishing
+      # new connection is slow.
+      # Default is 0. the idle connections are not closed by default.
+      MinIdleConns: 5
+      # Maximum number of idle connections.
+      # Default is 0. the idle connections are not closed by default.
+      MaxIdleConns: 10
+      # Maximum number of connections allocated by the pool at a given time.
+      # When zero, there is no limit on the number of connections in the pool.
+      MaxActiveConns: 40
+      # ConnMaxIdleTime is the maximum amount of time a connection may be idle.
+      # Should be less than server's timeout.
+      # Expired connections may be closed lazily before reuse.
+      # If d <= 0, connections are not closed due to a connection's idle time.
+      # Default is 30 minutes. -1 disables idle timeout check.
+      ConnMaxIdleTime: 30m
+      # ConnMaxLifetime is the maximum amount of time a connection may be reused.
+      # Expired connections may be closed lazily before reuse.
+      # If <= 0, connections are not closed due to a connection's age.
+      # Default is to not close idle connections.
+      ConnMaxLifetime: -1
+      # Enable TLS server authentication using the default system bundle.
+      EnableTLS: false
+      # Disable set-lib on connect. Default is false.
+      DisableIndentity: false
+      # Add suffix to client name. Default is empty.
+      IdentitySuffix: ""

  # Instance caches auth middleware instances, gettable by domain or ID.
-#  Instance:
+  Instance:
    # Connector must be enabled above.
    # When connector is empty, this cache will be disabled.
-#    Connector: "memory"
-#    MaxAge: 1h
-#    LastUsage: 10m
-#
-    # Log enables cache-specific logging. Default to error log to stdout when omitted.
-#    Log:
-#      Level: debug
-#      AddSource: true
-#      Formatter:
-#        Format: text
+    Connector: ""
+    MaxAge: 1h
+    LastUsage: 10m
+    # Log enables cache-specific logging. Default to error log to stderr when omitted.
+    Log:
+      Level: error
+      AddSource: true
+      Formatter:
+        Format: text
+  # Milestones caches instance milestone state, gettable by instance ID
+  Milestones:
+    Connector: ""
+    MaxAge: 1h
+    LastUsage: 10m
+    Log:
+      Level: error
+      AddSource: true
+      Formatter:
+        Format: text

 Machine:
  # Cloud-hosted VMs need to specify their metadata endpoint so that the machine can be uniquely identified.
--- a/cmd/mirror/projections.go
+++ b/cmd/mirror/projections.go
@@ -25,7 +25,7 @@ import (
 	auth_view "github.com/zitadel/zitadel/internal/auth/repository/eventsourcing/view"
 	"github.com/zitadel/zitadel/internal/authz"
 	authz_es "github.com/zitadel/zitadel/internal/authz/repository/eventsourcing/eventstore"
-	"github.com/zitadel/zitadel/internal/cache"
+	"github.com/zitadel/zitadel/internal/cache/connector"
 	"github.com/zitadel/zitadel/internal/command"
 	"github.com/zitadel/zitadel/internal/config/systemdefaults"
 	crypto_db "github.com/zitadel/zitadel/internal/crypto/database"
@@ -72,7 +72,7 @@ type ProjectionsConfig struct {
 	EncryptionKeys *encryption.EncryptionKeyConfig
 	SystemAPIUsers map[string]*internal_authz.SystemAPIUser
 	Eventstore     *eventstore.Config
-	Caches         *cache.CachesConfig
+	Caches         *connector.CachesConfig

 	Admin admin_es.Config
 	Auth  auth_es.Config
@@ -128,13 +128,16 @@ func projections(

 	sessionTokenVerifier := internal_authz.SessionTokenVerifier(keys.OIDC)

+	cacheConnectors, err := connector.StartConnectors(config.Caches, client)
+	logging.OnError(err).Fatal("unable to start caches")
+
 	queries, err := query.StartQueries(
 		ctx,
 		es,
 		esV4.Querier,
 		client,
 		client,
-		config.Caches,
+		cacheConnectors,
 		config.Projections,
 		config.SystemDefaults,
 		keys.IDPConfig,
@@ -161,9 +164,9 @@ func projections(
 		DisplayName:    config.WebAuthNName,
 		ExternalSecure: config.ExternalSecure,
 	}
-	commands, err := command.StartCommands(
+	commands, err := command.StartCommands(ctx,
 		es,
-		config.Caches,
+		cacheConnectors,
 		config.SystemDefaults,
 		config.InternalAuthZ.RolePermissionMappings,
 		staticStorage,
--- a/cmd/setup/03.go
+++ b/cmd/setup/03.go
@@ -9,6 +9,7 @@ import (
 	"golang.org/x/text/language"

 	"github.com/zitadel/zitadel/internal/api/authz"
+	"github.com/zitadel/zitadel/internal/cache/connector"
 	"github.com/zitadel/zitadel/internal/command"
 	"github.com/zitadel/zitadel/internal/config/systemdefaults"
 	"github.com/zitadel/zitadel/internal/crypto"
@@ -64,8 +65,9 @@ func (mig *FirstInstance) Execute(ctx context.Context, _ eventstore.Event) error
 		return err
 	}

-	cmd, err := command.StartCommands(mig.es,
-		nil,
+	cmd, err := command.StartCommands(ctx,
+		mig.es,
+		connector.Connectors{},
 		mig.defaults,
 		mig.zitadelRoles,
 		nil,
--- a/cmd/setup/config.go
+++ b/cmd/setup/config.go
@@ -15,7 +15,7 @@ import (
 	internal_authz "github.com/zitadel/zitadel/internal/api/authz"
 	"github.com/zitadel/zitadel/internal/api/oidc"
 	"github.com/zitadel/zitadel/internal/api/ui/login"
-	"github.com/zitadel/zitadel/internal/cache"
+	"github.com/zitadel/zitadel/internal/cache/connector"
 	"github.com/zitadel/zitadel/internal/command"
 	"github.com/zitadel/zitadel/internal/config/hook"
 	"github.com/zitadel/zitadel/internal/config/systemdefaults"
@@ -31,7 +31,7 @@ import (
 type Config struct {
 	ForMirror       bool
 	Database        database.Config
-	Caches          *cache.CachesConfig
+	Caches          *connector.CachesConfig
 	SystemDefaults  systemdefaults.SystemDefaults
 	InternalAuthZ   internal_authz.Config
 	ExternalDomain  string
--- a/cmd/setup/config_change.go
+++ b/cmd/setup/config_change.go
@@ -3,6 +3,7 @@ package setup
 import (
 	"context"

+	"github.com/zitadel/zitadel/internal/cache/connector"
 	"github.com/zitadel/zitadel/internal/command"
 	"github.com/zitadel/zitadel/internal/config/systemdefaults"
 	"github.com/zitadel/zitadel/internal/eventstore"
@@ -31,9 +32,9 @@ func (mig *externalConfigChange) Check(lastRun map[string]interface{}) bool {
 }

 func (mig *externalConfigChange) Execute(ctx context.Context, _ eventstore.Event) error {
-	cmd, err := command.StartCommands(
+	cmd, err := command.StartCommands(ctx,
 		mig.es,
-		nil,
+		connector.Connectors{},
 		mig.defaults,
 		nil,
 		nil,
--- a/cmd/setup/setup.go
+++ b/cmd/setup/setup.go
@@ -22,6 +22,7 @@ import (
 	auth_view "github.com/zitadel/zitadel/internal/auth/repository/eventsourcing/view"
 	"github.com/zitadel/zitadel/internal/authz"
 	authz_es "github.com/zitadel/zitadel/internal/authz/repository/eventsourcing/eventstore"
+	"github.com/zitadel/zitadel/internal/cache/connector"
 	"github.com/zitadel/zitadel/internal/command"
 	cryptoDB "github.com/zitadel/zitadel/internal/crypto/database"
 	"github.com/zitadel/zitadel/internal/database"
@@ -346,13 +347,17 @@ func initProjections(
 	}

 	sessionTokenVerifier := internal_authz.SessionTokenVerifier(keys.OIDC)
+
+	cacheConnectors, err := connector.StartConnectors(config.Caches, queryDBClient)
+	logging.OnError(err).Fatal("unable to start caches")
+
 	queries, err := query.StartQueries(
 		ctx,
 		eventstoreClient,
 		eventstoreV4.Querier,
 		queryDBClient,
 		projectionDBClient,
-		config.Caches,
+		cacheConnectors,
 		config.Projections,
 		config.SystemDefaults,
 		keys.IDPConfig,
@@ -394,9 +399,9 @@ func initProjections(
 	permissionCheck := func(ctx context.Context, permission, orgID, resourceID string) (err error) {
 		return internal_authz.CheckPermission(ctx, authZRepo, config.InternalAuthZ.RolePermissionMappings, permission, orgID, resourceID)
 	}
-	commands, err := command.StartCommands(
+	commands, err := command.StartCommands(ctx,
 		eventstoreClient,
-		config.Caches,
+		cacheConnectors,
 		config.SystemDefaults,
 		config.InternalAuthZ.RolePermissionMappings,
 		staticStorage,
--- a/cmd/start/config.go
+++ b/cmd/start/config.go
@@ -18,7 +18,7 @@ import (
 	"github.com/zitadel/zitadel/internal/api/ui/console"
 	"github.com/zitadel/zitadel/internal/api/ui/login"
 	auth_es "github.com/zitadel/zitadel/internal/auth/repository/eventsourcing"
-	"github.com/zitadel/zitadel/internal/cache"
+	"github.com/zitadel/zitadel/internal/cache/connector"
 	"github.com/zitadel/zitadel/internal/command"
 	"github.com/zitadel/zitadel/internal/config/hook"
 	"github.com/zitadel/zitadel/internal/config/network"
@@ -49,7 +49,7 @@ type Config struct {
 	HTTP1HostHeader     string
 	WebAuthNName        string
 	Database            database.Config
-	Caches              *cache.CachesConfig
+	Caches              *connector.CachesConfig
 	Tracing             tracing.Config
 	Metrics             metrics.Config
 	Profiler            profiler.Config
--- a/cmd/start/start.go
+++ b/cmd/start/start.go
@@ -69,6 +69,7 @@ import (
 	"github.com/zitadel/zitadel/internal/authz"
 	authz_repo "github.com/zitadel/zitadel/internal/authz/repository"
 	authz_es "github.com/zitadel/zitadel/internal/authz/repository/eventsourcing/eventstore"
+	"github.com/zitadel/zitadel/internal/cache/connector"
 	"github.com/zitadel/zitadel/internal/command"
 	"github.com/zitadel/zitadel/internal/crypto"
 	cryptoDB "github.com/zitadel/zitadel/internal/crypto/database"
@@ -177,6 +178,10 @@ func startZitadel(ctx context.Context, config *Config, masterKey string, server
 	}))

 	sessionTokenVerifier := internal_authz.SessionTokenVerifier(keys.OIDC)
+	cacheConnectors, err := connector.StartConnectors(config.Caches, queryDBClient)
+	if err != nil {
+		return fmt.Errorf("unable to start caches: %w", err)
+	}

 	queries, err := query.StartQueries(
 		ctx,
@@ -184,7 +189,7 @@ func startZitadel(ctx context.Context, config *Config, masterKey string, server
 		eventstoreV4.Querier,
 		queryDBClient,
 		projectionDBClient,
-		config.Caches,
+		cacheConnectors,
 		config.Projections,
 		config.SystemDefaults,
 		keys.IDPConfig,
@@ -222,9 +227,9 @@ func startZitadel(ctx context.Context, config *Config, masterKey string, server
 		DisplayName:    config.WebAuthNName,
 		ExternalSecure: config.ExternalSecure,
 	}
-	commands, err := command.StartCommands(
+	commands, err := command.StartCommands(ctx,
 		eventstoreClient,
-		config.Caches,
+		cacheConnectors,
 		config.SystemDefaults,
 		config.InternalAuthZ.RolePermissionMappings,
 		storage,