perf(cache): pgx pool connector (#8703)

# Which Problems Are Solved

Cache implementation using a PGX connection pool.

# How the Problems Are Solved

Defines a new schema `cache` in the zitadel database.
A table for string keys and a table for objects is defined.
For postgreSQL, tables are unlogged and partitioned by cache name for
performance.

Cockroach does not have unlogged tables and partitioning is an
enterprise feature that uses alternative syntax combined with sharding.
Regular tables are used here.

# Additional Changes

- `postgres.Config` can return a pxg pool. See following discussion

# Additional Context

- Part of https://github.com/zitadel/zitadel/issues/8648
- Closes https://github.com/zitadel/zitadel/issues/8647

---------

Co-authored-by: Silvan <silvan.reusser@gmail.com>

internal/query/cache.go

@@ -10,6 +10,8 @@ import (
 	"github.com/zitadel/zitadel/internal/cache"
 	"github.com/zitadel/zitadel/internal/cache/gomap"
 	"github.com/zitadel/zitadel/internal/cache/noop"
+	"github.com/zitadel/zitadel/internal/cache/pg"
+	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/eventstore"
 )
 
@@ -18,14 +20,14 @@ type Caches struct {
 	instance   cache.Cache[instanceIndex, string, *authzInstance]
 }
 
-func startCaches(background context.Context, conf *cache.CachesConfig) (_ *Caches, err error) {
+func startCaches(background context.Context, conf *cache.CachesConfig, client *database.DB) (_ *Caches, err error) {
 	caches := &Caches{
 		instance: noop.NewCache[instanceIndex, string, *authzInstance](),
 	}
 	if conf == nil {
 		return caches, nil
 	}
-	caches.connectors, err = startCacheConnectors(background, conf)
+	caches.connectors, err = startCacheConnectors(background, conf, client)
 	if err != nil {
 		return nil, err
 	}
@@ -39,20 +41,30 @@ func startCaches(background context.Context, conf *cache.CachesConfig) (_ *Cache
 }
 
 type cacheConnectors struct {
-	memory *cache.AutoPruneConfig
-	// pool   *pgxpool.Pool
+	memory   *cache.AutoPruneConfig
+	postgres *pgxPoolCacheConnector
 }
 
-func startCacheConnectors(_ context.Context, conf *cache.CachesConfig) (*cacheConnectors, error) {
+type pgxPoolCacheConnector struct {
+	*cache.AutoPruneConfig
+	client *database.DB
+}
+
+func startCacheConnectors(_ context.Context, conf *cache.CachesConfig, client *database.DB) (_ *cacheConnectors, err error) {
 	connectors := new(cacheConnectors)
 	if conf.Connectors.Memory.Enabled {
 		connectors.memory = &conf.Connectors.Memory.AutoPrune
 	}
-
+	if conf.Connectors.Postgres.Enabled {
+		connectors.postgres = &pgxPoolCacheConnector{
+			AutoPruneConfig: &conf.Connectors.Postgres.AutoPrune,
+			client:          client,
+		}
+	}
 	return connectors, nil
 }
 
-func startCache[I, K comparable, V cache.Entry[I, K]](background context.Context, indices []I, name string, conf *cache.CacheConfig, connectors *cacheConnectors) (cache.Cache[I, K, V], error) {
+func startCache[I ~int, K ~string, V cache.Entry[I, K]](background context.Context, indices []I, name string, conf *cache.CacheConfig, connectors *cacheConnectors) (cache.Cache[I, K, V], error) {
 	if conf == nil || conf.Connector == "" {
 		return noop.NewCache[I, K, V](), nil
 	}
@@ -61,12 +73,15 @@ func startCache[I, K comparable, V cache.Entry[I, K]](background context.Context
 		connectors.memory.StartAutoPrune(background, c, name)
 		return c, nil
 	}
-
-	/* TODO
-	if strings.EqualFold(conf.Connector, "sql") && connectors.pool != nil {
-		return ...
+	if strings.EqualFold(conf.Connector, "postgres") && connectors.postgres != nil {
+		client := connectors.postgres.client
+		c, err := pg.NewCache[I, K, V](background, name, *conf, indices, client.Pool, client.Type())
+		if err != nil {
+			return nil, fmt.Errorf("query start cache: %w", err)
+		}
+		connectors.postgres.StartAutoPrune(background, c, name)
+		return c, nil
 	}
-	*/
 
 	return nil, fmt.Errorf("cache connector %q not enabled", conf.Connector)
 }

internal/query/instance.go

@@ -435,71 +435,71 @@ func prepareInstanceDomainQuery(ctx context.Context, db prepareDatabase) (sq.Sel
 }
 
 type authzInstance struct {
-	id                  string
-	iamProjectID        string
-	consoleID           string
-	consoleAppID        string
-	defaultLang         language.Tag
-	defaultOrgID        string
-	csp                 csp
-	enableImpersonation bool
-	block               *bool
-	auditLogRetention   *time.Duration
-	features            feature.Features
-	externalDomains     database.TextArray[string]
-	trustedDomains      database.TextArray[string]
+	ID              string                     `json:"id,omitempty"`
+	IAMProjectID    string                     `json:"iam_project_id,omitempty"`
+	ConsoleID       string                     `json:"console_id,omitempty"`
+	ConsoleAppID    string                     `json:"console_app_id,omitempty"`
+	DefaultLang     language.Tag               `json:"default_lang,omitempty"`
+	DefaultOrgID    string                     `json:"default_org_id,omitempty"`
+	CSP             csp                        `json:"csp,omitempty"`
+	Impersonation   bool                       `json:"impersonation,omitempty"`
+	IsBlocked       *bool                      `json:"is_blocked,omitempty"`
+	LogRetention    *time.Duration             `json:"log_retention,omitempty"`
+	Feature         feature.Features           `json:"feature,omitempty"`
+	ExternalDomains database.TextArray[string] `json:"external_domains,omitempty"`
+	TrustedDomains  database.TextArray[string] `json:"trusted_domains,omitempty"`
 }
 
 type csp struct {
-	enableIframeEmbedding bool
-	allowedOrigins        database.TextArray[string]
+	EnableIframeEmbedding bool                       `json:"enable_iframe_embedding,omitempty"`
+	AllowedOrigins        database.TextArray[string] `json:"allowed_origins,omitempty"`
 }
 
 func (i *authzInstance) InstanceID() string {
-	return i.id
+	return i.ID
 }
 
 func (i *authzInstance) ProjectID() string {
-	return i.iamProjectID
+	return i.IAMProjectID
 }
 
 func (i *authzInstance) ConsoleClientID() string {
-	return i.consoleID
+	return i.ConsoleID
 }
 
 func (i *authzInstance) ConsoleApplicationID() string {
-	return i.consoleAppID
+	return i.ConsoleAppID
 }
 
 func (i *authzInstance) DefaultLanguage() language.Tag {
-	return i.defaultLang
+	return i.DefaultLang
 }
 
 func (i *authzInstance) DefaultOrganisationID() string {
-	return i.defaultOrgID
+	return i.DefaultOrgID
 }
 
 func (i *authzInstance) SecurityPolicyAllowedOrigins() []string {
-	if !i.csp.enableIframeEmbedding {
+	if !i.CSP.EnableIframeEmbedding {
 		return nil
 	}
-	return i.csp.allowedOrigins
+	return i.CSP.AllowedOrigins
 }
 
 func (i *authzInstance) EnableImpersonation() bool {
-	return i.enableImpersonation
+	return i.Impersonation
 }
 
 func (i *authzInstance) Block() *bool {
-	return i.block
+	return i.IsBlocked
 }
 
 func (i *authzInstance) AuditLogRetention() *time.Duration {
-	return i.auditLogRetention
+	return i.LogRetention
 }
 
 func (i *authzInstance) Features() feature.Features {
-	return i.features
+	return i.Feature
 }
 
 var errPublicDomain = "public domain %q not trusted"
@@ -509,7 +509,7 @@ func (i *authzInstance) checkDomain(instanceDomain, publicDomain string) error {
 	if publicDomain == "" || instanceDomain == publicDomain {
 		return nil
 	}
-	if !slices.Contains(i.trustedDomains, publicDomain) {
+	if !slices.Contains(i.TrustedDomains, publicDomain) {
 		return zerrors.ThrowNotFound(fmt.Errorf(errPublicDomain, publicDomain), "QUERY-IuGh1", "Errors.IAM.NotFound")
 	}
 	return nil
@@ -519,9 +519,9 @@ func (i *authzInstance) checkDomain(instanceDomain, publicDomain string) error {
 func (i *authzInstance) Keys(index instanceIndex) []string {
 	switch index {
 	case instanceIndexByID:
-		return []string{i.id}
+		return []string{i.ID}
 	case instanceIndexByHost:
-		return i.externalDomains
+		return i.ExternalDomains
 	default:
 		return nil
 	}
@@ -539,20 +539,20 @@ func scanAuthzInstance() (*authzInstance, func(row *sql.Row) error) {
 			features              []byte
 		)
 		err := row.Scan(
-			&instance.id,
-			&instance.defaultOrgID,
-			&instance.iamProjectID,
-			&instance.consoleID,
-			&instance.consoleAppID,
+			&instance.ID,
+			&instance.DefaultOrgID,
+			&instance.IAMProjectID,
+			&instance.ConsoleID,
+			&instance.ConsoleAppID,
 			&lang,
 			&enableIframeEmbedding,
-			&instance.csp.allowedOrigins,
+			&instance.CSP.AllowedOrigins,
 			&enableImpersonation,
 			&auditLogRetention,
 			&block,
 			&features,
-			&instance.externalDomains,
-			&instance.trustedDomains,
+			&instance.ExternalDomains,
+			&instance.TrustedDomains,
 		)
 		if errors.Is(err, sql.ErrNoRows) {
 			return zerrors.ThrowNotFound(nil, "QUERY-1kIjX", "Errors.IAM.NotFound")
@@ -560,19 +560,19 @@ func scanAuthzInstance() (*authzInstance, func(row *sql.Row) error) {
 		if err != nil {
 			return zerrors.ThrowInternal(err, "QUERY-d3fas", "Errors.Internal")
 		}
-		instance.defaultLang = language.Make(lang)
+		instance.DefaultLang = language.Make(lang)
 		if auditLogRetention.Valid {
-			instance.auditLogRetention = &auditLogRetention.Duration
+			instance.LogRetention = &auditLogRetention.Duration
 		}
 		if block.Valid {
-			instance.block = &block.Bool
+			instance.IsBlocked = &block.Bool
 		}
-		instance.csp.enableIframeEmbedding = enableIframeEmbedding.Bool
-		instance.enableImpersonation = enableImpersonation.Bool
+		instance.CSP.EnableIframeEmbedding = enableIframeEmbedding.Bool
+		instance.Impersonation = enableImpersonation.Bool
 		if len(features) == 0 {
 			return nil
 		}
-		if err = json.Unmarshal(features, &instance.features); err != nil {
+		if err = json.Unmarshal(features, &instance.Feature); err != nil {
 			return zerrors.ThrowInternal(err, "QUERY-Po8ki", "Errors.Internal")
 		}
 		return nil
@@ -598,10 +598,12 @@ func (c *Caches) registerInstanceInvalidation() {
 	})
 }
 
-type instanceIndex int16
+type instanceIndex int
 
-//go:generate enumer -type instanceIndex
+//go:generate enumer -type instanceIndex -linecomment
 const (
-	instanceIndexByID instanceIndex = iota
+	// Empty line comment ensures empty string for unspecified value
+	instanceIndexUnspecified instanceIndex = iota //
+	instanceIndexByID
 	instanceIndexByHost
 )

internal/query/instanceindex_enumer.go

@@ -1,4 +1,4 @@
-// Code generated by "enumer -type instanceIndex"; DO NOT EDIT.
+// Code generated by "enumer -type instanceIndex -linecomment"; DO NOT EDIT.
 
 package query
 
@@ -9,7 +9,7 @@ import (
 
 const _instanceIndexName = "instanceIndexByIDinstanceIndexByHost"
 
-var _instanceIndexIndex = [...]uint8{0, 17, 36}
+var _instanceIndexIndex = [...]uint8{0, 0, 17, 36}
 
 const _instanceIndexLowerName = "instanceindexbyidinstanceindexbyhost"
 
@@ -24,13 +24,16 @@ func (i instanceIndex) String() string {
 // Re-run the stringer command to generate them again.
 func _instanceIndexNoOp() {
 	var x [1]struct{}
-	_ = x[instanceIndexByID-(0)]
-	_ = x[instanceIndexByHost-(1)]
+	_ = x[instanceIndexUnspecified-(0)]
+	_ = x[instanceIndexByID-(1)]
+	_ = x[instanceIndexByHost-(2)]
 }
 
-var _instanceIndexValues = []instanceIndex{instanceIndexByID, instanceIndexByHost}
+var _instanceIndexValues = []instanceIndex{instanceIndexUnspecified, instanceIndexByID, instanceIndexByHost}
 
 var _instanceIndexNameToValueMap = map[string]instanceIndex{
+	_instanceIndexName[0:0]:        instanceIndexUnspecified,
+	_instanceIndexLowerName[0:0]:   instanceIndexUnspecified,
 	_instanceIndexName[0:17]:       instanceIndexByID,
 	_instanceIndexLowerName[0:17]:  instanceIndexByID,
 	_instanceIndexName[17:36]:      instanceIndexByHost,
@@ -38,6 +41,7 @@ var _instanceIndexNameToValueMap = map[string]instanceIndex{
 }
 
 var _instanceIndexNames = []string{
+	_instanceIndexName[0:0],
 	_instanceIndexName[0:17],
 	_instanceIndexName[17:36],
 }

internal/query/query.go

@@ -89,7 +89,7 @@ func StartQueries(
 	if startProjections {
 		projection.Start(ctx)
 	}
-	repo.caches, err = startCaches(ctx, caches)
+	repo.caches, err = startCaches(ctx, caches, querySqlClient)
 	if err != nil {
 		return nil, err
 	}