TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-11 19:17:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf(cache): pgx pool connector (#8703)

Browse Source 
# Which Problems Are Solved

Cache implementation using a PGX connection pool.

# How the Problems Are Solved

Defines a new schema `cache` in the zitadel database.
A table for string keys and a table for objects is defined.
For postgreSQL, tables are unlogged and partitioned by cache name for
performance.

Cockroach does not have unlogged tables and partitioning is an
enterprise feature that uses alternative syntax combined with sharding.
Regular tables are used here.

# Additional Changes

- `postgres.Config` can return a pxg pool. See following discussion

# Additional Context

- Part of https://github.com/zitadel/zitadel/issues/8648
- Closes https://github.com/zitadel/zitadel/issues/8647

---------

Co-authored-by: Silvan <silvan.reusser@gmail.com>
This commit is contained in:
Tim Möhlmann
2024-10-04 16:15:41 +03:00
committed by GitHub
parent bee0744d46
commit 25dc7bfe72
29 changed files with 1034 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
cmd/setup/34.go Normal file
View File

@@ -0,0 +1,37 @@
package setup
import (
"context"
_ "embed"
"fmt"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore"
)
var (
//go:embed 34/cockroach/34_cache_schema.sql
addCacheSchemaCockroach string
//go:embed 34/postgres/34_cache_schema.sql
addCacheSchemaPostgres string
)
type AddCacheSchema struct {
dbClient *database.DB
}
func (mig *AddCacheSchema) Execute(ctx context.Context, _ eventstore.Event) (err error) {
switch mig.dbClient.Type() {
case "cockroach":
_, err = mig.dbClient.ExecContext(ctx, addCacheSchemaCockroach)
case "postgres":
_, err = mig.dbClient.ExecContext(ctx, addCacheSchemaPostgres)
default:
err = fmt.Errorf("add cache schema: unsupported db type %q", mig.dbClient.Type())
}
return err
}
func (mig *AddCacheSchema) String() string {
return "34_add_cache_schema"
}

27
cmd/setup/34/cockroach/34_cache_schema.sql Normal file
View File

@@ -0,0 +1,27 @@
create schema if not exists cache;
create table if not exists cache.objects (
cache_name varchar not null,
id uuid not null default gen_random_uuid(),
created_at timestamptz not null default now(),
last_used_at timestamptz not null default now(),
payload jsonb not null,
primary key(cache_name, id)
);
create table if not exists cache.string_keys(
cache_name varchar not null check (cache_name <> ''),
index_id integer not null check (index_id > 0),
index_key varchar not null check (index_key <> ''),
object_id uuid not null,
primary key (cache_name, index_id, index_key),
constraint fk_object
foreign key(cache_name, object_id)
references cache.objects(cache_name, id)
on delete cascade
);
create index if not exists string_keys_object_id_idx
on cache.string_keys (cache_name, object_id); -- for delete cascade

29
cmd/setup/34/postgres/34_cache_schema.sql Normal file
View File

@@ -0,0 +1,29 @@
create schema if not exists cache;
create unlogged table if not exists cache.objects (
cache_name varchar not null,
id uuid not null default gen_random_uuid(),
created_at timestamptz not null default now(),
last_used_at timestamptz not null default now(),
payload jsonb not null,
primary key(cache_name, id)
)
partition by list (cache_name);
create unlogged table if not exists cache.string_keys(
cache_name varchar not null check (cache_name <> ''),
index_id integer not null check (index_id > 0),
index_key varchar not null check (index_key <> ''),
object_id uuid not null,
primary key (cache_name, index_id, index_key),
constraint fk_object
foreign key(cache_name, object_id)
references cache.objects(cache_name, id)
on delete cascade
)
partition by list (cache_name);
create index if not exists string_keys_object_id_idx
on cache.string_keys (cache_name, object_id); -- for delete cascade

59
cmd/setup/config.go
View File

@@ -90,36 +90,37 @@ func MustNewConfig(v *viper.Viper) *Config {
}
type Steps struct {
s1ProjectionTable *ProjectionTable
s2AssetsTable *AssetTable
FirstInstance *FirstInstance
s5LastFailed *LastFailed
s6OwnerRemoveColumns *OwnerRemoveColumns
s7LogstoreTables *LogstoreTables
s8AuthTokens *AuthTokenIndexes
CorrectCreationDate *CorrectCreationDate
s12AddOTPColumns *AddOTPColumns
s13FixQuotaProjection *FixQuotaConstraints
s14NewEventsTable *NewEventsTable
s15CurrentStates *CurrentProjectionState
s16UniqueConstraintsLower *UniqueConstraintToLower
s17AddOffsetToUniqueConstraints *AddOffsetToCurrentStates
s18AddLowerFieldsToLoginNames *AddLowerFieldsToLoginNames
s19AddCurrentStatesIndex *AddCurrentSequencesIndex
s20AddByUserSessionIndex *AddByUserIndexToSession
s21AddBlockFieldToLimits *AddBlockFieldToLimits
s22ActiveInstancesIndex *ActiveInstanceEvents
s23CorrectGlobalUniqueConstraints *CorrectGlobalUniqueConstraints
s24AddActorToAuthTokens *AddActorToAuthTokens
s25User11AddLowerFieldsToVerifiedEmail *User11AddLowerFieldsToVerifiedEmail
s26AuthUsers3 *AuthUsers3
s27IDPTemplate6SAMLNameIDFormat *IDPTemplate6SAMLNameIDFormat
s28AddFieldTable *AddFieldTable
s29FillFieldsForProjectGrant *FillFieldsForProjectGrant
s30FillFieldsForOrgDomainVerified *FillFieldsForOrgDomainVerified
s31AddAggregateIndexToFields *AddAggregateIndexToFields
s32AddAuthSessionID *AddAuthSessionID
s1ProjectionTable *ProjectionTable
s2AssetsTable *AssetTable
FirstInstance *FirstInstance
s5LastFailed *LastFailed
s6OwnerRemoveColumns *OwnerRemoveColumns
s7LogstoreTables *LogstoreTables
s8AuthTokens *AuthTokenIndexes
CorrectCreationDate *CorrectCreationDate
s12AddOTPColumns *AddOTPColumns
s13FixQuotaProjection *FixQuotaConstraints
s14NewEventsTable *NewEventsTable
s15CurrentStates *CurrentProjectionState
s16UniqueConstraintsLower *UniqueConstraintToLower
s17AddOffsetToUniqueConstraints *AddOffsetToCurrentStates
s18AddLowerFieldsToLoginNames *AddLowerFieldsToLoginNames
s19AddCurrentStatesIndex *AddCurrentSequencesIndex
s20AddByUserSessionIndex *AddByUserIndexToSession
s21AddBlockFieldToLimits *AddBlockFieldToLimits
s22ActiveInstancesIndex *ActiveInstanceEvents
s23CorrectGlobalUniqueConstraints *CorrectGlobalUniqueConstraints
s24AddActorToAuthTokens *AddActorToAuthTokens
s25User11AddLowerFieldsToVerifiedEmail *User11AddLowerFieldsToVerifiedEmail
s26AuthUsers3 *AuthUsers3
s27IDPTemplate6SAMLNameIDFormat *IDPTemplate6SAMLNameIDFormat
s28AddFieldTable *AddFieldTable
s29FillFieldsForProjectGrant *FillFieldsForProjectGrant
s30FillFieldsForOrgDomainVerified *FillFieldsForOrgDomainVerified
s31AddAggregateIndexToFields *AddAggregateIndexToFields
s32AddAuthSessionID *AddAuthSessionID
s33SMSConfigs3TwilioAddVerifyServiceSid *SMSConfigs3TwilioAddVerifyServiceSid
s34AddCacheSchema *AddCacheSchema
}
func MustNewSteps(v *viper.Viper) *Steps {

2
cmd/setup/setup.go
View File

@@ -162,6 +162,7 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string)
steps.s31AddAggregateIndexToFields = &AddAggregateIndexToFields{dbClient: esPusherDBClient}
steps.s32AddAuthSessionID = &AddAuthSessionID{dbClient: esPusherDBClient}
steps.s33SMSConfigs3TwilioAddVerifyServiceSid = &SMSConfigs3TwilioAddVerifyServiceSid{dbClient: esPusherDBClient}
steps.s34AddCacheSchema = &AddCacheSchema{dbClient: queryDBClient}
err = projection.Create(ctx, projectionDBClient, eventstoreClient, config.Projections, nil, nil, nil)
logging.OnError(err).Fatal("unable to start projections")
@@ -204,6 +205,7 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string)
steps.s26AuthUsers3,
steps.s29FillFieldsForProjectGrant,
steps.s30FillFieldsForOrgDomainVerified,
steps.s34AddCacheSchema,
} {
mustExecuteMigration(ctx, eventstoreClient, step, "migration failed")
}