TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-11 21:27:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: projections auto create their tables (#3324)

Browse Source 
* begin init checks for projections

* first projection checks

* debug notification providers with query fixes

* more projections and first index

* more projections

* more projections

* finish projections

* fix tests (remove db name)

* create tables in setup

* fix logging / error handling

* add tenant to views

* rename tenant to instance_id

* add instance_id to all projections

* add instance_id to all queries

* correct instance_id on projections

* add instance_id to failed_events

* use separate context for instance

* implement features projection

* implement features projection

* remove unique constraint from setup when migration failed

* add error to failed setup event

* add instance_id to primary keys

* fix IAM projection

* remove old migrations folder

* fix keysFromYAML test
This commit is contained in:
Livio Amstutz
2022-03-23 09:02:39 +01:00
committed by GitHub
parent 9e13b70a3d
commit 56b916a2b0
400 changed files with 6508 additions and 8890 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/admin/initialise/sql/09_events_table.sql
View File

@@ -12,13 +12,13 @@ CREATE TABLE eventstore.events (
, editor_user TEXT NOT NULL
, editor_service TEXT NOT NULL
, resource_owner TEXT NOT NULL
, tenant TEXT
, instance_id TEXT
, PRIMARY KEY (event_sequence DESC) USING HASH WITH BUCKET_COUNT = 10
, INDEX agg_type_agg_id (aggregate_type, aggregate_id)
, INDEX agg_type (aggregate_type)
, INDEX agg_type_seq (aggregate_type, event_sequence DESC)
STORING (id, event_type, aggregate_id, aggregate_version, previous_aggregate_sequence, creation_date, event_data, editor_user, editor_service, resource_owner, tenant, previous_aggregate_type_sequence)
STORING (id, event_type, aggregate_id, aggregate_version, previous_aggregate_sequence, creation_date, event_data, editor_user, editor_service, resource_owner, instance_id, previous_aggregate_type_sequence)
, INDEX max_sequence (aggregate_type, aggregate_id, event_sequence DESC)
, CONSTRAINT previous_sequence_unique UNIQUE (previous_aggregate_sequence DESC)
, CONSTRAINT prev_agg_type_seq_unique UNIQUE(previous_aggregate_type_sequence)

2
cmd/admin/key/key_test.go
View File

@@ -155,7 +155,7 @@ func Test_keysFromYAML(t *testing.T) {
if tt.res.err != nil && !tt.res.err(err) {
t.Errorf("got wrong err: %v ", err)
}
assert.EqualValues(t, got, tt.res.keys)
assert.ElementsMatch(t, got, tt.res.keys)
})
}
}

34
cmd/admin/setup/01_projection_tables.go Normal file
View File

@@ -0,0 +1,34 @@
package setup
import (
"context"
"database/sql"
_ "embed"
)
var (
//go:embed 01_sql/adminapi.sql
createAdminViews string
//go:embed 01_sql/auth.sql
createAuthViews string
//go:embed 01_sql/authz.sql
createAuthzViews string
//go:embed 01_sql/notification.sql
createNotificationViews string
//go:embed 01_sql/projections.sql
createProjections string
)
type ProjectionTable struct {
dbClient *sql.DB
}
func (mig *ProjectionTable) Execute(ctx context.Context) error {
stmt := createAdminViews + createAuthViews + createAuthzViews + createNotificationViews + createProjections
_, err := mig.dbClient.ExecContext(ctx, stmt)
return err
}
func (mig *ProjectionTable) String() string {
return "01_tables"
}

54
cmd/admin/setup/01_sql/adminapi.sql Normal file
View File

@@ -0,0 +1,54 @@
CREATE SCHEMA adminapi;
CREATE TABLE adminapi.locks (
locker_id TEXT,
locked_until TIMESTAMPTZ(3),
projection_name TEXT,
PRIMARY KEY (projection_name)
);
CREATE TABLE adminapi.current_sequences (
projection_name TEXT,
aggregate_type TEXT,
current_sequence BIGINT,
timestamp TIMESTAMPTZ,
PRIMARY KEY (projection_name, aggregate_type)
);
CREATE TABLE adminapi.failed_events (
projection_name TEXT,
failed_sequence BIGINT,
failure_count SMALLINT,
error TEXT,
PRIMARY KEY (projection_name, failed_sequence)
);
CREATE TABLE adminapi.styling (
aggregate_id STRING NOT NULL,
creation_date TIMESTAMPTZ NULL,
change_date TIMESTAMPTZ NULL,
label_policy_state INT2 NOT NULL DEFAULT 0:::INT2,
sequence INT8 NULL,
primary_color STRING NULL,
background_color STRING NULL,
warn_color STRING NULL,
font_color STRING NULL,
primary_color_dark STRING NULL,
background_color_dark STRING NULL,
warn_color_dark STRING NULL,
font_color_dark STRING NULL,
logo_url STRING NULL,
icon_url STRING NULL,
logo_dark_url STRING NULL,
icon_dark_url STRING NULL,
font_url STRING NULL,
err_msg_popup BOOL NULL,
disable_watermark BOOL NULL,
hide_login_name_suffix BOOL NULL,
instance_id STRING NOT NULL,
PRIMARY KEY (aggregate_id, label_policy_state)
);

222
cmd/admin/setup/01_sql/auth.sql Normal file
View File

@@ -0,0 +1,222 @@
CREATE SCHEMA auth;
CREATE TABLE auth.locks (
locker_id TEXT,
locked_until TIMESTAMPTZ(3),
projection_name TEXT,
PRIMARY KEY (projection_name)
);
CREATE TABLE auth.current_sequences (
projection_name TEXT,
aggregate_type TEXT,
current_sequence BIGINT,
timestamp TIMESTAMPTZ,
PRIMARY KEY (projection_name, aggregate_type)
);
CREATE TABLE auth.failed_events (
projection_name TEXT,
failed_sequence BIGINT,
failure_count SMALLINT,
error TEXT,
PRIMARY KEY (projection_name, failed_sequence)
);
CREATE TABLE auth.users (
id STRING NULL,
creation_date TIMESTAMPTZ NULL,
change_date TIMESTAMPTZ NULL,
resource_owner STRING NULL,
user_state INT2 NULL,
password_set BOOL NULL,
password_change_required BOOL NULL,
password_change TIMESTAMPTZ NULL,
last_login TIMESTAMPTZ NULL,
user_name STRING NULL,
login_names STRING[] NULL,
preferred_login_name STRING NULL,
first_name STRING NULL,
last_name STRING NULL,
nick_name STRING NULL,
display_name STRING NULL,
preferred_language STRING NULL,
gender INT2 NULL,
email STRING NULL,
is_email_verified BOOL NULL,
phone STRING NULL,
is_phone_verified BOOL NULL,
country STRING NULL,
locality STRING NULL,
postal_code STRING NULL,
region STRING NULL,
street_address STRING NULL,
otp_state INT2 NULL,
mfa_max_set_up INT2 NULL,
mfa_init_skipped TIMESTAMPTZ NULL,
sequence INT8 NULL,
init_required BOOL NULL,
username_change_required BOOL NULL,
machine_name STRING NULL,
machine_description STRING NULL,
user_type STRING NULL,
u2f_tokens BYTES NULL,
passwordless_tokens BYTES NULL,
avatar_key STRING NULL,
passwordless_init_required BOOL NULL,
password_init_required BOOL NULL,
instance_id STRING NULL,
PRIMARY KEY (id)
);
CREATE TABLE auth.user_sessions (
creation_date TIMESTAMPTZ NULL,
change_date TIMESTAMPTZ NULL,
resource_owner STRING NULL,
state INT2 NULL,
user_agent_id STRING NULL,
user_id STRING NULL,
user_name STRING NULL,
password_verification TIMESTAMPTZ NULL,
second_factor_verification TIMESTAMPTZ NULL,
multi_factor_verification TIMESTAMPTZ NULL,
sequence INT8 NULL,
second_factor_verification_type INT2 NULL,
multi_factor_verification_type INT2 NULL,
user_display_name STRING NULL,
login_name STRING NULL,
external_login_verification TIMESTAMPTZ NULL,
selected_idp_config_id STRING NULL,
passwordless_verification TIMESTAMPTZ NULL,
avatar_key STRING NULL,
instance_id STRING NULL,
PRIMARY KEY (user_agent_id, user_id)
);
CREATE TABLE auth.user_external_idps (
external_user_id STRING NOT NULL,
idp_config_id STRING NOT NULL,
user_id STRING NULL,
idp_name STRING NULL,
user_display_name STRING NULL,
creation_date TIMESTAMPTZ NULL,
change_date TIMESTAMPTZ NULL,
sequence INT8 NULL,
resource_owner STRING NULL,
instance_id STRING NULL,
PRIMARY KEY (external_user_id, idp_config_id)
);
CREATE TABLE auth.tokens (
id STRING NOT NULL,
creation_date TIMESTAMPTZ NULL,
change_date TIMESTAMPTZ NULL,
resource_owner STRING NULL,
application_id STRING NULL,
user_agent_id STRING NULL,
user_id STRING NULL,
expiration TIMESTAMPTZ NULL,
sequence INT8 NULL,
scopes STRING[] NULL,
audience STRING[] NULL,
preferred_language STRING NULL,
refresh_token_id STRING NULL,
instance_id STRING NULL,
PRIMARY KEY (id),
INDEX user_user_agent_idx (user_id, user_agent_id)
);
CREATE TABLE auth.refresh_tokens (
id STRING NOT NULL,
creation_date TIMESTAMPTZ NULL,
change_date TIMESTAMPTZ NULL,
resource_owner STRING NULL,
token STRING NULL,
client_id STRING NOT NULL,
user_agent_id STRING NOT NULL,
user_id STRING NOT NULL,
auth_time TIMESTAMPTZ NULL,
idle_expiration TIMESTAMPTZ NULL,
expiration TIMESTAMPTZ NULL,
sequence INT8 NULL,
scopes STRING[] NULL,
audience STRING[] NULL,
amr STRING[] NULL,
instance_id STRING NULL,
PRIMARY KEY (id),
UNIQUE INDEX unique_client_user_index (client_id ASC, user_agent_id ASC, user_id ASC)
);
CREATE TABLE auth.org_project_mapping (
org_id STRING NOT NULL,
project_id STRING NOT NULL,
project_grant_id STRING NULL,
instance_id STRING NULL,
PRIMARY KEY (org_id, project_id)
);
CREATE TABLE auth.idp_providers (
aggregate_id STRING NOT NULL,
idp_config_id STRING NOT NULL,
creation_date TIMESTAMPTZ NULL,
change_date TIMESTAMPTZ NULL,
sequence INT8 NULL,
name STRING NULL,
idp_config_type INT2 NULL,
idp_provider_type INT2 NULL,
idp_state INT2 NULL,
styling_type INT2 NULL,
instance_id STRING NULL,
PRIMARY KEY (aggregate_id, idp_config_id)
);
CREATE TABLE auth.idp_configs (
idp_config_id STRING NOT NULL,
creation_date TIMESTAMPTZ NULL,
change_date TIMESTAMPTZ NULL,
sequence INT8 NULL,
aggregate_id STRING NULL,
name STRING NULL,
idp_state INT2 NULL,
idp_provider_type INT2 NULL,
is_oidc BOOL NULL,
oidc_client_id STRING NULL,
oidc_client_secret JSONB NULL,
oidc_issuer STRING NULL,
oidc_scopes STRING[] NULL,
oidc_idp_display_name_mapping INT2 NULL,
oidc_idp_username_mapping INT2 NULL,
styling_type INT2 NULL,
oauth_authorization_endpoint STRING NULL,
oauth_token_endpoint STRING NULL,
auto_register BOOL NULL,
jwt_endpoint STRING NULL,
jwt_keys_endpoint STRING NULL,
jwt_header_name STRING NULL,
instance_id STRING NULL,
PRIMARY KEY (idp_config_id)
);
CREATE TABLE auth.auth_requests (
id STRING NOT NULL,
request JSONB NULL,
code STRING NULL,
request_type INT2 NULL,
creation_date TIMESTAMPTZ NULL,
change_date TIMESTAMPTZ NULL,
instance_id STRING NULL,
PRIMARY KEY (id),
INDEX auth_code_idx (code)
);

44
cmd/admin/setup/01_sql/authz.sql Normal file
View File

@@ -0,0 +1,44 @@
CREATE SCHEMA authz;
CREATE TABLE authz.locks (
locker_id TEXT,
locked_until TIMESTAMPTZ(3),
projection_name TEXT,
PRIMARY KEY (projection_name)
);
CREATE TABLE authz.current_sequences (
projection_name TEXT,
aggregate_type TEXT,
current_sequence BIGINT,
timestamp TIMESTAMPTZ,
PRIMARY KEY (projection_name, aggregate_type)
);
CREATE TABLE authz.failed_events (
projection_name TEXT,
failed_sequence BIGINT,
failure_count SMALLINT,
error TEXT,
PRIMARY KEY (projection_name, failed_sequence)
);
CREATE TABLE authz.user_memberships (
user_id STRING NOT NULL,
member_type INT2 NOT NULL,
aggregate_id STRING NOT NULL,
object_id STRING NOT NULL,
roles STRING[] NULL,
display_name STRING NULL,
resource_owner STRING NULL,
resource_owner_name STRING NULL,
creation_date TIMESTAMPTZ NULL,
change_date TIMESTAMPTZ NULL,
sequence INT8 NULL,
instance_id STRING NULL,
PRIMARY KEY (user_id, member_type, aggregate_id, object_id)
);

52
cmd/admin/setup/01_sql/notification.sql Normal file
View File

@@ -0,0 +1,52 @@
CREATE SCHEMA notification;
CREATE TABLE notification.locks (
locker_id TEXT,
locked_until TIMESTAMPTZ(3),
projection_name TEXT,
PRIMARY KEY (projection_name)
);
CREATE TABLE notification.current_sequences (
projection_name TEXT,
aggregate_type TEXT,
current_sequence BIGINT,
timestamp TIMESTAMPTZ,
PRIMARY KEY (projection_name, aggregate_type)
);
CREATE TABLE notification.failed_events (
projection_name TEXT,
failed_sequence BIGINT,
failure_count SMALLINT,
error TEXT,
PRIMARY KEY (projection_name, failed_sequence)
);
CREATE TABLE notification.notify_users (
id STRING NOT NULL,
creation_date TIMESTAMPTZ NULL,
change_date TIMESTAMPTZ NULL,
resource_owner STRING NULL,
user_name STRING NULL,
first_name STRING NULL,
last_name STRING NULL,
nick_name STRING NULL,
display_name STRING NULL,
preferred_language STRING NULL,
gender INT2 NULL,
last_email STRING NULL,
verified_email STRING NULL,
last_phone STRING NULL,
verified_phone STRING NULL,
sequence INT8 NULL,
password_set BOOL NULL,
login_names STRING NULL,
preferred_login_name STRING NULL,
instance_id STRING NULL,
PRIMARY KEY (id)
);

26
cmd/admin/setup/01_sql/projections.sql Normal file
View File

@@ -0,0 +1,26 @@
CREATE TABLE projections.locks (
locker_id TEXT,
locked_until TIMESTAMPTZ(3),
projection_name TEXT,
PRIMARY KEY (projection_name)
);
CREATE TABLE projections.current_sequences (
projection_name TEXT,
aggregate_type TEXT,
current_sequence BIGINT,
timestamp TIMESTAMPTZ,
PRIMARY KEY (projection_name, aggregate_type)
);
CREATE TABLE projections.failed_events (
projection_name TEXT,
failed_sequence BIGINT,
failure_count SMALLINT,
error TEXT,
instance_id TEXT,
PRIMARY KEY (projection_name, failed_sequence, instance_id)
);

13
cmd/admin/setup/config.go Normal file
View File

@@ -0,0 +1,13 @@
package setup
import (
"github.com/caos/zitadel/internal/database"
)
type Config struct {
Database database.Config
}
type Steps struct {
S1ProjectionTable *ProjectionTable
}

42
cmd/admin/setup/setup.go
View File

@@ -1,10 +1,22 @@
package setup
import (
"bytes"
"context"
_ "embed"
"github.com/caos/logging"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/caos/zitadel/internal/database"
"github.com/caos/zitadel/internal/eventstore"
"github.com/caos/zitadel/internal/migration"
)
var (
//go:embed steps.yaml
defaultSteps []byte
)
func New() *cobra.Command {
@@ -14,9 +26,33 @@ func New() *cobra.Command {
Long: `sets up data to start ZITADEL.
Requirements:
- cockroachdb`,
RunE: func(cmd *cobra.Command, args []string) error {
logging.Info("hello world")
return nil
Run: func(cmd *cobra.Command, args []string) {
config := new(Config)
err := viper.Unmarshal(config)
logging.OnError(err).Fatal("unable to read config")
v := viper.New()
v.SetConfigType("yaml")
err = v.ReadConfig(bytes.NewBuffer(defaultSteps))
logging.OnError(err).Fatal("unable to read setup steps")
steps := new(Steps)
err = v.Unmarshal(steps)
logging.OnError(err).Fatal("unable to read steps")
setup(config, steps)
},
}
}
func setup(config *Config, steps *Steps) {
dbClient, err := database.Connect(config.Database)
logging.OnError(err).Fatal("unable to connect to database")
eventstoreClient, err := eventstore.Start(dbClient)
logging.OnError(err).Fatal("unable to start eventstore")
steps.S1ProjectionTable = &ProjectionTable{dbClient: dbClient}
migration.Migrate(context.Background(), eventstoreClient, steps.S1ProjectionTable)
}

0
cmd/admin/setup/steps.yaml Normal file
View File

3
cmd/admin/start/start.go
View File

@@ -41,7 +41,6 @@ import (
"github.com/caos/zitadel/internal/crypto"
cryptoDB "github.com/caos/zitadel/internal/crypto/database"
"github.com/caos/zitadel/internal/database"
"github.com/caos/zitadel/internal/domain"
"github.com/caos/zitadel/internal/eventstore"
"github.com/caos/zitadel/internal/id"
"github.com/caos/zitadel/internal/notification"
@@ -308,7 +307,7 @@ func shutdownServer(ctx context.Context, server *http.Server) error {
//TODO:!!??!!
func consoleClientID(ctx context.Context, queries *query.Queries) (string, error) {
iam, err := queries.IAMByID(ctx, domain.IAMID)
iam, err := queries.IAM(ctx)
if err != nil {
return "", err
}