fix: enable env vars in setup steps (and deprecate admin subcommand) (#3871)

* fix: enable env vars in setup steps (and deprecate admin subcommand)

* fix tests and error text
This commit is contained in:
Livio Spring
2022-06-27 12:32:34 +02:00
committed by GitHub
parent 30f553dea1
commit 12d4d3ea0b
53 changed files with 44 additions and 31 deletions

32
cmd/setup/01.go Normal file
View File

@@ -0,0 +1,32 @@
package setup
import (
"context"
"database/sql"
_ "embed"
)
var (
//go:embed 01_sql/adminapi.sql
createAdminViews string
//go:embed 01_sql/auth.sql
createAuthViews string
//go:embed 01_sql/notification.sql
createNotificationViews string
//go:embed 01_sql/projections.sql
createProjections string
)
type ProjectionTable struct {
dbClient *sql.DB
}
func (mig *ProjectionTable) Execute(ctx context.Context) error {
stmt := createAdminViews + createAuthViews + createNotificationViews + createProjections
_, err := mig.dbClient.ExecContext(ctx, stmt)
return err
}
func (mig *ProjectionTable) String() string {
return "01_tables"
}

View File

@@ -0,0 +1,57 @@
CREATE SCHEMA adminapi;
CREATE TABLE adminapi.locks (
locker_id TEXT,
locked_until TIMESTAMPTZ(3),
view_name TEXT,
instance_id TEXT NOT NULL,
PRIMARY KEY (view_name, instance_id)
);
CREATE TABLE adminapi.current_sequences (
view_name TEXT,
current_sequence BIGINT,
event_timestamp TIMESTAMPTZ,
last_successful_spooler_run TIMESTAMPTZ,
instance_id TEXT NOT NULL,
PRIMARY KEY (view_name, instance_id)
);
CREATE TABLE adminapi.failed_events (
view_name TEXT,
failed_sequence BIGINT,
failure_count SMALLINT,
err_msg TEXT,
instance_id TEXT NOT NULL,
PRIMARY KEY (view_name, failed_sequence, instance_id)
);
CREATE TABLE adminapi.styling (
aggregate_id STRING NOT NULL,
creation_date TIMESTAMPTZ NULL,
change_date TIMESTAMPTZ NULL,
label_policy_state INT2 NOT NULL DEFAULT 0:::INT2,
sequence INT8 NULL,
primary_color STRING NULL,
background_color STRING NULL,
warn_color STRING NULL,
font_color STRING NULL,
primary_color_dark STRING NULL,
background_color_dark STRING NULL,
warn_color_dark STRING NULL,
font_color_dark STRING NULL,
logo_url STRING NULL,
icon_url STRING NULL,
logo_dark_url STRING NULL,
icon_dark_url STRING NULL,
font_url STRING NULL,
err_msg_popup BOOL NULL,
disable_watermark BOOL NULL,
hide_login_name_suffix BOOL NULL,
instance_id STRING NOT NULL,
PRIMARY KEY (aggregate_id, label_policy_state, instance_id)
);

226
cmd/setup/01_sql/auth.sql Normal file
View File

@@ -0,0 +1,226 @@
CREATE SCHEMA auth;
CREATE TABLE auth.locks (
locker_id TEXT,
locked_until TIMESTAMPTZ(3),
view_name TEXT,
instance_id TEXT NOT NULL,
PRIMARY KEY (view_name, instance_id)
);
CREATE TABLE auth.current_sequences (
view_name TEXT,
current_sequence BIGINT,
event_timestamp TIMESTAMPTZ,
last_successful_spooler_run TIMESTAMPTZ,
instance_id TEXT NOT NULL,
PRIMARY KEY (view_name, instance_id)
);
CREATE TABLE auth.failed_events (
view_name TEXT,
failed_sequence BIGINT,
failure_count SMALLINT,
err_msg TEXT,
instance_id TEXT NOT NULL,
PRIMARY KEY (view_name, failed_sequence, instance_id)
);
CREATE TABLE auth.users (
id STRING NULL,
creation_date TIMESTAMPTZ NULL,
change_date TIMESTAMPTZ NULL,
resource_owner STRING NULL,
user_state INT2 NULL,
password_set BOOL NULL,
password_change_required BOOL NULL,
password_change TIMESTAMPTZ NULL,
last_login TIMESTAMPTZ NULL,
user_name STRING NULL,
login_names STRING[] NULL,
preferred_login_name STRING NULL,
first_name STRING NULL,
last_name STRING NULL,
nick_name STRING NULL,
display_name STRING NULL,
preferred_language STRING NULL,
gender INT2 NULL,
email STRING NULL,
is_email_verified BOOL NULL,
phone STRING NULL,
is_phone_verified BOOL NULL,
country STRING NULL,
locality STRING NULL,
postal_code STRING NULL,
region STRING NULL,
street_address STRING NULL,
otp_state INT2 NULL,
mfa_max_set_up INT2 NULL,
mfa_init_skipped TIMESTAMPTZ NULL,
sequence INT8 NULL,
init_required BOOL NULL,
username_change_required BOOL NULL,
machine_name STRING NULL,
machine_description STRING NULL,
user_type STRING NULL,
u2f_tokens BYTES NULL,
passwordless_tokens BYTES NULL,
avatar_key STRING NULL,
passwordless_init_required BOOL NULL,
password_init_required BOOL NULL,
instance_id STRING NOT NULL,
PRIMARY KEY (id, instance_id)
);
CREATE TABLE auth.user_sessions (
creation_date TIMESTAMPTZ NULL,
change_date TIMESTAMPTZ NULL,
resource_owner STRING NULL,
state INT2 NULL,
user_agent_id STRING NULL,
user_id STRING NULL,
user_name STRING NULL,
password_verification TIMESTAMPTZ NULL,
second_factor_verification TIMESTAMPTZ NULL,
multi_factor_verification TIMESTAMPTZ NULL,
sequence INT8 NULL,
second_factor_verification_type INT2 NULL,
multi_factor_verification_type INT2 NULL,
user_display_name STRING NULL,
login_name STRING NULL,
external_login_verification TIMESTAMPTZ NULL,
selected_idp_config_id STRING NULL,
passwordless_verification TIMESTAMPTZ NULL,
avatar_key STRING NULL,
instance_id STRING NOT NULL,
PRIMARY KEY (user_agent_id, user_id, instance_id)
);
CREATE TABLE auth.user_external_idps (
external_user_id STRING NOT NULL,
idp_config_id STRING NOT NULL,
user_id STRING NULL,
idp_name STRING NULL,
user_display_name STRING NULL,
creation_date TIMESTAMPTZ NULL,
change_date TIMESTAMPTZ NULL,
sequence INT8 NULL,
resource_owner STRING NULL,
instance_id STRING NOT NULL,
PRIMARY KEY (external_user_id, idp_config_id, instance_id)
);
CREATE TABLE auth.tokens (
id STRING NOT NULL,
creation_date TIMESTAMPTZ NULL,
change_date TIMESTAMPTZ NULL,
resource_owner STRING NULL,
application_id STRING NULL,
user_agent_id STRING NULL,
user_id STRING NULL,
expiration TIMESTAMPTZ NULL,
sequence INT8 NULL,
scopes STRING[] NULL,
audience STRING[] NULL,
preferred_language STRING NULL,
refresh_token_id STRING NULL,
is_pat BOOL NOT NULL DEFAULT false,
instance_id STRING NOT NULL,
PRIMARY KEY (id, instance_id),
INDEX user_user_agent_idx (user_id, user_agent_id)
);
CREATE TABLE auth.refresh_tokens (
id STRING NOT NULL,
creation_date TIMESTAMPTZ NULL,
change_date TIMESTAMPTZ NULL,
resource_owner STRING NULL,
token STRING NULL,
client_id STRING NOT NULL,
user_agent_id STRING NOT NULL,
user_id STRING NOT NULL,
auth_time TIMESTAMPTZ NULL,
idle_expiration TIMESTAMPTZ NULL,
expiration TIMESTAMPTZ NULL,
sequence INT8 NULL,
scopes STRING[] NULL,
audience STRING[] NULL,
amr STRING[] NULL,
instance_id STRING NOT NULL,
PRIMARY KEY (id, instance_id),
UNIQUE INDEX unique_client_user_index (client_id, user_agent_id, user_id)
);
CREATE TABLE auth.org_project_mapping (
org_id STRING NOT NULL,
project_id STRING NOT NULL,
project_grant_id STRING NULL,
instance_id STRING NOT NULL,
PRIMARY KEY (org_id, project_id, instance_id)
);
CREATE TABLE auth.idp_providers (
aggregate_id STRING NOT NULL,
idp_config_id STRING NOT NULL,
creation_date TIMESTAMPTZ NULL,
change_date TIMESTAMPTZ NULL,
sequence INT8 NULL,
name STRING NULL,
idp_config_type INT2 NULL,
idp_provider_type INT2 NULL,
idp_state INT2 NULL,
styling_type INT2 NULL,
instance_id STRING NOT NULL,
PRIMARY KEY (aggregate_id, idp_config_id, instance_id)
);
CREATE TABLE auth.idp_configs (
idp_config_id STRING NOT NULL,
creation_date TIMESTAMPTZ NULL,
change_date TIMESTAMPTZ NULL,
sequence INT8 NULL,
aggregate_id STRING NULL,
name STRING NULL,
idp_state INT2 NULL,
idp_provider_type INT2 NULL,
is_oidc BOOL NULL,
oidc_client_id STRING NULL,
oidc_client_secret JSONB NULL,
oidc_issuer STRING NULL,
oidc_scopes STRING[] NULL,
oidc_idp_display_name_mapping INT2 NULL,
oidc_idp_username_mapping INT2 NULL,
styling_type INT2 NULL,
oauth_authorization_endpoint STRING NULL,
oauth_token_endpoint STRING NULL,
auto_register BOOL NULL,
jwt_endpoint STRING NULL,
jwt_keys_endpoint STRING NULL,
jwt_header_name STRING NULL,
instance_id STRING NOT NULL,
PRIMARY KEY (idp_config_id, instance_id)
);
CREATE TABLE auth.auth_requests (
id STRING NOT NULL,
request JSONB NULL,
code STRING NULL,
request_type INT2 NULL,
creation_date TIMESTAMPTZ NULL,
change_date TIMESTAMPTZ NULL,
instance_id STRING NOT NULL,
PRIMARY KEY (id, instance_id),
INDEX auth_code_idx (code)
);

View File

@@ -0,0 +1,55 @@
CREATE SCHEMA notification;
CREATE TABLE notification.locks (
locker_id TEXT,
locked_until TIMESTAMPTZ(3),
view_name TEXT,
instance_id TEXT NOT NULL,
PRIMARY KEY (view_name, instance_id)
);
CREATE TABLE notification.current_sequences (
view_name TEXT,
current_sequence BIGINT,
event_timestamp TIMESTAMPTZ,
last_successful_spooler_run TIMESTAMPTZ,
instance_id TEXT NOT NULL,
PRIMARY KEY (view_name, instance_id)
);
CREATE TABLE notification.failed_events (
view_name TEXT,
failed_sequence BIGINT,
failure_count SMALLINT,
err_msg TEXT,
instance_id TEXT NOT NULL,
PRIMARY KEY (view_name, failed_sequence, instance_id)
);
CREATE TABLE notification.notify_users (
id STRING NOT NULL,
creation_date TIMESTAMPTZ NULL,
change_date TIMESTAMPTZ NULL,
resource_owner STRING NULL,
user_name STRING NULL,
first_name STRING NULL,
last_name STRING NULL,
nick_name STRING NULL,
display_name STRING NULL,
preferred_language STRING NULL,
gender INT2 NULL,
last_email STRING NULL,
verified_email STRING NULL,
last_phone STRING NULL,
verified_phone STRING NULL,
sequence INT8 NULL,
password_set BOOL NULL,
login_names STRING NULL,
preferred_login_name STRING NULL,
instance_id STRING NULL,
PRIMARY KEY (id)
);

View File

@@ -0,0 +1,28 @@
CREATE TABLE projections.locks (
locker_id TEXT,
locked_until TIMESTAMPTZ(3),
projection_name TEXT,
instance_id TEXT NOT NULL,
PRIMARY KEY (projection_name, instance_id)
);
CREATE TABLE projections.current_sequences (
projection_name TEXT,
aggregate_type TEXT,
current_sequence BIGINT,
instance_id TEXT NOT NULL,
timestamp TIMESTAMPTZ,
PRIMARY KEY (projection_name, aggregate_type, instance_id)
);
CREATE TABLE projections.failed_events (
projection_name TEXT,
failed_sequence BIGINT,
failure_count SMALLINT,
error TEXT,
instance_id TEXT NOT NULL,
PRIMARY KEY (projection_name, failed_sequence, instance_id)
);

36
cmd/setup/02.go Normal file
View File

@@ -0,0 +1,36 @@
package setup
import (
"context"
"database/sql"
)
const (
createAssets = `
CREATE TABLE system.assets (
instance_id TEXT,
asset_type TEXT,
resource_owner TEXT,
name TEXT,
content_type TEXT,
hash TEXT AS (md5(data)) STORED,
data BYTES,
updated_at TIMESTAMPTZ,
PRIMARY KEY (instance_id, resource_owner, name)
);
`
)
type AssetTable struct {
dbClient *sql.DB
}
func (mig *AssetTable) Execute(ctx context.Context) error {
_, err := mig.dbClient.ExecContext(ctx, createAssets)
return err
}
func (mig *AssetTable) String() string {
return "02_assets"
}

106
cmd/setup/03.go Normal file
View File

@@ -0,0 +1,106 @@
package setup
import (
"context"
"database/sql"
"fmt"
"strings"
"golang.org/x/text/language"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/command"
"github.com/zitadel/zitadel/internal/config/systemdefaults"
"github.com/zitadel/zitadel/internal/crypto"
crypto_db "github.com/zitadel/zitadel/internal/crypto/database"
"github.com/zitadel/zitadel/internal/eventstore"
)
type DefaultInstance struct {
InstanceName string
CustomDomain string
DefaultLanguage language.Tag
Org command.OrgSetup
instanceSetup command.InstanceSetup
userEncryptionKey *crypto.KeyConfig
smtpEncryptionKey *crypto.KeyConfig
masterKey string
db *sql.DB
es *eventstore.Eventstore
defaults systemdefaults.SystemDefaults
zitadelRoles []authz.RoleMapping
externalDomain string
externalSecure bool
externalPort uint16
}
func (mig *DefaultInstance) Execute(ctx context.Context) error {
keyStorage, err := crypto_db.NewKeyStorage(mig.db, mig.masterKey)
if err != nil {
return fmt.Errorf("cannot start key storage: %w", err)
}
if err = verifyKey(mig.userEncryptionKey, keyStorage); err != nil {
return err
}
userAlg, err := crypto.NewAESCrypto(mig.userEncryptionKey, keyStorage)
if err != nil {
return err
}
if err = verifyKey(mig.smtpEncryptionKey, keyStorage); err != nil {
return err
}
smtpEncryption, err := crypto.NewAESCrypto(mig.smtpEncryptionKey, keyStorage)
if err != nil {
return err
}
cmd, err := command.StartCommands(mig.es,
mig.defaults,
mig.zitadelRoles,
nil,
nil,
mig.externalDomain,
mig.externalSecure,
mig.externalPort,
nil,
nil,
smtpEncryption,
nil,
userAlg,
nil,
nil)
if err != nil {
return err
}
mig.instanceSetup.InstanceName = mig.InstanceName
mig.instanceSetup.CustomDomain = mig.CustomDomain
mig.instanceSetup.DefaultLanguage = mig.DefaultLanguage
mig.instanceSetup.Org = mig.Org
mig.instanceSetup.Org.Human.Email.Address = strings.TrimSpace(mig.instanceSetup.Org.Human.Email.Address)
if mig.instanceSetup.Org.Human.Email.Address == "" {
mig.instanceSetup.Org.Human.Email.Address = "admin@" + mig.instanceSetup.CustomDomain
}
_, _, err = cmd.SetUpInstance(ctx, &mig.instanceSetup)
return err
}
func (mig *DefaultInstance) String() string {
return "03_default_instance"
}
func verifyKey(key *crypto.KeyConfig, storage crypto.KeyStorage) (err error) {
_, err = crypto.LoadKey(key.EncryptionKeyID, storage)
if err == nil {
return nil
}
k, err := crypto.NewKey(key.EncryptionKeyID)
if err != nil {
return err
}
return storage.CreateKeys(k)
}

85
cmd/setup/config.go Normal file
View File

@@ -0,0 +1,85 @@
package setup
import (
"bytes"
"strings"
"github.com/mitchellh/mapstructure"
"github.com/spf13/viper"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/command"
"github.com/zitadel/zitadel/internal/config/hook"
"github.com/zitadel/zitadel/internal/config/systemdefaults"
"github.com/zitadel/zitadel/internal/crypto"
"github.com/zitadel/zitadel/internal/database"
)
type Config struct {
Database database.Config
SystemDefaults systemdefaults.SystemDefaults
InternalAuthZ authz.Config
ExternalDomain string
ExternalPort uint16
ExternalSecure bool
Log *logging.Config
EncryptionKeys *encryptionKeyConfig
DefaultInstance command.InstanceSetup
}
func MustNewConfig(v *viper.Viper) *Config {
config := new(Config)
err := v.Unmarshal(config,
viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
hook.Base64ToBytesHookFunc(),
hook.TagToLanguageHookFunc(),
mapstructure.StringToTimeDurationHookFunc(),
mapstructure.StringToSliceHookFunc(","),
)),
)
logging.OnError(err).Fatal("unable to read default config")
err = config.Log.SetLogger()
logging.OnError(err).Fatal("unable to set logger")
return config
}
type Steps struct {
s1ProjectionTable *ProjectionTable
s2AssetsTable *AssetTable
S3DefaultInstance *DefaultInstance
}
type encryptionKeyConfig struct {
User *crypto.KeyConfig
SMTP *crypto.KeyConfig
}
func MustNewSteps(v *viper.Viper) *Steps {
v.AutomaticEnv()
v.SetEnvPrefix("ZITADEL")
v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
v.SetConfigType("yaml")
err := v.ReadConfig(bytes.NewBuffer(defaultSteps))
logging.OnError(err).Fatal("unable to read setup steps")
for _, file := range stepFiles {
v.SetConfigFile(file)
err := v.MergeInConfig()
logging.WithFields("file", file).OnError(err).Warn("unable to read setup file")
}
steps := new(Steps)
err = v.Unmarshal(steps,
viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
hook.Base64ToBytesHookFunc(),
hook.TagToLanguageHookFunc(),
mapstructure.StringToTimeDurationHookFunc(),
mapstructure.StringToSliceHookFunc(","),
)),
)
logging.OnError(err).Fatal("unable to read steps")
return steps
}

96
cmd/setup/setup.go Normal file
View File

@@ -0,0 +1,96 @@
package setup
import (
"context"
_ "embed"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/cmd/key"
"github.com/zitadel/zitadel/cmd/tls"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/migration"
)
var (
//go:embed steps.yaml
defaultSteps []byte
stepFiles []string
)
func New() *cobra.Command {
cmd := &cobra.Command{
Use: "setup",
Short: "setup ZITADEL instance",
Long: `sets up data to start ZITADEL.
Requirements:
- cockroachdb`,
Run: func(cmd *cobra.Command, args []string) {
err := tls.ModeFromFlag(cmd)
logging.OnError(err).Fatal("invalid tlsMode")
config := MustNewConfig(viper.GetViper())
steps := MustNewSteps(viper.New())
masterKey, err := key.MasterKey(cmd)
logging.OnError(err).Panic("No master key provided")
Setup(config, steps, masterKey)
},
}
Flags(cmd)
return cmd
}
func Flags(cmd *cobra.Command) {
cmd.PersistentFlags().StringArrayVar(&stepFiles, "steps", nil, "paths to step files to overwrite default steps")
key.AddMasterKeyFlag(cmd)
tls.AddTLSModeFlag(cmd)
}
func Setup(config *Config, steps *Steps, masterKey string) {
dbClient, err := database.Connect(config.Database)
logging.OnError(err).Fatal("unable to connect to database")
eventstoreClient, err := eventstore.Start(dbClient)
logging.OnError(err).Fatal("unable to start eventstore")
migration.RegisterMappers(eventstoreClient)
steps.s1ProjectionTable = &ProjectionTable{dbClient: dbClient}
steps.s2AssetsTable = &AssetTable{dbClient: dbClient}
steps.S3DefaultInstance.instanceSetup = config.DefaultInstance
steps.S3DefaultInstance.userEncryptionKey = config.EncryptionKeys.User
steps.S3DefaultInstance.smtpEncryptionKey = config.EncryptionKeys.SMTP
steps.S3DefaultInstance.masterKey = masterKey
steps.S3DefaultInstance.db = dbClient
steps.S3DefaultInstance.es = eventstoreClient
steps.S3DefaultInstance.defaults = config.SystemDefaults
steps.S3DefaultInstance.zitadelRoles = config.InternalAuthZ.RolePermissionMappings
steps.S3DefaultInstance.externalDomain = config.ExternalDomain
steps.S3DefaultInstance.externalSecure = config.ExternalSecure
steps.S3DefaultInstance.externalPort = config.ExternalPort
ctx := context.Background()
err = migration.Migrate(ctx, eventstoreClient, steps.s1ProjectionTable)
logging.OnError(err).Fatal("unable to migrate step 1")
err = migration.Migrate(ctx, eventstoreClient, steps.s2AssetsTable)
logging.OnError(err).Fatal("unable to migrate step 2")
err = migration.Migrate(ctx, eventstoreClient, steps.S3DefaultInstance)
logging.OnError(err).Fatal("unable to migrate step 3")
}
func initSteps(v *viper.Viper, files ...string) func() {
return func() {
for _, file := range files {
v.SetConfigFile(file)
err := v.MergeInConfig()
logging.WithFields("file", file).OnError(err).Warn("unable to read setup file")
}
}
}

22
cmd/setup/steps.yaml Normal file
View File

@@ -0,0 +1,22 @@
S3DefaultInstance:
InstanceName: Localhost
CustomDomain: localhost
DefaultLanguage: en
Org:
Name: ZITADEL
Human:
UserName: zitadel-admin
FirstName: ZITADEL
LastName: Admin
NickName:
DisplayName:
Email:
Address: #autogenerated if empty. uses domain from config and prefixes admin@. for example: admin@domain.tdl
Verified: true
PreferredLanguage: en
Gender:
Phone:
Number:
Verified:
Password: Password1!
PasswordChangeRequired: true