feat(operator): zitadel and database operator (#1208)

* feat(operator): add base for zitadel operator

* fix(operator): changed pipeline to release operator

* fix(operator): fmt with only one parameter

* fix(operator): corrected workflow job name

* fix(zitadelctl): added restore and backuplist command

* fix(zitadelctl): scale for restore

* chore(container): use scratch for deploy container

* fix(zitadelctl): limit image to scratch

* fix(migration): added migration scripts for newer version

* fix(operator): changed handling of kubeconfig in operator logic

* fix(operator): changed handling of secrets in operator logic

* fix(operator): use new version of zitadel

* fix(operator): added path for migrations

* fix(operator): delete doublets of migration scripts

* fix(operator): delete subpaths and integrate logic into init container

* fix(operator): corrected path in dockerfile for local migrations

* fix(operator): added migrations for cockroachdb-secure

* fix(operator): delete logic for ambassador module

* fix(operator): added read and write secret commands

* fix(operator): correct and align operator pipeline with zitadel pipeline

* fix(operator): correct yaml error in operator pipeline

* fix(operator): correct action name in operator pipeline

* fix(operator): correct case-sensitive filename in operator pipeline

* fix(operator): upload artifacts from buildx output

* fix(operator): corrected attribute spelling error

* fix(operator): combined jobs for operator binary and image

* fix(operator): added missing comma in operator pipeline

* fix(operator): added codecov for operator image

* fix(operator): added codecov for operator image

* fix(testing): code changes for testing and several unit-tests (#1009)

* fix(operator): usage of interface of kubernetes client for testing and several unit-tests

* fix(operator): several unit-tests

* fix(operator): several unit-tests

* fix(operator): changed order for the operator logic

* fix(operator): added version of zitadelctl from semantic release

* fix(operator): corrected function call with version of zitadelctl

* fix(operator): corrected function call with version of zitadelctl

* fix(operator): add check output to operator release pipeline

* fix(operator): set --short length everywhere to 12

* fix(operator): zitadel setup in job instead of exec with several unit tests

* fix(operator): fixes to combine newest zitadel and testing branch

* fix(operator): corrected path in Dockerfile

* fix(operator): fixed unit-test that was ignored during changes

* fix(operator): fixed unit-test that was ignored during changes

* fix(operator): corrected Dockerfile to correctly use env variable

* fix(operator): quickfix takeoff deployment

* fix(operator): corrected the clusterrolename in the applied artifacts

* fix: update secure migrations

* fix(operator): migrations (#1057)

* fix(operator): copied migrations from orbos repository

* fix(operator): newest migrations

* chore: use cockroach-secure

* fix: rename migration

* fix: remove insecure cockroach migrations

Co-authored-by: Stefan Benz <stefan@caos.ch>

* fix: finalize labels

* fix(operator): cli logging concurrent and fixe deployment of operator during restore

* fix: finalize labels and cli commands

* fix: restore

* chore: cockroachdb is always secure

* chore: use orbos consistent-labels latest commit

* test: make tests compatible with new labels

* fix: default to sa token for start command

* fix: use cockroachdb v12.02

* fix: don't delete flyway user

* test: fix migration test

* fix: use correct table qualifiers

* fix: don't alter sequence ownership

* fix: upgrade flyway

* fix: change ownership of all dbs and tables to admin user

* fix: change defaultdb user

* fix: treat clientid status codes >= 400 as errors

* fix: reconcile specified ZITADEL version, not binary version

* fix: add ca-certs

* fix: use latest orbos code

* fix: use orbos with fixed race condition

* fix: use latest ORBOS code

* fix: use latest ORBOS code

* fix: make migration and scaling around restoring work

* fix(operator): move zitadel operator

* chore(migrations): include owner change migration

* feat(db): add code base for database operator

* fix(db): change used image registry for database operator

* fix(db): generated mock

* fix(db): add accidentally ignored file

* fix(db): add cockroachdb backup image to pipeline

* fix(db): correct pipeline and image versions

* fix(db): correct version of used orbos

* fix(db): correct database import

* fix(db): go mod tidy

* fix(db): use new version for orbos

* fix(migrations): include migrations into zitadelctl binary (#1211)

* fix(db): use statik to integrate migrations into binary

* fix(migrations): corrections unit tests and pipeline for integrated migrations into zitadelctl binary

* fix(migrations): correction in dockerfile for pipeline build

* fix(migrations): correction in dockerfile for pipeline build

* fix(migrations):  dockerfile changes for cache optimization

* fix(database): correct used part-of label in database operator

* fix(database): correct used selectable label in zitadel operator

* fix(operator): correct lables for user secrets in zitadel operator

* fix(operator): correct lables for service test in zitadel operator

* fix: don't enable database features for user operations (#1227)

* fix: don't enable database features for user operations

* fix: omit database feature for connection info adapter

* fix: use latest orbos version

* fix: update ORBOS (#1240)

Co-authored-by: Florian Forster <florian@caos.ch>
Co-authored-by: Elio Bischof <eliobischof@gmail.com>

migrations/cockroach/V1.0__databases.sql

@@ -6,14 +6,14 @@ CREATE DATABASE authz;
 CREATE DATABASE eventstore;
 
 
-CREATE USER eventstore;
+CREATE USER eventstore WITH PASSWORD ${eventstorepassword};
 GRANT SELECT, INSERT ON DATABASE eventstore TO eventstore;
 
-CREATE USER management;
+CREATE USER management WITH PASSWORD ${managementpassword};
 GRANT SELECT, INSERT, UPDATE, DELETE ON DATABASE management TO management;
 GRANT SELECT, INSERT ON DATABASE eventstore TO management;
 
-CREATE USER adminapi;
+CREATE USER adminapi WITH PASSWORD ${adminapipassword};
 GRANT SELECT, INSERT, UPDATE, DELETE, DROP ON DATABASE adminapi TO adminapi;
 GRANT SELECT, INSERT ON DATABASE eventstore TO adminapi;
 GRANT SELECT, INSERT, UPDATE, DROP, DELETE  ON DATABASE auth TO adminapi;
@@ -21,15 +21,15 @@ GRANT SELECT, INSERT, UPDATE, DROP, DELETE ON DATABASE authz TO adminapi;
 GRANT SELECT, INSERT, UPDATE, DROP, DELETE ON DATABASE management TO adminapi;
 GRANT SELECT, INSERT, UPDATE, DROP, DELETE ON DATABASE notification TO adminapi;
 
-CREATE USER auth;
+CREATE USER auth WITH PASSWORD ${authpassword};
 GRANT SELECT, INSERT, UPDATE, DELETE ON DATABASE auth TO auth;
 GRANT SELECT, INSERT ON DATABASE eventstore TO auth;
 
-CREATE USER notification;
+CREATE USER notification WITH PASSWORD ${notificationpassword};
 GRANT SELECT, INSERT, UPDATE, DELETE ON DATABASE notification TO notification;
 GRANT SELECT, INSERT ON DATABASE eventstore TO notification;
 
-CREATE USER authz;
+CREATE USER authz WITH PASSWORD ${authzpassword};
 GRANT SELECT, INSERT, UPDATE, DELETE ON DATABASE authz TO authz;
 GRANT SELECT, INSERT ON DATABASE eventstore TO authz;
 GRANT SELECT, INSERT, UPDATE ON DATABASE auth TO authz;

migrations/cockroach/V1.10__user_machine_keys.sql

@@ -1,3 +1,4 @@
+
 ALTER TABLE management.users ADD COLUMN machine_name STRING, ADD COLUMN machine_description STRING, ADD COLUMN user_type STRING;
 ALTER TABLE adminapi.users ADD COLUMN machine_name STRING, ADD COLUMN machine_description STRING, ADD COLUMN user_type STRING;
 ALTER TABLE auth.users ADD COLUMN machine_name STRING, ADD COLUMN machine_description STRING, ADD COLUMN user_type STRING;
@@ -12,4 +13,4 @@ CREATE TABLE management.machine_keys (
     creation_date TIMESTAMPTZ,
 
     PRIMARY KEY (id, user_id)
-)
+)

migrations/cockroach/V1.14__auth_loginpolicy.sql

@@ -1,3 +1,4 @@
+
 ALTER TABLE adminapi.idp_configs ADD COLUMN oidc_idp_display_name_mapping SMALLINT;
 ALTER TABLE adminapi.idp_configs ADD COLUMN oidc_idp_username_mapping SMALLINT;
 
@@ -27,7 +28,6 @@ CREATE TABLE auth.idp_configs (
     PRIMARY KEY (idp_config_id)
 );
 
-
 CREATE TABLE auth.login_policies (
     aggregate_id TEXT,
 
@@ -58,7 +58,6 @@ CREATE TABLE auth.idp_providers (
     PRIMARY KEY (aggregate_id, idp_config_id)
 );
 
-
 CREATE TABLE auth.user_external_idps (
     external_user_id TEXT,
     idp_config_id TEXT,
@@ -88,7 +87,7 @@ CREATE TABLE management.user_external_idps (
 
     PRIMARY KEY (external_user_id, idp_config_id)
 );
-
+ 
 CREATE TABLE adminapi.user_external_idps (
     idp_config_id TEXT,
     external_user_id TEXT,
@@ -102,4 +101,4 @@ CREATE TABLE adminapi.user_external_idps (
     resource_owner TEXT,
 
     PRIMARY KEY (external_user_id, idp_config_id)
-);
+);

migrations/cockroach/V1.1__eventstore.sql

@@ -1,3 +1,4 @@
+
 CREATE SEQUENCE eventstore.event_seq;
 
 GRANT UPDATE ON TABLE eventstore.event_seq TO management;
@@ -27,3 +28,4 @@ CREATE TABLE eventstore.events (
     INDEX agg_type_agg_id (aggregate_type, aggregate_id),
     CONSTRAINT previous_sequence_unique UNIQUE (previous_sequence DESC)
 );
+ALTER SEQUENCE eventstore.event_seq OWNED BY eventstore.events.event_sequence;

migrations/cockroach/V1.27__database_and_table_ownership.sql

@@ -0,0 +1,82 @@
+ALTER DATABASE adminapi OWNER TO admin;
+ALTER DATABASE auth OWNER TO admin;
+ALTER DATABASE authz OWNER TO admin;
+ALTER DATABASE eventstore OWNER TO admin;
+ALTER DATABASE management OWNER TO admin;
+ALTER DATABASE notification OWNER TO admin;
+ALTER DATABASE defaultdb OWNER TO admin;
+ALTER DATABASE postgres OWNER TO admin;
+ALTER TABLE eventstore.events OWNER TO admin;
+ALTER TABLE management.locks OWNER TO admin;
+ALTER TABLE management.current_sequences OWNER TO admin;
+ALTER TABLE management.failed_events OWNER TO admin;
+ALTER TABLE management.projects OWNER TO admin;
+ALTER TABLE management.project_grants OWNER TO admin;
+ALTER TABLE management.project_roles OWNER TO admin;
+ALTER TABLE management.project_members OWNER TO admin;
+ALTER TABLE management.project_grant_members OWNER TO admin;
+ALTER TABLE management.applications OWNER TO admin;
+ALTER TABLE management.users OWNER TO admin;
+ALTER TABLE management.user_grants OWNER TO admin;
+ALTER TABLE management.org_domains OWNER TO admin;
+ALTER TABLE auth.locks OWNER TO admin;
+ALTER TABLE auth.current_sequences OWNER TO admin;
+ALTER TABLE auth.failed_events OWNER TO admin;
+ALTER TABLE auth.auth_requests OWNER TO admin;
+ALTER TABLE auth.users OWNER TO admin;
+ALTER TABLE auth.user_sessions OWNER TO admin;
+ALTER TABLE auth.tokens OWNER TO admin;
+ALTER TABLE notification.locks OWNER TO admin;
+ALTER TABLE notification.current_sequences OWNER TO admin;
+ALTER TABLE notification.failed_events OWNER TO admin;
+ALTER TABLE notification.notify_users OWNER TO admin;
+ALTER TABLE adminapi.orgs OWNER TO admin;
+ALTER TABLE adminapi.failed_events OWNER TO admin;
+ALTER TABLE adminapi.locks OWNER TO admin;
+ALTER TABLE adminapi.current_sequences OWNER TO admin;
+ALTER TABLE adminapi.iam_members OWNER TO admin;
+ALTER TABLE management.orgs OWNER TO admin;
+ALTER TABLE management.org_members OWNER TO admin;
+ALTER TABLE auth.keys OWNER TO admin;
+ALTER TABLE auth.applications OWNER TO admin;
+ALTER TABLE auth.user_grants OWNER TO admin;
+ALTER TABLE auth.orgs OWNER TO admin;
+ALTER TABLE authz.locks OWNER TO admin;
+ALTER TABLE authz.current_sequences OWNER TO admin;
+ALTER TABLE authz.failed_events OWNER TO admin;
+ALTER TABLE authz.user_grants OWNER TO admin;
+ALTER TABLE authz.applications OWNER TO admin;
+ALTER TABLE authz.orgs OWNER TO admin;
+ALTER TABLE management.user_memberships OWNER TO admin;
+ALTER TABLE adminapi.users OWNER TO admin;
+ALTER TABLE adminapi.idp_configs OWNER TO admin;
+ALTER TABLE management.idp_configs OWNER TO admin;
+ALTER TABLE adminapi.login_policies OWNER TO admin;
+ALTER TABLE management.login_policies OWNER TO admin;
+ALTER TABLE adminapi.idp_providers OWNER TO admin;
+ALTER TABLE management.idp_providers OWNER TO admin;
+ALTER TABLE management.machine_keys OWNER TO admin;
+ALTER TABLE auth.user_memberships OWNER TO admin;
+ALTER TABLE auth.machine_keys OWNER TO admin;
+ALTER TABLE auth.idp_configs OWNER TO admin;
+ALTER TABLE auth.login_policies OWNER TO admin;
+ALTER TABLE auth.idp_providers OWNER TO admin;
+ALTER TABLE auth.user_external_idps OWNER TO admin;
+ALTER TABLE management.user_external_idps OWNER TO admin;
+ALTER TABLE adminapi.user_external_idps OWNER TO admin;
+ALTER TABLE auth.password_complexity_policies OWNER TO admin;
+ALTER TABLE adminapi.password_complexity_policies OWNER TO admin;
+ALTER TABLE management.password_complexity_policies OWNER TO admin;
+ALTER TABLE auth.password_age_policies OWNER TO admin;
+ALTER TABLE adminapi.password_age_policies OWNER TO admin;
+ALTER TABLE management.password_age_policies OWNER TO admin;
+ALTER TABLE auth.password_lockout_policies OWNER TO admin;
+ALTER TABLE adminapi.password_lockout_policies OWNER TO admin;
+ALTER TABLE management.password_lockout_policies OWNER TO admin;
+ALTER TABLE management.org_iam_policies OWNER TO admin;
+ALTER TABLE auth.org_iam_policies OWNER TO admin;
+ALTER TABLE adminapi.org_iam_policies OWNER TO admin;
+ALTER TABLE auth.project_roles OWNER TO admin;
+ALTER TABLE adminapi.label_policies OWNER TO admin;
+ALTER TABLE management.label_policies OWNER TO admin;
+ALTER TABLE defaultdb.flyway_schema_history OWNER TO admin;

migrations/cockroach/V1.28__templates.sql

@@ -75,4 +75,9 @@ GRANT SELECT ON TABLE management.mail_texts TO notification;
 
 
 ALTER TABLE management.project_roles ADD COLUMN change_date TIMESTAMPTZ;
-ALTER TABLE auth.project_roles ADD COLUMN change_date TIMESTAMPTZ;
+ALTER TABLE auth.project_roles ADD COLUMN change_date TIMESTAMPTZ;
+
+ALTER TABLE management.mail_texts OWNER TO admin;
+ALTER TABLE adminapi.mail_texts OWNER TO admin;
+ALTER TABLE management.mail_templates OWNER TO admin;
+ALTER TABLE adminapi.mail_templates OWNER TO admin;

migrations/cockroach/clean_cluster.go

@@ -1,5 +0,0 @@
-//+build ignore
-
-package migrations
-
-//go:generate flyway -url=jdbc:postgresql://cockroachdb-public:26257/defaultdb -user=root -password= -locations=filesystem:./ clean

migrations/cockroach/clean_local.go

@@ -1,5 +0,0 @@
-//+build ignore
-
-package migrations
-
-//go:generate flyway -url=jdbc:postgresql://localhost:26257/defaultdb -user=root -password= -locations=filesystem:./ clean

migrations/cockroach/migrate_cluster.go

@@ -1,5 +0,0 @@
-//+build ignore
-
-package migrations
-
-//go:generate flyway -url=jdbc:postgresql://cockroachdb-public:26257/defaultdb -user=root -password= -locations=filesystem:./ migrate

migrations/cockroach/migrate_local.go

@@ -1,5 +0,0 @@
-//+build ignore
-
-package migrations
-
-//go:generate flyway -url=jdbc:postgresql://localhost:26257/defaultdb -user=root -password= -locations=filesystem:./ migrate

migrations/testfiles/1.2__not.sql

@@ -0,0 +1 @@
+not

migrations/testfiles/V1.1__test.sql

@@ -0,0 +1 @@
+test

migrations/testfiles/V1.2__test2.sql

@@ -0,0 +1 @@
+test2