TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-10-21 14:30:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(operator): zitadel and database operator (#1208)

Browse Source 
* feat(operator): add base for zitadel operator

* fix(operator): changed pipeline to release operator

* fix(operator): fmt with only one parameter

* fix(operator): corrected workflow job name

* fix(zitadelctl): added restore and backuplist command

* fix(zitadelctl): scale for restore

* chore(container): use scratch for deploy container

* fix(zitadelctl): limit image to scratch

* fix(migration): added migration scripts for newer version

* fix(operator): changed handling of kubeconfig in operator logic

* fix(operator): changed handling of secrets in operator logic

* fix(operator): use new version of zitadel

* fix(operator): added path for migrations

* fix(operator): delete doublets of migration scripts

* fix(operator): delete subpaths and integrate logic into init container

* fix(operator): corrected path in dockerfile for local migrations

* fix(operator): added migrations for cockroachdb-secure

* fix(operator): delete logic for ambassador module

* fix(operator): added read and write secret commands

* fix(operator): correct and align operator pipeline with zitadel pipeline

* fix(operator): correct yaml error in operator pipeline

* fix(operator): correct action name in operator pipeline

* fix(operator): correct case-sensitive filename in operator pipeline

* fix(operator): upload artifacts from buildx output

* fix(operator): corrected attribute spelling error

* fix(operator): combined jobs for operator binary and image

* fix(operator): added missing comma in operator pipeline

* fix(operator): added codecov for operator image

* fix(operator): added codecov for operator image

* fix(testing): code changes for testing and several unit-tests (#1009)

* fix(operator): usage of interface of kubernetes client for testing and several unit-tests

* fix(operator): several unit-tests

* fix(operator): several unit-tests

* fix(operator): changed order for the operator logic

* fix(operator): added version of zitadelctl from semantic release

* fix(operator): corrected function call with version of zitadelctl

* fix(operator): corrected function call with version of zitadelctl

* fix(operator): add check output to operator release pipeline

* fix(operator): set --short length everywhere to 12

* fix(operator): zitadel setup in job instead of exec with several unit tests

* fix(operator): fixes to combine newest zitadel and testing branch

* fix(operator): corrected path in Dockerfile

* fix(operator): fixed unit-test that was ignored during changes

* fix(operator): fixed unit-test that was ignored during changes

* fix(operator): corrected Dockerfile to correctly use env variable

* fix(operator): quickfix takeoff deployment

* fix(operator): corrected the clusterrolename in the applied artifacts

* fix: update secure migrations

* fix(operator): migrations (#1057)

* fix(operator): copied migrations from orbos repository

* fix(operator): newest migrations

* chore: use cockroach-secure

* fix: rename migration

* fix: remove insecure cockroach migrations

Co-authored-by: Stefan Benz <stefan@caos.ch>

* fix: finalize labels

* fix(operator): cli logging concurrent and fixe deployment of operator during restore

* fix: finalize labels and cli commands

* fix: restore

* chore: cockroachdb is always secure

* chore: use orbos consistent-labels latest commit

* test: make tests compatible with new labels

* fix: default to sa token for start command

* fix: use cockroachdb v12.02

* fix: don't delete flyway user

* test: fix migration test

* fix: use correct table qualifiers

* fix: don't alter sequence ownership

* fix: upgrade flyway

* fix: change ownership of all dbs and tables to admin user

* fix: change defaultdb user

* fix: treat clientid status codes >= 400 as errors

* fix: reconcile specified ZITADEL version, not binary version

* fix: add ca-certs

* fix: use latest orbos code

* fix: use orbos with fixed race condition

* fix: use latest ORBOS code

* fix: use latest ORBOS code

* fix: make migration and scaling around restoring work

* fix(operator): move zitadel operator

* chore(migrations): include owner change migration

* feat(db): add code base for database operator

* fix(db): change used image registry for database operator

* fix(db): generated mock

* fix(db): add accidentally ignored file

* fix(db): add cockroachdb backup image to pipeline

* fix(db): correct pipeline and image versions

* fix(db): correct version of used orbos

* fix(db): correct database import

* fix(db): go mod tidy

* fix(db): use new version for orbos

* fix(migrations): include migrations into zitadelctl binary (#1211)

* fix(db): use statik to integrate migrations into binary

* fix(migrations): corrections unit tests and pipeline for integrated migrations into zitadelctl binary

* fix(migrations): correction in dockerfile for pipeline build

* fix(migrations): correction in dockerfile for pipeline build

* fix(migrations):  dockerfile changes for cache optimization

* fix(database): correct used part-of label in database operator

* fix(database): correct used selectable label in zitadel operator

* fix(operator): correct lables for user secrets in zitadel operator

* fix(operator): correct lables for service test in zitadel operator

* fix: don't enable database features for user operations (#1227)

* fix: don't enable database features for user operations

* fix: omit database feature for connection info adapter

* fix: use latest orbos version

* fix: update ORBOS (#1240)

Co-authored-by: Florian Forster <florian@caos.ch>
Co-authored-by: Elio Bischof <eliobischof@gmail.com>
This commit is contained in:
Stefan Benz
2021-02-05 19:28:12 +01:00
committed by GitHub
parent 3428046ffa
commit ad25f35539
210 changed files with 18515 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
cmd/zitadelctl/cmds/backup.go Normal file
View File

@@ -0,0 +1,73 @@
package cmds
import (
"github.com/caos/orbos/pkg/kubernetes"
"github.com/caos/zitadel/operator/api"
"github.com/caos/zitadel/operator/start"
"github.com/spf13/cobra"
"io/ioutil"
)
func BackupCommand(rv RootValues) *cobra.Command {
var (
kubeconfig string
backup string
cmd = &cobra.Command{
Use: "backup",
Short: "Instant backup",
Long: "Instant backup",
}
)
flags := cmd.Flags()
flags.StringVar(&kubeconfig, "kubeconfig", "~/.kube/config", "Kubeconfig of cluster where the backup should be done")
flags.StringVar(&backup, "backup", "", "Name used for backup folder")
cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
_, monitor, orbConfig, gitClient, version, errFunc, err := rv()
if err != nil {
return err
}
defer func() {
err = errFunc(err)
}()
if err := gitClient.Configure(orbConfig.URL, []byte(orbConfig.Repokey)); err != nil {
return err
}
if err := gitClient.Clone(); err != nil {
return err
}
found, err := api.ExistsDatabaseYml(gitClient)
if err != nil {
return err
}
if found {
value, err := ioutil.ReadFile(kubeconfig)
if err != nil {
monitor.Error(err)
return nil
}
kubeconfigStr := string(value)
k8sClient := kubernetes.NewK8sClient(monitor, &kubeconfigStr)
if k8sClient.Available() {
if err := start.Backup(
monitor,
orbConfig.Path,
k8sClient,
backup,
&version,
); err != nil {
return err
}
}
}
return nil
}
return cmd
}

54
cmd/zitadelctl/cmds/backuplist.go Normal file
View File

@@ -0,0 +1,54 @@
package cmds
import (
"fmt"
"sort"
"github.com/caos/zitadel/pkg/databases"
"github.com/spf13/cobra"
)
func BackupListCommand(rv RootValues) *cobra.Command {
var (
cmd = &cobra.Command{
Use: "backuplist",
Short: "Get a list of all backups",
Long: "Get a list of all backups",
}
)
cmd.RunE = func(cmd *cobra.Command, args []string) error {
_, monitor, orbConfig, gitClient, _, errFunc, err := rv()
if err != nil {
return err
}
defer func() {
err = errFunc(err)
}()
if err := gitClient.Configure(orbConfig.URL, []byte(orbConfig.Repokey)); err != nil {
monitor.Error(err)
return nil
}
if err := gitClient.Clone(); err != nil {
monitor.Error(err)
return nil
}
backups, err := databases.ListBackups(monitor, gitClient)
if err != nil {
monitor.Error(err)
return nil
}
sort.Slice(backups, func(i, j int) bool {
return backups[i] > backups[j]
})
for _, backup := range backups {
fmt.Println(backup)
}
return nil
}
return cmd
}

57
cmd/zitadelctl/cmds/readsecret.go Normal file
View File

@@ -0,0 +1,57 @@
package cmds
import (
"os"
"github.com/caos/orbos/pkg/secret"
"github.com/caos/zitadel/operator/secrets"
"github.com/spf13/cobra"
)
func ReadSecretCommand(rv RootValues) *cobra.Command {
return &cobra.Command{
Use: "readsecret [path]",
Short: "Print a secrets decrypted value to stdout",
Long: "Print a secrets decrypted value to stdout.\nIf no path is provided, a secret can interactively be chosen from a list of all possible secrets",
Args: cobra.MaximumNArgs(1),
Example: `zitadelctl readsecret zitadel.emailappkey > ~/emailappkey`,
RunE: func(cmd *cobra.Command, args []string) error {
_, monitor, orbConfig, gitClient, _, errFunc, err := rv()
if err != nil {
return err
}
defer func() {
err = errFunc(err)
}()
if err := gitClient.Configure(orbConfig.URL, []byte(orbConfig.Repokey)); err != nil {
return err
}
if err := gitClient.Clone(); err != nil {
return err
}
path := ""
if len(args) > 0 {
path = args[0]
}
value, err := secret.Read(
monitor,
gitClient,
path,
secrets.GetAllSecretsFunc(orbConfig))
if err != nil {
monitor.Error(err)
return nil
}
if _, err := os.Stdout.Write([]byte(value)); err != nil {
monitor.Error(err)
return nil
}
return nil
},
}
}

100
cmd/zitadelctl/cmds/restore.go Normal file
View File

@@ -0,0 +1,100 @@
package cmds
import (
"errors"
"io/ioutil"
"github.com/caos/zitadel/operator/helpers"
"github.com/caos/orbos/pkg/kubernetes"
"github.com/caos/zitadel/operator/start"
"github.com/caos/zitadel/pkg/databases"
"github.com/manifoldco/promptui"
"github.com/spf13/cobra"
)
func RestoreCommand(rv RootValues) *cobra.Command {
var (
backup string
kubeconfig string
cmd = &cobra.Command{
Use: "restore",
Short: "Restore from backup",
Long: "Restore from backup",
}
)
flags := cmd.Flags()
flags.StringVar(&backup, "backup", "", "Backup used for db restore")
flags.StringVar(&kubeconfig, "kubeconfig", "~/.kube/config", "Kubeconfig for ZITADEL operator deployment")
cmd.RunE = func(cmd *cobra.Command, args []string) error {
_, monitor, orbConfig, gitClient, version, errFunc, err := rv()
if err != nil {
return err
}
defer func() {
err = errFunc(err)
}()
kubeconfig = helpers.PruneHome(kubeconfig)
if err := gitClient.Configure(orbConfig.URL, []byte(orbConfig.Repokey)); err != nil {
monitor.Error(err)
return nil
}
if err := gitClient.Clone(); err != nil {
monitor.Error(err)
return nil
}
value, err := ioutil.ReadFile(kubeconfig)
if err != nil {
monitor.Error(err)
return nil
}
kubeconfigStr := string(value)
k8sClient := kubernetes.NewK8sClient(monitor, &kubeconfigStr)
if k8sClient.Available() {
list, err := databases.ListBackups(monitor, gitClient)
if err != nil {
monitor.Error(err)
return nil
}
if backup == "" {
prompt := promptui.Select{
Label: "Select backup to restore",
Items: list,
}
_, result, err := prompt.Run()
if err != nil {
monitor.Error(err)
return nil
}
backup = result
}
existing := false
for _, listedBackup := range list {
if listedBackup == backup {
existing = true
}
}
if !existing {
monitor.Error(errors.New("chosen backup is not existing"))
return nil
}
if err := start.Restore(monitor, gitClient, orbConfig, k8sClient, backup, &version); err != nil {
monitor.Error(err)
}
return nil
}
return nil
}
return cmd
}

72
cmd/zitadelctl/cmds/root.go Normal file
View File

@@ -0,0 +1,72 @@
package cmds
import (
"context"
"github.com/caos/orbos/mntr"
"github.com/caos/orbos/pkg/git"
"github.com/caos/orbos/pkg/orb"
"github.com/caos/zitadel/operator/helpers"
"github.com/spf13/cobra"
)
type RootValues func() (context.Context, mntr.Monitor, *orb.Orb, *git.Client, string, errFunc, error)
type errFunc func(err error) error
func RootCommand(version string) (*cobra.Command, RootValues) {
var (
verbose bool
orbConfigPath string
)
cmd := &cobra.Command{
Use: "zitadelctl [flags]",
Short: "Interact with your IAM orbs",
Long: `zitadelctl launches zitadel and simplifies common tasks such as updating your kubeconfig.
Participate in our community on https://github.com/caos/orbos
and visit our website at https://caos.ch`,
Example: `$ mkdir -p ~/.orb
$ cat > ~/.orb/myorb << EOF
> url: git@github.com:me/my-orb.git
> masterkey: "$(gopass my-secrets/orbs/myorb/masterkey)"
> repokey: |
> $(cat ~/.ssh/myorbrepo | sed s/^/\ \ /g)
> EOF
$ orbctl -f ~/.orb/myorb [command]
`,
}
flags := cmd.PersistentFlags()
flags.StringVarP(&orbConfigPath, "orbconfig", "f", "~/.orb/config", "Path to the file containing the orbs git repo URL, deploy key and the master key for encrypting and decrypting secrets")
flags.BoolVar(&verbose, "verbose", false, "Print debug levelled logs")
return cmd, func() (context.Context, mntr.Monitor, *orb.Orb, *git.Client, string, errFunc, error) {
monitor := mntr.Monitor{
OnInfo: mntr.LogMessage,
OnChange: mntr.LogMessage,
OnError: mntr.LogError,
}
if verbose {
monitor = monitor.Verbose()
}
prunedPath := helpers.PruneHome(orbConfigPath)
orbConfig, err := orb.ParseOrbConfig(prunedPath)
if err != nil {
orbConfig = &orb.Orb{Path: prunedPath}
return nil, mntr.Monitor{}, nil, nil, "", nil, err
}
ctx := context.Background()
return ctx, monitor, orbConfig, git.New(ctx, monitor, "orbos", "orbos@caos.ch"), version, func(err error) error {
if err != nil {
monitor.Error(err)
}
return nil
}, nil
}
}

82
cmd/zitadelctl/cmds/start.go Normal file
View File

@@ -0,0 +1,82 @@
package cmds
import (
"github.com/caos/orbos/pkg/kubernetes"
"github.com/caos/zitadel/operator/helpers"
"github.com/caos/zitadel/operator/start"
"github.com/spf13/cobra"
)
func StartOperator(rv RootValues) *cobra.Command {
var (
kubeconfig string
cmd = &cobra.Command{
Use: "operator",
Short: "Launch a ZITADEL operator",
Long: "Ensures a desired state of ZITADEL",
}
)
flags := cmd.Flags()
flags.StringVar(&kubeconfig, "kubeconfig", "", "Kubeconfig for ZITADEL operator deployment")
cmd.RunE = func(cmd *cobra.Command, args []string) error {
_, monitor, orbConfig, _, version, errFunc, err := rv()
if err != nil {
return err
}
defer func() {
err = errFunc(err)
}()
kubeconfig = helpers.PruneHome(kubeconfig)
k8sClient, err := kubernetes.NewK8sClientWithPath(monitor, kubeconfig)
if err != nil {
monitor.Error(err)
return nil
}
if k8sClient.Available() {
if err := start.Operator(monitor, orbConfig.Path, k8sClient, &version); err != nil {
monitor.Error(err)
return nil
}
}
return nil
}
return cmd
}
func StartDatabase(rv RootValues) *cobra.Command {
var (
kubeconfig string
cmd = &cobra.Command{
Use: "database",
Short: "Launch a database operator",
Long: "Ensures a desired state of the database",
}
)
flags := cmd.Flags()
flags.StringVar(&kubeconfig, "kubeconfig", "", "kubeconfig used by zitadel operator")
cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
_, monitor, orbConfig, _, version, errFunc, err := rv()
if err != nil {
return err
}
defer func() {
err = errFunc(err)
}()
k8sClient, err := kubernetes.NewK8sClientWithPath(monitor, kubeconfig)
if err != nil {
return err
}
if k8sClient.Available() {
return start.Database(monitor, orbConfig.Path, k8sClient, &version)
}
return nil
}
return cmd
}

127
cmd/zitadelctl/cmds/takeoff.go Normal file
View File

@@ -0,0 +1,127 @@
package cmds
import (
orbdb "github.com/caos/zitadel/operator/database/kinds/orb"
"io/ioutil"
"github.com/caos/zitadel/operator/helpers"
"github.com/caos/orbos/mntr"
"github.com/caos/orbos/pkg/git"
"github.com/caos/orbos/pkg/kubernetes"
"github.com/caos/zitadel/operator/api"
"github.com/caos/zitadel/operator/zitadel/kinds/orb"
"github.com/spf13/cobra"
)
func TakeoffCommand(rv RootValues) *cobra.Command {
var (
kubeconfig string
cmd = &cobra.Command{
Use: "takeoff",
Short: "Launch a ZITADEL operator on the orb",
Long: "Ensures a desired state of the resources on the orb",
}
)
flags := cmd.Flags()
flags.StringVar(&kubeconfig, "kubeconfig", "~/.kube/config", "Kubeconfig for ZITADEL operator deployment")
cmd.RunE = func(cmd *cobra.Command, args []string) error {
_, monitor, orbConfig, gitClient, _, errFunc, err := rv()
if err != nil {
return err
}
defer func() {
err = errFunc(err)
}()
kubeconfig = helpers.PruneHome(kubeconfig)
if err := gitClient.Configure(orbConfig.URL, []byte(orbConfig.Repokey)); err != nil {
monitor.Error(err)
return nil
}
if err := gitClient.Clone(); err != nil {
monitor.Error(err)
return nil
}
value, err := ioutil.ReadFile(kubeconfig)
if err != nil {
monitor.Error(err)
return nil
}
kubeconfigStr := string(value)
if err := deployOperator(
monitor,
gitClient,
&kubeconfigStr,
); err != nil {
monitor.Error(err)
}
if err := deployDatabase(
monitor,
gitClient,
&kubeconfigStr,
); err != nil {
monitor.Error(err)
}
return nil
}
return cmd
}
func deployOperator(monitor mntr.Monitor, gitClient *git.Client, kubeconfig *string) error {
found, err := api.ExistsZitadelYml(gitClient)
if err != nil {
return err
}
if !found {
monitor.Info("No ZITADEL operator deployed as no zitadel.yml present")
return nil
}
if found {
k8sClient := kubernetes.NewK8sClient(monitor, kubeconfig)
if k8sClient.Available() {
desiredTree, err := api.ReadZitadelYml(gitClient)
if err != nil {
return err
}
if err := orb.Reconcile(monitor, desiredTree, true)(k8sClient); err != nil {
return err
}
}
}
return nil
}
func deployDatabase(monitor mntr.Monitor, gitClient *git.Client, kubeconfig *string) error {
found, err := api.ExistsDatabaseYml(gitClient)
if err != nil {
return err
}
if found {
k8sClient := kubernetes.NewK8sClient(monitor, kubeconfig)
if k8sClient.Available() {
tree, err := api.ReadDatabaseYml(gitClient)
if err != nil {
return err
}
if err := orbdb.Reconcile(
monitor,
tree)(k8sClient); err != nil {
return err
}
} else {
monitor.Info("Failed to connect to k8s")
}
}
return nil
}

115
cmd/zitadelctl/cmds/writesecret.go Normal file
View File

@@ -0,0 +1,115 @@
package cmds
import (
"errors"
"io/ioutil"
"os"
"github.com/caos/orbos/pkg/secret"
"github.com/caos/zitadel/operator/secrets"
"github.com/spf13/cobra"
)
func WriteSecretCommand(rv RootValues) *cobra.Command {
var (
value string
file string
stdin bool
cmd = &cobra.Command{
Use: "writesecret [path]",
Short: "Encrypt a secret and push it to the repository",
Long: "Encrypt a secret and push it to the repository.\nIf no path is provided, a secret can interactively be chosen from a list of all possible secrets",
Args: cobra.MaximumNArgs(1),
Example: `orbctl writesecret mystaticprovider.bootstrapkey --file ~/.ssh/my-orb-bootstrap
orbctl writesecret mygceprovider.google_application_credentials_value --value "$(cat $GOOGLE_APPLICATION_CREDENTIALS)" `,
}
)
flags := cmd.Flags()
flags.StringVar(&value, "value", "", "Secret value to encrypt")
flags.StringVarP(&file, "file", "s", "", "File containing the value to encrypt")
flags.BoolVar(&stdin, "stdin", false, "Value to encrypt is read from standard input")
cmd.RunE = func(cmd *cobra.Command, args []string) error {
_, monitor, orbConfig, gitClient, _, errFunc, err := rv()
if err != nil {
return err
}
defer func() {
err = errFunc(err)
}()
s, err := key(value, file, stdin)
if err != nil {
monitor.Error(err)
return nil
}
if err := gitClient.Configure(orbConfig.URL, []byte(orbConfig.Repokey)); err != nil {
monitor.Error(err)
return nil
}
if err := gitClient.Clone(); err != nil {
monitor.Error(err)
return nil
}
path := ""
if len(args) > 0 {
path = args[0]
}
if err := secret.Write(
monitor,
gitClient,
path,
s,
secrets.GetAllSecretsFunc(orbConfig),
secrets.PushFunc(),
); err != nil {
monitor.Error(err)
}
return nil
}
return cmd
}
func key(value string, file string, stdin bool) (string, error) {
channels := 0
if value != "" {
channels++
}
if file != "" {
channels++
}
if stdin {
channels++
}
if channels != 1 {
return "", errors.New("Key must be provided eighter by value or by file path or by standard input")
}
if value != "" {
return value, nil
}
readFunc := func() ([]byte, error) {
return ioutil.ReadFile(file)
}
if stdin {
readFunc = func() ([]byte, error) {
return ioutil.ReadAll(os.Stdin)
}
}
key, err := readFunc()
if err != nil {
panic(err)
}
return string(key), err
}