feat: comprehensive sentry instrumentation (#2023)

* feat: comprehensive sentry instrumentation

* test: pass

* fix: only fetch zitadel dsn in zitadel-operator

* chore: use dns for sentry environment as soon as parsed

* fix: trust ca certs

* ci: update orbos

* docs: add usage data explanation

* fix: dont send validation errors

* docs: improve ingestion data explanation

* style: rename flag --disable-ingestion to --disable-analytics

* fix: pass --disable-analytics flag to self deployments

* fix: destroy command for sentry

* fix: update orbos

* fix: only switch environment if analytics is enabled

* fix: ensure SENTRY_DSN is always set

* test: test empty sentry dsn

* ci: invalidate build caches

* chore: use zitadel-dev if no version is passed

* chore: combine dev releases in sentry

* refactor: only check for semrel if sentry is enabled

cmd/zitadel/main.go

@@ -3,7 +3,9 @@ package main
 import (
 	"context"
 	"flag"
+	"fmt"
 	"os"
+	"regexp"
 	"strconv"
 	"time"
 
@@ -41,6 +43,9 @@ import (
 	"github.com/caos/zitadel/openapi"
 )
 
+// build argument
+var version = "dev"
+
 type Config struct {
 	Log            logging.Config
 	Tracing        tracing.TracingConfig
@@ -94,8 +99,16 @@ const (
 
 func main() {
 	enableSentry, _ := strconv.ParseBool(os.Getenv("SENTRY_USAGE"))
+
 	if enableSentry {
-		err := sentry.Init(sentry.ClientOptions{})
+		sentryVersion := version
+		if !regexp.MustCompile("^v?[0-9]+.[0-9]+.[0-9]$").Match([]byte(version)) {
+			sentryVersion = "dev"
+		}
+		err := sentry.Init(sentry.ClientOptions{
+			Environment: os.Getenv("SENTRY_ENVIRONMENT"),
+			Release:     fmt.Sprintf("zitadel-%s", sentryVersion),
+		})
 		if err != nil {
 			logging.Log("MAIN-Gnzjw").WithError(err).Fatal("sentry init failed")
 		}

cmd/zitadelctl/cmds/backup.go

@@ -21,10 +21,7 @@ func BackupCommand(getRv GetRootValues) *cobra.Command {
 	flags.StringVar(&backup, "backup", "", "Name used for backup folder")
 
 	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
-		rv, err := getRv()
-		if err != nil {
-			return err
-		}
+		rv := getRv("backup", map[string]interface{}{"backup": backup}, "")
 		defer func() {
 			err = rv.ErrFunc(err)
 		}()

cmd/zitadelctl/cmds/backuplist.go

@@ -4,10 +4,11 @@ import (
 	"fmt"
 	"sort"
 
+	"github.com/spf13/cobra"
+
 	"github.com/caos/orbos/pkg/kubernetes/cli"
 
 	"github.com/caos/zitadel/pkg/databases"
-	"github.com/spf13/cobra"
 )
 
 func BackupListCommand(getRv GetRootValues) *cobra.Command {
@@ -19,11 +20,8 @@ func BackupListCommand(getRv GetRootValues) *cobra.Command {
 		}
 	)
 
-	cmd.RunE = func(cmd *cobra.Command, args []string) error {
-		rv, err := getRv()
-		if err != nil {
-			return err
-		}
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		rv := getRv("backuplist", nil, "")
 		defer func() {
 			err = rv.ErrFunc(err)
 		}()
@@ -41,15 +39,13 @@ func BackupListCommand(getRv GetRootValues) *cobra.Command {
 		if rv.Gitops {
 			backupsT, err := databases.GitOpsListBackups(monitor, gitClient, k8sClient)
 			if err != nil {
-				monitor.Error(err)
-				return nil
+				return err
 			}
 			backups = backupsT
 		} else {
 			backupsT, err := databases.CrdListBackups(monitor, k8sClient)
 			if err != nil {
-				monitor.Error(err)
-				return nil
+				return err
 			}
 			backups = backupsT
 		}

cmd/zitadelctl/cmds/configure.go

@@ -3,6 +3,8 @@ package cmds
 import (
 	"errors"
 
+	"github.com/caos/orbos/mntr"
+
 	"github.com/caos/orbos/pkg/tree"
 
 	"github.com/caos/orbos/pkg/cfg"
@@ -35,13 +37,13 @@ func ConfigCommand(getRv GetRootValues, ghClientID, ghClientSecret string) *cobr
 
 	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
 
-		rv, _ := getRv()
+		rv := getRv("configure", map[string]interface{}{"masterkey": newMasterKey != "", "newRepoURL": newRepoURL}, "")
 		defer func() {
 			err = rv.ErrFunc(err)
 		}()
 
 		if !rv.Gitops {
-			return errors.New("configure command is only supported with the --gitops flag")
+			return mntr.ToUserError(errors.New("configure command is only supported with the --gitops flag"))
 		}
 
 		if err := orb.Reconfigure(rv.Ctx, rv.Monitor, rv.OrbConfig, newRepoURL, newMasterKey, rv.GitClient, ghClientID, ghClientSecret); err != nil {

cmd/zitadelctl/cmds/destroy.go

@@ -46,10 +46,7 @@ func TeardownCommand(getRv GetRootValues) *cobra.Command {
 
 	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
 
-		rv, err := getRv()
-		if err != nil {
-			return err
-		}
+		rv := getRv("destroy", nil, "")
 		defer func() {
 			err = rv.ErrFunc(err)
 		}()

cmd/zitadelctl/cmds/readsecret.go

@@ -18,11 +18,14 @@ func ReadSecretCommand(getRv GetRootValues) *cobra.Command {
 		Long:    "Print a secrets decrypted value to stdout.\nIf no path is provided, a secret can interactively be chosen from a list of all possible secrets",
 		Args:    cobra.MaximumNArgs(1),
 		Example: `zitadelctl readsecret database.bucket.serviceaccountjson.encrypted > ~/googlecloudstoragesa.json`,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			rv, err := getRv()
-			if err != nil {
-				return err
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+
+			path := ""
+			if len(args) > 0 {
+				path = args[0]
 			}
+
+			rv := getRv("readsecret", map[string]interface{}{"path": path}, "")
 			defer func() {
 				err = rv.ErrFunc(err)
 			}()
@@ -31,11 +34,6 @@ func ReadSecretCommand(getRv GetRootValues) *cobra.Command {
 			orbConfig := rv.OrbConfig
 			gitClient := rv.GitClient
 
-			path := ""
-			if len(args) > 0 {
-				path = args[0]
-			}
-
 			k8sClient, err := cli.Client(monitor, orbConfig, gitClient, rv.Kubeconfig, rv.Gitops, true)
 			if err != nil && !rv.Gitops {
 				return err
@@ -47,13 +45,11 @@ func ReadSecretCommand(getRv GetRootValues) *cobra.Command {
 				secrets.GetAllSecretsFunc(monitor, path == "", rv.Gitops, gitClient, k8sClient, orbConfig),
 			)
 			if err != nil {
-				monitor.Error(err)
-				return nil
+				return err
 			}
 
 			if _, err := os.Stdout.Write([]byte(value)); err != nil {
-				monitor.Error(err)
-				return nil
+				return err
 			}
 			return nil
 		},

cmd/zitadelctl/cmds/restore.go

@@ -2,8 +2,11 @@ package cmds
 
 import (
 	"errors"
+
 	"github.com/caos/zitadel/pkg/zitadel"
 
+	"github.com/caos/orbos/mntr"
+
 	"github.com/caos/orbos/pkg/kubernetes/cli"
 
 	"github.com/caos/zitadel/pkg/databases"
@@ -24,11 +27,8 @@ func RestoreCommand(getRv GetRootValues) *cobra.Command {
 	flags := cmd.Flags()
 	flags.StringVar(&backup, "backup", "", "Backup used for db restore")
 
-	cmd.RunE = func(cmd *cobra.Command, args []string) error {
-		rv, err := getRv()
-		if err != nil {
-			return err
-		}
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		rv := getRv("restore", map[string]interface{}{"backup": backup}, "")
 		defer func() {
 			err = rv.ErrFunc(err)
 		}()
@@ -48,15 +48,13 @@ func RestoreCommand(getRv GetRootValues) *cobra.Command {
 		if rv.Gitops {
 			listT, err := databases.GitOpsListBackups(monitor, gitClient, k8sClient)
 			if err != nil {
-				monitor.Error(err)
-				return nil
+				return err
 			}
 			list = listT
 		} else {
 			listT, err := databases.CrdListBackups(monitor, k8sClient)
 			if err != nil {
-				monitor.Error(err)
-				return nil
+				return err
 			}
 			list = listT
 		}
@@ -69,8 +67,7 @@ func RestoreCommand(getRv GetRootValues) *cobra.Command {
 
 			_, result, err := prompt.Run()
 			if err != nil {
-				monitor.Error(err)
-				return nil
+				return err
 			}
 			backup = result
 		}
@@ -82,18 +79,15 @@ func RestoreCommand(getRv GetRootValues) *cobra.Command {
 		}
 
 		if !existing {
-			monitor.Error(errors.New("chosen backup is not existing"))
-			return nil
+			return mntr.ToUserError(errors.New("chosen backup is not existing"))
 		}
 
 		if rv.Gitops {
 			if err := zitadel.GitOpsClearMigrateRestore(monitor, gitClient, orbConfig, k8sClient, backup, &version); err != nil {
-				monitor.Error(err)
 				return err
 			}
 		} else {
 			if err := zitadel.CrdClearMigrateRestore(monitor, k8sClient, backup, &version); err != nil {
-				monitor.Error(err)
 				return err
 			}
 

cmd/zitadelctl/cmds/root.go

@@ -22,20 +22,15 @@ type RootValues struct {
 	ErrFunc    errFunc
 }
 
-type GetRootValues func() (*RootValues, error)
+type GetRootValues func(command string, tags map[string]interface{}, component string, moreComponents ...string) *RootValues
 
 type errFunc func(err error) error
 
-func RootCommand(version string) (*cobra.Command, GetRootValues) {
+func RootCommand(version string, monitor mntr.Monitor) (*cobra.Command, GetRootValues) {
 
 	var (
-		ctx     = context.Background()
-		monitor = mntr.Monitor{
-			OnInfo:   mntr.LogMessage,
-			OnChange: mntr.LogMessage,
-			OnError:  mntr.LogError,
-		}
-		rv = &RootValues{
+		ctx = context.Background()
+		rv  = &RootValues{
 			Ctx:     ctx,
 			Version: version,
 			ErrFunc: func(err error) error {
@@ -46,8 +41,9 @@ func RootCommand(version string) (*cobra.Command, GetRootValues) {
 				return nil
 			},
 		}
-		orbConfigPath string
-		verbose       bool
+		orbConfigPath    string
+		verbose          bool
+		disableAnalytics bool
 	)
 	cmd := &cobra.Command{
 		Use:   "zitadelctl [flags]",
@@ -77,8 +73,9 @@ $ zitadelctl --gitops -f ~/.orb/myorb [command]
 	flags.StringVarP(&orbConfigPath, "orbconfig", "f", "~/.orb/config", "Path to the file containing the orbs git repo URL, deploy key and the master key for encrypting and decrypting secrets")
 	flags.StringVarP(&rv.Kubeconfig, "kubeconfig", "k", "~/.kube/config", "Path to the kubeconfig file to the cluster zitadelctl should target")
 	flags.BoolVar(&verbose, "verbose", false, "Print debug levelled logs")
+	flags.BoolVar(&disableAnalytics, "disable-analytics", false, "Don't help CAOS AG to improve ZITADEL by sending them errors and usage data")
 
-	return cmd, func() (*RootValues, error) {
+	return cmd, func(command string, tags map[string]interface{}, component string, moreComponents ...string) *RootValues {
 
 		if verbose {
 			monitor = monitor.Verbose()
@@ -88,15 +85,31 @@ $ zitadelctl --gitops -f ~/.orb/myorb [command]
 		rv.Kubeconfig = helpers.PruneHome(rv.Kubeconfig)
 		rv.GitClient = git.New(ctx, monitor, "zitadel", "orbos@caos.ch")
 
-		var err error
 		if rv.Gitops {
 			prunedPath := helpers.PruneHome(orbConfigPath)
-			rv.OrbConfig, err = orb.ParseOrbConfig(prunedPath)
+			rv.OrbConfig, _ = orb.ParseOrbConfig(prunedPath)
 			if rv.OrbConfig == nil {
 				rv.OrbConfig = &orb.Orb{Path: prunedPath}
 			}
 		}
 
-		return rv, err
+		env := "unknown"
+		if orbID, err := rv.OrbConfig.ID(); err == nil {
+			env = orbID
+		}
+
+		if component == "" {
+			component = "zitadelctl"
+		}
+
+		if !disableAnalytics {
+			if err := mntr.Ingest(rv.Monitor, "zitadel", version, env, component, moreComponents...); err != nil {
+				panic(err)
+			}
+		}
+
+		rv.Monitor.WithFields(map[string]interface{}{"command": command, "gitops": rv.Gitops}).WithFields(tags).CaptureMessage("zitadelctl invoked")
+
+		return rv
 	}
 }

cmd/zitadelctl/cmds/start.go

@@ -19,11 +19,8 @@ func StartOperator(getRv GetRootValues) *cobra.Command {
 	flags := cmd.Flags()
 	flags.StringVar(&metricsAddr, "metrics-addr", "", "The address the metric endpoint binds to.")
 
-	cmd.RunE = func(cmd *cobra.Command, args []string) error {
-		rv, err := getRv()
-		if err != nil {
-			return err
-		}
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		rv := getRv("operator", nil, "zitadel-operator", "zitadel")
 		defer func() {
 			err = rv.ErrFunc(err)
 		}()
@@ -63,10 +60,7 @@ func StartDatabase(getRv GetRootValues) *cobra.Command {
 	flags.StringVar(&metricsAddr, "metrics-addr", "", "The address the metric endpoint binds to.")
 
 	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
-		rv, err := getRv()
-		if err != nil {
-			return err
-		}
+		rv := getRv("database", nil, "database-operator")
 		defer func() {
 			err = rv.ErrFunc(err)
 		}()

cmd/zitadelctl/cmds/takeoff.go

@@ -23,11 +23,8 @@ func TakeoffCommand(getRv GetRootValues) *cobra.Command {
 		}
 	)
 
-	cmd.RunE = func(cmd *cobra.Command, args []string) error {
-		rv, err := getRv()
-		if err != nil {
-			return err
-		}
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		rv := getRv("takeoff", nil, "")
 		defer func() {
 			err = rv.ErrFunc(err)
 		}()
@@ -49,7 +46,6 @@ func TakeoffCommand(getRv GetRootValues) *cobra.Command {
 		}
 
 		if err := kubernetes.EnsureCaosSystemNamespace(monitor, k8sClient); err != nil {
-			monitor.Info("failed to apply common resources into k8s-cluster")
 			return err
 		}
 
@@ -61,7 +57,6 @@ func TakeoffCommand(getRv GetRootValues) *cobra.Command {
 			}
 
 			if err := kubernetes.EnsureOrbconfigSecret(monitor, k8sClient, orbConfigBytes); err != nil {
-				monitor.Info("failed to apply configuration resources into k8s-cluster")
 				return err
 			}
 		}
@@ -73,45 +68,23 @@ func TakeoffCommand(getRv GetRootValues) *cobra.Command {
 			rv.Version,
 			rv.Gitops,
 		); err != nil {
-			monitor.Error(err)
+			return err
 		}
 
-		if err := deployDatabase(
+		return deployDatabase(
 			monitor,
 			gitClient,
 			k8sClient,
 			rv.Version,
 			rv.Gitops,
-		); err != nil {
-			monitor.Error(err)
-		}
-		return nil
+		)
 	}
 	return cmd
 }
 
 func deployOperator(monitor mntr.Monitor, gitClient *git.Client, k8sClient kubernetes.ClientInt, version string, gitops bool) error {
-	if gitops {
-		if gitClient.Exists(git.ZitadelFile) {
+	if !gitops {
 
-			desiredTree, err := gitClient.ReadTree(git.ZitadelFile)
-			if err != nil {
-				return err
-			}
-			desired, err := orbzit.ParseDesiredV0(desiredTree)
-			if err != nil {
-				return err
-			}
-			spec := desired.Spec
-
-			// at takeoff the artifacts have to be applied
-			spec.SelfReconciling = true
-			rec, _ := orbzit.Reconcile(monitor, spec, gitops)
-			if err := rec(k8sClient); err != nil {
-				return err
-			}
-		}
-	} else {
 		// at takeoff the artifacts have to be applied
 		spec := &orbzit.Spec{
 			Version:         version,
@@ -119,35 +92,33 @@ func deployOperator(monitor mntr.Monitor, gitClient *git.Client, k8sClient kuber
 		}
 
 		rec, _ := orbzit.Reconcile(monitor, spec, gitops)
-		if err := rec(k8sClient); err != nil {
-			return err
-		}
+		return rec(k8sClient)
 	}
 
-	return nil
+	if !gitClient.Exists(git.ZitadelFile) {
+		monitor.WithField("file", git.ZitadelFile).Info("File not found in git, skipping deployment")
+		return nil
+	}
+
+	desiredTree, err := gitClient.ReadTree(git.ZitadelFile)
+	if err != nil {
+		return err
+	}
+	desired, err := orbzit.ParseDesiredV0(desiredTree)
+	if err != nil {
+		return err
+	}
+	spec := desired.Spec
+
+	// at takeoff the artifacts have to be applied
+	spec.SelfReconciling = true
+	rec, _ := orbzit.Reconcile(monitor, spec, gitops)
+	return rec(k8sClient)
 }
 
 func deployDatabase(monitor mntr.Monitor, gitClient *git.Client, k8sClient kubernetes.ClientInt, version string, gitops bool) error {
-	if gitops {
-		if gitClient.Exists(git.DatabaseFile) {
-			desiredTree, err := gitClient.ReadTree(git.DatabaseFile)
-			if err != nil {
-				return err
-			}
-			desired, err := orbdb.ParseDesiredV0(desiredTree)
-			if err != nil {
-				return err
-			}
-			spec := desired.Spec
+	if !gitops {
 
-			// at takeoff the artifacts have to be applied
-			spec.SelfReconciling = true
-			rec, _ := orbdb.Reconcile(monitor, desired.Spec, gitops)
-			if err := rec(k8sClient); err != nil {
-				return err
-			}
-		}
-	} else {
 		// at takeoff the artifacts have to be applied
 		spec := &orbdb.Spec{
 			Version:         version,
@@ -155,9 +126,25 @@ func deployDatabase(monitor mntr.Monitor, gitClient *git.Client, k8sClient kuber
 		}
 
 		rec, _ := orbdb.Reconcile(monitor, spec, gitops)
-		if err := rec(k8sClient); err != nil {
-			return err
-		}
+		return rec(k8sClient)
 	}
-	return nil
+
+	if !gitClient.Exists(git.DatabaseFile) {
+		monitor.WithField("file", git.DatabaseFile).Info("File not found in git, skipping deployment")
+		return nil
+	}
+	desiredTree, err := gitClient.ReadTree(git.DatabaseFile)
+	if err != nil {
+		return err
+	}
+	desired, err := orbdb.ParseDesiredV0(desiredTree)
+	if err != nil {
+		return err
+	}
+	spec := desired.Spec
+
+	// at takeoff the artifacts have to be applied
+	spec.SelfReconciling = true
+	rec, _ := orbdb.Reconcile(monitor, spec, gitops)
+	return rec(k8sClient)
 }

cmd/zitadelctl/cmds/writesecret.go

@@ -35,11 +35,14 @@ cat ~/googlecloudstoragesa.json | zitadelctl writesecret database.bucket.service
 	flags.StringVarP(&file, "file", "s", "", "File containing the value to encrypt")
 	flags.BoolVar(&stdin, "stdin", false, "Value to encrypt is read from standard input")
 
-	cmd.RunE = func(cmd *cobra.Command, args []string) error {
-		rv, err := getRv()
-		if err != nil {
-			return err
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+
+		path := ""
+		if len(args) > 0 {
+			path = args[0]
 		}
+
+		rv := getRv("writesecret", map[string]interface{}{"path": path, "value": value != "", "file": file, "stdin": stdin}, "")
 		defer func() {
 			err = rv.ErrFunc(err)
 		}()
@@ -50,13 +53,7 @@ cat ~/googlecloudstoragesa.json | zitadelctl writesecret database.bucket.service
 
 		s, err := key(value, file, stdin)
 		if err != nil {
-			monitor.Error(err)
-			return nil
-		}
-
-		path := ""
-		if len(args) > 0 {
-			path = args[0]
+			return err
 		}
 
 		k8sClient, err := cli.Client(monitor, orbConfig, gitClient, rv.Kubeconfig, rv.Gitops, true)
@@ -64,7 +61,7 @@ cat ~/googlecloudstoragesa.json | zitadelctl writesecret database.bucket.service
 			return err
 		}
 
-		if err := secret.Write(
+		return secret.Write(
 			monitor,
 			k8sClient,
 			path,
@@ -73,10 +70,7 @@ cat ~/googlecloudstoragesa.json | zitadelctl writesecret database.bucket.service
 			fmt.Sprintf(rv.Version),
 			secrets.GetAllSecretsFunc(monitor, path != "", rv.Gitops, gitClient, k8sClient, orbConfig),
 			secrets.PushFunc(monitor, rv.Gitops, gitClient, k8sClient),
-		); err != nil {
-			monitor.Error(err)
-		}
-		return nil
+		)
 	}
 	return cmd
 }

cmd/zitadelctl/main.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/caos/orbos/mntr"
+
 	"github.com/caos/zitadel/cmd/zitadelctl/cmds"
 )
 
@@ -14,7 +16,16 @@ var (
 )
 
 func main() {
-	rootCmd, rootValues := cmds.RootCommand(Version)
+	monitor := mntr.Monitor{
+		OnInfo:         mntr.LogMessage,
+		OnChange:       mntr.LogMessage,
+		OnError:        mntr.LogError,
+		OnRecoverPanic: mntr.LogPanic,
+	}
+
+	defer func() { monitor.RecoverPanic(recover()) }()
+
+	rootCmd, rootValues := cmds.RootCommand(Version, monitor)
 	rootCmd.Version = fmt.Sprintf("%s\n", Version)
 
 	rootCmd.AddCommand(