Merge branch 'main' into next

# Conflicts: # .releaserc.js
2025-08-11 18:07:31 +00:00 · 2023-07-21 07:47:54 +02:00
parent 5060c7463a ccc903a13e
commit e2644cf076
407 changed files with 35021 additions and 23014 deletions
--- a/cmd/defaults.yaml
+++ b/cmd/defaults.yaml
@@ -14,6 +14,29 @@ Tracing:
  Fraction: 1.0
  MetricPrefix: zitadel

+Telemetry:
+  # As long as Enabled is true, ZITADEL tries to send usage data to the configured Telemetry.Endpoints.
+  # Data is projected by ZITADEL even if Enabled is false.
+  # This means that switching this to true makes ZITADEL try to send past data.
+  Enabled: false
+  # Push telemetry data to all these endpoints at least once using an HTTP POST request.
+  # If one endpoint returns an unsuccessful response code or times out,
+  # ZITADEL retries to push the data point to all configured endpoints until it succeeds.
+  # Configure delivery guarantees and intervals in the section Projections.Customizations.Telemetry
+  # The endpoints can be reconfigured at runtime.
+  # Ten redirects are followed.
+  # If you change this configuration at runtime, remaining data that is not successfully delivered to the old endpoints is sent to the new endpoints.
+  Endpoints:
+    - https://httpbin.org/post
+  # These headers are sent with every request to the configured endpoints.
+  Headers:
+  # single-value: "single-value"
+  # multi-value:
+  #   - "multi-value-1"
+  #   - "multi-value-2"
+  # The maximum number of data points that are queried before they are sent to the configured endpoints.
+  Limit: 100 # ZITADEL_TELEMETRY_LIMIT
+
 # Port ZITADEL will listen on
 Port: 8080
 # Port ZITADEL is exposed on, it can differ from port e.g. if you proxy the traffic
@@ -169,17 +192,29 @@ Projections:
      BulkLimit: 2000
    # The Notifications projection is used for sending emails and SMS to users
    Notifications:
-      # As notification projections don't result in database statements, retries don't have an effect
+      # As notification projections don't result in database statements, retries don't have any effects
      MaxFailureCount: 0
    # The NotificationsQuotas projection is used for calling quota webhooks
    NotificationsQuotas:
-      # Delivery guarantee requirements are probably higher for quota webhooks
+      # In case of failed deliveries, ZITADEL retries to send the data points to the configured endpoints, but only for active instances.
+      # An instance is active, as long as there are projected events on the instance, that are not older than the HandleActiveInstances duration.
+      # Delivery guarantee requirements are higher for quota webhooks
      # Defaults to 45 days
      HandleActiveInstances: 1080h
-      # As quota notification projections don't result in database statements, retries don't have an effect
+      # As quota notification projections don't result in database statements, retries don't have any effects
      MaxFailureCount: 0
-      # Quota notifications are not so time critical. Setting RequeueEvery every five minutes doesn't annoy the db too much.
+      # Quota notifications are not so time critical. Setting RequeueEvery every five minutes doesn't annoy the database too much.
      RequeueEvery: 300s
+    Telemetry:
+      # In case of failed deliveries, ZITADEL retries to send the data points to the configured endpoints, but only for active instances.
+      # An instance is active, as long as there are projected events on the instance, that are not older than the HandleActiveInstances duration.
+      # Telemetry delivery guarantee requirements are a bit higher than normal data projections, as they are not interactively retryable.
+      # Defaults to 15 days
+      HandleActiveInstances: 360h
+      # As sending telemetry data doesn't result in database statements, retries don't have any effects
+      MaxFailureCount: 0
+      # Telemetry data synchronization is not time critical. Setting RequeueEvery to 55 minutes doesn't annoy the database too much.
+      RequeueEvery: 3300s

 Auth:
  SearchLimit: 1000
@@ -235,6 +270,8 @@ OIDC:
      Path: /oauth/v2/keys
    DeviceAuth:
      Path: /oauth/v2/device_authorization
+  DefaultLoginURLV2: "/login?authRequest="
+  DefaultLogoutURLV2: "/logout?post_logout_redirect="

 SAML:
  ProviderConfig:
@@ -320,6 +357,51 @@ SystemDefaults:
    PasswordSaltCost: 14
    MachineKeySize: 2048
    ApplicationKeySize: 2048
+  PasswordHasher:
+  # Set hasher configuration for user passwords.
+  # Passwords previously hashed with a different algorithm
+  # or cost are automatically re-hashed using this config,
+  # upon password validation or update.
+    Hasher:
+      Algorithm: "bcrypt"
+      Cost: 14
+
+    # Other supported Hasher configs:
+
+    # Hasher:
+    #   Algorithm: "argon2i"
+    #   Time: 3
+    #   Memory: 32768
+    #   Threads: 4
+
+    # Hasher:
+    #   Algorithm: "argon2id"
+    #   Time: 1
+    #   Memory: 65536
+    #   Threads: 4
+
+    # Hasher:
+    #   Algorithm: "scrypt"
+    #   Cost: 15
+    
+    # Verifiers enable the possibility of verifying
+    # passwords that are previously hashed using another
+    # algorithm then the Hasher.
+    # This can be used when migrating from one algorithm to another,
+    # or when importing users with hashed passwords.
+    # There is no need to enable a Verifier of the same algorithm
+    # as the Hasher.
+    #
+    # The format of the encoded hash strings must comply
+    # with https://github.com/P-H-C/phc-string-format/blob/master/phc-sf-spec.md
+    # https://passlib.readthedocs.io/en/stable/modular_crypt_format.html
+    #
+    # Supported verifiers: (uncomment to enable)
+    # Verifiers:
+    #   - "argon2" # verifier for both argon2i and argon2id.
+    #   - "bcrypt"
+    #   - "md5"
+    #   - "scrypt"
  Multifactors:
    OTP:
      # If this is empty, the issuer is the requested domain
@@ -1037,6 +1119,10 @@ InternalAuthZ:
        - "org.create"
        - "policy.read"
        - "user.self.delete"
+    - Role: "ORG_USER_SELF_MANAGER"
+      Permissions:
+        - "policy.read"
+        - "user.self.delete"
    - Role: "PROJECT_OWNER_GLOBAL"
      Permissions:
        - "org.global.read"
--- a/cmd/ready/config.go
+++ b/cmd/ready/config.go
@@ -0,0 +1,34 @@
+package ready
+
+import (
+	"time"
+
+	"github.com/mitchellh/mapstructure"
+	"github.com/spf13/viper"
+	"github.com/zitadel/logging"
+
+	"github.com/zitadel/zitadel/internal/config/hook"
+)
+
+type Config struct {
+	Log  *logging.Config
+	Port uint16
+}
+
+func MustNewConfig(v *viper.Viper) *Config {
+	config := new(Config)
+	err := v.Unmarshal(config,
+		viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
+			hook.Base64ToBytesHookFunc(),
+			mapstructure.StringToTimeDurationHookFunc(),
+			mapstructure.StringToTimeHookFunc(time.RFC3339),
+			mapstructure.StringToSliceHookFunc(","),
+		)),
+	)
+	logging.OnError(err).Fatal("unable to read default config")
+
+	err = config.Log.SetLogger()
+	logging.OnError(err).Fatal("unable to set logger")
+
+	return config
+}
--- a/cmd/ready/ready.go
+++ b/cmd/ready/ready.go
@@ -0,0 +1,37 @@
+package ready
+
+import (
+	"net"
+	"net/http"
+	"os"
+	"strconv"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"github.com/zitadel/logging"
+)
+
+func New() *cobra.Command {
+	return &cobra.Command{
+		Use:   "ready",
+		Short: "Checks if zitadel is ready",
+		Long:  "Checks if zitadel is ready",
+		Run: func(cmd *cobra.Command, args []string) {
+			config := MustNewConfig(viper.GetViper())
+			if !ready(config) {
+				os.Exit(1)
+			}
+		},
+	}
+}
+
+func ready(config *Config) bool {
+	res, err := http.Get("http://" + net.JoinHostPort("localhost", strconv.Itoa(int(config.Port))) + "/debug/ready")
+	if err != nil {
+		logging.WithError(err).Warn("ready check failed")
+		return false
+	}
+	defer res.Body.Close()
+	logging.WithFields("status", res.StatusCode).Warn("ready check failed")
+	return res.StatusCode == 200
+}
--- a/cmd/setup/03.go
+++ b/cmd/setup/03.go
@@ -88,6 +88,9 @@ func (mig *FirstInstance) Execute(ctx context.Context) error {
 		nil,
 		nil,
 		nil,
+		0,
+		0,
+		0,
 	)
 	if err != nil {
 		return err
--- a/cmd/setup/config_change.go
+++ b/cmd/setup/config_change.go
@@ -17,6 +17,7 @@ type externalConfigChange struct {
 	currentExternalDomain string
 	currentExternalSecure bool
 	currentExternalPort   uint16
+	defaults              systemdefaults.SystemDefaults
 }

 func (mig *externalConfigChange) SetLastExecution(lastRun map[string]interface{}) {
@@ -35,7 +36,7 @@ func (mig *externalConfigChange) Check() bool {
 func (mig *externalConfigChange) Execute(ctx context.Context) error {
 	cmd, err := command.StartCommands(
 		mig.es,
-		systemdefaults.SystemDefaults{},
+		mig.defaults,
 		nil,
 		nil,
 		nil,
@@ -53,6 +54,9 @@ func (mig *externalConfigChange) Execute(ctx context.Context) error {
 		nil,
 		nil,
 		nil,
+		0,
+		0,
+		0,
 	)

 	if err != nil {
--- a/cmd/setup/setup.go
+++ b/cmd/setup/setup.go
@@ -104,6 +104,7 @@ func Setup(config *Config, steps *Steps, masterKey string) {
 			ExternalDomain: config.ExternalDomain,
 			ExternalPort:   config.ExternalPort,
 			ExternalSecure: config.ExternalSecure,
+			defaults:       config.SystemDefaults,
 		},
 		&projectionTables{
 			es:      eventstoreClient,
--- a/cmd/start/config.go
+++ b/cmd/start/config.go
@@ -1,6 +1,8 @@
 package start

 import (
+	"encoding/json"
+	"reflect"
 	"time"

 	"github.com/mitchellh/mapstructure"
@@ -25,6 +27,7 @@ import (
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/id"
 	"github.com/zitadel/zitadel/internal/logstore"
+	"github.com/zitadel/zitadel/internal/notification/handlers"
 	"github.com/zitadel/zitadel/internal/query/projection"
 	static_config "github.com/zitadel/zitadel/internal/static/config"
 	metrics "github.com/zitadel/zitadel/internal/telemetry/metrics/config"
@@ -58,13 +61,14 @@ type Config struct {
 	EncryptionKeys    *encryptionKeyConfig
 	DefaultInstance   command.InstanceSetup
 	AuditLogRetention time.Duration
-	SystemAPIUsers    map[string]*internal_authz.SystemAPIUser
+	SystemAPIUsers    SystemAPIUsers
 	CustomerPortal    string
 	Machine           *id.Config
 	Actions           *actions.Config
 	Eventstore        *eventstore.Config
 	LogStore          *logstore.Configs
 	Quotas            *QuotasConfig
+	Telemetry         *handlers.TelemetryPusherConfig
 }

 type QuotasConfig struct {
@@ -83,6 +87,7 @@ func MustNewConfig(v *viper.Viper) *Config {
 			mapstructure.StringToSliceHookFunc(","),
 			database.DecodeHook,
 			actions.HTTPConfigDecodeHook,
+			systemAPIUsersDecodeHook,
 		)),
 	)
 	logging.OnError(err).Fatal("unable to read config")
@@ -114,3 +119,22 @@ type encryptionKeyConfig struct {
 	CSRFCookieKeyID      string
 	UserAgentCookieKeyID string
 }
+
+type SystemAPIUsers map[string]*internal_authz.SystemAPIUser
+
+func systemAPIUsersDecodeHook(from, to reflect.Value) (any, error) {
+	if to.Type() != reflect.TypeOf(SystemAPIUsers{}) {
+		return from.Interface(), nil
+	}
+
+	data, ok := from.Interface().(string)
+	if !ok {
+		return from.Interface(), nil
+	}
+	users := make(SystemAPIUsers)
+	err := json.Unmarshal([]byte(data), &users)
+	if err != nil {
+		return nil, err
+	}
+	return users, nil
+}
--- a/cmd/start/start.go
+++ b/cmd/start/start.go
@@ -13,6 +13,8 @@ import (
 	"time"

 	clockpkg "github.com/benbjohnson/clock"
+	"github.com/common-nighthawk/go-figure"
+	"github.com/fatih/color"
 	"github.com/gorilla/mux"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -22,6 +24,7 @@ import (
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/h2c"

+	"github.com/zitadel/zitadel/cmd/build"
 	"github.com/zitadel/zitadel/cmd/key"
 	cmd_tls "github.com/zitadel/zitadel/cmd/tls"
 	"github.com/zitadel/zitadel/internal/actions"
@@ -32,6 +35,7 @@ import (
 	"github.com/zitadel/zitadel/internal/api/grpc/admin"
 	"github.com/zitadel/zitadel/internal/api/grpc/auth"
 	"github.com/zitadel/zitadel/internal/api/grpc/management"
+	oidc_v2 "github.com/zitadel/zitadel/internal/api/grpc/oidc/v2"
 	"github.com/zitadel/zitadel/internal/api/grpc/session/v2"
 	"github.com/zitadel/zitadel/internal/api/grpc/settings/v2"
 	"github.com/zitadel/zitadel/internal/api/grpc/system"
@@ -110,6 +114,8 @@ type Server struct {
 }

 func startZitadel(config *Config, masterKey string, server chan<- *Server) error {
+	showBasicInformation(config)
+
 	ctx := context.Background()

 	dbClient, err := database.Connect(config.Database, false)
@@ -192,6 +198,9 @@ func startZitadel(config *Config, masterKey string, server chan<- *Server) error
 		&http.Client{},
 		permissionCheck,
 		sessionTokenVerifier,
+		config.OIDC.DefaultAccessTokenLifetime,
+		config.OIDC.DefaultRefreshTokenExpiration,
+		config.OIDC.DefaultRefreshTokenIdleExpiration,
 	)
 	if err != nil {
 		return fmt.Errorf("cannot start commands: %w", err)
@@ -207,14 +216,14 @@ func startZitadel(config *Config, masterKey string, server chan<- *Server) error
 		return err
 	}

-	usageReporter := logstore.UsageReporterFunc(commands.ReportUsage)
+	usageReporter := logstore.UsageReporterFunc(commands.ReportQuotaUsage)
 	actionsLogstoreSvc := logstore.New(queries, usageReporter, actionsExecutionDBEmitter, actionsExecutionStdoutEmitter)
 	if actionsLogstoreSvc.Enabled() {
 		logging.Warn("execution logs are currently in beta")
 	}
 	actions.SetLogstoreService(actionsLogstoreSvc)

-	notification.Start(ctx, config.Projections.Customizations["notifications"], config.Projections.Customizations["notificationsquotas"], config.ExternalPort, config.ExternalSecure, commands, queries, eventstoreClient, assets.AssetAPIFromDomain(config.ExternalSecure, config.ExternalPort), config.SystemDefaults.Notifications.FileSystemPath, keys.User, keys.SMTP, keys.SMS)
+	notification.Start(ctx, config.Projections.Customizations["notifications"], config.Projections.Customizations["notificationsquotas"], config.Projections.Customizations["telemetry"], *config.Telemetry, config.ExternalDomain, config.ExternalPort, config.ExternalSecure, commands, queries, eventstoreClient, assets.AssetAPIFromDomain(config.ExternalSecure, config.ExternalPort), config.SystemDefaults.Notifications.FileSystemPath, keys.User, keys.SMTP, keys.SMS)

 	router := mux.NewRouter()
 	tlsConfig, err := config.TLS.Config()
@@ -344,6 +353,7 @@ func startAPIs(
 	if err := apis.RegisterService(ctx, session.CreateServer(commands, queries, permissionCheck)); err != nil {
 		return err
 	}
+
 	if err := apis.RegisterService(ctx, settings.CreateServer(commands, queries, config.ExternalSecure)); err != nil {
 		return err
 	}
@@ -397,6 +407,11 @@ func startAPIs(
 	apis.RegisterHandlerOnPrefix(login.HandlerPrefix, l.Handler())
 	apis.HandleFunc(login.EndpointDeviceAuth, login.RedirectDeviceAuthToPrefix)

+	// After OIDC provider so that the callback endpoint can be used
+	if err := apis.RegisterService(ctx, oidc_v2.CreateServer(commands, queries, oidcProvider, config.ExternalSecure)); err != nil {
+		return err
+	}
+
 	// handle grpc at last to be able to handle the root, because grpc and gateway require a lot of different prefixes
 	apis.RouteGRPC()
 	return nil
@@ -444,3 +459,29 @@ func shutdownServer(ctx context.Context, server *http.Server) error {
 	logging.New().Info("server shutdown gracefully")
 	return nil
 }
+
+func showBasicInformation(startConfig *Config) {
+	fmt.Println(color.MagentaString(figure.NewFigure("Zitadel", "", true).String()))
+	http := "http"
+	if startConfig.TLS.Enabled || startConfig.ExternalSecure {
+		http = "https"
+	}
+
+	consoleURL := fmt.Sprintf("%s://%s:%v/ui/console\n", http, startConfig.ExternalDomain, startConfig.ExternalPort)
+	healthCheckURL := fmt.Sprintf("%s://%s:%v/debug/healthz\n", http, startConfig.ExternalDomain, startConfig.ExternalPort)
+
+	insecure := !startConfig.TLS.Enabled && !startConfig.ExternalSecure
+
+	fmt.Printf(" ===============================================================\n\n")
+	fmt.Printf(" Version          : %s\n", build.Version())
+	fmt.Printf(" TLS enabled      : %v\n", startConfig.TLS.Enabled)
+	fmt.Printf(" External Secure  : %v\n", startConfig.ExternalSecure)
+	fmt.Printf(" Console URL      : %s", color.BlueString(consoleURL))
+	fmt.Printf(" Health Check URL : %s", color.BlueString(healthCheckURL))
+	if insecure {
+		fmt.Printf("\n %s: you're using plain http without TLS. Be aware this is \n", color.RedString("Warning"))
+		fmt.Printf(" not a secure setup and should only be used for test systems.         \n")
+		fmt.Printf(" Visit: %s    \n", color.CyanString("https://zitadel.com/docs/self-hosting/manage/tls_modes"))
+	}
+	fmt.Printf("\n ===============================================================\n\n")
+}
--- a/cmd/zitadel.go
+++ b/cmd/zitadel.go
@@ -15,6 +15,7 @@ import (
 	"github.com/zitadel/zitadel/cmd/build"
 	"github.com/zitadel/zitadel/cmd/initialise"
 	"github.com/zitadel/zitadel/cmd/key"
+	"github.com/zitadel/zitadel/cmd/ready"
 	"github.com/zitadel/zitadel/cmd/setup"
 	"github.com/zitadel/zitadel/cmd/start"
 )
@@ -55,6 +56,7 @@ func New(out io.Writer, in io.Reader, args []string, server chan<- *start.Server
 		start.NewStartFromInit(server),
 		start.NewStartFromSetup(server),
 		key.New(),
+		ready.New(),
 	)

 	cmd.InitDefaultVersionFlag()