chore: move the go code into a subfolder

apps/api/internal/telemetry/http_handler.go

@@ -0,0 +1,34 @@
+package telemetry
+
+import (
+	"net/http"
+	"strings"
+
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+
+	"github.com/zitadel/zitadel/internal/telemetry/metrics"
+)
+
+func shouldNotIgnore(endpoints ...string) func(r *http.Request) bool {
+	return func(r *http.Request) bool {
+		for _, endpoint := range endpoints {
+			if strings.HasPrefix(r.URL.RequestURI(), endpoint) {
+				return false
+			}
+		}
+		return true
+	}
+}
+
+func TelemetryHandler(handler http.Handler, ignoredEndpoints ...string) http.Handler {
+	return otelhttp.NewHandler(handler,
+		"zitadel",
+		otelhttp.WithFilter(shouldNotIgnore(ignoredEndpoints...)),
+		otelhttp.WithPublicEndpoint(),
+		otelhttp.WithSpanNameFormatter(spanNameFormatter),
+		otelhttp.WithMeterProvider(metrics.GetMetricsProvider()))
+}
+
+func spanNameFormatter(_ string, r *http.Request) string {
+	return strings.Split(r.RequestURI, "?")[0]
+}

apps/api/internal/telemetry/metrics/config/config.go

@@ -0,0 +1,37 @@
+package config
+
+import (
+	"github.com/zitadel/zitadel/internal/telemetry/metrics"
+	"github.com/zitadel/zitadel/internal/telemetry/metrics/otel"
+	"github.com/zitadel/zitadel/internal/zerrors"
+)
+
+type Config struct {
+	Type   string
+	Config map[string]interface{} `mapstructure:",remain"`
+}
+
+var meter = map[string]func(map[string]interface{}) error{
+	"otel": otel.NewTracerFromConfig,
+	"none": registerNoopMetrics,
+	"":     registerNoopMetrics,
+}
+
+func (c *Config) NewMeter() error {
+	// When using start-from-init or start-from-setup the metric provider
+	// was already set in the setup phase and the start phase must not overwrite it.
+	if metrics.M != nil {
+		return nil
+	}
+	t, ok := meter[c.Type]
+	if !ok {
+		return zerrors.ThrowInternalf(nil, "METER-Dfqsx", "config type %s not supported", c.Type)
+	}
+
+	return t(c.Config)
+}
+
+func registerNoopMetrics(rawConfig map[string]interface{}) (err error) {
+	metrics.M = &metrics.NoopMetrics{}
+	return nil
+}

apps/api/internal/telemetry/metrics/http_handler.go

@@ -0,0 +1,147 @@
+package metrics
+
+import (
+	"context"
+	"net/http"
+	"strings"
+
+	"go.opentelemetry.io/otel/attribute"
+)
+
+const (
+	RequestCounter               = "http.server.request_count"
+	RequestCountDescription      = "Request counter"
+	TotalRequestCounter          = "http.server.total_request_count"
+	TotalRequestDescription      = "Total return code counter"
+	ReturnCodeCounter            = "http.server.return_code_counter"
+	ReturnCodeCounterDescription = "Return code counter"
+	Method                       = "method"
+	URI                          = "uri"
+	ReturnCode                   = "return_code"
+)
+
+type Handler struct {
+	handler http.Handler
+	methods []MetricType
+	filters []Filter
+}
+
+type MetricType int32
+
+const (
+	MetricTypeTotalCount MetricType = iota
+	MetricTypeStatusCode
+	MetricTypeRequestCount
+)
+
+type StatusRecorder struct {
+	http.ResponseWriter
+	RequestURI *string
+	Status     int
+}
+
+func (r *StatusRecorder) WriteHeader(status int) {
+	r.Status = status
+	r.ResponseWriter.WriteHeader(status)
+}
+
+type Filter func(*http.Request) bool
+
+func NewMetricsHandler(handler http.Handler, metricMethods []MetricType, ignoredEndpoints ...string) http.Handler {
+	h := Handler{
+		handler: handler,
+		methods: metricMethods,
+	}
+	if len(ignoredEndpoints) > 0 {
+		h.filters = append(h.filters, shouldNotIgnore(ignoredEndpoints...))
+	}
+	return &h
+}
+
+type key int
+
+const requestURI key = iota
+
+func SetRequestURIPattern(ctx context.Context, pattern string) {
+	uri, ok := ctx.Value(requestURI).(*string)
+	if !ok {
+		return
+	}
+	*uri = pattern
+}
+
+// ServeHTTP serves HTTP requests (http.Handler)
+func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if len(h.methods) == 0 {
+		h.handler.ServeHTTP(w, r)
+		return
+	}
+	for _, f := range h.filters {
+		if !f(r) {
+			// Simply pass through to the handler if a filter rejects the request
+			h.handler.ServeHTTP(w, r)
+			return
+		}
+	}
+	uri := strings.Split(r.RequestURI, "?")[0]
+	recorder := &StatusRecorder{
+		ResponseWriter: w,
+		RequestURI:     &uri,
+		Status:         200,
+	}
+	r = r.WithContext(context.WithValue(r.Context(), requestURI, &uri))
+	h.handler.ServeHTTP(recorder, r)
+	if h.containsMetricsMethod(MetricTypeRequestCount) {
+		RegisterRequestCounter(recorder, r)
+	}
+	if h.containsMetricsMethod(MetricTypeTotalCount) {
+		RegisterTotalRequestCounter(r)
+	}
+	if h.containsMetricsMethod(MetricTypeStatusCode) {
+		RegisterRequestCodeCounter(recorder, r)
+	}
+}
+
+func (h *Handler) containsMetricsMethod(method MetricType) bool {
+	for _, m := range h.methods {
+		if m == method {
+			return true
+		}
+	}
+	return false
+}
+
+func RegisterRequestCounter(recorder *StatusRecorder, r *http.Request) {
+	var labels = map[string]attribute.Value{
+		URI:    attribute.StringValue(*recorder.RequestURI),
+		Method: attribute.StringValue(r.Method),
+	}
+	RegisterCounter(RequestCounter, RequestCountDescription)
+	AddCount(r.Context(), RequestCounter, 1, labels)
+}
+
+func RegisterTotalRequestCounter(r *http.Request) {
+	RegisterCounter(TotalRequestCounter, TotalRequestDescription)
+	AddCount(r.Context(), TotalRequestCounter, 1, nil)
+}
+
+func RegisterRequestCodeCounter(recorder *StatusRecorder, r *http.Request) {
+	var labels = map[string]attribute.Value{
+		URI:        attribute.StringValue(*recorder.RequestURI),
+		Method:     attribute.StringValue(r.Method),
+		ReturnCode: attribute.IntValue(recorder.Status),
+	}
+	RegisterCounter(ReturnCodeCounter, ReturnCodeCounterDescription)
+	AddCount(r.Context(), ReturnCodeCounter, 1, labels)
+}
+
+func shouldNotIgnore(endpoints ...string) func(r *http.Request) bool {
+	return func(r *http.Request) bool {
+		for _, endpoint := range endpoints {
+			if strings.HasPrefix(r.URL.RequestURI(), endpoint) {
+				return false
+			}
+		}
+		return true
+	}
+}

apps/api/internal/telemetry/metrics/metrics.go

@@ -0,0 +1,87 @@
+package metrics
+
+import (
+	"context"
+	"net/http"
+
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+)
+
+const (
+	ActiveSessionCounter            = "zitadel.active_session_counter"
+	ActiveSessionCounterDescription = "Active session counter"
+	SpoolerDivCounter               = "zitadel.spooler_div_milliseconds"
+	SpoolerDivCounterDescription    = "Spooler div from last successful run to now in milliseconds"
+	Database                        = "database"
+	ViewName                        = "view_name"
+)
+
+type Metrics interface {
+	GetExporter() http.Handler
+	GetMetricsProvider() metric.MeterProvider
+	RegisterCounter(name, description string) error
+	AddCount(ctx context.Context, name string, value int64, labels map[string]attribute.Value) error
+	AddHistogramMeasurement(ctx context.Context, name string, value float64, labels map[string]attribute.Value) error
+	RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64Callback) error
+	RegisterValueObserver(name, description string, callbackFunc metric.Int64Callback) error
+	RegisterHistogram(name, description, unit string, buckets []float64) error
+}
+
+var M Metrics
+
+func GetExporter() http.Handler {
+	if M == nil {
+		return nil
+	}
+	return M.GetExporter()
+}
+
+func GetMetricsProvider() metric.MeterProvider {
+	if M == nil {
+		return nil
+	}
+	return M.GetMetricsProvider()
+}
+
+func RegisterCounter(name, description string) error {
+	if M == nil {
+		return nil
+	}
+	return M.RegisterCounter(name, description)
+}
+
+func AddCount(ctx context.Context, name string, value int64, labels map[string]attribute.Value) error {
+	if M == nil {
+		return nil
+	}
+	return M.AddCount(ctx, name, value, labels)
+}
+
+func AddHistogramMeasurement(ctx context.Context, name string, value float64, labels map[string]attribute.Value) error {
+	if M == nil {
+		return nil
+	}
+	return M.AddHistogramMeasurement(ctx, name, value, labels)
+}
+
+func RegisterHistogram(name, description, unit string, buckets []float64) error {
+	if M == nil {
+		return nil
+	}
+	return M.RegisterHistogram(name, description, unit, buckets)
+}
+
+func RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64Callback) error {
+	if M == nil {
+		return nil
+	}
+	return M.RegisterUpDownSumObserver(name, description, callbackFunc)
+}
+
+func RegisterValueObserver(name, description string, callbackFunc metric.Int64Callback) error {
+	if M == nil {
+		return nil
+	}
+	return M.RegisterValueObserver(name, description, callbackFunc)
+}

apps/api/internal/telemetry/metrics/mock.go

@@ -0,0 +1,95 @@
+package metrics
+
+import (
+	"context"
+	"net/http"
+	"sync"
+
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+)
+
+// MockMetrics implements the metrics.Metrics interface for testing
+type MockMetrics struct {
+	mu              sync.RWMutex
+	histogramValues map[string][]float64
+	counterValues   map[string]int64
+	histogramLabels map[string][]map[string]attribute.Value
+	counterLabels   map[string][]map[string]attribute.Value
+}
+
+var _ Metrics = new(MockMetrics)
+
+// NewMockMetrics creates a new Metrics instance for testing
+func NewMockMetrics() *MockMetrics {
+	return &MockMetrics{
+		histogramValues: make(map[string][]float64),
+		counterValues:   make(map[string]int64),
+		histogramLabels: make(map[string][]map[string]attribute.Value),
+		counterLabels:   make(map[string][]map[string]attribute.Value),
+	}
+}
+
+func (m *MockMetrics) GetExporter() http.Handler {
+	return nil
+}
+
+func (m *MockMetrics) GetMetricsProvider() metric.MeterProvider {
+	return nil
+}
+
+func (m *MockMetrics) RegisterCounter(name, description string) error {
+	return nil
+}
+
+func (m *MockMetrics) AddCount(ctx context.Context, name string, value int64, labels map[string]attribute.Value) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.counterValues[name] += value
+	m.counterLabels[name] = append(m.counterLabels[name], labels)
+	return nil
+}
+
+func (m *MockMetrics) AddHistogramMeasurement(ctx context.Context, name string, value float64, labels map[string]attribute.Value) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.histogramValues[name] = append(m.histogramValues[name], value)
+	m.histogramLabels[name] = append(m.histogramLabels[name], labels)
+	return nil
+}
+
+func (m *MockMetrics) RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64Callback) error {
+	return nil
+}
+
+func (m *MockMetrics) RegisterValueObserver(name, description string, callbackFunc metric.Int64Callback) error {
+	return nil
+}
+
+func (m *MockMetrics) RegisterHistogram(name, description, unit string, buckets []float64) error {
+	return nil
+}
+
+func (m *MockMetrics) GetHistogramValues(name string) []float64 {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.histogramValues[name]
+}
+
+func (m *MockMetrics) GetHistogramLabels(name string) []map[string]attribute.Value {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.histogramLabels[name]
+}
+
+func (m *MockMetrics) GetCounterValue(name string) int64 {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.counterValues[name]
+}
+
+func (m *MockMetrics) GetCounterLabels(name string) []map[string]attribute.Value {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.counterLabels[name]
+}

apps/api/internal/telemetry/metrics/noop.go

@@ -0,0 +1,45 @@
+package metrics
+
+import (
+	"context"
+	"net/http"
+
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+)
+
+type NoopMetrics struct{}
+
+var _ Metrics = new(NoopMetrics)
+
+func (n *NoopMetrics) GetExporter() http.Handler {
+	return nil
+}
+
+func (n *NoopMetrics) GetMetricsProvider() metric.MeterProvider {
+	return nil
+}
+
+func (n *NoopMetrics) RegisterCounter(name, description string) error {
+	return nil
+}
+
+func (n *NoopMetrics) AddCount(ctx context.Context, name string, value int64, labels map[string]attribute.Value) error {
+	return nil
+}
+
+func (n *NoopMetrics) AddHistogramMeasurement(ctx context.Context, name string, value float64, labels map[string]attribute.Value) error {
+	return nil
+}
+
+func (n *NoopMetrics) RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64Callback) error {
+	return nil
+}
+
+func (n *NoopMetrics) RegisterValueObserver(name, description string, callbackFunc metric.Int64Callback) error {
+	return nil
+}
+
+func (n *NoopMetrics) RegisterHistogram(name, description, unit string, buckets []float64) error {
+	return nil
+}

apps/api/internal/telemetry/metrics/otel/config.go

@@ -0,0 +1,20 @@
+package otel
+
+import (
+	"github.com/zitadel/zitadel/internal/telemetry/metrics"
+)
+
+type Config struct {
+	MeterName string
+}
+
+func NewTracerFromConfig(rawConfig map[string]interface{}) (err error) {
+	c := new(Config)
+	c.MeterName, _ = rawConfig["metername"].(string)
+	return c.NewMetrics()
+}
+
+func (c *Config) NewMetrics() (err error) {
+	metrics.M, err = NewMetrics(c.MeterName)
+	return err
+}

apps/api/internal/telemetry/metrics/otel/open_telemetry.go

@@ -0,0 +1,163 @@
+package otel
+
+import (
+	"context"
+	"net/http"
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/prometheus"
+	"go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/sdk/instrumentation"
+	sdk_metric "go.opentelemetry.io/otel/sdk/metric"
+
+	"github.com/zitadel/zitadel/internal/telemetry/metrics"
+	otel_resource "github.com/zitadel/zitadel/internal/telemetry/otel"
+	"github.com/zitadel/zitadel/internal/zerrors"
+)
+
+type Metrics struct {
+	Provider          metric.MeterProvider
+	Meter             metric.Meter
+	Counters          sync.Map
+	UpDownSumObserver sync.Map
+	ValueObservers    sync.Map
+	Histograms        sync.Map
+}
+
+func NewMetrics(meterName string) (metrics.Metrics, error) {
+	resource, err := otel_resource.ResourceWithService("ZITADEL")
+	if err != nil {
+		return nil, err
+	}
+	exporter, err := prometheus.New()
+	if err != nil {
+		return &Metrics{}, err
+	}
+	// create a view to filter out unwanted attributes
+	view := sdk_metric.NewView(
+		sdk_metric.Instrument{
+			Scope: instrumentation.Scope{Name: otelhttp.ScopeName},
+		},
+		sdk_metric.Stream{
+			AttributeFilter: attribute.NewAllowKeysFilter("http.method", "http.status_code", "http.target"),
+		},
+	)
+	meterProvider := sdk_metric.NewMeterProvider(
+		sdk_metric.WithReader(exporter),
+		sdk_metric.WithResource(resource),
+		sdk_metric.WithView(view),
+	)
+	return &Metrics{
+		Provider: meterProvider,
+		Meter:    meterProvider.Meter(meterName),
+	}, nil
+}
+
+func (m *Metrics) GetExporter() http.Handler {
+	return promhttp.Handler()
+}
+
+func (m *Metrics) GetMetricsProvider() metric.MeterProvider {
+	return m.Provider
+}
+
+func (m *Metrics) RegisterCounter(name, description string) error {
+	if _, exists := m.Counters.Load(name); exists {
+		return nil
+	}
+	counter, err := m.Meter.Int64Counter(name, metric.WithDescription(description))
+	if err != nil {
+		return err
+	}
+	m.Counters.Store(name, counter)
+	return nil
+}
+
+func (m *Metrics) AddCount(ctx context.Context, name string, value int64, labels map[string]attribute.Value) error {
+	counter, exists := m.Counters.Load(name)
+	if !exists {
+		return zerrors.ThrowNotFound(nil, "METER-4u8fs", "Errors.Metrics.Counter.NotFound")
+	}
+	counter.(metric.Int64Counter).Add(ctx, value, MapToAddOption(labels)...)
+	return nil
+}
+
+func (m *Metrics) AddHistogramMeasurement(ctx context.Context, name string, value float64, labels map[string]attribute.Value) error {
+	histogram, exists := m.Histograms.Load(name)
+	if !exists {
+		return zerrors.ThrowNotFound(nil, "METER-5wwb1", "Errors.Metrics.Histogram.NotFound")
+	}
+	histogram.(metric.Float64Histogram).Record(ctx, value, MapToRecordOption(labels)...)
+	return nil
+}
+
+func (m *Metrics) RegisterHistogram(name, description, unit string, buckets []float64) error {
+	if _, exists := m.Histograms.Load(name); exists {
+		return nil
+	}
+
+	histogram, err := m.Meter.Float64Histogram(name,
+		metric.WithDescription(description),
+		metric.WithUnit(unit),
+		metric.WithExplicitBucketBoundaries(buckets...),
+	)
+	if err != nil {
+		return err
+	}
+
+	m.Histograms.Store(name, histogram)
+	return nil
+}
+
+func (m *Metrics) RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64Callback) error {
+	if _, exists := m.UpDownSumObserver.Load(name); exists {
+		return nil
+	}
+
+	counter, err := m.Meter.Int64ObservableUpDownCounter(name, metric.WithInt64Callback(callbackFunc), metric.WithDescription(description))
+	if err != nil {
+		return err
+	}
+
+	m.UpDownSumObserver.Store(name, counter)
+	return nil
+}
+
+func (m *Metrics) RegisterValueObserver(name, description string, callbackFunc metric.Int64Callback) error {
+	if _, exists := m.UpDownSumObserver.Load(name); exists {
+		return nil
+	}
+
+	gauge, err := m.Meter.Int64ObservableGauge(name, metric.WithInt64Callback(callbackFunc), metric.WithDescription(description))
+	if err != nil {
+		return err
+	}
+
+	m.UpDownSumObserver.Store(name, gauge)
+	return nil
+}
+
+func MapToAddOption(labels map[string]attribute.Value) []metric.AddOption {
+	return []metric.AddOption{metric.WithAttributes(labelsToAttributes(labels)...)}
+}
+
+func MapToRecordOption(labels map[string]attribute.Value) []metric.RecordOption {
+	return []metric.RecordOption{metric.WithAttributes(labelsToAttributes(labels)...)}
+}
+
+func labelsToAttributes(labels map[string]attribute.Value) []attribute.KeyValue {
+	if labels == nil {
+		return nil
+	}
+	attributes := make([]attribute.KeyValue, 0, len(labels))
+	for key, value := range labels {
+		attributes = append(attributes, attribute.KeyValue{
+			Key:   attribute.Key(key),
+			Value: value,
+		})
+	}
+	return attributes
+}

apps/api/internal/telemetry/otel/resource.go

@@ -0,0 +1,22 @@
+package otel
+
+import (
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/sdk/resource"
+	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
+
+	"github.com/zitadel/zitadel/cmd/build"
+)
+
+func ResourceWithService(serviceName string) (*resource.Resource, error) {
+	attributes := []attribute.KeyValue{
+		semconv.ServiceNameKey.String(serviceName),
+	}
+	if build.Version() != "" {
+		attributes = append(attributes, semconv.ServiceVersionKey.String(build.Version()))
+	}
+	return resource.Merge(
+		resource.Default(),
+		resource.NewWithAttributes("", attributes...),
+	)
+}

apps/api/internal/telemetry/profiler/config/config.go

@@ -0,0 +1,30 @@
+package config
+
+import (
+	"github.com/zitadel/zitadel/internal/telemetry/profiler/google"
+	"github.com/zitadel/zitadel/internal/zerrors"
+)
+
+type Config struct {
+	Type   string
+	Config map[string]interface{} `mapstructure:",remain"`
+}
+
+var profiler = map[string]func(map[string]interface{}) error{
+	"google": google.NewProfiler,
+	"none":   NoProfiler,
+	"":       NoProfiler,
+}
+
+func (c *Config) NewProfiler() error {
+	t, ok := profiler[c.Type]
+	if !ok {
+		return zerrors.ThrowInternalf(nil, "PROFI-Dfqsx", "config type %s not supported", c.Type)
+	}
+
+	return t(c.Config)
+}
+
+func NoProfiler(_ map[string]interface{}) error {
+	return nil
+}

apps/api/internal/telemetry/profiler/google/profiler.go

@@ -0,0 +1,26 @@
+package google
+
+import (
+	"cloud.google.com/go/profiler"
+
+	"github.com/zitadel/zitadel/cmd/build"
+)
+
+type Config struct {
+	ProjectID string
+}
+
+func NewProfiler(rawConfig map[string]interface{}) (err error) {
+	c := new(Config)
+	c.ProjectID, _ = rawConfig["projectid"].(string)
+	return c.NewProfiler()
+}
+
+func (c *Config) NewProfiler() (err error) {
+	cfg := profiler.Config{
+		Service:        "zitadel",
+		ServiceVersion: build.Version(),
+		ProjectID:      c.ProjectID,
+	}
+	return profiler.Start(cfg)
+}

apps/api/internal/telemetry/tracing/caller.go

@@ -0,0 +1,22 @@
+package tracing
+
+import (
+	"runtime"
+
+	"github.com/zitadel/logging"
+)
+
+func GetCaller() string {
+	fpcs := make([]uintptr, 1)
+	n := runtime.Callers(3, fpcs)
+	if n == 0 {
+		logging.WithFields("logID", "TRACE-rWjfC").Debug("no caller")
+		return ""
+	}
+	caller := runtime.FuncForPC(fpcs[0] - 1)
+	if caller == nil {
+		logging.WithFields("logID", "TRACE-25POw").Debug("caller was nil")
+		return ""
+	}
+	return caller.Name()
+}

apps/api/internal/telemetry/tracing/config/config.go

@@ -0,0 +1,34 @@
+package config
+
+import (
+	"github.com/zitadel/zitadel/internal/telemetry/tracing/google"
+	"github.com/zitadel/zitadel/internal/telemetry/tracing/log"
+	"github.com/zitadel/zitadel/internal/telemetry/tracing/otel"
+	"github.com/zitadel/zitadel/internal/zerrors"
+)
+
+type Config struct {
+	Type   string
+	Config map[string]interface{} `mapstructure:",remain"`
+}
+
+func (c *Config) NewTracer() error {
+	t, ok := tracer[c.Type]
+	if !ok {
+		return zerrors.ThrowInternalf(nil, "TRACE-dsbjh", "config type %s not supported", c.Type)
+	}
+
+	return t(c.Config)
+}
+
+var tracer = map[string]func(map[string]interface{}) error{
+	"otel":   otel.NewTracerFromConfig,
+	"google": google.NewTracer,
+	"log":    log.NewTracer,
+	"none":   NoTracer,
+	"":       NoTracer,
+}
+
+func NoTracer(_ map[string]interface{}) error {
+	return nil
+}

apps/api/internal/telemetry/tracing/google/google_tracer.go

@@ -0,0 +1,41 @@
+package google
+
+import (
+	texporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace"
+	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
+
+	"github.com/zitadel/zitadel/internal/telemetry/tracing"
+	"github.com/zitadel/zitadel/internal/telemetry/tracing/otel"
+)
+
+type Config struct {
+	ProjectID   string
+	Fraction    float64
+	ServiceName string
+}
+
+func NewTracer(rawConfig map[string]interface{}) (err error) {
+	c := new(Config)
+	c.ProjectID, _ = rawConfig["projectid"].(string)
+	c.ServiceName, _ = rawConfig["servicename"].(string)
+	c.Fraction, err = otel.FractionFromConfig(rawConfig["fraction"])
+	if err != nil {
+		return err
+	}
+	return c.NewTracer()
+}
+
+type Tracer struct {
+	otel.Tracer
+}
+
+func (c *Config) NewTracer() error {
+	sampler := otel.NewSampler(sdk_trace.TraceIDRatioBased(c.Fraction))
+	exporter, err := texporter.New(texporter.WithProjectID(c.ProjectID))
+	if err != nil {
+		return err
+	}
+
+	tracing.T, err = otel.NewTracer(sampler, exporter, c.ServiceName)
+	return err
+}

apps/api/internal/telemetry/tracing/log/config.go

@@ -0,0 +1,39 @@
+package log
+
+import (
+	stdout "go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
+	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
+
+	"github.com/zitadel/zitadel/internal/telemetry/tracing"
+	"github.com/zitadel/zitadel/internal/telemetry/tracing/otel"
+)
+
+type Config struct {
+	Fraction    float64
+	ServiceName string
+}
+
+func NewTracer(rawConfig map[string]interface{}) (err error) {
+	c := new(Config)
+	c.Fraction, err = otel.FractionFromConfig(rawConfig["fraction"])
+	c.ServiceName, _ = rawConfig["servicename"].(string)
+	if err != nil {
+		return err
+	}
+	return c.NewTracer()
+}
+
+type Tracer struct {
+	otel.Tracer
+}
+
+func (c *Config) NewTracer() error {
+	sampler := otel.NewSampler(sdk_trace.TraceIDRatioBased(c.Fraction))
+	exporter, err := stdout.New(stdout.WithPrettyPrint())
+	if err != nil {
+		return err
+	}
+
+	tracing.T, err = otel.NewTracer(sampler, exporter, c.ServiceName)
+	return err
+}

apps/api/internal/telemetry/tracing/otel/config.go

@@ -0,0 +1,77 @@
+package otel
+
+import (
+	"context"
+	"strconv"
+
+	otlpgrpc "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
+	api_trace "go.opentelemetry.io/otel/trace"
+
+	"github.com/zitadel/zitadel/internal/telemetry/tracing"
+	"github.com/zitadel/zitadel/internal/zerrors"
+)
+
+type Config struct {
+	Fraction    float64
+	Endpoint    string
+	ServiceName string
+}
+
+func NewTracerFromConfig(rawConfig map[string]interface{}) (err error) {
+	c := new(Config)
+	c.Endpoint, _ = rawConfig["endpoint"].(string)
+	c.ServiceName, _ = rawConfig["servicename"].(string)
+	c.Fraction, err = FractionFromConfig(rawConfig["fraction"])
+	if err != nil {
+		return err
+	}
+	return c.NewTracer()
+}
+
+func FractionFromConfig(i interface{}) (float64, error) {
+	if i == nil {
+		return 0, nil
+	}
+	switch fraction := i.(type) {
+	case float64:
+		return fraction, nil
+	case int:
+		return float64(fraction), nil
+	case string:
+		f, err := strconv.ParseFloat(fraction, 64)
+		if err != nil {
+			return 0, zerrors.ThrowInternal(err, "OTEL-SAfe1", "could not map fraction")
+		}
+		return f, nil
+	default:
+		return 0, zerrors.ThrowInternal(nil, "OTEL-Dd2s", "could not map fraction, unknown type")
+	}
+}
+
+func (c *Config) NewTracer() error {
+	sampler := NewSampler(sdk_trace.TraceIDRatioBased(c.Fraction))
+	exporter, err := otlpgrpc.New(context.Background(), otlpgrpc.WithEndpoint(c.Endpoint), otlpgrpc.WithInsecure())
+	if err != nil {
+		return err
+	}
+
+	tracing.T, err = NewTracer(sampler, exporter, c.ServiceName)
+	return err
+}
+
+// NewSampler returns a sampler decorator which behaves differently,
+// based on the parent of the span. If the span has no parent and is of kind server,
+// the decorated sampler is used to make sampling decision.
+// If the span has a parent, depending on whether the parent is remote and whether it
+// is sampled, one of the following samplers will apply:
+//   - remote parent sampled -> always sample
+//   - remote parent not sampled -> sample based on the decorated sampler (fraction based)
+//   - local parent sampled -> always sample
+//   - local parent not sampled -> never sample
+func NewSampler(sampler sdk_trace.Sampler) sdk_trace.Sampler {
+	return sdk_trace.ParentBased(
+		tracing.SpanKindBased(sampler, api_trace.SpanKindServer),
+		sdk_trace.WithRemoteParentNotSampled(sampler),
+	)
+}

apps/api/internal/telemetry/tracing/otel/open_telemetry.go

@@ -0,0 +1,77 @@
+package otel
+
+import (
+	"context"
+	"net/http"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/propagation"
+	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
+	api_trace "go.opentelemetry.io/otel/trace"
+
+	otel_resource "github.com/zitadel/zitadel/internal/telemetry/otel"
+	"github.com/zitadel/zitadel/internal/telemetry/tracing"
+)
+
+type Tracer struct {
+	Exporter api_trace.Tracer
+	sampler  sdk_trace.Sampler
+}
+
+func NewTracer(sampler sdk_trace.Sampler, exporter sdk_trace.SpanExporter, serviceName string) (*Tracer, error) {
+	resource, err := otel_resource.ResourceWithService(serviceName)
+	if err != nil {
+		return nil, err
+	}
+
+	tp := sdk_trace.NewTracerProvider(
+		sdk_trace.WithSampler(sampler),
+		sdk_trace.WithBatcher(exporter),
+		sdk_trace.WithResource(resource),
+	)
+
+	otel.SetTracerProvider(tp)
+	tc := propagation.TraceContext{}
+	otel.SetTextMapPropagator(tc)
+
+	return &Tracer{Exporter: tp.Tracer(""), sampler: sampler}, nil
+}
+
+func (t *Tracer) Sampler() sdk_trace.Sampler {
+	return t.sampler
+}
+
+func (t *Tracer) NewServerInterceptorSpan(ctx context.Context, name string) (context.Context, *tracing.Span) {
+	return t.newSpanFromName(ctx, name, api_trace.WithSpanKind(api_trace.SpanKindServer))
+}
+
+func (t *Tracer) NewServerSpan(ctx context.Context, caller string) (context.Context, *tracing.Span) {
+	return t.newSpan(ctx, caller, api_trace.WithSpanKind(api_trace.SpanKindServer))
+}
+
+func (t *Tracer) NewClientInterceptorSpan(ctx context.Context, name string) (context.Context, *tracing.Span) {
+	return t.newSpanFromName(ctx, name, api_trace.WithSpanKind(api_trace.SpanKindClient))
+}
+
+func (t *Tracer) NewClientSpan(ctx context.Context, caller string) (context.Context, *tracing.Span) {
+	return t.newSpan(ctx, caller, api_trace.WithSpanKind(api_trace.SpanKindClient))
+}
+
+func (t *Tracer) NewSpan(ctx context.Context, caller string) (context.Context, *tracing.Span) {
+	return t.newSpan(ctx, caller)
+}
+
+func (t *Tracer) newSpan(ctx context.Context, caller string, options ...api_trace.SpanStartOption) (context.Context, *tracing.Span) {
+	return t.newSpanFromName(ctx, caller, options...)
+}
+
+func (t *Tracer) newSpanFromName(ctx context.Context, name string, options ...api_trace.SpanStartOption) (context.Context, *tracing.Span) {
+	ctx, span := t.Exporter.Start(ctx, name, options...)
+	return ctx, tracing.CreateSpan(span)
+}
+
+func (t *Tracer) NewSpanHTTP(r *http.Request, caller string) (*http.Request, *tracing.Span) {
+	ctx, span := t.NewSpan(r.Context(), caller)
+	r = r.WithContext(ctx)
+	return r, span
+}

apps/api/internal/telemetry/tracing/sampler.go

@@ -0,0 +1,46 @@
+package tracing
+
+import (
+	"fmt"
+	"slices"
+
+	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/trace"
+)
+
+type spanKindSampler struct {
+	sampler sdk_trace.Sampler
+	kinds   []trace.SpanKind
+}
+
+// ShouldSample implements the [sdk_trace.Sampler] interface.
+// It will not sample any spans which do not match the configured span kinds.
+// For spans which do match, the decorated sampler is used to make the sampling decision.
+func (sk spanKindSampler) ShouldSample(p sdk_trace.SamplingParameters) sdk_trace.SamplingResult {
+	psc := trace.SpanContextFromContext(p.ParentContext)
+	if !slices.Contains(sk.kinds, p.Kind) {
+		return sdk_trace.SamplingResult{
+			Decision:   sdk_trace.Drop,
+			Tracestate: psc.TraceState(),
+		}
+	}
+	s := sk.sampler.ShouldSample(p)
+	return s
+}
+
+func (sk spanKindSampler) Description() string {
+	return fmt.Sprintf("SpanKindBased{sampler:%s,kinds:%v}",
+		sk.sampler.Description(),
+		sk.kinds,
+	)
+}
+
+// SpanKindBased returns a sampler decorator which behaves differently, based on the kind of the span.
+// If the span kind does not match one of the configured kinds, it will not be sampled.
+// If the span kind matches, the decorated sampler is used to make sampling decision.
+func SpanKindBased(sampler sdk_trace.Sampler, kinds ...trace.SpanKind) sdk_trace.Sampler {
+	return spanKindSampler{
+		sampler: sampler,
+		kinds:   kinds,
+	}
+}

apps/api/internal/telemetry/tracing/sampler_test.go

@@ -0,0 +1,80 @@
+package tracing
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/trace"
+)
+
+func TestSpanKindBased(t *testing.T) {
+	type args struct {
+		sampler sdk_trace.Sampler
+		kinds   []trace.SpanKind
+	}
+	type want struct {
+		description string
+		sampled     int
+	}
+	tests := []struct {
+		name string
+		args args
+		want want
+	}{
+		{
+			"never sample, no sample",
+			args{
+				sampler: sdk_trace.NeverSample(),
+				kinds:   []trace.SpanKind{trace.SpanKindServer},
+			},
+			want{
+				description: "SpanKindBased{sampler:AlwaysOffSampler,kinds:[server]}",
+				sampled:     0,
+			},
+		},
+		{
+			"always sample, no kind, no sample",
+			args{
+				sampler: sdk_trace.AlwaysSample(),
+				kinds:   nil,
+			},
+			want{
+				description: "SpanKindBased{sampler:AlwaysOnSampler,kinds:[]}",
+				sampled:     0,
+			},
+		},
+		{
+			"always sample, 2 kinds, 2 samples",
+			args{
+				sampler: sdk_trace.AlwaysSample(),
+				kinds:   []trace.SpanKind{trace.SpanKindServer, trace.SpanKindClient},
+			},
+			want{
+				description: "SpanKindBased{sampler:AlwaysOnSampler,kinds:[server client]}",
+				sampled:     2,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sampler := SpanKindBased(tt.args.sampler, tt.args.kinds...)
+			assert.Equal(t, tt.want.description, sampler.Description())
+
+			p := sdk_trace.NewTracerProvider(sdk_trace.WithSampler(sampler))
+			tr := p.Tracer("test")
+
+			var sampled int
+			for i := trace.SpanKindUnspecified; i <= trace.SpanKindConsumer; i++ {
+				ctx := context.Background()
+				_, span := tr.Start(ctx, "test", trace.WithSpanKind(i))
+				if span.SpanContext().IsSampled() {
+					sampled++
+				}
+			}
+
+			assert.Equal(t, tt.want.sampled, sampled)
+		})
+	}
+}

apps/api/internal/telemetry/tracing/span.go

@@ -0,0 +1,46 @@
+package tracing
+
+import (
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
+
+	"github.com/zitadel/zitadel/internal/api/grpc/gerrors"
+)
+
+type Span struct {
+	span trace.Span
+	opts []trace.SpanEndOption
+}
+
+func CreateSpan(span trace.Span) *Span {
+	return &Span{span: span, opts: []trace.SpanEndOption{}}
+}
+
+func (s *Span) End() {
+	if s.span == nil {
+		return
+	}
+
+	s.span.End(s.opts...)
+}
+
+func (s *Span) EndWithError(err error) {
+	s.SetStatusByError(err)
+	s.End()
+}
+
+func (s *Span) SetStatusByError(err error) {
+	if s.span == nil {
+		return
+	}
+	if err != nil {
+		// trace.WithErrorStatus(codes.Error)
+		s.span.RecordError(err)
+		s.span.SetAttributes(
+			attribute.KeyValue{},
+		)
+	}
+
+	code, msg, id, _ := gerrors.ExtractZITADELError(err)
+	s.span.SetAttributes(attribute.Int("grpc_code", int(code)), attribute.String("grpc_msg", msg), attribute.String("error_id", id))
+}

apps/api/internal/telemetry/tracing/tracing.go

@@ -0,0 +1,81 @@
+package tracing
+
+import (
+	"context"
+	"net/http"
+
+	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
+	api_trace "go.opentelemetry.io/otel/trace"
+)
+
+type Tracer interface {
+	NewSpan(ctx context.Context, caller string) (context.Context, *Span)
+	NewClientSpan(ctx context.Context, caller string) (context.Context, *Span)
+	NewServerSpan(ctx context.Context, caller string) (context.Context, *Span)
+	NewClientInterceptorSpan(ctx context.Context, name string) (context.Context, *Span)
+	NewServerInterceptorSpan(ctx context.Context, name string) (context.Context, *Span)
+	NewSpanHTTP(r *http.Request, caller string) (*http.Request, *Span)
+	Sampler() sdk_trace.Sampler
+}
+
+var T Tracer
+
+func Sampler() sdk_trace.Sampler {
+	if T == nil {
+		return sdk_trace.NeverSample()
+	}
+	return T.Sampler()
+}
+
+func NewSpan(ctx context.Context) (context.Context, *Span) {
+	if T == nil {
+		return ctx, CreateSpan(nil)
+	}
+	return T.NewSpan(ctx, GetCaller())
+}
+
+func NewNamedSpan(ctx context.Context, name string) (context.Context, *Span) {
+	if T == nil {
+		return ctx, CreateSpan(nil)
+	}
+	return T.NewSpan(ctx, name)
+}
+
+func NewClientSpan(ctx context.Context) (context.Context, *Span) {
+	if T == nil {
+		return ctx, CreateSpan(nil)
+	}
+	return T.NewClientSpan(ctx, GetCaller())
+}
+
+func NewServerSpan(ctx context.Context) (context.Context, *Span) {
+	if T == nil {
+		return ctx, CreateSpan(nil)
+	}
+	return T.NewServerSpan(ctx, GetCaller())
+}
+
+func NewClientInterceptorSpan(ctx context.Context) (context.Context, *Span) {
+	if T == nil {
+		return ctx, CreateSpan(nil)
+	}
+	return T.NewClientInterceptorSpan(ctx, GetCaller())
+}
+
+func NewServerInterceptorSpan(ctx context.Context) (context.Context, *Span) {
+	if T == nil {
+		return ctx, CreateSpan(nil)
+	}
+	return T.NewServerInterceptorSpan(ctx, GetCaller())
+}
+
+func NewSpanHTTP(r *http.Request) (*http.Request, *Span) {
+	if T == nil {
+		return r, CreateSpan(nil)
+	}
+	return T.NewSpanHTTP(r, GetCaller())
+}
+
+func TraceIDFromCtx(ctx context.Context) string {
+	return api_trace.SpanFromContext(ctx).SpanContext().TraceID().String()
+}