feat: metrics (#1024)

* refactor: switch from opencensus to opentelemetry

* tempo works as designed nooooot

* fix: log traceids

* with grafana agent

* fix: http tracing

* fix: cleanup files

* chore: remove todo

* fix: bad test

* fix: ignore methods in grpc interceptors

* fix: remove test log

* clean up

* typo

* fix(config): configure tracing endpoint

* fix(span): add error id to span

* feat: metrics package

* feat: metrics package

* fix: counter

* fix: metric

* try metrics

* fix: coutner metrics

* fix: active sessin counter

* fix: active sessin counter

* fix: change current Sequence table

* fix: change current Sequence table

* fix: current sequences

* fix: spooler div metrics

* fix: console view

* fix: merge master

* fix: Last spool run on search result instead of eventtimestamp

* fix: go mod

* Update console/src/assets/i18n/de.json

Co-authored-by: Livio Amstutz <livio.a@gmail.com>

* fix: pr review

* fix: map

* update oidc pkg

* fix: handlers

* fix: value observer

* fix: remove fmt

* fix: handlers

* fix: tests

* fix: handler minimum cycle duration 1s

* fix(spooler): handler channel buffer

* fix interceptors

Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
Co-authored-by: Livio Amstutz <livio.a@gmail.com>

internal/telemetry/http_handler.go

@@ -0,0 +1,33 @@
+package telemetry
+
+import (
+	"github.com/caos/zitadel/internal/telemetry/metrics"
+	"net/http"
+	"strings"
+
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+)
+
+func shouldNotIgnore(endpoints ...string) func(r *http.Request) bool {
+	return func(r *http.Request) bool {
+		for _, endpoint := range endpoints {
+			if strings.HasPrefix(r.URL.RequestURI(), endpoint) {
+				return false
+			}
+		}
+		return true
+	}
+}
+
+func TelemetryHandler(handler http.Handler, ignoredEndpoints ...string) http.Handler {
+	return otelhttp.NewHandler(handler,
+		"zitadel",
+		otelhttp.WithFilter(shouldNotIgnore(ignoredEndpoints...)),
+		otelhttp.WithPublicEndpoint(),
+		otelhttp.WithSpanNameFormatter(spanNameFormatter),
+		otelhttp.WithMeterProvider(metrics.GetMetricsProvider()))
+}
+
+func spanNameFormatter(_ string, r *http.Request) string {
+	return r.Host + r.URL.EscapedPath()
+}

internal/telemetry/metrics/config/config.go

@@ -0,0 +1,65 @@
+package config
+
+import (
+	"encoding/json"
+
+	"github.com/caos/zitadel/internal/errors"
+	"github.com/caos/zitadel/internal/telemetry/metrics"
+	"github.com/caos/zitadel/internal/telemetry/metrics/otel"
+)
+
+type MetricsConfig struct {
+	Type   string
+	Config metrics.Config
+}
+
+var meter = map[string]func() metrics.Config{
+	"otel": func() metrics.Config { return &otel.Config{} },
+	"none": func() metrics.Config { return &NoMetrics{} },
+	"":     func() metrics.Config { return &NoMetrics{} },
+}
+
+func (c *MetricsConfig) UnmarshalJSON(data []byte) error {
+	var rc struct {
+		Type   string
+		Config json.RawMessage
+	}
+
+	if err := json.Unmarshal(data, &rc); err != nil {
+		return errors.ThrowInternal(err, "METER-4M9so", "error parsing config")
+	}
+
+	c.Type = rc.Type
+
+	var err error
+	c.Config, err = newMetricsConfig(c.Type, rc.Config)
+	if err != nil {
+		return err
+	}
+
+	return c.Config.NewMetrics()
+}
+
+func newMetricsConfig(tracerType string, configData []byte) (metrics.Config, error) {
+	t, ok := meter[tracerType]
+	if !ok {
+		return nil, errors.ThrowInternalf(nil, "METER-3M0ps", "config type %s not supported", tracerType)
+	}
+
+	metricsConfig := t()
+	if len(configData) == 0 {
+		return metricsConfig, nil
+	}
+
+	if err := json.Unmarshal(configData, metricsConfig); err != nil {
+		return nil, errors.ThrowInternal(err, "METER-4M9sf", "Could not read config: %v")
+	}
+
+	return metricsConfig, nil
+}
+
+type NoMetrics struct{}
+
+func (_ *NoMetrics) NewMetrics() error {
+	return nil
+}

internal/telemetry/metrics/http_handler.go

@@ -0,0 +1,128 @@
+package metrics
+
+import (
+	"net/http"
+	"strings"
+)
+
+const (
+	RequestCounter               = "http.server.request_count"
+	RequestCountDescription      = "Request counter"
+	TotalRequestCounter          = "http.server.total_request_count"
+	TotalRequestDescription      = "Total return code counter"
+	ReturnCodeCounter            = "http.server.return_code_counter"
+	ReturnCodeCounterDescription = "Return code counter"
+	Method                       = "method"
+	URI                          = "uri"
+	ReturnCode                   = "return_code"
+)
+
+type Handler struct {
+	handler http.Handler
+	methods []MetricType
+	filters []Filter
+}
+
+type MetricType int32
+
+const (
+	MetricTypeTotalCount MetricType = iota
+	MetricTypeStatusCode
+	MetricTypeRequestCount
+)
+
+type StatusRecorder struct {
+	http.ResponseWriter
+	Status int
+}
+
+func (r *StatusRecorder) WriteHeader(status int) {
+	r.Status = status
+	r.ResponseWriter.WriteHeader(status)
+}
+
+type Filter func(*http.Request) bool
+
+func NewMetricsHandler(handler http.Handler, metricMethods []MetricType, ignoredEndpoints ...string) http.Handler {
+	h := Handler{
+		handler: handler,
+		methods: metricMethods,
+	}
+	if len(ignoredEndpoints) > 0 {
+		h.filters = append(h.filters, shouldNotIgnore(ignoredEndpoints...))
+	}
+	return &h
+}
+
+// ServeHTTP serves HTTP requests (http.Handler)
+func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if len(h.methods) == 0 {
+		h.handler.ServeHTTP(w, r)
+		return
+	}
+	for _, f := range h.filters {
+		if !f(r) {
+			// Simply pass through to the handler if a filter rejects the request
+			h.handler.ServeHTTP(w, r)
+			return
+		}
+	}
+	recorder := &StatusRecorder{
+		ResponseWriter: w,
+		Status:         200,
+	}
+	h.handler.ServeHTTP(recorder, r)
+	if h.containsMetricsMethod(MetricTypeRequestCount) {
+		RegisterRequestCounter(r)
+	}
+	if h.containsMetricsMethod(MetricTypeTotalCount) {
+		RegisterTotalRequestCounter(r)
+	}
+	if h.containsMetricsMethod(MetricTypeStatusCode) {
+		RegisterRequestCodeCounter(recorder, r)
+	}
+}
+
+func (h *Handler) containsMetricsMethod(method MetricType) bool {
+	for _, m := range h.methods {
+		if m == method {
+			return true
+		}
+	}
+	return false
+}
+
+func RegisterRequestCounter(r *http.Request) {
+	var labels = map[string]interface{}{
+		URI:    strings.Split(r.RequestURI, "?")[0],
+		Method: r.Method,
+	}
+	RegisterCounter(RequestCounter, RequestCountDescription)
+	AddCount(r.Context(), RequestCounter, 1, labels)
+}
+
+func RegisterTotalRequestCounter(r *http.Request) {
+	RegisterCounter(TotalRequestCounter, TotalRequestDescription)
+	AddCount(r.Context(), TotalRequestCounter, 1, nil)
+}
+
+func RegisterRequestCodeCounter(recorder *StatusRecorder, r *http.Request) {
+	var labels = map[string]interface{}{
+		URI:        strings.Split(r.RequestURI, "?")[0],
+		Method:     r.Method,
+		ReturnCode: recorder.Status,
+	}
+	RegisterCounter(ReturnCodeCounter, ReturnCodeCounterDescription)
+	AddCount(r.Context(), ReturnCodeCounter, 1, labels)
+}
+
+func shouldNotIgnore(endpoints ...string) func(r *http.Request) bool {
+	return func(r *http.Request) bool {
+		for _, endpoint := range endpoints {
+			if strings.HasPrefix(r.URL.RequestURI(), endpoint) {
+				return false
+			}
+		}
+		return true
+	}
+}

internal/telemetry/metrics/metrics.go

@@ -0,0 +1,73 @@
+package metrics
+
+import (
+	"context"
+	"go.opentelemetry.io/otel/api/metric"
+	"net/http"
+)
+
+const (
+	ActiveSessionCounter            = "zitadel.active_session_counter"
+	ActiveSessionCounterDescription = "Active session counter"
+	SpoolerDivCounter               = "zitadel.spooler_div_milliseconds"
+	SpoolerDivCounterDescription    = "Spooler div from last successful run to now in milliseconds"
+	Database                        = "database"
+	ViewName                        = "view_name"
+)
+
+type Metrics interface {
+	GetExporter() http.Handler
+	GetMetricsProvider() metric.MeterProvider
+	RegisterCounter(name, description string) error
+	AddCount(ctx context.Context, name string, value int64, labels map[string]interface{}) error
+	RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64ObserverFunc) error
+	RegisterValueObserver(name, description string, callbackFunc metric.Int64ObserverFunc) error
+}
+
+type Config interface {
+	NewMetrics() error
+}
+
+var M Metrics
+
+func GetExporter() http.Handler {
+	if M == nil {
+		return nil
+	}
+	return M.GetExporter()
+}
+
+func GetMetricsProvider() metric.MeterProvider {
+	if M == nil {
+		return nil
+	}
+	return M.GetMetricsProvider()
+}
+
+func RegisterCounter(name, description string) error {
+	if M == nil {
+		return nil
+	}
+	return M.RegisterCounter(name, description)
+}
+
+func AddCount(ctx context.Context, name string, value int64, labels map[string]interface{}) error {
+	if M == nil {
+		return nil
+	}
+	return M.AddCount(ctx, name, value, labels)
+}
+
+func RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64ObserverFunc) error {
+	if M == nil {
+		return nil
+	}
+	return M.RegisterUpDownSumObserver(name, description, callbackFunc)
+}
+
+func RegisterValueObserver(name, description string, callbackFunc metric.Int64ObserverFunc) error {
+	if M == nil {
+		return nil
+	}
+	return M.RegisterValueObserver(name, description, callbackFunc)
+}

internal/telemetry/metrics/otel/config.go

@@ -0,0 +1,14 @@
+package otel
+
+import (
+	"github.com/caos/zitadel/internal/telemetry/metrics"
+)
+
+type Config struct {
+	MeterName string
+}
+
+func (c *Config) NewMetrics() (err error) {
+	metrics.M, err = NewMetrics(c.MeterName)
+	return err
+}

internal/telemetry/metrics/otel/open_telemetry.go

@@ -0,0 +1,92 @@
+package otel
+
+import (
+	"context"
+	caos_errs "github.com/caos/zitadel/internal/errors"
+	"github.com/caos/zitadel/internal/telemetry/metrics"
+	"go.opentelemetry.io/otel/api/metric"
+	"go.opentelemetry.io/otel/exporters/metric/prometheus"
+	"go.opentelemetry.io/otel/label"
+	"net/http"
+	"sync"
+)
+
+type Metrics struct {
+	Exporter          *prometheus.Exporter
+	Meter             metric.Meter
+	Counters          sync.Map
+	UpDownSumObserver sync.Map
+	ValueObservers    sync.Map
+}
+
+func NewMetrics(meterName string) (metrics.Metrics, error) {
+	exporter, err := prometheus.NewExportPipeline(
+		prometheus.Config{},
+	)
+	if err != nil {
+		return &Metrics{}, err
+	}
+	return &Metrics{
+		Exporter: exporter,
+		Meter:    exporter.MeterProvider().Meter(meterName),
+	}, nil
+}
+
+func (m *Metrics) GetExporter() http.Handler {
+	return m.Exporter
+}
+
+func (m *Metrics) GetMetricsProvider() metric.MeterProvider {
+	return m.Exporter.MeterProvider()
+}
+
+func (m *Metrics) RegisterCounter(name, description string) error {
+	if _, exists := m.Counters.Load(name); exists {
+		return nil
+	}
+	counter := metric.Must(m.Meter).NewInt64Counter(name, metric.WithDescription(description))
+	m.Counters.Store(name, counter)
+	return nil
+}
+
+func (m *Metrics) AddCount(ctx context.Context, name string, value int64, labels map[string]interface{}) error {
+	counter, exists := m.Counters.Load(name)
+	if !exists {
+		return caos_errs.ThrowNotFound(nil, "METER-4u8fs", "Errors.Metrics.Counter.NotFound")
+	}
+	counter.(metric.Int64Counter).Add(ctx, value, MapToKeyValue(labels)...)
+	return nil
+}
+
+func (m *Metrics) RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64ObserverFunc) error {
+	if _, exists := m.UpDownSumObserver.Load(name); exists {
+		return nil
+	}
+	sumObserver := metric.Must(m.Meter).NewInt64UpDownSumObserver(
+		name, callbackFunc, metric.WithDescription(description))
+
+	m.UpDownSumObserver.Store(name, sumObserver)
+	return nil
+}
+
+func (m *Metrics) RegisterValueObserver(name, description string, callbackFunc metric.Int64ObserverFunc) error {
+	if _, exists := m.UpDownSumObserver.Load(name); exists {
+		return nil
+	}
+	sumObserver := metric.Must(m.Meter).NewInt64ValueObserver(
+		name, callbackFunc, metric.WithDescription(description))
+
+	m.UpDownSumObserver.Store(name, sumObserver)
+	return nil
+}
+
+func MapToKeyValue(labels map[string]interface{}) []label.KeyValue {
+	if labels == nil {
+		return nil
+	}
+	keyValues := make([]label.KeyValue, 0, len(labels))
+	for key, value := range labels {
+		keyValues = append(keyValues, label.Any(key, value))
+	}
+	return keyValues
+}

internal/telemetry/tracing/caller.go

@@ -0,0 +1,22 @@
+package tracing
+
+import (
+	"runtime"
+
+	"github.com/caos/logging"
+)
+
+func GetCaller() string {
+	fpcs := make([]uintptr, 1)
+	n := runtime.Callers(3, fpcs)
+	if n == 0 {
+		logging.Log("TRACE-rWjfC").Debug("no caller")
+		return ""
+	}
+	caller := runtime.FuncForPC(fpcs[0] - 1)
+	if caller == nil {
+		logging.Log("TRACE-25POw").Debug("caller was nil")
+		return ""
+	}
+	return caller.Name()
+}

internal/telemetry/tracing/config/config.go

@@ -0,0 +1,69 @@
+package config
+
+import (
+	"encoding/json"
+
+	"github.com/caos/zitadel/internal/errors"
+	"github.com/caos/zitadel/internal/telemetry/tracing"
+	"github.com/caos/zitadel/internal/telemetry/tracing/google"
+	"github.com/caos/zitadel/internal/telemetry/tracing/log"
+	"github.com/caos/zitadel/internal/telemetry/tracing/otel"
+)
+
+type TracingConfig struct {
+	Type   string
+	Config tracing.Config
+}
+
+var tracer = map[string]func() tracing.Config{
+	"otel":   func() tracing.Config { return &otel.Config{} },
+	"google": func() tracing.Config { return &google.Config{} },
+	"log":    func() tracing.Config { return &log.Config{} },
+	"none":   func() tracing.Config { return &NoTracing{} },
+	"":       func() tracing.Config { return &NoTracing{} },
+}
+
+func (c *TracingConfig) UnmarshalJSON(data []byte) error {
+	var rc struct {
+		Type   string
+		Config json.RawMessage
+	}
+
+	if err := json.Unmarshal(data, &rc); err != nil {
+		return errors.ThrowInternal(err, "TRACE-vmjS", "error parsing config")
+	}
+
+	c.Type = rc.Type
+
+	var err error
+	c.Config, err = newTracingConfig(c.Type, rc.Config)
+	if err != nil {
+		return err
+	}
+
+	return c.Config.NewTracer()
+}
+
+func newTracingConfig(tracerType string, configData []byte) (tracing.Config, error) {
+	t, ok := tracer[tracerType]
+	if !ok {
+		return nil, errors.ThrowInternalf(nil, "TRACE-HMEJ", "config type %s not supported", tracerType)
+	}
+
+	tracingConfig := t()
+	if len(configData) == 0 {
+		return tracingConfig, nil
+	}
+
+	if err := json.Unmarshal(configData, tracingConfig); err != nil {
+		return nil, errors.ThrowInternal(err, "TRACE-1tSS", "Could not read config: %v")
+	}
+
+	return tracingConfig, nil
+}
+
+type NoTracing struct{}
+
+func (_ *NoTracing) NewTracer() error {
+	return nil
+}

internal/telemetry/tracing/google/google_tracer.go

@@ -0,0 +1,43 @@
+package google
+
+import (
+	"os"
+	"strings"
+
+	texporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace"
+	"github.com/caos/zitadel/internal/errors"
+	"github.com/caos/zitadel/internal/telemetry/tracing"
+	"github.com/caos/zitadel/internal/telemetry/tracing/otel"
+	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+type Config struct {
+	ProjectID    string
+	MetricPrefix string
+	Fraction     float64
+}
+
+type Tracer struct {
+	otel.Tracer
+}
+
+func (c *Config) NewTracer() error {
+	if !envIsSet() {
+		return errors.ThrowInvalidArgument(nil, "GOOGL-sdh3a", "env not properly set, GOOGLE_APPLICATION_CREDENTIALS is misconfigured or missing")
+	}
+
+	sampler := sdk_trace.ParentBased(sdk_trace.TraceIDRatioBased(c.Fraction))
+	exporter, err := texporter.NewExporter(texporter.WithProjectID(c.ProjectID))
+	if err != nil {
+		return err
+	}
+
+	tracing.T = &Tracer{Tracer: *(otel.NewTracer(c.MetricPrefix, sampler, exporter))}
+
+	return nil
+}
+
+func envIsSet() bool {
+	gAuthCred := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")
+	return strings.Contains(gAuthCred, ".json")
+}

internal/telemetry/tracing/log/config.go

@@ -0,0 +1,28 @@
+package log
+
+import (
+	"github.com/caos/zitadel/internal/telemetry/tracing"
+	"github.com/caos/zitadel/internal/telemetry/tracing/otel"
+	"go.opentelemetry.io/otel/exporters/stdout"
+	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+type Config struct {
+	Fraction     float64
+	MetricPrefix string
+}
+
+type Tracer struct {
+	otel.Tracer
+}
+
+func (c *Config) NewTracer() error {
+	sampler := sdk_trace.ParentBased(sdk_trace.TraceIDRatioBased(c.Fraction))
+	exporter, err := stdout.NewExporter(stdout.WithPrettyPrint())
+	if err != nil {
+		return err
+	}
+
+	tracing.T = &Tracer{Tracer: *(otel.NewTracer(c.MetricPrefix, sampler, exporter))}
+	return nil
+}

internal/telemetry/tracing/otel/config.go

@@ -0,0 +1,24 @@
+package otel
+
+import (
+	"github.com/caos/zitadel/internal/telemetry/tracing"
+	"go.opentelemetry.io/otel/exporters/otlp"
+	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+type Config struct {
+	Fraction     float64
+	MetricPrefix string
+	Endpoint     string
+}
+
+func (c *Config) NewTracer() error {
+	sampler := sdk_trace.ParentBased(sdk_trace.TraceIDRatioBased(c.Fraction))
+	exporter, err := otlp.NewExporter(otlp.WithAddress(c.Endpoint), otlp.WithInsecure())
+	if err != nil {
+		return err
+	}
+
+	tracing.T = NewTracer(c.MetricPrefix, sampler, exporter)
+	return nil
+}

internal/telemetry/tracing/otel/open_telemetry.go

@@ -0,0 +1,70 @@
+package otel
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/caos/zitadel/internal/telemetry/tracing"
+	"go.opentelemetry.io/otel/api/global"
+	api_trace "go.opentelemetry.io/otel/api/trace"
+	"go.opentelemetry.io/otel/propagators"
+	"go.opentelemetry.io/otel/sdk/export/trace"
+	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+type Tracer struct {
+	Exporter api_trace.Tracer
+	sampler  sdk_trace.Sampler
+}
+
+func NewTracer(name string, sampler sdk_trace.Sampler, exporter trace.SpanExporter) *Tracer {
+	tp := sdk_trace.NewTracerProvider(
+		sdk_trace.WithConfig(sdk_trace.Config{DefaultSampler: sampler}),
+		sdk_trace.WithSyncer(exporter),
+	)
+
+	global.SetTracerProvider(tp)
+	tc := propagators.TraceContext{}
+	global.SetTextMapPropagator(tc)
+
+	return &Tracer{Exporter: tp.Tracer(name), sampler: sampler}
+}
+
+func (t *Tracer) Sampler() sdk_trace.Sampler {
+	return t.sampler
+}
+
+func (t *Tracer) NewServerInterceptorSpan(ctx context.Context, name string) (context.Context, *tracing.Span) {
+	return t.newSpanFromName(ctx, name, api_trace.WithSpanKind(api_trace.SpanKindServer))
+}
+
+func (t *Tracer) NewServerSpan(ctx context.Context, caller string) (context.Context, *tracing.Span) {
+	return t.newSpan(ctx, caller, api_trace.WithSpanKind(api_trace.SpanKindServer))
+}
+
+func (t *Tracer) NewClientInterceptorSpan(ctx context.Context, name string) (context.Context, *tracing.Span) {
+	return t.newSpanFromName(ctx, name, api_trace.WithSpanKind(api_trace.SpanKindClient))
+}
+
+func (t *Tracer) NewClientSpan(ctx context.Context, caller string) (context.Context, *tracing.Span) {
+	return t.newSpan(ctx, caller, api_trace.WithSpanKind(api_trace.SpanKindClient))
+}
+
+func (t *Tracer) NewSpan(ctx context.Context, caller string) (context.Context, *tracing.Span) {
+	return t.newSpan(ctx, caller)
+}
+
+func (t *Tracer) newSpan(ctx context.Context, caller string, options ...api_trace.SpanOption) (context.Context, *tracing.Span) {
+	return t.newSpanFromName(ctx, caller, options...)
+}
+
+func (t *Tracer) newSpanFromName(ctx context.Context, name string, options ...api_trace.SpanOption) (context.Context, *tracing.Span) {
+	ctx, span := t.Exporter.Start(ctx, name, options...)
+	return ctx, tracing.CreateSpan(span)
+}
+
+func (t *Tracer) NewSpanHTTP(r *http.Request, caller string) (*http.Request, *tracing.Span) {
+	ctx, span := t.NewSpan(r.Context(), caller)
+	r = r.WithContext(ctx)
+	return r, span
+}

internal/telemetry/tracing/span.go

@@ -0,0 +1,44 @@
+package tracing
+
+import (
+	"context"
+
+	grpc_errs "github.com/caos/zitadel/internal/api/grpc/errors"
+	api_trace "go.opentelemetry.io/otel/api/trace"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/label"
+)
+
+type Span struct {
+	span api_trace.Span
+	opts []api_trace.SpanOption
+}
+
+func CreateSpan(span api_trace.Span) *Span {
+	return &Span{span: span, opts: []api_trace.SpanOption{}}
+}
+
+func (s *Span) End() {
+	if s.span == nil {
+		return
+	}
+
+	s.span.End(s.opts...)
+}
+
+func (s *Span) EndWithError(err error) {
+	s.SetStatusByError(err)
+	s.End()
+}
+
+func (s *Span) SetStatusByError(err error) {
+	if s.span == nil {
+		return
+	}
+	if err != nil {
+		s.span.RecordError(context.TODO(), err, api_trace.WithErrorStatus(codes.Error))
+	}
+
+	code, msg, id, _ := grpc_errs.ExtractCaosError(err)
+	s.span.SetAttributes(label.Uint32("grpc_code", uint32(code)), label.String("grpc_msg", msg), label.String("error_id", id))
+}

internal/telemetry/tracing/tracing.go

@@ -0,0 +1,85 @@
+package tracing
+
+import (
+	"context"
+	"net/http"
+
+	api_trace "go.opentelemetry.io/otel/api/trace"
+	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+type Tracer interface {
+	NewSpan(ctx context.Context, caller string) (context.Context, *Span)
+	NewClientSpan(ctx context.Context, caller string) (context.Context, *Span)
+	NewServerSpan(ctx context.Context, caller string) (context.Context, *Span)
+	NewClientInterceptorSpan(ctx context.Context, name string) (context.Context, *Span)
+	NewServerInterceptorSpan(ctx context.Context, name string) (context.Context, *Span)
+	NewSpanHTTP(r *http.Request, caller string) (*http.Request, *Span)
+	Sampler() sdk_trace.Sampler
+}
+
+type Config interface {
+	NewTracer() error
+}
+
+var T Tracer
+
+func Sampler() sdk_trace.Sampler {
+	if T == nil {
+		return sdk_trace.NeverSample()
+	}
+	return T.Sampler()
+}
+
+func NewSpan(ctx context.Context) (context.Context, *Span) {
+	if T == nil {
+		return ctx, CreateSpan(nil)
+	}
+	return T.NewSpan(ctx, GetCaller())
+}
+
+func NewNamedSpan(ctx context.Context, name string) (context.Context, *Span) {
+	if T == nil {
+		return ctx, CreateSpan(nil)
+	}
+	return T.NewSpan(ctx, name)
+}
+
+func NewClientSpan(ctx context.Context) (context.Context, *Span) {
+	if T == nil {
+		return ctx, CreateSpan(nil)
+	}
+	return T.NewClientSpan(ctx, GetCaller())
+}
+
+func NewServerSpan(ctx context.Context) (context.Context, *Span) {
+	if T == nil {
+		return ctx, CreateSpan(nil)
+	}
+	return T.NewServerSpan(ctx, GetCaller())
+}
+
+func NewClientInterceptorSpan(ctx context.Context) (context.Context, *Span) {
+	if T == nil {
+		return ctx, CreateSpan(nil)
+	}
+	return T.NewClientInterceptorSpan(ctx, GetCaller())
+}
+
+func NewServerInterceptorSpan(ctx context.Context) (context.Context, *Span) {
+	if T == nil {
+		return ctx, CreateSpan(nil)
+	}
+	return T.NewServerInterceptorSpan(ctx, GetCaller())
+}
+
+func NewSpanHTTP(r *http.Request) (*http.Request, *Span) {
+	if T == nil {
+		return r, CreateSpan(nil)
+	}
+	return T.NewSpanHTTP(r, GetCaller())
+}
+
+func TraceIDFromCtx(ctx context.Context) string {
+	return api_trace.SpanFromContext(ctx).SpanContext().TraceID.String()
+}