feat: metrics (#1024)

* refactor: switch from opencensus to opentelemetry

* tempo works as designed nooooot

* fix: log traceids

* with grafana agent

* fix: http tracing

* fix: cleanup files

* chore: remove todo

* fix: bad test

* fix: ignore methods in grpc interceptors

* fix: remove test log

* clean up

* typo

* fix(config): configure tracing endpoint

* fix(span): add error id to span

* feat: metrics package

* feat: metrics package

* fix: counter

* fix: metric

* try metrics

* fix: coutner metrics

* fix: active sessin counter

* fix: active sessin counter

* fix: change current Sequence table

* fix: change current Sequence table

* fix: current sequences

* fix: spooler div metrics

* fix: console view

* fix: merge master

* fix: Last spool run on search result instead of eventtimestamp

* fix: go mod

* Update console/src/assets/i18n/de.json

Co-authored-by: Livio Amstutz <livio.a@gmail.com>

* fix: pr review

* fix: map

* update oidc pkg

* fix: handlers

* fix: value observer

* fix: remove fmt

* fix: handlers

* fix: tests

* fix: handler minimum cycle duration 1s

* fix(spooler): handler channel buffer

* fix interceptors

Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
Co-authored-by: Livio Amstutz <livio.a@gmail.com>

internal/telemetry/tracing/caller.go

@@ -0,0 +1,22 @@
+package tracing
+
+import (
+	"runtime"
+
+	"github.com/caos/logging"
+)
+
+func GetCaller() string {
+	fpcs := make([]uintptr, 1)
+	n := runtime.Callers(3, fpcs)
+	if n == 0 {
+		logging.Log("TRACE-rWjfC").Debug("no caller")
+		return ""
+	}
+	caller := runtime.FuncForPC(fpcs[0] - 1)
+	if caller == nil {
+		logging.Log("TRACE-25POw").Debug("caller was nil")
+		return ""
+	}
+	return caller.Name()
+}

internal/telemetry/tracing/config/config.go

@@ -0,0 +1,69 @@
+package config
+
+import (
+	"encoding/json"
+
+	"github.com/caos/zitadel/internal/errors"
+	"github.com/caos/zitadel/internal/telemetry/tracing"
+	"github.com/caos/zitadel/internal/telemetry/tracing/google"
+	"github.com/caos/zitadel/internal/telemetry/tracing/log"
+	"github.com/caos/zitadel/internal/telemetry/tracing/otel"
+)
+
+type TracingConfig struct {
+	Type   string
+	Config tracing.Config
+}
+
+var tracer = map[string]func() tracing.Config{
+	"otel":   func() tracing.Config { return &otel.Config{} },
+	"google": func() tracing.Config { return &google.Config{} },
+	"log":    func() tracing.Config { return &log.Config{} },
+	"none":   func() tracing.Config { return &NoTracing{} },
+	"":       func() tracing.Config { return &NoTracing{} },
+}
+
+func (c *TracingConfig) UnmarshalJSON(data []byte) error {
+	var rc struct {
+		Type   string
+		Config json.RawMessage
+	}
+
+	if err := json.Unmarshal(data, &rc); err != nil {
+		return errors.ThrowInternal(err, "TRACE-vmjS", "error parsing config")
+	}
+
+	c.Type = rc.Type
+
+	var err error
+	c.Config, err = newTracingConfig(c.Type, rc.Config)
+	if err != nil {
+		return err
+	}
+
+	return c.Config.NewTracer()
+}
+
+func newTracingConfig(tracerType string, configData []byte) (tracing.Config, error) {
+	t, ok := tracer[tracerType]
+	if !ok {
+		return nil, errors.ThrowInternalf(nil, "TRACE-HMEJ", "config type %s not supported", tracerType)
+	}
+
+	tracingConfig := t()
+	if len(configData) == 0 {
+		return tracingConfig, nil
+	}
+
+	if err := json.Unmarshal(configData, tracingConfig); err != nil {
+		return nil, errors.ThrowInternal(err, "TRACE-1tSS", "Could not read config: %v")
+	}
+
+	return tracingConfig, nil
+}
+
+type NoTracing struct{}
+
+func (_ *NoTracing) NewTracer() error {
+	return nil
+}

internal/telemetry/tracing/google/google_tracer.go

@@ -0,0 +1,43 @@
+package google
+
+import (
+	"os"
+	"strings"
+
+	texporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace"
+	"github.com/caos/zitadel/internal/errors"
+	"github.com/caos/zitadel/internal/telemetry/tracing"
+	"github.com/caos/zitadel/internal/telemetry/tracing/otel"
+	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+type Config struct {
+	ProjectID    string
+	MetricPrefix string
+	Fraction     float64
+}
+
+type Tracer struct {
+	otel.Tracer
+}
+
+func (c *Config) NewTracer() error {
+	if !envIsSet() {
+		return errors.ThrowInvalidArgument(nil, "GOOGL-sdh3a", "env not properly set, GOOGLE_APPLICATION_CREDENTIALS is misconfigured or missing")
+	}
+
+	sampler := sdk_trace.ParentBased(sdk_trace.TraceIDRatioBased(c.Fraction))
+	exporter, err := texporter.NewExporter(texporter.WithProjectID(c.ProjectID))
+	if err != nil {
+		return err
+	}
+
+	tracing.T = &Tracer{Tracer: *(otel.NewTracer(c.MetricPrefix, sampler, exporter))}
+
+	return nil
+}
+
+func envIsSet() bool {
+	gAuthCred := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")
+	return strings.Contains(gAuthCred, ".json")
+}

internal/telemetry/tracing/log/config.go

@@ -0,0 +1,28 @@
+package log
+
+import (
+	"github.com/caos/zitadel/internal/telemetry/tracing"
+	"github.com/caos/zitadel/internal/telemetry/tracing/otel"
+	"go.opentelemetry.io/otel/exporters/stdout"
+	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+type Config struct {
+	Fraction     float64
+	MetricPrefix string
+}
+
+type Tracer struct {
+	otel.Tracer
+}
+
+func (c *Config) NewTracer() error {
+	sampler := sdk_trace.ParentBased(sdk_trace.TraceIDRatioBased(c.Fraction))
+	exporter, err := stdout.NewExporter(stdout.WithPrettyPrint())
+	if err != nil {
+		return err
+	}
+
+	tracing.T = &Tracer{Tracer: *(otel.NewTracer(c.MetricPrefix, sampler, exporter))}
+	return nil
+}

internal/telemetry/tracing/otel/config.go

@@ -0,0 +1,24 @@
+package otel
+
+import (
+	"github.com/caos/zitadel/internal/telemetry/tracing"
+	"go.opentelemetry.io/otel/exporters/otlp"
+	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+type Config struct {
+	Fraction     float64
+	MetricPrefix string
+	Endpoint     string
+}
+
+func (c *Config) NewTracer() error {
+	sampler := sdk_trace.ParentBased(sdk_trace.TraceIDRatioBased(c.Fraction))
+	exporter, err := otlp.NewExporter(otlp.WithAddress(c.Endpoint), otlp.WithInsecure())
+	if err != nil {
+		return err
+	}
+
+	tracing.T = NewTracer(c.MetricPrefix, sampler, exporter)
+	return nil
+}

internal/telemetry/tracing/otel/open_telemetry.go

@@ -0,0 +1,70 @@
+package otel
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/caos/zitadel/internal/telemetry/tracing"
+	"go.opentelemetry.io/otel/api/global"
+	api_trace "go.opentelemetry.io/otel/api/trace"
+	"go.opentelemetry.io/otel/propagators"
+	"go.opentelemetry.io/otel/sdk/export/trace"
+	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+type Tracer struct {
+	Exporter api_trace.Tracer
+	sampler  sdk_trace.Sampler
+}
+
+func NewTracer(name string, sampler sdk_trace.Sampler, exporter trace.SpanExporter) *Tracer {
+	tp := sdk_trace.NewTracerProvider(
+		sdk_trace.WithConfig(sdk_trace.Config{DefaultSampler: sampler}),
+		sdk_trace.WithSyncer(exporter),
+	)
+
+	global.SetTracerProvider(tp)
+	tc := propagators.TraceContext{}
+	global.SetTextMapPropagator(tc)
+
+	return &Tracer{Exporter: tp.Tracer(name), sampler: sampler}
+}
+
+func (t *Tracer) Sampler() sdk_trace.Sampler {
+	return t.sampler
+}
+
+func (t *Tracer) NewServerInterceptorSpan(ctx context.Context, name string) (context.Context, *tracing.Span) {
+	return t.newSpanFromName(ctx, name, api_trace.WithSpanKind(api_trace.SpanKindServer))
+}
+
+func (t *Tracer) NewServerSpan(ctx context.Context, caller string) (context.Context, *tracing.Span) {
+	return t.newSpan(ctx, caller, api_trace.WithSpanKind(api_trace.SpanKindServer))
+}
+
+func (t *Tracer) NewClientInterceptorSpan(ctx context.Context, name string) (context.Context, *tracing.Span) {
+	return t.newSpanFromName(ctx, name, api_trace.WithSpanKind(api_trace.SpanKindClient))
+}
+
+func (t *Tracer) NewClientSpan(ctx context.Context, caller string) (context.Context, *tracing.Span) {
+	return t.newSpan(ctx, caller, api_trace.WithSpanKind(api_trace.SpanKindClient))
+}
+
+func (t *Tracer) NewSpan(ctx context.Context, caller string) (context.Context, *tracing.Span) {
+	return t.newSpan(ctx, caller)
+}
+
+func (t *Tracer) newSpan(ctx context.Context, caller string, options ...api_trace.SpanOption) (context.Context, *tracing.Span) {
+	return t.newSpanFromName(ctx, caller, options...)
+}
+
+func (t *Tracer) newSpanFromName(ctx context.Context, name string, options ...api_trace.SpanOption) (context.Context, *tracing.Span) {
+	ctx, span := t.Exporter.Start(ctx, name, options...)
+	return ctx, tracing.CreateSpan(span)
+}
+
+func (t *Tracer) NewSpanHTTP(r *http.Request, caller string) (*http.Request, *tracing.Span) {
+	ctx, span := t.NewSpan(r.Context(), caller)
+	r = r.WithContext(ctx)
+	return r, span
+}

internal/telemetry/tracing/span.go

@@ -0,0 +1,44 @@
+package tracing
+
+import (
+	"context"
+
+	grpc_errs "github.com/caos/zitadel/internal/api/grpc/errors"
+	api_trace "go.opentelemetry.io/otel/api/trace"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/label"
+)
+
+type Span struct {
+	span api_trace.Span
+	opts []api_trace.SpanOption
+}
+
+func CreateSpan(span api_trace.Span) *Span {
+	return &Span{span: span, opts: []api_trace.SpanOption{}}
+}
+
+func (s *Span) End() {
+	if s.span == nil {
+		return
+	}
+
+	s.span.End(s.opts...)
+}
+
+func (s *Span) EndWithError(err error) {
+	s.SetStatusByError(err)
+	s.End()
+}
+
+func (s *Span) SetStatusByError(err error) {
+	if s.span == nil {
+		return
+	}
+	if err != nil {
+		s.span.RecordError(context.TODO(), err, api_trace.WithErrorStatus(codes.Error))
+	}
+
+	code, msg, id, _ := grpc_errs.ExtractCaosError(err)
+	s.span.SetAttributes(label.Uint32("grpc_code", uint32(code)), label.String("grpc_msg", msg), label.String("error_id", id))
+}

internal/telemetry/tracing/tracing.go

@@ -0,0 +1,85 @@
+package tracing
+
+import (
+	"context"
+	"net/http"
+
+	api_trace "go.opentelemetry.io/otel/api/trace"
+	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+type Tracer interface {
+	NewSpan(ctx context.Context, caller string) (context.Context, *Span)
+	NewClientSpan(ctx context.Context, caller string) (context.Context, *Span)
+	NewServerSpan(ctx context.Context, caller string) (context.Context, *Span)
+	NewClientInterceptorSpan(ctx context.Context, name string) (context.Context, *Span)
+	NewServerInterceptorSpan(ctx context.Context, name string) (context.Context, *Span)
+	NewSpanHTTP(r *http.Request, caller string) (*http.Request, *Span)
+	Sampler() sdk_trace.Sampler
+}
+
+type Config interface {
+	NewTracer() error
+}
+
+var T Tracer
+
+func Sampler() sdk_trace.Sampler {
+	if T == nil {
+		return sdk_trace.NeverSample()
+	}
+	return T.Sampler()
+}
+
+func NewSpan(ctx context.Context) (context.Context, *Span) {
+	if T == nil {
+		return ctx, CreateSpan(nil)
+	}
+	return T.NewSpan(ctx, GetCaller())
+}
+
+func NewNamedSpan(ctx context.Context, name string) (context.Context, *Span) {
+	if T == nil {
+		return ctx, CreateSpan(nil)
+	}
+	return T.NewSpan(ctx, name)
+}
+
+func NewClientSpan(ctx context.Context) (context.Context, *Span) {
+	if T == nil {
+		return ctx, CreateSpan(nil)
+	}
+	return T.NewClientSpan(ctx, GetCaller())
+}
+
+func NewServerSpan(ctx context.Context) (context.Context, *Span) {
+	if T == nil {
+		return ctx, CreateSpan(nil)
+	}
+	return T.NewServerSpan(ctx, GetCaller())
+}
+
+func NewClientInterceptorSpan(ctx context.Context) (context.Context, *Span) {
+	if T == nil {
+		return ctx, CreateSpan(nil)
+	}
+	return T.NewClientInterceptorSpan(ctx, GetCaller())
+}
+
+func NewServerInterceptorSpan(ctx context.Context) (context.Context, *Span) {
+	if T == nil {
+		return ctx, CreateSpan(nil)
+	}
+	return T.NewServerInterceptorSpan(ctx, GetCaller())
+}
+
+func NewSpanHTTP(r *http.Request) (*http.Request, *Span) {
+	if T == nil {
+		return r, CreateSpan(nil)
+	}
+	return T.NewSpanHTTP(r, GetCaller())
+}
+
+func TraceIDFromCtx(ctx context.Context) string {
+	return api_trace.SpanFromContext(ctx).SpanContext().TraceID.String()
+}