fix(tracing): from opencensus to opentelemetry (#937)

* refactor: switch from opencensus to opentelemetry

* tempo works as designed nooooot

* fix: log traceids

* with grafana agent

* fix: http tracing

* fix: cleanup files

* chore: remove todo

* fix: bad test

* fix: ignore methods in grpc interceptors

* fix: remove test log

* clean up

* typo

* fix(config): configure tracing endpoint

* fix(span): add error id to span

internal/api/grpc/server/middleware/auth_interceptor.go

@@ -20,15 +20,14 @@ func AuthorizationInterceptor(verifier *authz.TokenVerifier, authConfig authz.Co
 }
 
 func authorize(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler, verifier *authz.TokenVerifier, authConfig authz.Config) (_ interface{}, err error) {
-	ctx, span := tracing.NewServerInterceptorSpan(ctx)
-	defer func() { span.EndWithError(err) }()
-
 	authOpt, needsToken := verifier.CheckAuthMethod(info.FullMethod)
 	if !needsToken {
-		span.End()
 		return handler(ctx, req)
 	}
 
+	ctx, span := tracing.NewServerInterceptorSpan(ctx)
+	defer func() { span.EndWithError(err) }()
+
 	authToken := grpc_util.GetAuthorizationHeader(ctx)
 	if authToken == "" {
 		return nil, status.Error(codes.Unauthenticated, "auth header missing")

internal/api/grpc/server/middleware/tracing.go

@@ -4,45 +4,30 @@ import (
 	"context"
 	"strings"
 
-	"go.opencensus.io/plugin/ocgrpc"
-	"go.opencensus.io/trace"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/stats"
-
 	grpc_utils "github.com/caos/zitadel/internal/api/grpc"
-	"github.com/caos/zitadel/internal/tracing"
+	grpc_trace "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
+	"google.golang.org/grpc"
 )
 
 type GRPCMethod string
 
-func TracingStatsServer(ignoredMethods ...GRPCMethod) grpc.ServerOption {
-	return grpc.StatsHandler(
-		&tracingServerHandler{
-			ignoredMethods,
-			ocgrpc.ServerHandler{
-				StartOptions: trace.StartOptions{
-					Sampler:  tracing.Sampler(),
-					SpanKind: trace.SpanKindServer,
-				},
-			},
-		},
-	)
+func DefaultTracingServer() grpc.UnaryServerInterceptor {
+	return TracingServer(grpc_utils.Healthz, grpc_utils.Readiness, grpc_utils.Validation)
 }
 
-func DefaultTracingStatsServer() grpc.ServerOption {
-	return TracingStatsServer(grpc_utils.Healthz, grpc_utils.Readiness, grpc_utils.Validation)
-}
+func TracingServer(ignoredMethods ...GRPCMethod) grpc.UnaryServerInterceptor {
+	return func(
+		ctx context.Context,
+		req interface{},
+		info *grpc.UnaryServerInfo,
+		handler grpc.UnaryHandler,
+	) (interface{}, error) {
 
-type tracingServerHandler struct {
-	IgnoredMethods []GRPCMethod
-	ocgrpc.ServerHandler
-}
-
-func (s *tracingServerHandler) TagRPC(ctx context.Context, tagInfo *stats.RPCTagInfo) context.Context {
-	for _, method := range s.IgnoredMethods {
-		if strings.HasSuffix(tagInfo.FullMethodName, string(method)) {
-			return ctx
+		for _, ignoredMethod := range ignoredMethods {
+			if strings.HasSuffix(info.FullMethod, string(ignoredMethod)) {
+				return handler(ctx, req)
+			}
 		}
+		return grpc_trace.UnaryServerInterceptor()(ctx, req, info, handler)
 	}
-	return s.ServerHandler.TagRPC(ctx, tagInfo)
 }

internal/api/grpc/server/middleware/tracing_test.go

@@ -1,71 +0,0 @@
-package middleware
-
-import (
-	"context"
-	"testing"
-
-	"go.opencensus.io/plugin/ocgrpc"
-	"go.opencensus.io/trace"
-	"google.golang.org/grpc/stats"
-)
-
-func Test_tracingServerHandler_TagRPC(t *testing.T) {
-	type fields struct {
-		IgnoredMethods []GRPCMethod
-		ServerHandler  ocgrpc.ServerHandler
-	}
-	type args struct {
-		ctx     context.Context
-		tagInfo *stats.RPCTagInfo
-	}
-	type res struct {
-		wantSpan bool
-	}
-	tests := []struct {
-		name   string
-		fields fields
-		args   args
-		res    res
-	}{
-		{
-			"ignored method",
-			fields{
-				IgnoredMethods: []GRPCMethod{"ignore"},
-				ServerHandler:  ocgrpc.ServerHandler{},
-			},
-			args{
-				ctx: context.Background(),
-				tagInfo: &stats.RPCTagInfo{
-					FullMethodName: "ignore",
-				},
-			},
-			res{false},
-		},
-		{
-			"tag",
-			fields{
-				IgnoredMethods: []GRPCMethod{"ignore"},
-				ServerHandler:  ocgrpc.ServerHandler{},
-			},
-			args{
-				ctx: context.Background(),
-				tagInfo: &stats.RPCTagInfo{
-					FullMethodName: "tag",
-				},
-			},
-			res{true},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			s := &tracingServerHandler{
-				IgnoredMethods: tt.fields.IgnoredMethods,
-				ServerHandler:  tt.fields.ServerHandler,
-			}
-			got := s.TagRPC(tt.args.ctx, tt.args.tagInfo)
-			if (trace.FromContext(got) != nil) != tt.res.wantSpan {
-				t.Errorf("TagRPC() = %v, want %v", got, tt.res.wantSpan)
-			}
-		})
-	}
-}