Merge branch 'master' into new-eventstore

# Conflicts:
#	go.sum

internal/api/grpc/admin/administrator.go

@@ -10,7 +10,7 @@ import (
 )
 
 func (s *Server) GetViews(ctx context.Context, _ *empty.Empty) (_ *admin.Views, err error) {
-	views, err := s.administrator.GetViews(ctx)
+	views, err := s.administrator.GetViews()
 	if err != nil {
 		return nil, err
 	}

internal/api/grpc/admin/administrator_converter.go

@@ -26,14 +26,17 @@ func failedEventsFromModel(failedEvents []*view_model.FailedEvent) []*admin.Fail
 }
 
 func viewFromModel(view *view_model.View) *admin.View {
-	timestamp, err := ptypes.TimestampProto(view.CurrentTimestamp)
+	eventTimestamp, err := ptypes.TimestampProto(view.EventTimestamp)
+	logging.Log("GRPC-KSo03").OnError(err).Debug("unable to parse timestamp")
+	lastSpool, err := ptypes.TimestampProto(view.EventTimestamp)
 	logging.Log("GRPC-KSo03").OnError(err).Debug("unable to parse timestamp")
 
 	return &admin.View{
-		Database:          view.Database,
-		ViewName:          view.ViewName,
-		ProcessedSequence: view.CurrentSequence,
-		ViewTimestamp:     timestamp,
+		Database:                 view.Database,
+		ViewName:                 view.ViewName,
+		ProcessedSequence:        view.CurrentSequence,
+		EventTimestamp:           eventTimestamp,
+		LastSuccessfulSpoolerRun: lastSpool,
 	}
 }
 

internal/api/grpc/admin/login_policy_converter.go

@@ -13,6 +13,7 @@ func loginPolicyToModel(policy *admin.DefaultLoginPolicyRequest) *iam_model.Logi
 		AllowExternalIdp:      policy.AllowExternalIdp,
 		AllowRegister:         policy.AllowRegister,
 		ForceMFA:              policy.ForceMfa,
+		PasswordlessType:      passwordlessTypeToModel(policy.PasswordlessType),
 	}
 }
 
@@ -28,6 +29,7 @@ func loginPolicyFromModel(policy *iam_model.LoginPolicy) *admin.DefaultLoginPoli
 		AllowExternalIdp:      policy.AllowExternalIdp,
 		AllowRegister:         policy.AllowRegister,
 		ForceMfa:              policy.ForceMFA,
+		PasswordlessType:      passwordlessTypeFromModel(policy.PasswordlessType),
 		CreationDate:          creationDate,
 		ChangeDate:            changeDate,
 	}
@@ -45,6 +47,7 @@ func loginPolicyViewFromModel(policy *iam_model.LoginPolicyView) *admin.DefaultL
 		AllowExternalIdp:      policy.AllowExternalIDP,
 		AllowRegister:         policy.AllowRegister,
 		ForceMfa:              policy.ForceMFA,
+		PasswordlessType:      passwordlessTypeFromModel(policy.PasswordlessType),
 		CreationDate:          creationDate,
 		ChangeDate:            changeDate,
 	}
@@ -145,6 +148,24 @@ func secondFactorTypeToModel(mfaType *admin.SecondFactor) iam_model.SecondFactor
 	}
 }
 
+func passwordlessTypeFromModel(passwordlessType iam_model.PasswordlessType) admin.PasswordlessType {
+	switch passwordlessType {
+	case iam_model.PasswordlessTypeAllowed:
+		return admin.PasswordlessType_PASSWORDLESSTYPE_ALLOWED
+	default:
+		return admin.PasswordlessType_PASSWORDLESSTYPE_NOT_ALLOWED
+	}
+}
+
+func passwordlessTypeToModel(passwordlessType admin.PasswordlessType) iam_model.PasswordlessType {
+	switch passwordlessType {
+	case admin.PasswordlessType_PASSWORDLESSTYPE_ALLOWED:
+		return iam_model.PasswordlessTypeAllowed
+	default:
+		return iam_model.PasswordlessTypeNotAllowed
+	}
+}
+
 func multiFactorResultFromModel(result *iam_model.MultiFactorsSearchResponse) *admin.MultiFactorsResult {
 	converted := make([]admin.MultiFactorType, len(result.Result))
 	for i, mfaType := range result.Result {

internal/api/grpc/admin/probes.go

@@ -2,7 +2,6 @@ package admin
 
 import (
 	"context"
-
 	"github.com/golang/protobuf/ptypes/empty"
 )
 

internal/api/grpc/auth/user.go

@@ -2,7 +2,6 @@ package auth
 
 import (
 	"context"
-
 	"github.com/golang/protobuf/ptypes/empty"
 
 	"github.com/caos/zitadel/pkg/grpc/auth"
@@ -54,7 +53,7 @@ func (s *Server) GetMyUserAddress(ctx context.Context, _ *empty.Empty) (*auth.Us
 }
 
 func (s *Server) GetMyMfas(ctx context.Context, _ *empty.Empty) (*auth.MultiFactors, error) {
-	mfas, err := s.repo.MyUserMfas(ctx)
+	mfas, err := s.repo.MyUserMFAs(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -144,7 +143,7 @@ func (s *Server) GetMyPasswordComplexityPolicy(ctx context.Context, _ *empty.Emp
 }
 
 func (s *Server) AddMfaOTP(ctx context.Context, _ *empty.Empty) (_ *auth.MfaOtpResponse, err error) {
-	otp, err := s.repo.AddMyMfaOTP(ctx)
+	otp, err := s.repo.AddMyMFAOTP(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -152,12 +151,42 @@ func (s *Server) AddMfaOTP(ctx context.Context, _ *empty.Empty) (_ *auth.MfaOtpR
 }
 
 func (s *Server) VerifyMfaOTP(ctx context.Context, request *auth.VerifyMfaOtp) (*empty.Empty, error) {
-	err := s.repo.VerifyMyMfaOTPSetup(ctx, request.Code)
+	err := s.repo.VerifyMyMFAOTPSetup(ctx, request.Code)
 	return &empty.Empty{}, err
 }
 
 func (s *Server) RemoveMfaOTP(ctx context.Context, _ *empty.Empty) (_ *empty.Empty, err error) {
-	s.repo.RemoveMyMfaOTP(ctx)
+	err = s.repo.RemoveMyMFAOTP(ctx)
+	return &empty.Empty{}, err
+}
+
+func (s *Server) AddMyMfaU2F(ctx context.Context, _ *empty.Empty) (_ *auth.WebAuthNResponse, err error) {
+	u2f, err := s.repo.AddMyMFAU2F(ctx)
+	return verifyWebAuthNFromModel(u2f), err
+}
+
+func (s *Server) VerifyMyMfaU2F(ctx context.Context, request *auth.VerifyWebAuthN) (*empty.Empty, error) {
+	err := s.repo.VerifyMyMFAU2FSetup(ctx, request.TokenName, request.PublicKeyCredential)
+	return &empty.Empty{}, err
+}
+
+func (s *Server) RemoveMyMfaU2F(ctx context.Context, id *auth.WebAuthNTokenID) (*empty.Empty, error) {
+	err := s.repo.RemoveMyMFAU2F(ctx, id.Id)
+	return &empty.Empty{}, err
+}
+
+func (s *Server) AddMyPasswordless(ctx context.Context, _ *empty.Empty) (_ *auth.WebAuthNResponse, err error) {
+	u2f, err := s.repo.AddMyPasswordless(ctx)
+	return verifyWebAuthNFromModel(u2f), err
+}
+
+func (s *Server) VerifyMyPasswordless(ctx context.Context, request *auth.VerifyWebAuthN) (*empty.Empty, error) {
+	err := s.repo.VerifyMyPasswordlessSetup(ctx, request.TokenName, request.PublicKeyCredential)
+	return &empty.Empty{}, err
+}
+
+func (s *Server) RemoveMyPasswordless(ctx context.Context, id *auth.WebAuthNTokenID) (*empty.Empty, error) {
+	err := s.repo.RemoveMyPasswordless(ctx, id.Id)
 	return &empty.Empty{}, err
 }
 

internal/api/grpc/auth/user_converter.go

@@ -12,7 +12,7 @@ import (
 
 	"github.com/caos/zitadel/internal/api/authz"
 	"github.com/caos/zitadel/internal/eventstore/models"
-	"github.com/caos/zitadel/internal/tracing"
+	"github.com/caos/zitadel/internal/telemetry/tracing"
 	usr_model "github.com/caos/zitadel/internal/user/model"
 	"github.com/caos/zitadel/pkg/grpc/auth"
 	"github.com/caos/zitadel/pkg/grpc/message"
@@ -358,11 +358,11 @@ func genderToModel(gender auth.Gender) usr_model.Gender {
 	}
 }
 
-func mfaStateFromModel(state usr_model.MfaState) auth.MFAState {
+func mfaStateFromModel(state usr_model.MFAState) auth.MFAState {
 	switch state {
-	case usr_model.MfaStateReady:
+	case usr_model.MFAStateReady:
 		return auth.MFAState_MFASTATE_READY
-	case usr_model.MfaStateNotReady:
+	case usr_model.MFAStateNotReady:
 		return auth.MFAState_MFASTATE_NOT_READY
 	default:
 		return auth.MFAState_MFASTATE_UNSPECIFIED
@@ -379,17 +379,18 @@ func mfasFromModel(mfas []*usr_model.MultiFactor) []*auth.MultiFactor {
 
 func mfaFromModel(mfa *usr_model.MultiFactor) *auth.MultiFactor {
 	return &auth.MultiFactor{
-		State: mfaStateFromModel(mfa.State),
-		Type:  mfaTypeFromModel(mfa.Type),
+		State:     mfaStateFromModel(mfa.State),
+		Type:      mfaTypeFromModel(mfa.Type),
+		Attribute: mfa.Attribute,
 	}
 }
 
-func mfaTypeFromModel(mfatype usr_model.MfaType) auth.MfaType {
-	switch mfatype {
-	case usr_model.MfaTypeOTP:
+func mfaTypeFromModel(mfaType usr_model.MFAType) auth.MfaType {
+	switch mfaType {
+	case usr_model.MFATypeOTP:
 		return auth.MfaType_MFATYPE_OTP
-	case usr_model.MfaTypeSMS:
-		return auth.MfaType_MFATYPE_SMS
+	case usr_model.MFATypeU2F:
+		return auth.MfaType_MFATYPE_U2F
 	default:
 		return auth.MfaType_MFATYPE_UNSPECIFIED
 	}
@@ -426,3 +427,11 @@ func userChangesToAPI(changes *usr_model.UserChanges) (_ []*auth.Change) {
 
 	return result
 }
+
+func verifyWebAuthNFromModel(u2f *usr_model.WebAuthNToken) *auth.WebAuthNResponse {
+	return &auth.WebAuthNResponse{
+		Id:        u2f.WebAuthNTokenID,
+		PublicKey: u2f.PublicKey,
+		State:     mfaStateFromModel(u2f.State),
+	}
+}

internal/api/grpc/management/login_policy_converter.go

@@ -13,6 +13,7 @@ func loginPolicyRequestToModel(policy *management.LoginPolicyRequest) *iam_model
 		AllowExternalIdp:      policy.AllowExternalIdp,
 		AllowRegister:         policy.AllowRegister,
 		ForceMFA:              policy.ForceMfa,
+		PasswordlessType:      passwordlessTypeToModel(policy.PasswordlessType),
 	}
 }
 
@@ -30,6 +31,7 @@ func loginPolicyFromModel(policy *iam_model.LoginPolicy) *management.LoginPolicy
 		CreationDate:          creationDate,
 		ChangeDate:            changeDate,
 		ForceMfa:              policy.ForceMFA,
+		PasswordlessType:      passwordlessTypeFromModel(policy.PasswordlessType),
 	}
 }
 
@@ -48,6 +50,7 @@ func loginPolicyViewFromModel(policy *iam_model.LoginPolicyView) *management.Log
 		CreationDate:          creationDate,
 		ChangeDate:            changeDate,
 		ForceMfa:              policy.ForceMFA,
+		PasswordlessType:      passwordlessTypeFromModel(policy.PasswordlessType),
 	}
 }
 
@@ -215,3 +218,21 @@ func multiFactorTypeToModel(mfaType *management.MultiFactor) iam_model.MultiFact
 		return iam_model.MultiFactorTypeUnspecified
 	}
 }
+
+func passwordlessTypeFromModel(passwordlessType iam_model.PasswordlessType) management.PasswordlessType {
+	switch passwordlessType {
+	case iam_model.PasswordlessTypeAllowed:
+		return management.PasswordlessType_PASSWORDLESSTYPE_ALLOWED
+	default:
+		return management.PasswordlessType_PASSWORDLESSTYPE_NOT_ALLOWED
+	}
+}
+
+func passwordlessTypeToModel(passwordlessType management.PasswordlessType) iam_model.PasswordlessType {
+	switch passwordlessType {
+	case management.PasswordlessType_PASSWORDLESSTYPE_ALLOWED:
+		return iam_model.PasswordlessTypeAllowed
+	default:
+		return iam_model.PasswordlessTypeNotAllowed
+	}
+}

internal/api/grpc/management/user.go

@@ -214,7 +214,7 @@ func (s *Server) RemoveExternalIDP(ctx context.Context, request *management.Exte
 }
 
 func (s *Server) GetUserMfas(ctx context.Context, userID *management.UserID) (*management.UserMultiFactors, error) {
-	mfas, err := s.user.UserMfas(ctx, userID.Id)
+	mfas, err := s.user.UserMFAs(ctx, userID.Id)
 	if err != nil {
 		return nil, err
 	}

internal/api/grpc/management/user_converter.go

@@ -572,22 +572,22 @@ func genderToModel(gender management.Gender) usr_model.Gender {
 	}
 }
 
-func mfaTypeFromModel(mfatype usr_model.MfaType) management.MfaType {
+func mfaTypeFromModel(mfatype usr_model.MFAType) management.MfaType {
 	switch mfatype {
-	case usr_model.MfaTypeOTP:
+	case usr_model.MFATypeOTP:
 		return management.MfaType_MFATYPE_OTP
-	case usr_model.MfaTypeSMS:
-		return management.MfaType_MFATYPE_SMS
+	case usr_model.MFATypeU2F:
+		return management.MfaType_MFATYPE_U2F
 	default:
 		return management.MfaType_MFATYPE_UNSPECIFIED
 	}
 }
 
-func mfaStateFromModel(state usr_model.MfaState) management.MFAState {
+func mfaStateFromModel(state usr_model.MFAState) management.MFAState {
 	switch state {
-	case usr_model.MfaStateReady:
+	case usr_model.MFAStateReady:
 		return management.MFAState_MFASTATE_READY
-	case usr_model.MfaStateNotReady:
+	case usr_model.MFAStateNotReady:
 		return management.MFAState_MFASTATE_NOT_READY
 	default:
 		return management.MFAState_MFASTATE_UNSPECIFIED

internal/api/grpc/server/gateway.go

@@ -14,7 +14,7 @@ import (
 	client_middleware "github.com/caos/zitadel/internal/api/grpc/client/middleware"
 	http_util "github.com/caos/zitadel/internal/api/http"
 	http_mw "github.com/caos/zitadel/internal/api/http/middleware"
-	"github.com/caos/zitadel/internal/tracing"
+	"github.com/caos/zitadel/internal/telemetry/tracing"
 )
 
 const (
@@ -129,7 +129,8 @@ func createDialOptions(g Gateway) []grpc.DialOption {
 }
 
 func addInterceptors(handler http.Handler, g Gateway) http.Handler {
-	handler = http_mw.DefaultTraceHandler(handler)
+	handler = http_mw.DefaultMetricsHandler(handler)
+	handler = http_mw.DefaultTelemetryHandler(handler)
 	handler = http_mw.NoCacheInterceptor(handler)
 	if interceptor, ok := g.(grpcGatewayCustomInterceptor); ok {
 		handler = interceptor.GatewayHTTPInterceptor(handler)

internal/api/grpc/server/middleware/auth_interceptor.go

@@ -10,7 +10,7 @@ import (
 	"github.com/caos/zitadel/internal/api/authz"
 	grpc_util "github.com/caos/zitadel/internal/api/grpc"
 	"github.com/caos/zitadel/internal/api/http"
-	"github.com/caos/zitadel/internal/tracing"
+	"github.com/caos/zitadel/internal/telemetry/tracing"
 )
 
 func AuthorizationInterceptor(verifier *authz.TokenVerifier, authConfig authz.Config) grpc.UnaryServerInterceptor {

internal/api/grpc/server/middleware/metrics_interceptor.go

@@ -0,0 +1,86 @@
+package middleware
+
+import (
+	"context"
+	"strings"
+
+	"github.com/grpc-ecosystem/grpc-gateway/runtime"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/status"
+
+	_ "github.com/caos/zitadel/internal/statik"
+	"github.com/caos/zitadel/internal/telemetry/metrics"
+)
+
+const (
+	GrpcMethod                         = "grpc_method"
+	ReturnCode                         = "return_code"
+	GrpcRequestCounter                 = "grpc.server.request_counter"
+	GrpcRequestCounterDescription      = "Grpc request counter"
+	TotalGrpcRequestCounter            = "grpc.server.total_request_counter"
+	TotalGrpcRequestCounterDescription = "Total grpc request counter"
+	GrpcStatusCodeCounter              = "grpc.server.grpc_status_code"
+	GrpcStatusCodeCounterDescription   = "Grpc status code counter"
+)
+
+func MetricsHandler(metricTypes []metrics.MetricType, ignoredMethodSuffixes ...string) grpc.UnaryServerInterceptor {
+	return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
+		return RegisterMetrics(ctx, req, info, handler, metricTypes, ignoredMethodSuffixes...)
+	}
+}
+
+func RegisterMetrics(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler, metricTypes []metrics.MetricType, ignoredMethodSuffixes ...string) (_ interface{}, err error) {
+	if len(metricTypes) == 0 {
+		return handler(ctx, req)
+	}
+
+	for _, ignore := range ignoredMethodSuffixes {
+		if strings.HasSuffix(info.FullMethod, ignore) {
+			return handler(ctx, req)
+		}
+	}
+
+	resp, err := handler(ctx, req)
+	if containsMetricsMethod(metrics.MetricTypeRequestCount, metricTypes) {
+		RegisterGrpcRequestCounter(ctx, info)
+	}
+	if containsMetricsMethod(metrics.MetricTypeTotalCount, metricTypes) {
+		RegisterGrpcTotalRequestCounter(ctx)
+	}
+	if containsMetricsMethod(metrics.MetricTypeStatusCode, metricTypes) {
+		RegisterGrpcRequestCodeCounter(ctx, info, err)
+	}
+	return resp, err
+}
+
+func RegisterGrpcRequestCounter(ctx context.Context, info *grpc.UnaryServerInfo) {
+	var labels = map[string]interface{}{
+		GrpcMethod: info.FullMethod,
+	}
+	metrics.RegisterCounter(GrpcRequestCounter, GrpcRequestCounterDescription)
+	metrics.AddCount(ctx, GrpcRequestCounter, 1, labels)
+}
+
+func RegisterGrpcTotalRequestCounter(ctx context.Context) {
+	metrics.RegisterCounter(TotalGrpcRequestCounter, TotalGrpcRequestCounterDescription)
+	metrics.AddCount(ctx, TotalGrpcRequestCounter, 1, nil)
+}
+
+func RegisterGrpcRequestCodeCounter(ctx context.Context, info *grpc.UnaryServerInfo, err error) {
+	statusCode := status.Code(err)
+	var labels = map[string]interface{}{
+		GrpcMethod: info.FullMethod,
+		ReturnCode: runtime.HTTPStatusFromCode(statusCode),
+	}
+	metrics.RegisterCounter(GrpcStatusCodeCounter, GrpcStatusCodeCounterDescription)
+	metrics.AddCount(ctx, GrpcStatusCodeCounter, 1, labels)
+}
+
+func containsMetricsMethod(metricType metrics.MetricType, metricTypes []metrics.MetricType) bool {
+	for _, m := range metricTypes {
+		if m == metricType {
+			return true
+		}
+	}
+	return false
+}

internal/api/grpc/server/probes.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/caos/zitadel/internal/errors"
 	"github.com/caos/zitadel/internal/proto"
-	"github.com/caos/zitadel/internal/tracing"
+	"github.com/caos/zitadel/internal/telemetry/tracing"
 )
 
 type ValidationFunction func(ctx context.Context) error

internal/api/grpc/server/server.go

@@ -2,6 +2,8 @@ package server
 
 import (
 	"context"
+	grpc_api "github.com/caos/zitadel/internal/api/grpc"
+	"github.com/caos/zitadel/internal/telemetry/metrics"
 
 	"github.com/caos/logging"
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
@@ -11,7 +13,7 @@ import (
 	"github.com/caos/zitadel/internal/api/authz"
 	"github.com/caos/zitadel/internal/api/grpc/server/middleware"
 	"github.com/caos/zitadel/internal/api/http"
-	"github.com/caos/zitadel/internal/tracing"
+	"github.com/caos/zitadel/internal/telemetry/tracing"
 )
 
 const (
@@ -27,10 +29,12 @@ type Server interface {
 }
 
 func CreateServer(verifier *authz.TokenVerifier, authConfig authz.Config, lang language.Tag) *grpc.Server {
+	metricTypes := []metrics.MetricType{metrics.MetricTypeTotalCount, metrics.MetricTypeRequestCount, metrics.MetricTypeStatusCode}
 	return grpc.NewServer(
 		grpc.UnaryInterceptor(
 			grpc_middleware.ChainUnaryServer(
 				middleware.DefaultTracingServer(),
+				middleware.MetricsHandler(metricTypes, grpc_api.Probes...),
 				middleware.ErrorHandler(),
 				middleware.AuthorizationInterceptor(verifier, authConfig),
 				middleware.TranslationHandler(lang),