Merge branch 'master' into new-eventstore

# Conflicts:
#	go.mod
#	internal/admin/repository/eventsourcing/eventstore/iam.go
#	internal/authz/repository/eventsourcing/repository.go
#	internal/eventstore/eventstore.go
#	internal/setup/config.go
#	pkg/grpc/management/mock/management.proto.mock.go

internal/api/authz/authorization.go

@@ -14,33 +14,39 @@ const (
 	authenticated = "authenticated"
 )
 
-func CheckUserAuthorization(ctx context.Context, req interface{}, token, orgID string, verifier *TokenVerifier, authConfig Config, requiredAuthOption Option, method string) (_ context.Context, err error) {
+func CheckUserAuthorization(ctx context.Context, req interface{}, token, orgID string, verifier *TokenVerifier, authConfig Config, requiredAuthOption Option, method string) (ctxSetter func(context.Context) context.Context, err error) {
 	ctx, span := tracing.NewServerInterceptorSpan(ctx)
 	defer func() { span.EndWithError(err) }()
 
-	ctx, err = VerifyTokenAndWriteCtxData(ctx, token, orgID, verifier, method)
+	ctxData, err := VerifyTokenAndCreateCtxData(ctx, token, orgID, verifier, method)
 	if err != nil {
 		return nil, err
 	}
 
-	var perms []string
 	if requiredAuthOption.Permission == authenticated {
-		return ctx, nil
+		return func(parent context.Context) context.Context {
+			return context.WithValue(parent, dataKey, ctxData)
+		}, nil
 	}
 
-	ctx, perms, err = getUserMethodPermissions(ctx, verifier, requiredAuthOption.Permission, authConfig)
+	requestedPermissions, allPermissions, err := getUserMethodPermissions(ctx, verifier, requiredAuthOption.Permission, authConfig, ctxData)
 	if err != nil {
 		return nil, err
 	}
 
 	ctx, userPermissionSpan := tracing.NewNamedSpan(ctx, "checkUserPermissions")
-	err = checkUserPermissions(req, perms, requiredAuthOption)
+	err = checkUserPermissions(req, requestedPermissions, requiredAuthOption)
 	userPermissionSpan.EndWithError(err)
 	if err != nil {
 		return nil, err
 	}
 
-	return ctx, nil
+	return func(parent context.Context) context.Context {
+		parent = context.WithValue(parent, dataKey, ctxData)
+		parent = context.WithValue(parent, allPermissionsKey, allPermissions)
+		parent = context.WithValue(parent, requestPermissionsKey, requestedPermissions)
+		return parent
+	}, nil
 }
 
 func checkUserPermissions(req interface{}, userPerms []string, authOpt Option) error {

internal/api/authz/context.go

@@ -36,29 +36,36 @@ type Grant struct {
 	Roles []string
 }
 
-func VerifyTokenAndWriteCtxData(ctx context.Context, token, orgID string, t *TokenVerifier, method string) (_ context.Context, err error) {
+func VerifyTokenAndCreateCtxData(ctx context.Context, token, orgID string, t *TokenVerifier, method string) (_ CtxData, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() { span.EndWithError(err) }()
 
 	if orgID != "" {
 		err = t.ExistsOrg(ctx, orgID)
 		if err != nil {
-			return nil, errors.ThrowPermissionDenied(nil, "AUTH-Bs7Ds", "Organisation doesn't exist")
+			return CtxData{}, errors.ThrowPermissionDenied(nil, "AUTH-Bs7Ds", "Organisation doesn't exist")
 		}
 	}
 
 	userID, clientID, agentID, prefLang, err := verifyAccessToken(ctx, token, t, method)
 	if err != nil {
-		return nil, err
+		return CtxData{}, err
 	}
 	projectID, origins, err := t.ProjectIDAndOriginsByClientID(ctx, clientID)
 	if err != nil {
-		return nil, errors.ThrowPermissionDenied(err, "AUTH-GHpw2", "could not read projectid by clientid")
+		return CtxData{}, errors.ThrowPermissionDenied(err, "AUTH-GHpw2", "could not read projectid by clientid")
 	}
 	if err := checkOrigin(ctx, origins); err != nil {
-		return nil, err
+		return CtxData{}, err
 	}
-	return context.WithValue(ctx, dataKey, CtxData{UserID: userID, OrgID: orgID, ProjectID: projectID, AgentID: agentID, PreferredLanguage: prefLang}), nil
+	return CtxData{
+		UserID:            userID,
+		OrgID:             orgID,
+		ProjectID:         projectID,
+		AgentID:           agentID,
+		PreferredLanguage: prefLang,
+	}, nil
+
 }
 
 func SetCtxData(ctx context.Context, ctxData CtxData) context.Context {

internal/api/authz/permissions.go

@@ -7,29 +7,29 @@ import (
 	"github.com/caos/zitadel/internal/telemetry/tracing"
 )
 
-func getUserMethodPermissions(ctx context.Context, t *TokenVerifier, requiredPerm string, authConfig Config) (_ context.Context, _ []string, err error) {
+func getUserMethodPermissions(ctx context.Context, t *TokenVerifier, requiredPerm string, authConfig Config, ctxData CtxData) (requestedPermissions, allPermissions []string, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() { span.EndWithError(err) }()
 
-	ctxData := GetCtxData(ctx)
 	if ctxData.IsZero() {
 		return nil, nil, errors.ThrowUnauthenticated(nil, "AUTH-rKLWEH", "context missing")
 	}
+
+	ctx = context.WithValue(ctx, dataKey, ctxData)
 	grant, err := t.ResolveGrant(ctx)
 	if err != nil {
 		return nil, nil, err
 	}
 	if grant == nil {
-		return context.WithValue(ctx, requestPermissionsKey, []string{}), []string{}, nil
+		return requestedPermissions, nil, nil
 	}
-	requestPermissions, allPermissions := mapGrantToPermissions(requiredPerm, grant, authConfig)
-	ctx = context.WithValue(ctx, allPermissionsKey, allPermissions)
-	return context.WithValue(ctx, requestPermissionsKey, requestPermissions), requestPermissions, nil
+	requestedPermissions, allPermissions = mapGrantToPermissions(requiredPerm, grant, authConfig)
+	return requestedPermissions, allPermissions, nil
 }
 
-func mapGrantToPermissions(requiredPerm string, grant *Grant, authConfig Config) ([]string, []string) {
-	requestPermissions := make([]string, 0)
-	allPermissions := make([]string, 0)
+func mapGrantToPermissions(requiredPerm string, grant *Grant, authConfig Config) (requestPermissions, allPermissions []string) {
+	requestPermissions = make([]string, 0)
+	allPermissions = make([]string, 0)
 	for _, role := range grant.Roles {
 		requestPermissions, allPermissions = mapRoleToPerm(requiredPerm, role, authConfig, requestPermissions, allPermissions)
 	}

internal/api/authz/permissions_test.go

@@ -49,7 +49,7 @@ func equalStringArray(a, b []string) bool {
 
 func Test_GetUserMethodPermissions(t *testing.T) {
 	type args struct {
-		ctx          context.Context
+		ctxData      CtxData
 		verifier     *TokenVerifier
 		requiredPerm string
 		authConfig   Config
@@ -64,7 +64,7 @@ func Test_GetUserMethodPermissions(t *testing.T) {
 		{
 			name: "Empty Context",
 			args: args{
-				ctx: getTestCtx("", ""),
+				ctxData: CtxData{},
 				verifier: Start(&testVerifier{grant: &Grant{
 					Roles: []string{"ORG_OWNER"},
 				}}),
@@ -89,7 +89,7 @@ func Test_GetUserMethodPermissions(t *testing.T) {
 		{
 			name: "No Grants",
 			args: args{
-				ctx:          getTestCtx("", ""),
+				ctxData:      CtxData{},
 				verifier:     Start(&testVerifier{grant: &Grant{}}),
 				requiredPerm: "project.read",
 				authConfig: Config{
@@ -110,9 +110,9 @@ func Test_GetUserMethodPermissions(t *testing.T) {
 		{
 			name: "Get Permissions",
 			args: args{
-				ctx: getTestCtx("userID", "orgID"),
+				ctxData: CtxData{UserID: "userID", OrgID: "orgID"},
 				verifier: Start(&testVerifier{grant: &Grant{
-					Roles: []string{"ORG_OWNER"},
+					Roles: []string{"IAM_OWNER"},
 				}}),
 				requiredPerm: "project.read",
 				authConfig: Config{
@@ -133,7 +133,7 @@ func Test_GetUserMethodPermissions(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			_, perms, err := getUserMethodPermissions(tt.args.ctx, tt.args.verifier, tt.args.requiredPerm, tt.args.authConfig)
+			_, perms, err := getUserMethodPermissions(context.Background(), tt.args.verifier, tt.args.requiredPerm, tt.args.authConfig, tt.args.ctxData)
 
 			if tt.wantErr && err == nil {
 				t.Errorf("got wrong result, should get err: actual: %v ", err)

internal/api/grpc/admin/administrator_converter.go

@@ -28,8 +28,8 @@ func failedEventsFromModel(failedEvents []*view_model.FailedEvent) []*admin.Fail
 func viewFromModel(view *view_model.View) *admin.View {
 	eventTimestamp, err := ptypes.TimestampProto(view.EventTimestamp)
 	logging.Log("GRPC-KSo03").OnError(err).Debug("unable to parse timestamp")
-	lastSpool, err := ptypes.TimestampProto(view.EventTimestamp)
-	logging.Log("GRPC-KSo03").OnError(err).Debug("unable to parse timestamp")
+	lastSpool, err := ptypes.TimestampProto(view.LastSuccessfulSpoolerRun)
+	logging.Log("GRPC-0oP87").OnError(err).Debug("unable to parse timestamp")
 
 	return &admin.View{
 		Database:                 view.Database,

internal/api/grpc/auth/user.go

@@ -2,6 +2,7 @@ package auth
 
 import (
 	"context"
+
 	"github.com/golang/protobuf/ptypes/empty"
 
 	"github.com/caos/zitadel/pkg/grpc/auth"
@@ -162,6 +163,9 @@ func (s *Server) RemoveMfaOTP(ctx context.Context, _ *empty.Empty) (_ *empty.Emp
 
 func (s *Server) AddMyMfaU2F(ctx context.Context, _ *empty.Empty) (_ *auth.WebAuthNResponse, err error) {
 	u2f, err := s.repo.AddMyMFAU2F(ctx)
+	if err != nil {
+		return nil, err
+	}
 	return verifyWebAuthNFromModel(u2f), err
 }
 
@@ -175,8 +179,19 @@ func (s *Server) RemoveMyMfaU2F(ctx context.Context, id *auth.WebAuthNTokenID) (
 	return &empty.Empty{}, err
 }
 
+func (s *Server) GetMyPasswordless(ctx context.Context, _ *empty.Empty) (_ *auth.WebAuthNTokens, err error) {
+	tokens, err := s.repo.GetMyPasswordless(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return webAuthNTokensFromModel(tokens), err
+}
+
 func (s *Server) AddMyPasswordless(ctx context.Context, _ *empty.Empty) (_ *auth.WebAuthNResponse, err error) {
 	u2f, err := s.repo.AddMyPasswordless(ctx)
+	if err != nil {
+		return nil, err
+	}
 	return verifyWebAuthNFromModel(u2f), err
 }
 

internal/api/grpc/auth/user_converter.go

@@ -436,3 +436,19 @@ func verifyWebAuthNFromModel(u2f *usr_model.WebAuthNToken) *auth.WebAuthNRespons
 		State:     mfaStateFromModel(u2f.State),
 	}
 }
+
+func webAuthNTokensFromModel(tokens []*usr_model.WebAuthNToken) *auth.WebAuthNTokens {
+	result := make([]*auth.WebAuthNToken, len(tokens))
+	for i, token := range tokens {
+		result[i] = webAuthNTokenFromModel(token)
+	}
+	return &auth.WebAuthNTokens{Tokens: result}
+}
+
+func webAuthNTokenFromModel(token *usr_model.WebAuthNToken) *auth.WebAuthNToken {
+	return &auth.WebAuthNToken{
+		Id:    token.WebAuthNTokenID,
+		Name:  token.WebAuthNTokenName,
+		State: mfaStateFromModel(token.State),
+	}
+}

internal/api/grpc/management/user.go

@@ -2,9 +2,11 @@ package management
 
 import (
 	"context"
+
+	"github.com/golang/protobuf/ptypes/empty"
+
 	"github.com/caos/zitadel/internal/api/authz"
 	"github.com/caos/zitadel/pkg/grpc/management"
-	"github.com/golang/protobuf/ptypes/empty"
 )
 
 func (s *Server) GetUserByID(ctx context.Context, id *management.UserID) (*management.UserView, error) {
@@ -226,6 +228,24 @@ func (s *Server) RemoveMfaOTP(ctx context.Context, userID *management.UserID) (*
 	return &empty.Empty{}, err
 }
 
+func (s *Server) RemoveMfaU2F(ctx context.Context, webAuthNTokenID *management.WebAuthNTokenID) (*empty.Empty, error) {
+	err := s.user.RemoveU2F(ctx, webAuthNTokenID.UserId, webAuthNTokenID.Id)
+	return &empty.Empty{}, err
+}
+
+func (s *Server) GetPasswordless(ctx context.Context, userID *management.UserID) (_ *management.WebAuthNTokens, err error) {
+	tokens, err := s.user.GetPasswordless(ctx, userID.Id)
+	if err != nil {
+		return nil, err
+	}
+	return webAuthNTokensFromModel(tokens), err
+}
+
+func (s *Server) RemovePasswordless(ctx context.Context, id *management.WebAuthNTokenID) (*empty.Empty, error) {
+	err := s.user.RemovePasswordless(ctx, id.UserId, id.Id)
+	return &empty.Empty{}, err
+}
+
 func (s *Server) SearchUserMemberships(ctx context.Context, in *management.UserMembershipSearchRequest) (*management.UserMembershipSearchResponse, error) {
 	request := userMembershipSearchRequestsToModel(in)
 	request.AppendUserIDQuery(in.UserId)

internal/api/grpc/management/user_converter.go

@@ -2,14 +2,15 @@ package management
 
 import (
 	"encoding/json"
+
 	"github.com/caos/logging"
-	"github.com/caos/zitadel/internal/model"
 	"github.com/golang/protobuf/ptypes"
 	"golang.org/x/text/language"
 	"google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/types/known/structpb"
 
 	"github.com/caos/zitadel/internal/eventstore/models"
+	"github.com/caos/zitadel/internal/model"
 	usr_model "github.com/caos/zitadel/internal/user/model"
 	"github.com/caos/zitadel/pkg/grpc/management"
 	"github.com/caos/zitadel/pkg/grpc/message"
@@ -504,6 +505,7 @@ func mfaFromModel(mfa *usr_model.MultiFactor) *management.UserMultiFactor {
 		State:     mfaStateFromModel(mfa.State),
 		Type:      mfaTypeFromModel(mfa.Type),
 		Attribute: mfa.Attribute,
+		Id:        mfa.ID,
 	}
 }
 
@@ -627,3 +629,19 @@ func userChangesToMgtAPI(changes *usr_model.UserChanges) (_ []*management.Change
 
 	return result
 }
+
+func webAuthNTokensFromModel(tokens []*usr_model.WebAuthNToken) *management.WebAuthNTokens {
+	result := make([]*management.WebAuthNToken, len(tokens))
+	for i, token := range tokens {
+		result[i] = webAuthNTokenFromModel(token)
+	}
+	return &management.WebAuthNTokens{Tokens: result}
+}
+
+func webAuthNTokenFromModel(token *usr_model.WebAuthNToken) *management.WebAuthNToken {
+	return &management.WebAuthNToken{
+		Id:    token.WebAuthNTokenID,
+		Name:  token.WebAuthNTokenName,
+		State: mfaStateFromModel(token.State),
+	}
+}

internal/api/grpc/server/middleware/auth_interceptor.go

@@ -25,20 +25,20 @@ func authorize(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo,
 		return handler(ctx, req)
 	}
 
-	ctx, span := tracing.NewServerInterceptorSpan(ctx)
+	authCtx, span := tracing.NewServerInterceptorSpan(ctx)
 	defer func() { span.EndWithError(err) }()
 
-	authToken := grpc_util.GetAuthorizationHeader(ctx)
+	authToken := grpc_util.GetAuthorizationHeader(authCtx)
 	if authToken == "" {
 		return nil, status.Error(codes.Unauthenticated, "auth header missing")
 	}
 
-	orgID := grpc_util.GetHeader(ctx, http.ZitadelOrgID)
+	orgID := grpc_util.GetHeader(authCtx, http.ZitadelOrgID)
 
-	ctx, err = authz.CheckUserAuthorization(ctx, req, authToken, orgID, verifier, authConfig, authOpt, info.FullMethod)
+	ctxSetter, err := authz.CheckUserAuthorization(authCtx, req, authToken, orgID, verifier, authConfig, authOpt, info.FullMethod)
 	if err != nil {
 		return nil, err
 	}
 	span.End()
-	return handler(ctx, req)
+	return handler(ctxSetter(ctx), req)
 }

internal/api/oidc/client.go

@@ -2,17 +2,16 @@ package oidc
 
 import (
 	"context"
-	"github.com/caos/zitadel/internal/auth_request/model"
 	"strings"
 
-	"golang.org/x/text/language"
-	"gopkg.in/square/go-jose.v2"
-
 	"github.com/caos/oidc/pkg/oidc"
 	"github.com/caos/oidc/pkg/op"
+	"golang.org/x/text/language"
+	"gopkg.in/square/go-jose.v2"
 
 	"github.com/caos/zitadel/internal/api/authz"
 	"github.com/caos/zitadel/internal/api/http"
+	"github.com/caos/zitadel/internal/auth_request/model"
 	"github.com/caos/zitadel/internal/crypto"
 	"github.com/caos/zitadel/internal/errors"
 	proj_model "github.com/caos/zitadel/internal/project/model"
@@ -155,7 +154,7 @@ func (o *OPStorage) GetUserinfoFromScopes(ctx context.Context, userID, applicati
 				roles = append(roles, strings.TrimPrefix(scope, ScopeProjectRolePrefix))
 			}
 			if strings.HasPrefix(scope, model.OrgDomainPrimaryScope) {
-				userInfo.AppendClaims(model.OrgDomainPrimaryScope, strings.TrimPrefix(scope, model.OrgDomainPrimaryScope))
+				userInfo.AppendClaims(model.OrgDomainPrimaryClaim, strings.TrimPrefix(scope, model.OrgDomainPrimaryScope))
 			}
 		}
 	}
@@ -180,7 +179,7 @@ func (o *OPStorage) GetPrivateClaimsFromScopes(ctx context.Context, userID, clie
 		if strings.HasPrefix(scope, ScopeProjectRolePrefix) {
 			roles = append(roles, strings.TrimPrefix(scope, ScopeProjectRolePrefix))
 		} else if strings.HasPrefix(scope, model.OrgDomainPrimaryScope) {
-			claims = map[string]interface{}{model.OrgDomainPrimaryScope: strings.TrimPrefix(scope, model.OrgDomainPrimaryScope)}
+			claims = appendClaim(claims, model.OrgDomainPrimaryClaim, strings.TrimPrefix(scope, model.OrgDomainPrimaryScope))
 		}
 	}
 	if len(roles) == 0 || clientID == "" {
@@ -191,7 +190,7 @@ func (o *OPStorage) GetPrivateClaimsFromScopes(ctx context.Context, userID, clie
 		return nil, err
 	}
 	if len(projectRoles) > 0 {
-		claims = map[string]interface{}{ClaimProjectRoles: projectRoles}
+		claims = appendClaim(claims, ClaimProjectRoles, projectRoles)
 	}
 	return claims, err
 }
@@ -240,3 +239,11 @@ func getGender(gender user_model.Gender) string {
 	}
 	return ""
 }
+
+func appendClaim(claims map[string]interface{}, claim string, value interface{}) map[string]interface{} {
+	if claims == nil {
+		claims = make(map[string]interface{})
+	}
+	claims[claim] = value
+	return claims
+}