feat: exchange gRPC server implementation to connectRPC (#10145)

# Which Problems Are Solved

The current maintained gRPC server in combination with a REST (grpc)
gateway is getting harder and harder to maintain. Additionally, there
have been and still are issues with supporting / displaying `oneOf`s
correctly.
We therefore decided to exchange the server implementation to
connectRPC, which apart from supporting connect as protocol, also also
"standard" gRCP clients as well as HTTP/1.1 / rest like clients, e.g.
curl directly call the server without any additional gateway.

# How the Problems Are Solved

- All v2 services are moved to connectRPC implementation. (v1 services
are still served as pure grpc servers)
- All gRPC server interceptors were migrated / copied to a corresponding
connectRPC interceptor.
- API.ListGrpcServices and API. ListGrpcMethods were changed to include
the connect services and endpoints.
- gRPC server reflection was changed to a `StaticReflector` using the
`ListGrpcServices` list.
- The `grpc.Server` interfaces was split into different combinations to
be able to handle the different cases (grpc server and prefixed gateway,
connect server with grpc gateway, connect server only, ...)
- Docs of services serving connectRPC only with no additional gateway
(instance, webkey, project, app, org v2 beta) are changed to expose that
- since the plugin is not yet available on buf, we download it using
`postinstall` hook of the docs

# Additional Changes

- WebKey service is added as v2 service (in addition to the current
v2beta)

# Additional Context

closes #9483

---------

Co-authored-by: Elio Bischof <elio@zitadel.com>

internal/api/grpc/app/v2beta/app.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 	"time"
 
+	"connectrpc.com/connect"
 	"google.golang.org/protobuf/types/known/timestamppb"
 
 	"github.com/zitadel/zitadel/internal/api/grpc/app/v2beta/convert"
@@ -13,15 +14,15 @@ import (
 	app "github.com/zitadel/zitadel/pkg/grpc/app/v2beta"
 )
 
-func (s *Server) CreateApplication(ctx context.Context, req *app.CreateApplicationRequest) (*app.CreateApplicationResponse, error) {
-	switch t := req.GetCreationRequestType().(type) {
+func (s *Server) CreateApplication(ctx context.Context, req *connect.Request[app.CreateApplicationRequest]) (*connect.Response[app.CreateApplicationResponse], error) {
+	switch t := req.Msg.GetCreationRequestType().(type) {
 	case *app.CreateApplicationRequest_ApiRequest:
-		apiApp, err := s.command.AddAPIApplication(ctx, convert.CreateAPIApplicationRequestToDomain(req.GetName(), req.GetProjectId(), req.GetId(), t.ApiRequest), "")
+		apiApp, err := s.command.AddAPIApplication(ctx, convert.CreateAPIApplicationRequestToDomain(req.Msg.GetName(), req.Msg.GetProjectId(), req.Msg.GetId(), t.ApiRequest), "")
 		if err != nil {
 			return nil, err
 		}
 
-		return &app.CreateApplicationResponse{
+		return connect.NewResponse(&app.CreateApplicationResponse{
 			AppId:        apiApp.AppID,
 			CreationDate: timestamppb.New(apiApp.ChangeDate),
 			CreationResponseType: &app.CreateApplicationResponse_ApiResponse{
@@ -30,10 +31,10 @@ func (s *Server) CreateApplication(ctx context.Context, req *app.CreateApplicati
 					ClientSecret: apiApp.ClientSecretString,
 				},
 			},
-		}, nil
+		}), nil
 
 	case *app.CreateApplicationRequest_OidcRequest:
-		oidcAppRequest, err := convert.CreateOIDCAppRequestToDomain(req.GetName(), req.GetProjectId(), req.GetOidcRequest())
+		oidcAppRequest, err := convert.CreateOIDCAppRequestToDomain(req.Msg.GetName(), req.Msg.GetProjectId(), req.Msg.GetOidcRequest())
 		if err != nil {
 			return nil, err
 		}
@@ -43,7 +44,7 @@ func (s *Server) CreateApplication(ctx context.Context, req *app.CreateApplicati
 			return nil, err
 		}
 
-		return &app.CreateApplicationResponse{
+		return connect.NewResponse(&app.CreateApplicationResponse{
 			AppId:        oidcApp.AppID,
 			CreationDate: timestamppb.New(oidcApp.ChangeDate),
 			CreationResponseType: &app.CreateApplicationResponse_OidcResponse{
@@ -54,10 +55,10 @@ func (s *Server) CreateApplication(ctx context.Context, req *app.CreateApplicati
 					ComplianceProblems: convert.ComplianceProblemsToLocalizedMessages(oidcApp.Compliance.Problems),
 				},
 			},
-		}, nil
+		}), nil
 
 	case *app.CreateApplicationRequest_SamlRequest:
-		samlAppRequest, err := convert.CreateSAMLAppRequestToDomain(req.GetName(), req.GetProjectId(), req.GetSamlRequest())
+		samlAppRequest, err := convert.CreateSAMLAppRequestToDomain(req.Msg.GetName(), req.Msg.GetProjectId(), req.Msg.GetSamlRequest())
 		if err != nil {
 			return nil, err
 		}
@@ -67,27 +68,27 @@ func (s *Server) CreateApplication(ctx context.Context, req *app.CreateApplicati
 			return nil, err
 		}
 
-		return &app.CreateApplicationResponse{
+		return connect.NewResponse(&app.CreateApplicationResponse{
 			AppId:        samlApp.AppID,
 			CreationDate: timestamppb.New(samlApp.ChangeDate),
 			CreationResponseType: &app.CreateApplicationResponse_SamlResponse{
 				SamlResponse: &app.CreateSAMLApplicationResponse{},
 			},
-		}, nil
+		}), nil
 	default:
 		return nil, zerrors.ThrowInvalidArgument(nil, "APP-0iiN46", "unknown app type")
 	}
 }
 
-func (s *Server) UpdateApplication(ctx context.Context, req *app.UpdateApplicationRequest) (*app.UpdateApplicationResponse, error) {
+func (s *Server) UpdateApplication(ctx context.Context, req *connect.Request[app.UpdateApplicationRequest]) (*connect.Response[app.UpdateApplicationResponse], error) {
 	var changedTime time.Time
 
-	if name := strings.TrimSpace(req.GetName()); name != "" {
+	if name := strings.TrimSpace(req.Msg.GetName()); name != "" {
 		updatedDetails, err := s.command.UpdateApplicationName(
 			ctx,
-			req.GetProjectId(),
+			req.Msg.GetProjectId(),
 			&domain.ChangeApp{
-				AppID:   req.GetId(),
+				AppID:   req.Msg.GetId(),
 				AppName: name,
 			},
 			"",
@@ -99,9 +100,9 @@ func (s *Server) UpdateApplication(ctx context.Context, req *app.UpdateApplicati
 		changedTime = updatedDetails.EventDate
 	}
 
-	switch t := req.GetUpdateRequestType().(type) {
+	switch t := req.Msg.GetUpdateRequestType().(type) {
 	case *app.UpdateApplicationRequest_ApiConfigurationRequest:
-		updatedAPIApp, err := s.command.UpdateAPIApplication(ctx, convert.UpdateAPIApplicationConfigurationRequestToDomain(req.GetId(), req.GetProjectId(), t.ApiConfigurationRequest), "")
+		updatedAPIApp, err := s.command.UpdateAPIApplication(ctx, convert.UpdateAPIApplicationConfigurationRequestToDomain(req.Msg.GetId(), req.Msg.GetProjectId(), t.ApiConfigurationRequest), "")
 		if err != nil {
 			return nil, err
 		}
@@ -109,7 +110,7 @@ func (s *Server) UpdateApplication(ctx context.Context, req *app.UpdateApplicati
 		changedTime = updatedAPIApp.ChangeDate
 
 	case *app.UpdateApplicationRequest_OidcConfigurationRequest:
-		oidcApp, err := convert.UpdateOIDCAppConfigRequestToDomain(req.GetId(), req.GetProjectId(), t.OidcConfigurationRequest)
+		oidcApp, err := convert.UpdateOIDCAppConfigRequestToDomain(req.Msg.GetId(), req.Msg.GetProjectId(), t.OidcConfigurationRequest)
 		if err != nil {
 			return nil, err
 		}
@@ -122,7 +123,7 @@ func (s *Server) UpdateApplication(ctx context.Context, req *app.UpdateApplicati
 		changedTime = updatedOIDCApp.ChangeDate
 
 	case *app.UpdateApplicationRequest_SamlConfigurationRequest:
-		samlApp, err := convert.UpdateSAMLAppConfigRequestToDomain(req.GetId(), req.GetProjectId(), t.SamlConfigurationRequest)
+		samlApp, err := convert.UpdateSAMLAppConfigRequestToDomain(req.Msg.GetId(), req.Msg.GetProjectId(), t.SamlConfigurationRequest)
 		if err != nil {
 			return nil, err
 		}
@@ -135,53 +136,53 @@ func (s *Server) UpdateApplication(ctx context.Context, req *app.UpdateApplicati
 		changedTime = updatedSAMLApp.ChangeDate
 	}
 
-	return &app.UpdateApplicationResponse{
+	return connect.NewResponse(&app.UpdateApplicationResponse{
 		ChangeDate: timestamppb.New(changedTime),
-	}, nil
+	}), nil
 }
 
-func (s *Server) DeleteApplication(ctx context.Context, req *app.DeleteApplicationRequest) (*app.DeleteApplicationResponse, error) {
-	details, err := s.command.RemoveApplication(ctx, req.GetProjectId(), req.GetId(), "")
+func (s *Server) DeleteApplication(ctx context.Context, req *connect.Request[app.DeleteApplicationRequest]) (*connect.Response[app.DeleteApplicationResponse], error) {
+	details, err := s.command.RemoveApplication(ctx, req.Msg.GetProjectId(), req.Msg.GetId(), "")
 	if err != nil {
 		return nil, err
 	}
 
-	return &app.DeleteApplicationResponse{
+	return connect.NewResponse(&app.DeleteApplicationResponse{
 		DeletionDate: timestamppb.New(details.EventDate),
-	}, nil
+	}), nil
 }
 
-func (s *Server) DeactivateApplication(ctx context.Context, req *app.DeactivateApplicationRequest) (*app.DeactivateApplicationResponse, error) {
-	details, err := s.command.DeactivateApplication(ctx, req.GetProjectId(), req.GetId(), "")
+func (s *Server) DeactivateApplication(ctx context.Context, req *connect.Request[app.DeactivateApplicationRequest]) (*connect.Response[app.DeactivateApplicationResponse], error) {
+	details, err := s.command.DeactivateApplication(ctx, req.Msg.GetProjectId(), req.Msg.GetId(), "")
 	if err != nil {
 		return nil, err
 	}
 
-	return &app.DeactivateApplicationResponse{
+	return connect.NewResponse(&app.DeactivateApplicationResponse{
 		DeactivationDate: timestamppb.New(details.EventDate),
-	}, nil
+	}), nil
 
 }
 
-func (s *Server) ReactivateApplication(ctx context.Context, req *app.ReactivateApplicationRequest) (*app.ReactivateApplicationResponse, error) {
-	details, err := s.command.ReactivateApplication(ctx, req.GetProjectId(), req.GetId(), "")
+func (s *Server) ReactivateApplication(ctx context.Context, req *connect.Request[app.ReactivateApplicationRequest]) (*connect.Response[app.ReactivateApplicationResponse], error) {
+	details, err := s.command.ReactivateApplication(ctx, req.Msg.GetProjectId(), req.Msg.GetId(), "")
 	if err != nil {
 		return nil, err
 	}
 
-	return &app.ReactivateApplicationResponse{
+	return connect.NewResponse(&app.ReactivateApplicationResponse{
 		ReactivationDate: timestamppb.New(details.EventDate),
-	}, nil
+	}), nil
 
 }
 
-func (s *Server) RegenerateClientSecret(ctx context.Context, req *app.RegenerateClientSecretRequest) (*app.RegenerateClientSecretResponse, error) {
+func (s *Server) RegenerateClientSecret(ctx context.Context, req *connect.Request[app.RegenerateClientSecretRequest]) (*connect.Response[app.RegenerateClientSecretResponse], error) {
 	var secret string
 	var changeDate time.Time
 
-	switch req.GetAppType().(type) {
+	switch req.Msg.GetAppType().(type) {
 	case *app.RegenerateClientSecretRequest_IsApi:
-		config, err := s.command.ChangeAPIApplicationSecret(ctx, req.GetProjectId(), req.GetApplicationId(), "")
+		config, err := s.command.ChangeAPIApplicationSecret(ctx, req.Msg.GetProjectId(), req.Msg.GetApplicationId(), "")
 		if err != nil {
 			return nil, err
 		}
@@ -189,7 +190,7 @@ func (s *Server) RegenerateClientSecret(ctx context.Context, req *app.Regenerate
 		changeDate = config.ChangeDate
 
 	case *app.RegenerateClientSecretRequest_IsOidc:
-		config, err := s.command.ChangeOIDCApplicationSecret(ctx, req.GetProjectId(), req.GetApplicationId(), "")
+		config, err := s.command.ChangeOIDCApplicationSecret(ctx, req.Msg.GetProjectId(), req.Msg.GetApplicationId(), "")
 		if err != nil {
 			return nil, err
 		}
@@ -201,8 +202,8 @@ func (s *Server) RegenerateClientSecret(ctx context.Context, req *app.Regenerate
 		return nil, zerrors.ThrowInvalidArgument(nil, "APP-aLWIzw", "unknown app type")
 	}
 
-	return &app.RegenerateClientSecretResponse{
+	return connect.NewResponse(&app.RegenerateClientSecretResponse{
 		ClientSecret: secret,
 		CreationDate: timestamppb.New(changeDate),
-	}, nil
+	}), nil
 }

internal/api/grpc/app/v2beta/app_key.go

@@ -4,14 +4,15 @@ import (
 	"context"
 	"strings"
 
+	"connectrpc.com/connect"
 	"google.golang.org/protobuf/types/known/timestamppb"
 
 	"github.com/zitadel/zitadel/internal/api/grpc/app/v2beta/convert"
 	app "github.com/zitadel/zitadel/pkg/grpc/app/v2beta"
 )
 
-func (s *Server) CreateApplicationKey(ctx context.Context, req *app.CreateApplicationKeyRequest) (*app.CreateApplicationKeyResponse, error) {
-	domainReq := convert.CreateAPIClientKeyRequestToDomain(req)
+func (s *Server) CreateApplicationKey(ctx context.Context, req *connect.Request[app.CreateApplicationKeyRequest]) (*connect.Response[app.CreateApplicationKeyResponse], error) {
+	domainReq := convert.CreateAPIClientKeyRequestToDomain(req.Msg)
 
 	appKey, err := s.command.AddApplicationKey(ctx, domainReq, "")
 	if err != nil {
@@ -23,25 +24,25 @@ func (s *Server) CreateApplicationKey(ctx context.Context, req *app.CreateApplic
 		return nil, err
 	}
 
-	return &app.CreateApplicationKeyResponse{
+	return connect.NewResponse(&app.CreateApplicationKeyResponse{
 		Id:           appKey.KeyID,
 		CreationDate: timestamppb.New(appKey.ChangeDate),
 		KeyDetails:   keyDetails,
-	}, nil
+	}), nil
 }
 
-func (s *Server) DeleteApplicationKey(ctx context.Context, req *app.DeleteApplicationKeyRequest) (*app.DeleteApplicationKeyResponse, error) {
+func (s *Server) DeleteApplicationKey(ctx context.Context, req *connect.Request[app.DeleteApplicationKeyRequest]) (*connect.Response[app.DeleteApplicationKeyResponse], error) {
 	deletionDetails, err := s.command.RemoveApplicationKey(ctx,
-		strings.TrimSpace(req.GetProjectId()),
-		strings.TrimSpace(req.GetApplicationId()),
-		strings.TrimSpace(req.GetId()),
-		strings.TrimSpace(req.GetOrganizationId()),
+		strings.TrimSpace(req.Msg.GetProjectId()),
+		strings.TrimSpace(req.Msg.GetApplicationId()),
+		strings.TrimSpace(req.Msg.GetId()),
+		strings.TrimSpace(req.Msg.GetOrganizationId()),
 	)
 	if err != nil {
 		return nil, err
 	}
 
-	return &app.DeleteApplicationKeyResponse{
+	return connect.NewResponse(&app.DeleteApplicationKeyResponse{
 		DeletionDate: timestamppb.New(deletionDetails.EventDate),
-	}, nil
+	}), nil
 }

internal/api/grpc/app/v2beta/query.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"strings"
 
+	"connectrpc.com/connect"
 	"google.golang.org/protobuf/types/known/timestamppb"
 
 	"github.com/zitadel/zitadel/internal/api/grpc/app/v2beta/convert"
@@ -12,19 +13,19 @@ import (
 	app "github.com/zitadel/zitadel/pkg/grpc/app/v2beta"
 )
 
-func (s *Server) GetApplication(ctx context.Context, req *app.GetApplicationRequest) (*app.GetApplicationResponse, error) {
-	res, err := s.query.AppByIDWithPermission(ctx, req.GetId(), false, s.checkPermission)
+func (s *Server) GetApplication(ctx context.Context, req *connect.Request[app.GetApplicationRequest]) (*connect.Response[app.GetApplicationResponse], error) {
+	res, err := s.query.AppByIDWithPermission(ctx, req.Msg.GetId(), false, s.checkPermission)
 	if err != nil {
 		return nil, err
 	}
 
-	return &app.GetApplicationResponse{
+	return connect.NewResponse(&app.GetApplicationResponse{
 		App: convert.AppToPb(res),
-	}, nil
+	}), nil
 }
 
-func (s *Server) ListApplications(ctx context.Context, req *app.ListApplicationsRequest) (*app.ListApplicationsResponse, error) {
-	queries, err := convert.ListApplicationsRequestToModel(s.systemDefaults, req)
+func (s *Server) ListApplications(ctx context.Context, req *connect.Request[app.ListApplicationsRequest]) (*connect.Response[app.ListApplicationsResponse], error) {
+	queries, err := convert.ListApplicationsRequestToModel(s.systemDefaults, req.Msg)
 	if err != nil {
 		return nil, err
 	}
@@ -34,32 +35,32 @@ func (s *Server) ListApplications(ctx context.Context, req *app.ListApplications
 		return nil, err
 	}
 
-	return &app.ListApplicationsResponse{
+	return connect.NewResponse(&app.ListApplicationsResponse{
 		Applications: convert.AppsToPb(res.Apps),
 		Pagination:   filter.QueryToPaginationPb(queries.SearchRequest, res.SearchResponse),
-	}, nil
+	}), nil
 }
 
-func (s *Server) GetApplicationKey(ctx context.Context, req *app.GetApplicationKeyRequest) (*app.GetApplicationKeyResponse, error) {
-	queries, err := convert.GetApplicationKeyQueriesRequestToDomain(req.GetOrganizationId(), req.GetProjectId(), req.GetApplicationId())
+func (s *Server) GetApplicationKey(ctx context.Context, req *connect.Request[app.GetApplicationKeyRequest]) (*connect.Response[app.GetApplicationKeyResponse], error) {
+	queries, err := convert.GetApplicationKeyQueriesRequestToDomain(req.Msg.GetOrganizationId(), req.Msg.GetProjectId(), req.Msg.GetApplicationId())
 	if err != nil {
 		return nil, err
 	}
 
-	key, err := s.query.GetAuthNKeyByIDWithPermission(ctx, true, strings.TrimSpace(req.GetId()), s.checkPermission, queries...)
+	key, err := s.query.GetAuthNKeyByIDWithPermission(ctx, true, strings.TrimSpace(req.Msg.GetId()), s.checkPermission, queries...)
 	if err != nil {
 		return nil, err
 	}
 
-	return &app.GetApplicationKeyResponse{
+	return connect.NewResponse(&app.GetApplicationKeyResponse{
 		Id:             key.ID,
 		CreationDate:   timestamppb.New(key.CreationDate),
 		ExpirationDate: timestamppb.New(key.Expiration),
-	}, nil
+	}), nil
 }
 
-func (s *Server) ListApplicationKeys(ctx context.Context, req *app.ListApplicationKeysRequest) (*app.ListApplicationKeysResponse, error) {
-	queries, err := convert.ListApplicationKeysRequestToDomain(s.systemDefaults, req)
+func (s *Server) ListApplicationKeys(ctx context.Context, req *connect.Request[app.ListApplicationKeysRequest]) (*connect.Response[app.ListApplicationKeysResponse], error) {
+	queries, err := convert.ListApplicationKeysRequestToDomain(s.systemDefaults, req.Msg)
 	if err != nil {
 		return nil, err
 	}
@@ -69,8 +70,8 @@ func (s *Server) ListApplicationKeys(ctx context.Context, req *app.ListApplicati
 		return nil, err
 	}
 
-	return &app.ListApplicationKeysResponse{
+	return connect.NewResponse(&app.ListApplicationKeysResponse{
 		Keys:       convert.ApplicationKeysToPb(res.AuthNKeys),
 		Pagination: filter.QueryToPaginationPb(queries.SearchRequest, res.SearchResponse),
-	}, nil
+	}), nil
 }

internal/api/grpc/app/v2beta/server.go

@@ -1,21 +1,23 @@
 package app
 
 import (
-	"google.golang.org/grpc"
+	"net/http"
+
+	"connectrpc.com/connect"
+	"google.golang.org/protobuf/reflect/protoreflect"
 
 	"github.com/zitadel/zitadel/internal/api/authz"
-	"github.com/zitadel/zitadel/internal/api/grpc/server"
 	"github.com/zitadel/zitadel/internal/command"
 	"github.com/zitadel/zitadel/internal/config/systemdefaults"
 	"github.com/zitadel/zitadel/internal/domain"
 	"github.com/zitadel/zitadel/internal/query"
 	app "github.com/zitadel/zitadel/pkg/grpc/app/v2beta"
+	"github.com/zitadel/zitadel/pkg/grpc/app/v2beta/appconnect"
 )
 
-var _ app.AppServiceServer = (*Server)(nil)
+var _ appconnect.AppServiceHandler = (*Server)(nil)
 
 type Server struct {
-	app.UnimplementedAppServiceServer
 	command         *command.Commands
 	query           *query.Queries
 	systemDefaults  systemdefaults.SystemDefaults
@@ -36,8 +38,12 @@ func CreateServer(
 	}
 }
 
-func (s *Server) RegisterServer(grpcServer *grpc.Server) {
-	app.RegisterAppServiceServer(grpcServer, s)
+func (s *Server) RegisterConnectServer(interceptors ...connect.Interceptor) (string, http.Handler) {
+	return appconnect.NewAppServiceHandler(s, connect.WithInterceptors(interceptors...))
+}
+
+func (s *Server) FileDescriptor() protoreflect.FileDescriptor {
+	return app.File_zitadel_app_v2beta_app_service_proto
 }
 
 func (s *Server) AppName() string {
@@ -51,7 +57,3 @@ func (s *Server) MethodPrefix() string {
 func (s *Server) AuthMethods() authz.MethodMapping {
 	return app.AppService_AuthMethods
 }
-
-func (s *Server) RegisterGateway() server.RegisterGatewayFunc {
-	return app.RegisterAppServiceHandler
-}