feat: exchange gRPC server implementation to connectRPC (#10145)

# Which Problems Are Solved

The current maintained gRPC server in combination with a REST (grpc)
gateway is getting harder and harder to maintain. Additionally, there
have been and still are issues with supporting / displaying `oneOf`s
correctly.
We therefore decided to exchange the server implementation to
connectRPC, which apart from supporting connect as protocol, also also
"standard" gRCP clients as well as HTTP/1.1 / rest like clients, e.g.
curl directly call the server without any additional gateway.

# How the Problems Are Solved

- All v2 services are moved to connectRPC implementation. (v1 services
are still served as pure grpc servers)
- All gRPC server interceptors were migrated / copied to a corresponding
connectRPC interceptor.
- API.ListGrpcServices and API. ListGrpcMethods were changed to include
the connect services and endpoints.
- gRPC server reflection was changed to a `StaticReflector` using the
`ListGrpcServices` list.
- The `grpc.Server` interfaces was split into different combinations to
be able to handle the different cases (grpc server and prefixed gateway,
connect server with grpc gateway, connect server only, ...)
- Docs of services serving connectRPC only with no additional gateway
(instance, webkey, project, app, org v2 beta) are changed to expose that
- since the plugin is not yet available on buf, we download it using
`postinstall` hook of the docs

# Additional Changes

- WebKey service is added as v2 service (in addition to the current
v2beta)

# Additional Context

closes #9483

---------

Co-authored-by: Elio Bischof <elio@zitadel.com>

internal/api/grpc/instance/v2beta/domain.go

@@ -3,48 +3,49 @@ package instance
 import (
 	"context"
 
+	"connectrpc.com/connect"
 	"google.golang.org/protobuf/types/known/timestamppb"
 
 	instance "github.com/zitadel/zitadel/pkg/grpc/instance/v2beta"
 )
 
-func (s *Server) AddCustomDomain(ctx context.Context, req *instance.AddCustomDomainRequest) (*instance.AddCustomDomainResponse, error) {
-	details, err := s.command.AddInstanceDomain(ctx, req.GetDomain())
+func (s *Server) AddCustomDomain(ctx context.Context, req *connect.Request[instance.AddCustomDomainRequest]) (*connect.Response[instance.AddCustomDomainResponse], error) {
+	details, err := s.command.AddInstanceDomain(ctx, req.Msg.GetDomain())
 	if err != nil {
 		return nil, err
 	}
-	return &instance.AddCustomDomainResponse{
+	return connect.NewResponse(&instance.AddCustomDomainResponse{
 		CreationDate: timestamppb.New(details.CreationDate),
-	}, nil
+	}), nil
 }
 
-func (s *Server) RemoveCustomDomain(ctx context.Context, req *instance.RemoveCustomDomainRequest) (*instance.RemoveCustomDomainResponse, error) {
-	details, err := s.command.RemoveInstanceDomain(ctx, req.GetDomain())
+func (s *Server) RemoveCustomDomain(ctx context.Context, req *connect.Request[instance.RemoveCustomDomainRequest]) (*connect.Response[instance.RemoveCustomDomainResponse], error) {
+	details, err := s.command.RemoveInstanceDomain(ctx, req.Msg.GetDomain())
 	if err != nil {
 		return nil, err
 	}
-	return &instance.RemoveCustomDomainResponse{
+	return connect.NewResponse(&instance.RemoveCustomDomainResponse{
 		DeletionDate: timestamppb.New(details.EventDate),
-	}, nil
+	}), nil
 }
 
-func (s *Server) AddTrustedDomain(ctx context.Context, req *instance.AddTrustedDomainRequest) (*instance.AddTrustedDomainResponse, error) {
-	details, err := s.command.AddTrustedDomain(ctx, req.GetDomain())
+func (s *Server) AddTrustedDomain(ctx context.Context, req *connect.Request[instance.AddTrustedDomainRequest]) (*connect.Response[instance.AddTrustedDomainResponse], error) {
+	details, err := s.command.AddTrustedDomain(ctx, req.Msg.GetDomain())
 	if err != nil {
 		return nil, err
 	}
-	return &instance.AddTrustedDomainResponse{
+	return connect.NewResponse(&instance.AddTrustedDomainResponse{
 		CreationDate: timestamppb.New(details.CreationDate),
-	}, nil
+	}), nil
 }
 
-func (s *Server) RemoveTrustedDomain(ctx context.Context, req *instance.RemoveTrustedDomainRequest) (*instance.RemoveTrustedDomainResponse, error) {
-	details, err := s.command.RemoveTrustedDomain(ctx, req.GetDomain())
+func (s *Server) RemoveTrustedDomain(ctx context.Context, req *connect.Request[instance.RemoveTrustedDomainRequest]) (*connect.Response[instance.RemoveTrustedDomainResponse], error) {
+	details, err := s.command.RemoveTrustedDomain(ctx, req.Msg.GetDomain())
 	if err != nil {
 		return nil, err
 	}
 
-	return &instance.RemoveTrustedDomainResponse{
+	return connect.NewResponse(&instance.RemoveTrustedDomainResponse{
 		DeletionDate: timestamppb.New(details.EventDate),
-	}, nil
+	}), nil
 }

internal/api/grpc/instance/v2beta/instance.go

@@ -3,30 +3,31 @@ package instance
 import (
 	"context"
 
+	"connectrpc.com/connect"
 	"google.golang.org/protobuf/types/known/timestamppb"
 
 	instance "github.com/zitadel/zitadel/pkg/grpc/instance/v2beta"
 )
 
-func (s *Server) DeleteInstance(ctx context.Context, request *instance.DeleteInstanceRequest) (*instance.DeleteInstanceResponse, error) {
-	obj, err := s.command.RemoveInstance(ctx, request.GetInstanceId())
+func (s *Server) DeleteInstance(ctx context.Context, request *connect.Request[instance.DeleteInstanceRequest]) (*connect.Response[instance.DeleteInstanceResponse], error) {
+	obj, err := s.command.RemoveInstance(ctx, request.Msg.GetInstanceId())
 	if err != nil {
 		return nil, err
 	}
 
-	return &instance.DeleteInstanceResponse{
+	return connect.NewResponse(&instance.DeleteInstanceResponse{
 		DeletionDate: timestamppb.New(obj.EventDate),
-	}, nil
+	}), nil
 
 }
 
-func (s *Server) UpdateInstance(ctx context.Context, request *instance.UpdateInstanceRequest) (*instance.UpdateInstanceResponse, error) {
-	obj, err := s.command.UpdateInstance(ctx, request.GetInstanceName())
+func (s *Server) UpdateInstance(ctx context.Context, request *connect.Request[instance.UpdateInstanceRequest]) (*connect.Response[instance.UpdateInstanceResponse], error) {
+	obj, err := s.command.UpdateInstance(ctx, request.Msg.GetInstanceName())
 	if err != nil {
 		return nil, err
 	}
 
-	return &instance.UpdateInstanceResponse{
+	return connect.NewResponse(&instance.UpdateInstanceResponse{
 		ChangeDate: timestamppb.New(obj.EventDate),
-	}, nil
+	}), nil
 }

internal/api/grpc/instance/v2beta/query.go

@@ -3,23 +3,25 @@ package instance
 import (
 	"context"
 
+	"connectrpc.com/connect"
+
 	filter "github.com/zitadel/zitadel/internal/api/grpc/filter/v2beta"
 	instance "github.com/zitadel/zitadel/pkg/grpc/instance/v2beta"
 )
 
-func (s *Server) GetInstance(ctx context.Context, _ *instance.GetInstanceRequest) (*instance.GetInstanceResponse, error) {
+func (s *Server) GetInstance(ctx context.Context, _ *connect.Request[instance.GetInstanceRequest]) (*connect.Response[instance.GetInstanceResponse], error) {
 	inst, err := s.query.Instance(ctx, true)
 	if err != nil {
 		return nil, err
 	}
 
-	return &instance.GetInstanceResponse{
+	return connect.NewResponse(&instance.GetInstanceResponse{
 		Instance: ToProtoObject(inst),
-	}, nil
+	}), nil
 }
 
-func (s *Server) ListInstances(ctx context.Context, req *instance.ListInstancesRequest) (*instance.ListInstancesResponse, error) {
-	queries, err := ListInstancesRequestToModel(req, s.systemDefaults)
+func (s *Server) ListInstances(ctx context.Context, req *connect.Request[instance.ListInstancesRequest]) (*connect.Response[instance.ListInstancesResponse], error) {
+	queries, err := ListInstancesRequestToModel(req.Msg, s.systemDefaults)
 	if err != nil {
 		return nil, err
 	}
@@ -29,14 +31,14 @@ func (s *Server) ListInstances(ctx context.Context, req *instance.ListInstancesR
 		return nil, err
 	}
 
-	return &instance.ListInstancesResponse{
+	return connect.NewResponse(&instance.ListInstancesResponse{
 		Instances:  InstancesToPb(instances.Instances),
 		Pagination: filter.QueryToPaginationPb(queries.SearchRequest, instances.SearchResponse),
-	}, nil
+	}), nil
 }
 
-func (s *Server) ListCustomDomains(ctx context.Context, req *instance.ListCustomDomainsRequest) (*instance.ListCustomDomainsResponse, error) {
-	queries, err := ListCustomDomainsRequestToModel(req, s.systemDefaults)
+func (s *Server) ListCustomDomains(ctx context.Context, req *connect.Request[instance.ListCustomDomainsRequest]) (*connect.Response[instance.ListCustomDomainsResponse], error) {
+	queries, err := ListCustomDomainsRequestToModel(req.Msg, s.systemDefaults)
 	if err != nil {
 		return nil, err
 	}
@@ -46,14 +48,14 @@ func (s *Server) ListCustomDomains(ctx context.Context, req *instance.ListCustom
 		return nil, err
 	}
 
-	return &instance.ListCustomDomainsResponse{
+	return connect.NewResponse(&instance.ListCustomDomainsResponse{
 		Domains:    DomainsToPb(domains.Domains),
 		Pagination: filter.QueryToPaginationPb(queries.SearchRequest, domains.SearchResponse),
-	}, nil
+	}), nil
 }
 
-func (s *Server) ListTrustedDomains(ctx context.Context, req *instance.ListTrustedDomainsRequest) (*instance.ListTrustedDomainsResponse, error) {
-	queries, err := ListTrustedDomainsRequestToModel(req, s.systemDefaults)
+func (s *Server) ListTrustedDomains(ctx context.Context, req *connect.Request[instance.ListTrustedDomainsRequest]) (*connect.Response[instance.ListTrustedDomainsResponse], error) {
+	queries, err := ListTrustedDomainsRequestToModel(req.Msg, s.systemDefaults)
 	if err != nil {
 		return nil, err
 	}
@@ -63,8 +65,8 @@ func (s *Server) ListTrustedDomains(ctx context.Context, req *instance.ListTrust
 		return nil, err
 	}
 
-	return &instance.ListTrustedDomainsResponse{
+	return connect.NewResponse(&instance.ListTrustedDomainsResponse{
 		TrustedDomain: trustedDomainsToPb(domains.Domains),
 		Pagination:    filter.QueryToPaginationPb(queries.SearchRequest, domains.SearchResponse),
-	}, nil
+	}), nil
 }

internal/api/grpc/instance/v2beta/server.go

@@ -1,7 +1,10 @@
 package instance
 
 import (
-	"google.golang.org/grpc"
+	"net/http"
+
+	"connectrpc.com/connect"
+	"google.golang.org/protobuf/reflect/protoreflect"
 
 	"github.com/zitadel/zitadel/internal/api/authz"
 	"github.com/zitadel/zitadel/internal/api/grpc/server"
@@ -9,12 +12,12 @@ import (
 	"github.com/zitadel/zitadel/internal/config/systemdefaults"
 	"github.com/zitadel/zitadel/internal/query"
 	instance "github.com/zitadel/zitadel/pkg/grpc/instance/v2beta"
+	"github.com/zitadel/zitadel/pkg/grpc/instance/v2beta/instanceconnect"
 )
 
-var _ instance.InstanceServiceServer = (*Server)(nil)
+var _ instanceconnect.InstanceServiceHandler = (*Server)(nil)
 
 type Server struct {
-	instance.UnimplementedInstanceServiceServer
 	command         *command.Commands
 	query           *query.Queries
 	systemDefaults  systemdefaults.SystemDefaults
@@ -39,8 +42,12 @@ func CreateServer(
 	}
 }
 
-func (s *Server) RegisterServer(grpcServer *grpc.Server) {
-	instance.RegisterInstanceServiceServer(grpcServer, s)
+func (s *Server) RegisterConnectServer(interceptors ...connect.Interceptor) (string, http.Handler) {
+	return instanceconnect.NewInstanceServiceHandler(s, connect.WithInterceptors(interceptors...))
+}
+
+func (s *Server) FileDescriptor() protoreflect.FileDescriptor {
+	return instance.File_zitadel_instance_v2beta_instance_service_proto
 }
 
 func (s *Server) AppName() string {