feat: trusted (instance) domains (#8369)

# Which Problems Are Solved

ZITADEL currently selects the instance context based on a HTTP header
(see https://github.com/zitadel/zitadel/issues/8279#issue-2399959845 and
checks it against the list of instance domains. Let's call it instance
or API domain.
For any context based URL (e.g. OAuth, OIDC, SAML endpoints, links in
emails, ...) the requested domain (instance domain) will be used. Let's
call it the public domain.
In cases of proxied setups, all exposed domains (public domains) require
the domain to be managed as instance domain.
This can either be done using the "ExternalDomain" in the runtime config
or via system API, which requires a validation through CustomerPortal on
zitadel.cloud.

# How the Problems Are Solved

- Two new headers / header list are added:
- `InstanceHostHeaders`: an ordered list (first sent wins), which will
be used to match the instance.
(For backward compatibility: the `HTTP1HostHeader`, `HTTP2HostHeader`
and `forwarded`, `x-forwarded-for`, `x-forwarded-host` are checked
afterwards as well)
- `PublicHostHeaders`: an ordered list (first sent wins), which will be
used as public host / domain. This will be checked against a list of
trusted domains on the instance.
- The middleware intercepts all requests to the API and passes a
`DomainCtx` object with the hosts and protocol into the context
(previously only a computed `origin` was passed)
- HTTP / GRPC server do not longer try to match the headers to instances
themself, but use the passed `http.DomainContext` in their interceptors.
- The `RequestedHost` and `RequestedDomain` from authz.Instance are
removed in favor of the `http.DomainContext`
- When authenticating to or signing out from Console UI, the current
`http.DomainContext(ctx).Origin` (already checked by instance
interceptor for validity) is used to compute and dynamically add a
`redirect_uri` and `post_logout_redirect_uri`.
- Gateway passes all configured host headers (previously only did
`x-zitadel-*`)
- Admin API allows to manage trusted domain

# Additional Changes

None

# Additional Context

- part of #8279 
- open topics: 
  - "single-instance" mode
  - Console UI

internal/api/grpc/server/gateway.go

@@ -5,6 +5,7 @@ import (
 	"crypto/tls"
 	"fmt"
 	"net/http"
+	"slices"
 	"strings"
 
 	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
@@ -19,10 +20,8 @@ import (
 	"google.golang.org/protobuf/proto"
 
 	client_middleware "github.com/zitadel/zitadel/internal/api/grpc/client/middleware"
-	"github.com/zitadel/zitadel/internal/api/grpc/server/middleware"
 	http_utils "github.com/zitadel/zitadel/internal/api/http"
 	http_mw "github.com/zitadel/zitadel/internal/api/http/middleware"
-	"github.com/zitadel/zitadel/internal/query"
 	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 )
 
@@ -57,26 +56,29 @@ var (
 		},
 	)
 
-	serveMuxOptions = []runtime.ServeMuxOption{
-		runtime.WithMarshalerOption(jsonMarshaler.ContentType(nil), jsonMarshaler),
-		runtime.WithMarshalerOption(mimeWildcard, jsonMarshaler),
-		runtime.WithMarshalerOption(runtime.MIMEWildcard, jsonMarshaler),
-		runtime.WithIncomingHeaderMatcher(headerMatcher),
-		runtime.WithOutgoingHeaderMatcher(runtime.DefaultHeaderMatcher),
-		runtime.WithForwardResponseOption(responseForwarder),
-		runtime.WithRoutingErrorHandler(httpErrorHandler),
+	serveMuxOptions = func(hostHeaders []string) []runtime.ServeMuxOption {
+		return []runtime.ServeMuxOption{
+			runtime.WithMarshalerOption(jsonMarshaler.ContentType(nil), jsonMarshaler),
+			runtime.WithMarshalerOption(mimeWildcard, jsonMarshaler),
+			runtime.WithMarshalerOption(runtime.MIMEWildcard, jsonMarshaler),
+			runtime.WithIncomingHeaderMatcher(headerMatcher(hostHeaders)),
+			runtime.WithOutgoingHeaderMatcher(runtime.DefaultHeaderMatcher),
+			runtime.WithForwardResponseOption(responseForwarder),
+			runtime.WithRoutingErrorHandler(httpErrorHandler),
+		}
 	}
 
-	headerMatcher = runtime.HeaderMatcherFunc(
-		func(header string) (string, bool) {
+	headerMatcher = func(hostHeaders []string) runtime.HeaderMatcherFunc {
+		customHeaders = slices.Compact(append(customHeaders, hostHeaders...))
+		return func(header string) (string, bool) {
 			for _, customHeader := range customHeaders {
 				if strings.HasPrefix(strings.ToLower(header), customHeader) {
 					return header, true
 				}
 			}
 			return runtime.DefaultHeaderMatcher(header)
-		},
-	)
+		}
+	}
 
 	responseForwarder = func(ctx context.Context, w http.ResponseWriter, resp proto.Message) error {
 		t, ok := resp.(CustomHTTPResponse)
@@ -90,14 +92,12 @@ var (
 
 type Gateway struct {
 	mux               *runtime.ServeMux
-	http1HostName     string
 	connection        *grpc.ClientConn
 	accessInterceptor *http_mw.AccessInterceptor
-	queries           *query.Queries
 }
 
 func (g *Gateway) Handler() http.Handler {
-	return addInterceptors(g.mux, g.http1HostName, g.accessInterceptor, g.queries)
+	return addInterceptors(g.mux, g.accessInterceptor)
 }
 
 type CustomHTTPResponse interface {
@@ -110,12 +110,11 @@ func CreateGatewayWithPrefix(
 	ctx context.Context,
 	g WithGatewayPrefix,
 	port uint16,
-	http1HostName string,
+	hostHeaders []string,
 	accessInterceptor *http_mw.AccessInterceptor,
-	queries *query.Queries,
 	tlsConfig *tls.Config,
 ) (http.Handler, string, error) {
-	runtimeMux := runtime.NewServeMux(serveMuxOptions...)
+	runtimeMux := runtime.NewServeMux(serveMuxOptions(hostHeaders)...)
 	opts := []grpc.DialOption{
 		grpc.WithTransportCredentials(grpcCredentials(tlsConfig)),
 		grpc.WithChainUnaryInterceptor(
@@ -131,13 +130,13 @@ func CreateGatewayWithPrefix(
 	if err != nil {
 		return nil, "", fmt.Errorf("failed to register grpc gateway: %w", err)
 	}
-	return addInterceptors(runtimeMux, http1HostName, accessInterceptor, queries), g.GatewayPathPrefix(), nil
+	return addInterceptors(runtimeMux, accessInterceptor), g.GatewayPathPrefix(), nil
 }
 
 func CreateGateway(
 	ctx context.Context,
 	port uint16,
-	http1HostName string,
+	hostHeaders []string,
 	accessInterceptor *http_mw.AccessInterceptor,
 	tlsConfig *tls.Config,
 ) (*Gateway, error) {
@@ -153,10 +152,9 @@ func CreateGateway(
 	if err != nil {
 		return nil, err
 	}
-	runtimeMux := runtime.NewServeMux(append(serveMuxOptions, runtime.WithHealthzEndpoint(healthpb.NewHealthClient(connection)))...)
+	runtimeMux := runtime.NewServeMux(append(serveMuxOptions(hostHeaders), runtime.WithHealthzEndpoint(healthpb.NewHealthClient(connection)))...)
 	return &Gateway{
 		mux:               runtimeMux,
-		http1HostName:     http1HostName,
 		connection:        connection,
 		accessInterceptor: accessInterceptor,
 	}, nil
@@ -195,12 +193,9 @@ func dial(ctx context.Context, port uint16, opts []grpc.DialOption) (*grpc.Clien
 
 func addInterceptors(
 	handler http.Handler,
-	http1HostName string,
 	accessInterceptor *http_mw.AccessInterceptor,
-	queries *query.Queries,
 ) http.Handler {
 	handler = http_mw.CallDurationHandler(handler)
-	handler = http1Host(handler, http1HostName)
 	handler = http_mw.CORSInterceptor(handler)
 	handler = http_mw.RobotsTagHandler(handler)
 	handler = http_mw.DefaultTelemetryHandler(handler)
@@ -215,18 +210,6 @@ func addInterceptors(
 	return handler
 }
 
-func http1Host(next http.Handler, http1HostName string) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		host, err := http_mw.HostFromRequest(r, http1HostName)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusForbidden)
-			return
-		}
-		r.Header.Set(middleware.HTTP1Host, host)
-		next.ServeHTTP(w, r)
-	})
-}
-
 func exhaustedCookieInterceptor(
 	next http.Handler,
 	accessInterceptor *http_mw.AccessInterceptor,

internal/api/grpc/server/middleware/access_interceptor.go

@@ -9,6 +9,7 @@ import (
 	"google.golang.org/grpc/status"
 
 	"github.com/zitadel/zitadel/internal/api/authz"
+	http_util "github.com/zitadel/zitadel/internal/api/http"
 	"github.com/zitadel/zitadel/internal/logstore"
 	"github.com/zitadel/zitadel/internal/logstore/record"
 	"github.com/zitadel/zitadel/internal/telemetry/tracing"
@@ -35,6 +36,7 @@ func AccessStorageInterceptor(svc *logstore.Service[*record.AccessLog]) grpc.Una
 
 		resMd, _ := metadata.FromOutgoingContext(ctx)
 		instance := authz.GetInstance(ctx)
+		domainCtx := http_util.DomainContext(ctx)
 
 		r := &record.AccessLog{
 			LogDate:         time.Now(),
@@ -45,8 +47,8 @@ func AccessStorageInterceptor(svc *logstore.Service[*record.AccessLog]) grpc.Una
 			ResponseHeaders: resMd,
 			InstanceID:      instance.InstanceID(),
 			ProjectID:       instance.ProjectID(),
-			RequestedDomain: instance.RequestedDomain(),
-			RequestedHost:   instance.RequestedHost(),
+			RequestedDomain: domainCtx.RequestedDomain(),
+			RequestedHost:   domainCtx.RequestedHost(),
 		}
 
 		svc.Handle(interceptorCtx, r)

internal/api/grpc/server/middleware/instance_interceptor.go

@@ -10,7 +10,6 @@ import (
 	"golang.org/x/text/language"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
 
 	"github.com/zitadel/zitadel/internal/api/authz"
@@ -24,15 +23,15 @@ const (
 	HTTP1Host = "x-zitadel-http1-host"
 )
 
-func InstanceInterceptor(verifier authz.InstanceVerifier, headerName, externalDomain string, explicitInstanceIdServices ...string) grpc.UnaryServerInterceptor {
+func InstanceInterceptor(verifier authz.InstanceVerifier, externalDomain string, explicitInstanceIdServices ...string) grpc.UnaryServerInterceptor {
 	translator, err := i18n.NewZitadelTranslator(language.English)
 	logging.OnError(err).Panic("unable to get translator")
 	return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
-		return setInstance(ctx, req, info, handler, verifier, headerName, externalDomain, translator, explicitInstanceIdServices...)
+		return setInstance(ctx, req, info, handler, verifier, externalDomain, translator, explicitInstanceIdServices...)
 	}
 }
 
-func setInstance(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler, verifier authz.InstanceVerifier, headerName, externalDomain string, translator *i18n.Translator, idFromRequestsServices ...string) (_ interface{}, err error) {
+func setInstance(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler, verifier authz.InstanceVerifier, externalDomain string, translator *i18n.Translator, idFromRequestsServices ...string) (_ interface{}, err error) {
 	interceptorCtx, span := tracing.NewServerInterceptorSpan(ctx)
 	defer func() { span.EndWithError(err) }()
 	for _, service := range idFromRequestsServices {
@@ -56,14 +55,15 @@ func setInstance(ctx context.Context, req interface{}, info *grpc.UnaryServerInf
 			return handler(authz.WithInstance(ctx, instance), req)
 		}
 	}
-	host, err := hostFromContext(interceptorCtx, headerName)
-	if err != nil {
-		return nil, status.Error(codes.NotFound, err.Error())
+	requestContext := zitadel_http.DomainContext(ctx)
+	if requestContext.InstanceHost == "" {
+		logging.WithFields("origin", requestContext.Origin(), "externalDomain", externalDomain).WithError(err).Error("unable to set instance")
+		return nil, status.Error(codes.NotFound, "no instanceHost specified")
 	}
-	instance, err := verifier.InstanceByHost(interceptorCtx, host)
+	instance, err := verifier.InstanceByHost(interceptorCtx, requestContext.InstanceHost, requestContext.PublicHost)
 	if err != nil {
-		origin := zitadel_http.ComposedOrigin(ctx)
-		logging.WithFields("origin", origin, "externalDomain", externalDomain).WithError(err).Error("unable to set instance")
+		origin := zitadel_http.DomainContext(ctx)
+		logging.WithFields("origin", requestContext.Origin(), "externalDomain", externalDomain).WithError(err).Error("unable to set instance")
 		zErr := new(zerrors.ZitadelError)
 		if errors.As(err, &zErr) {
 			zErr.SetMessage(translator.LocalizeFromCtx(ctx, zErr.GetMessage(), nil))
@@ -75,33 +75,3 @@ func setInstance(ctx context.Context, req interface{}, info *grpc.UnaryServerInf
 	span.End()
 	return handler(authz.WithInstance(ctx, instance), req)
 }
-
-func hostFromContext(ctx context.Context, headerName string) (string, error) {
-	md, ok := metadata.FromIncomingContext(ctx)
-	if !ok {
-		return "", fmt.Errorf("cannot read metadata")
-	}
-	host, ok := md[HTTP1Host]
-	if ok && len(host) == 1 {
-		if !isAllowedToSendHTTP1Header(md) {
-			return "", fmt.Errorf("no valid host header")
-		}
-		return host[0], nil
-	}
-	host, ok = md[headerName]
-	if !ok {
-		return "", fmt.Errorf("cannot find header: %v", headerName)
-	}
-	if len(host) != 1 {
-		return "", fmt.Errorf("invalid host header: %v", host)
-	}
-	return host[0], nil
-}
-
-// isAllowedToSendHTTP1Header check if the gRPC call was sent to `localhost`
-// this is only possible when calling the server directly running on localhost
-// or through the gRPC gateway
-func isAllowedToSendHTTP1Header(md metadata.MD) bool {
-	authority, ok := md[":authority"]
-	return ok && len(authority) == 1 && strings.Split(authority[0], ":")[0] == "localhost"
-}

internal/api/grpc/server/middleware/instance_interceptor_test.go

@@ -9,82 +9,19 @@ import (
 
 	"golang.org/x/text/language"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/metadata"
 
 	"github.com/zitadel/zitadel/internal/api/authz"
+	http_util "github.com/zitadel/zitadel/internal/api/http"
 	"github.com/zitadel/zitadel/internal/feature"
 )
 
-func Test_hostNameFromContext(t *testing.T) {
-	type args struct {
-		ctx        context.Context
-		headerName string
-	}
-	type res struct {
-		want string
-		err  bool
-	}
-	tests := []struct {
-		name string
-		args args
-		res  res
-	}{
-		{
-			"empty context, error",
-			args{
-				ctx:        context.Background(),
-				headerName: "header",
-			},
-			res{
-				want: "",
-				err:  true,
-			},
-		},
-		{
-			"header not found",
-			args{
-				ctx:        metadata.NewIncomingContext(context.Background(), nil),
-				headerName: "header",
-			},
-			res{
-				want: "",
-				err:  true,
-			},
-		},
-		{
-			"header not found",
-			args{
-				ctx:        metadata.NewIncomingContext(context.Background(), metadata.Pairs("header", "value")),
-				headerName: "header",
-			},
-			res{
-				want: "value",
-				err:  false,
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := hostFromContext(tt.args.ctx, tt.args.headerName)
-			if (err != nil) != tt.res.err {
-				t.Errorf("hostFromContext() error = %v, wantErr %v", err, tt.res.err)
-				return
-			}
-			if got != tt.res.want {
-				t.Errorf("hostFromContext() got = %v, want %v", got, tt.res.want)
-			}
-		})
-	}
-}
-
 func Test_setInstance(t *testing.T) {
 	type args struct {
-		ctx        context.Context
-		req        interface{}
-		info       *grpc.UnaryServerInfo
-		handler    grpc.UnaryHandler
-		verifier   authz.InstanceVerifier
-		headerName string
+		ctx      context.Context
+		req      interface{}
+		info     *grpc.UnaryServerInfo
+		handler  grpc.UnaryHandler
+		verifier authz.InstanceVerifier
 	}
 	type res struct {
 		want interface{}
@@ -108,10 +45,9 @@ func Test_setInstance(t *testing.T) {
 		{
 			"invalid host, error",
 			args{
-				ctx:        metadata.NewIncomingContext(context.Background(), metadata.Pairs("header", "host2")),
-				req:        &mockRequest{},
-				verifier:   &mockInstanceVerifier{"host"},
-				headerName: "header",
+				ctx:      http_util.WithDomainContext(context.Background(), &http_util.DomainCtx{InstanceHost: "host2"}),
+				req:      &mockRequest{},
+				verifier: &mockInstanceVerifier{instanceHost: "host"},
 			},
 			res{
 				want: nil,
@@ -121,10 +57,9 @@ func Test_setInstance(t *testing.T) {
 		{
 			"valid host",
 			args{
-				ctx:        metadata.NewIncomingContext(context.Background(), metadata.Pairs("header", "host")),
-				req:        &mockRequest{},
-				verifier:   &mockInstanceVerifier{"host"},
-				headerName: "header",
+				ctx:      http_util.WithDomainContext(context.Background(), &http_util.DomainCtx{InstanceHost: "host"}),
+				req:      &mockRequest{},
+				verifier: &mockInstanceVerifier{instanceHost: "host"},
 				handler: func(ctx context.Context, req interface{}) (interface{}, error) {
 					return req, nil
 				},
@@ -137,7 +72,7 @@ func Test_setInstance(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := setInstance(tt.args.ctx, tt.args.req, tt.args.info, tt.args.handler, tt.args.verifier, tt.args.headerName, "", nil)
+			got, err := setInstance(tt.args.ctx, tt.args.req, tt.args.info, tt.args.handler, tt.args.verifier, "", nil)
 			if (err != nil) != tt.res.err {
 				t.Errorf("setInstance() error = %v, wantErr %v", err, tt.res.err)
 				return
@@ -152,11 +87,18 @@ func Test_setInstance(t *testing.T) {
 type mockRequest struct{}
 
 type mockInstanceVerifier struct {
-	host string
+	instanceHost string
+	publicHost   string
 }
 
-func (m *mockInstanceVerifier) InstanceByHost(_ context.Context, host string) (authz.Instance, error) {
-	if host != m.host {
+func (m *mockInstanceVerifier) InstanceByHost(_ context.Context, instanceHost, publicHost string) (authz.Instance, error) {
+	if instanceHost != m.instanceHost {
+		return nil, fmt.Errorf("invalid host")
+	}
+	if publicHost == "" {
+		return &mockInstance{}, nil
+	}
+	if publicHost != instanceHost && publicHost != m.publicHost {
 		return nil, fmt.Errorf("invalid host")
 	}
 	return &mockInstance{}, nil
@@ -198,14 +140,6 @@ func (m *mockInstance) DefaultOrganisationID() string {
 	return "orgID"
 }
 
-func (m *mockInstance) RequestedDomain() string {
-	return "localhost"
-}
-
-func (m *mockInstance) RequestedHost() string {
-	return "localhost:8080"
-}
-
 func (m *mockInstance) SecurityPolicyAllowedOrigins() []string {
 	return nil
 }

internal/api/grpc/server/server.go

@@ -38,7 +38,6 @@ func CreateServer(
 	verifier authz.APITokenVerifier,
 	authConfig authz.Config,
 	queries *query.Queries,
-	hostHeaderName string,
 	externalDomain string,
 	tlsConfig *tls.Config,
 	accessSvc *logstore.Service[*record.AccessLog],
@@ -51,7 +50,7 @@ func CreateServer(
 				middleware.DefaultTracingServer(),
 				middleware.MetricsHandler(metricTypes, grpc_api.Probes...),
 				middleware.NoCacheInterceptor(),
-				middleware.InstanceInterceptor(queries, hostHeaderName, externalDomain, system_pb.SystemService_ServiceDesc.ServiceName, healthpb.Health_ServiceDesc.ServiceName),
+				middleware.InstanceInterceptor(queries, externalDomain, system_pb.SystemService_ServiceDesc.ServiceName, healthpb.Health_ServiceDesc.ServiceName),
 				middleware.AccessStorageInterceptor(accessSvc),
 				middleware.ErrorHandler(),
 				middleware.LimitsInterceptor(system_pb.SystemService_ServiceDesc.ServiceName),