feat(saml): allow setting nameid-format and alternative mapping for transient format (#7979)

# Which Problems Are Solved

ZITADEL currently always uses
`urn:oasis:names:tc:SAML:2.0:nameid-format:persistent` in SAML requests,
relying on the IdP to respect that flag and always return a peristent
nameid in order to be able to map the external user with an existing
user (idp link) in ZITADEL.
In case the IdP however returns a
`urn:oasis:names:tc:SAML:2.0:nameid-format:transient` (transient)
nameid, the attribute will differ between each request and it will not
be possible to match existing users.

# How the Problems Are Solved

This PR adds the following two options on SAML IdP:
- **nameIDFormat**: allows to set the nameid-format used in the SAML
Request
- **transientMappingAttributeName**: allows to set an attribute name,
which will be used instead of the nameid itself in case the returned
nameid-format is transient

# Additional Changes

To reduce impact on current installations, the `idp_templates6_saml`
table is altered with the two added columns by a setup job. New
installations will automatically get the table with the two columns
directly.
All idp unit tests are updated to use `expectEventstore` instead of the
deprecated `eventstoreExpect`.

# Additional Context

Closes #7483
Closes #7743

---------

Co-authored-by: peintnermax <max@caos.ch>
Co-authored-by: Stefan Benz <46600784+stebenz@users.noreply.github.com>

internal/api/grpc/admin/idp_converter.go

@@ -2,6 +2,7 @@ package admin
 
 import (
 	"github.com/crewjam/saml"
+	"github.com/muhlemmer/gu"
 
 	idp_grpc "github.com/zitadel/zitadel/internal/api/grpc/idp"
 	"github.com/zitadel/zitadel/internal/api/grpc/object"
@@ -469,24 +470,36 @@ func updateAppleProviderToCommand(req *admin_pb.UpdateAppleProviderRequest) comm
 }
 
 func addSAMLProviderToCommand(req *admin_pb.AddSAMLProviderRequest) command.SAMLProvider {
+	var nameIDFormat *domain.SAMLNameIDFormat
+	if req.NameIdFormat != nil {
+		nameIDFormat = gu.Ptr(idp_grpc.SAMLNameIDFormatToDomain(req.GetNameIdFormat()))
+	}
 	return command.SAMLProvider{
-		Name:              req.Name,
-		Metadata:          req.GetMetadataXml(),
-		MetadataURL:       req.GetMetadataUrl(),
-		Binding:           bindingToCommand(req.Binding),
-		WithSignedRequest: req.WithSignedRequest,
-		IDPOptions:        idp_grpc.OptionsToCommand(req.ProviderOptions),
+		Name:                          req.Name,
+		Metadata:                      req.GetMetadataXml(),
+		MetadataURL:                   req.GetMetadataUrl(),
+		Binding:                       bindingToCommand(req.Binding),
+		WithSignedRequest:             req.WithSignedRequest,
+		NameIDFormat:                  nameIDFormat,
+		TransientMappingAttributeName: req.GetTransientMappingAttributeName(),
+		IDPOptions:                    idp_grpc.OptionsToCommand(req.ProviderOptions),
 	}
 }
 
 func updateSAMLProviderToCommand(req *admin_pb.UpdateSAMLProviderRequest) command.SAMLProvider {
+	var nameIDFormat *domain.SAMLNameIDFormat
+	if req.NameIdFormat != nil {
+		nameIDFormat = gu.Ptr(idp_grpc.SAMLNameIDFormatToDomain(req.GetNameIdFormat()))
+	}
 	return command.SAMLProvider{
-		Name:              req.Name,
-		Metadata:          req.GetMetadataXml(),
-		MetadataURL:       req.GetMetadataUrl(),
-		Binding:           bindingToCommand(req.Binding),
-		WithSignedRequest: req.WithSignedRequest,
-		IDPOptions:        idp_grpc.OptionsToCommand(req.ProviderOptions),
+		Name:                          req.Name,
+		Metadata:                      req.GetMetadataXml(),
+		MetadataURL:                   req.GetMetadataUrl(),
+		Binding:                       bindingToCommand(req.Binding),
+		WithSignedRequest:             req.WithSignedRequest,
+		NameIDFormat:                  nameIDFormat,
+		TransientMappingAttributeName: req.GetTransientMappingAttributeName(),
+		IDPOptions:                    idp_grpc.OptionsToCommand(req.ProviderOptions),
 	}
 }
 

internal/api/grpc/idp/converter.go

@@ -2,6 +2,7 @@ package idp
 
 import (
 	"github.com/crewjam/saml"
+	"github.com/muhlemmer/gu"
 	"google.golang.org/protobuf/types/known/durationpb"
 
 	obj_grpc "github.com/zitadel/zitadel/internal/api/grpc/object"
@@ -339,6 +340,21 @@ func azureADTenantTypeToCommand(tenantType idp_pb.AzureADTenantType) azuread.Ten
 	}
 }
 
+func SAMLNameIDFormatToDomain(format idp_pb.SAMLNameIDFormat) domain.SAMLNameIDFormat {
+	switch format {
+	case idp_pb.SAMLNameIDFormat_SAML_NAME_ID_FORMAT_UNSPECIFIED:
+		return domain.SAMLNameIDFormatUnspecified
+	case idp_pb.SAMLNameIDFormat_SAML_NAME_ID_FORMAT_EMAIL_ADDRESS:
+		return domain.SAMLNameIDFormatEmailAddress
+	case idp_pb.SAMLNameIDFormat_SAML_NAME_ID_FORMAT_PERSISTENT:
+		return domain.SAMLNameIDFormatPersistent
+	case idp_pb.SAMLNameIDFormat_SAML_NAME_ID_FORMAT_TRANSIENT:
+		return domain.SAMLNameIDFormatTransient
+	default:
+		return domain.SAMLNameIDFormatUnspecified
+	}
+}
+
 func ProvidersToPb(providers []*query.IDPTemplate) []*idp_pb.Provider {
 	list := make([]*idp_pb.Provider, len(providers))
 	for i, provider := range providers {
@@ -639,11 +655,17 @@ func appleConfigToPb(providerConfig *idp_pb.ProviderConfig, template *query.Appl
 }
 
 func samlConfigToPb(providerConfig *idp_pb.ProviderConfig, template *query.SAMLIDPTemplate) {
+	nameIDFormat := idp_pb.SAMLNameIDFormat_SAML_NAME_ID_FORMAT_PERSISTENT
+	if template.NameIDFormat.Valid {
+		nameIDFormat = nameIDToPb(template.NameIDFormat.V)
+	}
 	providerConfig.Config = &idp_pb.ProviderConfig_Saml{
 		Saml: &idp_pb.SAMLConfig{
-			MetadataXml:       template.Metadata,
-			Binding:           bindingToPb(template.Binding),
-			WithSignedRequest: template.WithSignedRequest,
+			MetadataXml:                   template.Metadata,
+			Binding:                       bindingToPb(template.Binding),
+			WithSignedRequest:             template.WithSignedRequest,
+			NameIdFormat:                  nameIDFormat,
+			TransientMappingAttributeName: gu.Ptr(template.TransientMappingAttributeName),
 		},
 	}
 }
@@ -662,3 +684,18 @@ func bindingToPb(binding string) idp_pb.SAMLBinding {
 		return idp_pb.SAMLBinding_SAML_BINDING_UNSPECIFIED
 	}
 }
+
+func nameIDToPb(format domain.SAMLNameIDFormat) idp_pb.SAMLNameIDFormat {
+	switch format {
+	case domain.SAMLNameIDFormatUnspecified:
+		return idp_pb.SAMLNameIDFormat_SAML_NAME_ID_FORMAT_UNSPECIFIED
+	case domain.SAMLNameIDFormatEmailAddress:
+		return idp_pb.SAMLNameIDFormat_SAML_NAME_ID_FORMAT_EMAIL_ADDRESS
+	case domain.SAMLNameIDFormatPersistent:
+		return idp_pb.SAMLNameIDFormat_SAML_NAME_ID_FORMAT_PERSISTENT
+	case domain.SAMLNameIDFormatTransient:
+		return idp_pb.SAMLNameIDFormat_SAML_NAME_ID_FORMAT_TRANSIENT
+	default:
+		return idp_pb.SAMLNameIDFormat_SAML_NAME_ID_FORMAT_UNSPECIFIED
+	}
+}

internal/api/grpc/management/idp_converter.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/crewjam/saml"
+	"github.com/muhlemmer/gu"
 
 	"github.com/zitadel/zitadel/internal/api/authz"
 	idp_grpc "github.com/zitadel/zitadel/internal/api/grpc/idp"
@@ -462,24 +463,36 @@ func updateAppleProviderToCommand(req *mgmt_pb.UpdateAppleProviderRequest) comma
 }
 
 func addSAMLProviderToCommand(req *mgmt_pb.AddSAMLProviderRequest) command.SAMLProvider {
+	var nameIDFormat *domain.SAMLNameIDFormat
+	if req.NameIdFormat != nil {
+		nameIDFormat = gu.Ptr(idp_grpc.SAMLNameIDFormatToDomain(req.GetNameIdFormat()))
+	}
 	return command.SAMLProvider{
-		Name:              req.Name,
-		Metadata:          req.GetMetadataXml(),
-		MetadataURL:       req.GetMetadataUrl(),
-		Binding:           bindingToCommand(req.Binding),
-		WithSignedRequest: req.WithSignedRequest,
-		IDPOptions:        idp_grpc.OptionsToCommand(req.ProviderOptions),
+		Name:                          req.Name,
+		Metadata:                      req.GetMetadataXml(),
+		MetadataURL:                   req.GetMetadataUrl(),
+		Binding:                       bindingToCommand(req.Binding),
+		WithSignedRequest:             req.WithSignedRequest,
+		NameIDFormat:                  nameIDFormat,
+		TransientMappingAttributeName: req.GetTransientMappingAttributeName(),
+		IDPOptions:                    idp_grpc.OptionsToCommand(req.ProviderOptions),
 	}
 }
 
 func updateSAMLProviderToCommand(req *mgmt_pb.UpdateSAMLProviderRequest) command.SAMLProvider {
+	var nameIDFormat *domain.SAMLNameIDFormat
+	if req.NameIdFormat != nil {
+		nameIDFormat = gu.Ptr(idp_grpc.SAMLNameIDFormatToDomain(req.GetNameIdFormat()))
+	}
 	return command.SAMLProvider{
-		Name:              req.Name,
-		Metadata:          req.GetMetadataXml(),
-		MetadataURL:       req.GetMetadataUrl(),
-		Binding:           bindingToCommand(req.Binding),
-		WithSignedRequest: req.WithSignedRequest,
-		IDPOptions:        idp_grpc.OptionsToCommand(req.ProviderOptions),
+		Name:                          req.Name,
+		Metadata:                      req.GetMetadataXml(),
+		MetadataURL:                   req.GetMetadataUrl(),
+		Binding:                       bindingToCommand(req.Binding),
+		WithSignedRequest:             req.WithSignedRequest,
+		NameIDFormat:                  nameIDFormat,
+		TransientMappingAttributeName: req.GetTransientMappingAttributeName(),
+		IDPOptions:                    idp_grpc.OptionsToCommand(req.ProviderOptions),
 	}
 }
 

internal/api/grpc/user/v2/user_integration_test.go

@@ -1883,7 +1883,7 @@ func TestServer_StartIdentityProviderIntent(t *testing.T) {
 	orgResp := Tester.CreateOrganization(IamCTX, fmt.Sprintf("NotDefaultOrg%d", time.Now().UnixNano()), fmt.Sprintf("%d@mouse.com", time.Now().UnixNano()))
 	notDefaultOrgIdpID := Tester.AddOrgGenericOAuthProvider(t, CTX, orgResp.OrganizationId)
 	samlIdpID := Tester.AddSAMLProvider(t, CTX)
-	samlRedirectIdpID := Tester.AddSAMLRedirectProvider(t, CTX)
+	samlRedirectIdpID := Tester.AddSAMLRedirectProvider(t, CTX, "")
 	samlPostIdpID := Tester.AddSAMLPostProvider(t, CTX)
 	type args struct {
 		ctx context.Context