feat(saml): allow setting nameid-format and alternative mapping for transient format (#7979)

# Which Problems Are Solved

ZITADEL currently always uses
`urn:oasis:names:tc:SAML:2.0:nameid-format:persistent` in SAML requests,
relying on the IdP to respect that flag and always return a peristent
nameid in order to be able to map the external user with an existing
user (idp link) in ZITADEL.
In case the IdP however returns a
`urn:oasis:names:tc:SAML:2.0:nameid-format:transient` (transient)
nameid, the attribute will differ between each request and it will not
be possible to match existing users.

# How the Problems Are Solved

This PR adds the following two options on SAML IdP:
- **nameIDFormat**: allows to set the nameid-format used in the SAML
Request
- **transientMappingAttributeName**: allows to set an attribute name,
which will be used instead of the nameid itself in case the returned
nameid-format is transient

# Additional Changes

To reduce impact on current installations, the `idp_templates6_saml`
table is altered with the two added columns by a setup job. New
installations will automatically get the table with the two columns
directly.
All idp unit tests are updated to use `expectEventstore` instead of the
deprecated `eventstoreExpect`.

# Additional Context

Closes #7483
Closes #7743

---------

Co-authored-by: peintnermax <max@caos.ch>
Co-authored-by: Stefan Benz <46600784+stebenz@users.noreply.github.com>

internal/query/projection/idp_template.go

@@ -161,13 +161,15 @@ const (
 	ApplePrivateKeyCol = "private_key"
 	AppleScopesCol     = "scopes"
 
-	SAMLIDCol                = "idp_id"
-	SAMLInstanceIDCol        = "instance_id"
-	SAMLMetadataCol          = "metadata"
-	SAMLKeyCol               = "key"
-	SAMLCertificateCol       = "certificate"
-	SAMLBindingCol           = "binding"
-	SAMLWithSignedRequestCol = "with_signed_request"
+	SAMLIDCol                         = "idp_id"
+	SAMLInstanceIDCol                 = "instance_id"
+	SAMLMetadataCol                   = "metadata"
+	SAMLKeyCol                        = "key"
+	SAMLCertificateCol                = "certificate"
+	SAMLBindingCol                    = "binding"
+	SAMLWithSignedRequestCol          = "with_signed_request"
+	SAMLNameIDFormatCol               = "name_id_format"
+	SAMLTransientMappingAttributeName = "transient_mapping_attribute_name"
 )
 
 type idpTemplateProjection struct{}
@@ -367,6 +369,8 @@ func (*idpTemplateProjection) Init() *old_handler.Check {
 			handler.NewColumn(SAMLCertificateCol, handler.ColumnTypeBytes),
 			handler.NewColumn(SAMLBindingCol, handler.ColumnTypeText, handler.Nullable()),
 			handler.NewColumn(SAMLWithSignedRequestCol, handler.ColumnTypeBool, handler.Nullable()),
+			handler.NewColumn(SAMLNameIDFormatCol, handler.ColumnTypeEnum, handler.Nullable()),
+			handler.NewColumn(SAMLTransientMappingAttributeName, handler.ColumnTypeText, handler.Nullable()),
 		},
 			handler.NewPrimaryKey(SAMLInstanceIDCol, SAMLIDCol),
 			IDPTemplateSAMLSuffix,
@@ -1967,6 +1971,20 @@ func (p *idpTemplateProjection) reduceSAMLIDPAdded(event eventstore.Event) (*han
 		return nil, zerrors.ThrowInvalidArgumentf(nil, "HANDL-9s02m1", "reduce.wrong.event.type %v", []eventstore.EventType{org.SAMLIDPAddedEventType, instance.SAMLIDPAddedEventType})
 	}
 
+	columns := []handler.Column{
+		handler.NewCol(SAMLIDCol, idpEvent.ID),
+		handler.NewCol(SAMLInstanceIDCol, idpEvent.Aggregate().InstanceID),
+		handler.NewCol(SAMLMetadataCol, idpEvent.Metadata),
+		handler.NewCol(SAMLKeyCol, idpEvent.Key),
+		handler.NewCol(SAMLCertificateCol, idpEvent.Certificate),
+		handler.NewCol(SAMLBindingCol, idpEvent.Binding),
+		handler.NewCol(SAMLWithSignedRequestCol, idpEvent.WithSignedRequest),
+		handler.NewCol(SAMLTransientMappingAttributeName, idpEvent.TransientMappingAttributeName),
+	}
+	if idpEvent.NameIDFormat != nil {
+		columns = append(columns, handler.NewCol(SAMLNameIDFormatCol, *idpEvent.NameIDFormat))
+	}
+
 	return handler.NewMultiStatement(
 		&idpEvent,
 		handler.AddCreateStatement(
@@ -1989,15 +2007,7 @@ func (p *idpTemplateProjection) reduceSAMLIDPAdded(event eventstore.Event) (*han
 			},
 		),
 		handler.AddCreateStatement(
-			[]handler.Column{
-				handler.NewCol(SAMLIDCol, idpEvent.ID),
-				handler.NewCol(SAMLInstanceIDCol, idpEvent.Aggregate().InstanceID),
-				handler.NewCol(SAMLMetadataCol, idpEvent.Metadata),
-				handler.NewCol(SAMLKeyCol, idpEvent.Key),
-				handler.NewCol(SAMLCertificateCol, idpEvent.Certificate),
-				handler.NewCol(SAMLBindingCol, idpEvent.Binding),
-				handler.NewCol(SAMLWithSignedRequestCol, idpEvent.WithSignedRequest),
-			},
+			columns,
 			handler.WithTableSuffix(IDPTemplateSAMLSuffix),
 		),
 	), nil
@@ -2490,5 +2500,11 @@ func reduceSAMLIDPChangedColumns(idpEvent idp.SAMLIDPChangedEvent) []handler.Col
 	if idpEvent.WithSignedRequest != nil {
 		SAMLCols = append(SAMLCols, handler.NewCol(SAMLWithSignedRequestCol, *idpEvent.WithSignedRequest))
 	}
+	if idpEvent.NameIDFormat != nil {
+		SAMLCols = append(SAMLCols, handler.NewCol(SAMLNameIDFormatCol, *idpEvent.NameIDFormat))
+	}
+	if idpEvent.TransientMappingAttributeName != nil {
+		SAMLCols = append(SAMLCols, handler.NewCol(SAMLTransientMappingAttributeName, *idpEvent.TransientMappingAttributeName))
+	}
 	return SAMLCols
 }