feat(saml): allow setting nameid-format and alternative mapping for transient format (#7979)

# Which Problems Are Solved

ZITADEL currently always uses
`urn:oasis:names:tc:SAML:2.0:nameid-format:persistent` in SAML requests,
relying on the IdP to respect that flag and always return a peristent
nameid in order to be able to map the external user with an existing
user (idp link) in ZITADEL.
In case the IdP however returns a
`urn:oasis:names:tc:SAML:2.0:nameid-format:transient` (transient)
nameid, the attribute will differ between each request and it will not
be possible to match existing users.

# How the Problems Are Solved

This PR adds the following two options on SAML IdP:
- **nameIDFormat**: allows to set the nameid-format used in the SAML
Request
- **transientMappingAttributeName**: allows to set an attribute name,
which will be used instead of the nameid itself in case the returned
nameid-format is transient

# Additional Changes

To reduce impact on current installations, the `idp_templates6_saml`
table is altered with the two added columns by a setup job. New
installations will automatically get the table with the two columns
directly.
All idp unit tests are updated to use `expectEventstore` instead of the
deprecated `eventstoreExpect`.

# Additional Context

Closes #7483
Closes #7743

---------

Co-authored-by: peintnermax <max@caos.ch>
Co-authored-by: Stefan Benz <46600784+stebenz@users.noreply.github.com>

cmd/setup/27.go

@@ -0,0 +1,27 @@
+package setup
+
+import (
+	"context"
+	_ "embed"
+
+	"github.com/zitadel/zitadel/internal/database"
+	"github.com/zitadel/zitadel/internal/eventstore"
+)
+
+var (
+	//go:embed 27.sql
+	addSAMLNameIDFormat string
+)
+
+type IDPTemplate6SAMLNameIDFormat struct {
+	dbClient *database.DB
+}
+
+func (mig *IDPTemplate6SAMLNameIDFormat) Execute(ctx context.Context, _ eventstore.Event) error {
+	_, err := mig.dbClient.ExecContext(ctx, addSAMLNameIDFormat)
+	return err
+}
+
+func (mig *IDPTemplate6SAMLNameIDFormat) String() string {
+	return "26_idp_templates6_add_saml_name_id_format"
+}

cmd/setup/27.sql

@@ -0,0 +1,2 @@
+ALTER TABLE IF EXISTS projections.idp_templates6_saml ADD COLUMN IF NOT EXISTS name_id_format SMALLINT;
+ALTER TABLE IF EXISTS projections.idp_templates6_saml ADD COLUMN IF NOT EXISTS transient_mapping_attribute_name TEXT;

cmd/setup/config.go

@@ -109,6 +109,7 @@ type Steps struct {
 	s24AddActorToAuthTokens                *AddActorToAuthTokens
 	s25User11AddLowerFieldsToVerifiedEmail *User11AddLowerFieldsToVerifiedEmail
 	s26AuthUsers3                          *AuthUsers3
+	s27IDPTemplate6SAMLNameIDFormat        *IDPTemplate6SAMLNameIDFormat
 }
 
 func MustNewSteps(v *viper.Viper) *Steps {

cmd/setup/setup.go

@@ -139,6 +139,7 @@ func Setup(config *Config, steps *Steps, masterKey string) {
 	steps.s24AddActorToAuthTokens = &AddActorToAuthTokens{dbClient: queryDBClient}
 	steps.s25User11AddLowerFieldsToVerifiedEmail = &User11AddLowerFieldsToVerifiedEmail{dbClient: esPusherDBClient}
 	steps.s26AuthUsers3 = &AuthUsers3{dbClient: esPusherDBClient}
+	steps.s27IDPTemplate6SAMLNameIDFormat = &IDPTemplate6SAMLNameIDFormat{dbClient: esPusherDBClient}
 
 	err = projection.Create(ctx, projectionDBClient, eventstoreClient, config.Projections, nil, nil, nil)
 	logging.OnError(err).Fatal("unable to start projections")
@@ -190,6 +191,7 @@ func Setup(config *Config, steps *Steps, masterKey string) {
 		steps.s18AddLowerFieldsToLoginNames,
 		steps.s21AddBlockFieldToLimits,
 		steps.s25User11AddLowerFieldsToVerifiedEmail,
+		steps.s27IDPTemplate6SAMLNameIDFormat,
 	} {
 		mustExecuteMigration(ctx, eventstoreClient, step, "migration failed")
 	}