perf(oidc): optimize the introspection endpoint (#6909)

* get key by id and cache them

* userinfo from events for v2 tokens

* improve keyset caching

* concurrent token and client checks

* client and project in single query

* logging and otel

* drop owner_removed column on apps and authN tables

* userinfo and project roles in go routines

* get  oidc user info from projections and add actions

* add avatar URL

* some cleanup

* pull oidc work branch

* remove storage from server

* add config flag for experimental introspection

* legacy introspection flag

* drop owner_removed column on user projections

* drop owner_removed column on useer_metadata

* query userinfo unit test

* query introspection client test

* add user_grants to the userinfo query

* handle PAT scopes

* bring triggers back

* test instance keys query

* add userinfo unit tests

* unit test keys

* go mod tidy

* solve some bugs

* fix missing preferred login name

* do not run triggers in go routines, they seem to deadlock

* initialize the trigger handlers late with a sync.OnceValue

* Revert "do not run triggers in go routines, they seem to deadlock"

This reverts commit 2a03da2127.

* add missing translations

* chore: update go version for linting

* pin oidc version

* parse a global time location for query test

* fix linter complains

* upgrade go lint

* fix more linting issues

---------

Co-authored-by: Stefan Benz <46600784+stebenz@users.noreply.github.com>

internal/query/query.go

@@ -9,15 +9,16 @@ import (
 	"time"
 
 	"github.com/rakyll/statik/fs"
+	"github.com/zitadel/logging"
 	"golang.org/x/text/language"
 
 	"github.com/zitadel/zitadel/internal/api/authz"
-	internal_authz "github.com/zitadel/zitadel/internal/api/authz"
 	sd "github.com/zitadel/zitadel/internal/config/systemdefaults"
 	"github.com/zitadel/zitadel/internal/crypto"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/domain"
 	"github.com/zitadel/zitadel/internal/eventstore"
+	"github.com/zitadel/zitadel/internal/eventstore/handler/v2"
 	"github.com/zitadel/zitadel/internal/query/projection"
 	"github.com/zitadel/zitadel/internal/repository/action"
 	"github.com/zitadel/zitadel/internal/repository/authrequest"
@@ -32,15 +33,17 @@ import (
 	"github.com/zitadel/zitadel/internal/repository/session"
 	usr_repo "github.com/zitadel/zitadel/internal/repository/user"
 	"github.com/zitadel/zitadel/internal/repository/usergrant"
+	"github.com/zitadel/zitadel/internal/telemetry/tracing"
 )
 
 type Queries struct {
 	eventstore *eventstore.Eventstore
 	client     *database.DB
 
-	idpConfigEncryption  crypto.EncryptionAlgorithm
-	sessionTokenVerifier func(ctx context.Context, sessionToken string, sessionID string, tokenID string) (err error)
-	checkPermission      domain.PermissionCheck
+	keyEncryptionAlgorithm crypto.EncryptionAlgorithm
+	idpConfigEncryption    crypto.EncryptionAlgorithm
+	sessionTokenVerifier   func(ctx context.Context, sessionToken string, sessionID string, tokenID string) (err error)
+	checkPermission        domain.PermissionCheck
 
 	DefaultLanguage                     language.Tag
 	LoginDir                            http.FileSystem
@@ -65,7 +68,7 @@ func StartQueries(
 	sessionTokenVerifier func(ctx context.Context, sessionToken string, sessionID string, tokenID string) (err error),
 	permissionCheck func(q *Queries) domain.PermissionCheck,
 	defaultAuditLogRetention time.Duration,
-	systemAPIUsers map[string]*internal_authz.SystemAPIUser,
+	systemAPIUsers map[string]*authz.SystemAPIUser,
 ) (repo *Queries, err error) {
 	statikLoginFS, err := fs.NewWithNamespace("login")
 	if err != nil {
@@ -86,8 +89,16 @@ func StartQueries(
 		LoginTranslationFileContents:        make(map[string][]byte),
 		NotificationTranslationFileContents: make(map[string][]byte),
 		zitadelRoles:                        zitadelRoles,
+		keyEncryptionAlgorithm:              keyEncryptionAlgorithm,
+		idpConfigEncryption:                 idpConfigEncryption,
 		sessionTokenVerifier:                sessionTokenVerifier,
-		defaultAuditLogRetention:            defaultAuditLogRetention,
+		multifactors: domain.MultifactorConfigs{
+			OTP: domain.OTPConfig{
+				CryptoMFA: otpEncryption,
+				Issuer:    defaults.Multifactors.OTP.Issuer,
+			},
+		},
+		defaultAuditLogRetention: defaultAuditLogRetention,
 	}
 	iam_repo.RegisterEventMappers(repo.eventstore)
 	usr_repo.RegisterEventMappers(repo.eventstore)
@@ -103,14 +114,6 @@ func StartQueries(
 	quota.RegisterEventMappers(repo.eventstore)
 	limits.RegisterEventMappers(repo.eventstore)
 
-	repo.idpConfigEncryption = idpConfigEncryption
-	repo.multifactors = domain.MultifactorConfigs{
-		OTP: domain.OTPConfig{
-			CryptoMFA: otpEncryption,
-			Issuer:    defaults.Multifactors.OTP.Issuer,
-		},
-	}
-
 	repo.checkPermission = permissionCheck(repo)
 
 	err = projection.Create(ctx, sqlClient, es, projections, keyEncryptionAlgorithm, certEncryptionAlgorithm, systemAPIUsers)
@@ -145,3 +148,24 @@ func init() {
 		&authRequestByIDQuery,
 	)
 }
+
+// triggerBatch calls Trigger on every handler in a separate Go routine.
+// The returned context is the context returned by the Trigger that finishes last.
+func triggerBatch(ctx context.Context, handlers ...*handler.Handler) {
+	var wg sync.WaitGroup
+	wg.Add(len(handlers))
+
+	for _, h := range handlers {
+		go func(ctx context.Context, h *handler.Handler) {
+			name := h.ProjectionName()
+			_, traceSpan := tracing.NewNamedSpan(ctx, fmt.Sprintf("Trigger%s", name))
+			_, err := h.Trigger(ctx, handler.WithAwaitRunning())
+			logging.OnError(err).WithField("projection", name).Debug("trigger failed")
+			traceSpan.EndWithError(err)
+
+			wg.Done()
+		}(ctx, h)
+	}
+
+	wg.Wait()
+}