perf(oidc): optimize the introspection endpoint (#6909)

* get key by id and cache them

* userinfo from events for v2 tokens

* improve keyset caching

* concurrent token and client checks

* client and project in single query

* logging and otel

* drop owner_removed column on apps and authN tables

* userinfo and project roles in go routines

* get  oidc user info from projections and add actions

* add avatar URL

* some cleanup

* pull oidc work branch

* remove storage from server

* add config flag for experimental introspection

* legacy introspection flag

* drop owner_removed column on user projections

* drop owner_removed column on useer_metadata

* query userinfo unit test

* query introspection client test

* add user_grants to the userinfo query

* handle PAT scopes

* bring triggers back

* test instance keys query

* add userinfo unit tests

* unit test keys

* go mod tidy

* solve some bugs

* fix missing preferred login name

* do not run triggers in go routines, they seem to deadlock

* initialize the trigger handlers late with a sync.OnceValue

* Revert "do not run triggers in go routines, they seem to deadlock"

This reverts commit 2a03da2127.

* add missing translations

* chore: update go version for linting

* pin oidc version

* parse a global time location for query test

* fix linter complains

* upgrade go lint

* fix more linting issues

---------

Co-authored-by: Stefan Benz <46600784+stebenz@users.noreply.github.com>

internal/api/oidc/key.go

@@ -3,14 +3,17 @@ package oidc
 import (
 	"context"
 	"fmt"
+	"sync"
 	"time"
 
 	"github.com/go-jose/go-jose/v3"
+	"github.com/jonboulle/clockwork"
 	"github.com/zitadel/logging"
 	"github.com/zitadel/oidc/v3/pkg/op"
 
 	"github.com/zitadel/zitadel/internal/api/authz"
 	"github.com/zitadel/zitadel/internal/crypto"
+	"github.com/zitadel/zitadel/internal/domain"
 	"github.com/zitadel/zitadel/internal/errors"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/query"
@@ -19,6 +22,145 @@ import (
 	"github.com/zitadel/zitadel/internal/telemetry/tracing"
 )
 
+// keySetCache implements oidc.KeySet for Access Token verification.
+// Public Keys are cached in a 2-dimensional map of Instance ID and Key ID.
+// When a key is not present the queryKey function is called to obtain the key
+// from the database.
+type keySetCache struct {
+	mtx          sync.RWMutex
+	instanceKeys map[string]map[string]query.PublicKey
+	queryKey     func(ctx context.Context, keyID string, current time.Time) (query.PublicKey, error)
+	clock        clockwork.Clock
+}
+
+// newKeySet initializes a keySetCache and starts a purging Go routine,
+// which runs once every purgeInterval.
+// When the passed context is done, the purge routine will terminate.
+func newKeySet(background context.Context, purgeInterval time.Duration, queryKey func(ctx context.Context, keyID string, current time.Time) (query.PublicKey, error)) *keySetCache {
+	k := &keySetCache{
+		instanceKeys: make(map[string]map[string]query.PublicKey),
+		queryKey:     queryKey,
+		clock:        clockwork.FromContext(background), // defaults to real clock
+	}
+	go k.purgeOnInterval(background, k.clock.NewTicker(purgeInterval))
+	return k
+}
+
+func (k *keySetCache) purgeOnInterval(background context.Context, ticker clockwork.Ticker) {
+	defer ticker.Stop()
+	for {
+		select {
+		case <-background.Done():
+			return
+		case <-ticker.Chan():
+		}
+
+		// do the actual purging
+		k.mtx.Lock()
+		for instanceID, keys := range k.instanceKeys {
+			for keyID, key := range keys {
+				if key.Expiry().Before(k.clock.Now()) {
+					delete(keys, keyID)
+				}
+			}
+			if len(keys) == 0 {
+				delete(k.instanceKeys, instanceID)
+			}
+		}
+		k.mtx.Unlock()
+	}
+}
+
+func (k *keySetCache) setKey(instanceID, keyID string, key query.PublicKey) {
+	k.mtx.Lock()
+	defer k.mtx.Unlock()
+
+	if keys, ok := k.instanceKeys[instanceID]; ok {
+		keys[keyID] = key
+		return
+	}
+
+	k.instanceKeys[instanceID] = map[string]query.PublicKey{keyID: key}
+}
+
+func (k *keySetCache) getKey(ctx context.Context, keyID string) (_ *jose.JSONWebKey, err error) {
+	ctx, span := tracing.NewSpan(ctx)
+	defer func() { span.EndWithError(err) }()
+
+	instanceID := authz.GetInstance(ctx).InstanceID()
+
+	k.mtx.RLock()
+	key, ok := k.instanceKeys[instanceID][keyID]
+	k.mtx.RUnlock()
+
+	if ok {
+		if key.Expiry().After(k.clock.Now()) {
+			return jsonWebkey(key), nil
+		}
+		return nil, errors.ThrowInvalidArgument(nil, "OIDC-Zoh9E", "Errors.Key.ExpireBeforeNow")
+	}
+
+	key, err = k.queryKey(ctx, keyID, k.clock.Now())
+	if err != nil {
+		return nil, err
+	}
+	k.setKey(instanceID, keyID, key)
+	return jsonWebkey(key), nil
+}
+
+// VerifySignature implements the oidc.KeySet interface.
+func (k *keySetCache) VerifySignature(ctx context.Context, jws *jose.JSONWebSignature) (_ []byte, err error) {
+	ctx, span := tracing.NewSpan(ctx)
+	defer func() { span.EndWithError(err) }()
+
+	if len(jws.Signatures) != 1 {
+		return nil, errors.ThrowInvalidArgument(nil, "OIDC-Gid9s", "Errors.Token.Invalid")
+	}
+	key, err := k.getKey(ctx, jws.Signatures[0].Header.KeyID)
+	if err != nil {
+		return nil, err
+	}
+	return jws.Verify(key)
+}
+
+func jsonWebkey(key query.PublicKey) *jose.JSONWebKey {
+	return &jose.JSONWebKey{
+		KeyID:     key.ID(),
+		Algorithm: key.Algorithm(),
+		Use:       key.Use().String(),
+		Key:       key.Key(),
+	}
+}
+
+// keySetMap is a mapping of key IDs to public key data.
+type keySetMap map[string][]byte
+
+// getKey finds the keyID and parses the public key data
+// into a JSONWebKey.
+func (k keySetMap) getKey(keyID string) (*jose.JSONWebKey, error) {
+	pubKey, err := crypto.BytesToPublicKey(k[keyID])
+	if err != nil {
+		return nil, err
+	}
+	return &jose.JSONWebKey{
+		Key:   pubKey,
+		KeyID: keyID,
+		Use:   domain.KeyUsageSigning.String(),
+	}, nil
+}
+
+// VerifySignature implements the oidc.KeySet interface.
+func (k keySetMap) VerifySignature(ctx context.Context, jws *jose.JSONWebSignature) ([]byte, error) {
+	if len(jws.Signatures) != 1 {
+		return nil, errors.ThrowInvalidArgument(nil, "OIDC-Eeth6", "Errors.Token.Invalid")
+	}
+	key, err := k.getKey(jws.Signatures[0].Header.KeyID)
+	if err != nil {
+		return nil, err
+	}
+	return jws.Verify(key)
+}
+
 const (
 	locksTable = "projections.locks"
 	signingKey = "signing_key"