feat: token introspection, api clients and auth method private_key_jwt (#1276)

* introspect

* testingapplication key

* date

* client keys

* fix client keys

* fix client keys

* access tokens only for users

* AuthMethodPrivateKeyJWT

* client keys

* set introspection info correctly

* managae apis

* update oidc pkg

* cleanup

* merge msater

* set current sequence in migration

* set current sequence in migration

* set current sequence in migration

* Apply suggestions from code review

Co-authored-by: Fabi <38692350+fgerschwiler@users.noreply.github.com>

* DeleteAuthNKeysByObjectID

* ensure authn keys uptodate

* update oidc version

* merge master

* merge master

Co-authored-by: Fabi <38692350+fgerschwiler@users.noreply.github.com>

internal/auth/repository/eventsourcing/eventstore/org.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/caos/logging"
+
 	"github.com/caos/zitadel/internal/api/authz"
 	"github.com/caos/zitadel/internal/config/systemdefaults"
 	"github.com/caos/zitadel/internal/errors"
@@ -55,6 +56,14 @@ func (repo *OrgRepository) SearchOrgs(ctx context.Context, request *org_model.Or
 	return result, nil
 }
 
+func (repo *OrgRepository) OrgByPrimaryDomain(primaryDomain string) (*org_model.OrgView, error) {
+	org, err := repo.View.OrgByPrimaryDomain(primaryDomain)
+	if err != nil {
+		return nil, err
+	}
+	return model.OrgToModel(org), nil
+}
+
 func (repo *OrgRepository) RegisterOrg(ctx context.Context, register *auth_model.RegisterOrg) (*auth_model.RegisterOrg, error) {
 	pwPolicy, err := repo.View.PasswordComplexityPolicyByAggregateID(repo.SystemDefaults.IamID)
 	if err != nil {

internal/auth/repository/eventsourcing/eventstore/token.go

@@ -3,23 +3,26 @@ package eventstore
 import (
 	"context"
 	"strings"
+	"time"
 
 	"github.com/caos/logging"
+
 	auth_req_model "github.com/caos/zitadel/internal/auth_request/model"
 	"github.com/caos/zitadel/internal/errors"
 	"github.com/caos/zitadel/internal/eventstore/models"
+	proj_event "github.com/caos/zitadel/internal/project/repository/eventsourcing"
 	"github.com/caos/zitadel/internal/telemetry/tracing"
 	usr_model "github.com/caos/zitadel/internal/user/model"
 	user_event "github.com/caos/zitadel/internal/user/repository/eventsourcing"
 	"github.com/caos/zitadel/internal/user/repository/view/model"
-	"time"
 
 	"github.com/caos/zitadel/internal/auth/repository/eventsourcing/view"
 )
 
 type TokenRepo struct {
-	UserEvents *user_event.UserEventstore
-	View       *view.View
+	UserEvents    *user_event.UserEventstore
+	ProjectEvents *proj_event.ProjectEventstore
+	View          *view.View
 }
 
 func (repo *TokenRepo) CreateToken(ctx context.Context, agentID, clientID, userID string, audience, scopes []string, lifetime time.Duration) (*usr_model.Token, error) {

internal/auth/repository/eventsourcing/eventstore/user.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/caos/zitadel/internal/config/systemdefaults"
 	iam_es_model "github.com/caos/zitadel/internal/iam/repository/view/model"
+	key_model "github.com/caos/zitadel/internal/key/model"
 
 	"github.com/caos/logging"
 
@@ -14,6 +15,7 @@ import (
 	"github.com/caos/zitadel/internal/eventstore"
 	es_models "github.com/caos/zitadel/internal/eventstore/models"
 	"github.com/caos/zitadel/internal/eventstore/sdk"
+	key_view_model "github.com/caos/zitadel/internal/key/repository/view/model"
 	org_model "github.com/caos/zitadel/internal/org/model"
 	org_event "github.com/caos/zitadel/internal/org/repository/eventsourcing"
 	"github.com/caos/zitadel/internal/telemetry/tracing"
@@ -515,10 +517,10 @@ func checkIDs(ctx context.Context, obj es_models.ObjectRoot) error {
 	return nil
 }
 
-func (repo *UserRepo) MachineKeyByID(ctx context.Context, keyID string) (*model.MachineKeyView, error) {
-	key, err := repo.View.MachineKeyByID(keyID)
+func (repo *UserRepo) MachineKeyByID(ctx context.Context, keyID string) (*key_model.AuthNKeyView, error) {
+	key, err := repo.View.AuthNKeyByID(keyID)
 	if err != nil {
 		return nil, err
 	}
-	return usr_view_model.MachineKeyToModel(key), nil
+	return key_view_model.AuthNKeyToModel(key), nil
 }

internal/auth/repository/eventsourcing/handler/application.go

@@ -84,6 +84,8 @@ func (a *Application) Reduce(event *models.Event) (err error) {
 	case es_model.ApplicationChanged,
 		es_model.OIDCConfigAdded,
 		es_model.OIDCConfigChanged,
+		es_model.APIConfigAdded,
+		es_model.APIConfigChanged,
 		es_model.ApplicationDeactivated,
 		es_model.ApplicationReactivated:
 		err = app.SetData(event)

internal/auth/repository/eventsourcing/handler/authn_keys.go

@@ -0,0 +1,120 @@
+package handler
+
+import (
+	"time"
+
+	"github.com/caos/logging"
+
+	"github.com/caos/zitadel/internal/eventstore"
+	es_models "github.com/caos/zitadel/internal/eventstore/models"
+	"github.com/caos/zitadel/internal/eventstore/query"
+	"github.com/caos/zitadel/internal/eventstore/spooler"
+	key_model "github.com/caos/zitadel/internal/key/repository/view/model"
+	proj_model "github.com/caos/zitadel/internal/project/repository/eventsourcing/model"
+	user_model "github.com/caos/zitadel/internal/user/repository/eventsourcing/model"
+)
+
+const (
+	authnKeysTable = "auth.authn_keys"
+)
+
+type AuthNKeys struct {
+	handler
+	subscription *eventstore.Subscription
+}
+
+func newAuthNKeys(handler handler) *AuthNKeys {
+	h := &AuthNKeys{
+		handler: handler,
+	}
+
+	h.subscribe()
+
+	return h
+}
+
+func (k *AuthNKeys) subscribe() {
+	k.subscription = k.es.Subscribe(k.AggregateTypes()...)
+	go func() {
+		for event := range k.subscription.Events {
+			query.ReduceEvent(k, event)
+		}
+	}()
+}
+
+func (k *AuthNKeys) ViewModel() string {
+	return authnKeysTable
+}
+
+func (_ *AuthNKeys) AggregateTypes() []es_models.AggregateType {
+	return []es_models.AggregateType{user_model.UserAggregate, proj_model.ProjectAggregate}
+}
+
+func (k *AuthNKeys) CurrentSequence() (uint64, error) {
+	sequence, err := k.view.GetLatestAuthNKeySequence()
+	if err != nil {
+		return 0, err
+	}
+	return sequence.CurrentSequence, nil
+}
+
+func (k *AuthNKeys) EventQuery() (*es_models.SearchQuery, error) {
+	sequence, err := k.view.GetLatestAuthNKeySequence()
+	if err != nil {
+		return nil, err
+	}
+	return es_models.NewSearchQuery().
+		AggregateTypeFilter(k.AggregateTypes()...).
+		LatestSequenceFilter(sequence.CurrentSequence), nil
+}
+
+func (k *AuthNKeys) Reduce(event *es_models.Event) (err error) {
+	switch event.AggregateType {
+	case user_model.UserAggregate,
+		proj_model.ProjectAggregate:
+		err = k.processAuthNKeys(event)
+	}
+	return err
+}
+
+func (k *AuthNKeys) processAuthNKeys(event *es_models.Event) (err error) {
+	key := new(key_model.AuthNKeyView)
+	switch event.Type {
+	case user_model.MachineKeyAdded,
+		proj_model.ClientKeyAdded:
+		err = key.AppendEvent(event)
+		if key.ExpirationDate.Before(time.Now()) {
+			return k.view.ProcessedAuthNKeySequence(event)
+		}
+	case user_model.MachineKeyRemoved:
+		err = key.SetUserData(event)
+		if err != nil {
+			return err
+		}
+		return k.view.DeleteAuthNKey(key.ID, event)
+	case proj_model.ClientKeyRemoved:
+		err = key.SetClientData(event)
+		if err != nil {
+			return err
+		}
+		return k.view.DeleteAuthNKey(key.ID, event)
+	case user_model.UserRemoved,
+		proj_model.ApplicationRemoved:
+		return k.view.DeleteAuthNKeysByObjectID(event.AggregateID, event)
+	default:
+		return k.view.ProcessedAuthNKeySequence(event)
+	}
+	if err != nil {
+		return err
+	}
+	return k.view.PutAuthNKey(key, event)
+}
+
+func (k *AuthNKeys) OnError(event *es_models.Event, err error) error {
+	logging.LogWithFields("SPOOL-S9fe", "id", event.AggregateID).WithError(err).Warn("something went wrong in authn key handler")
+	return spooler.HandleError(event, err, k.view.GetLatestAuthNKeyFailedEvent, k.view.ProcessedAuthNKeyFailedEvent, k.view.ProcessedAuthNKeySequence, k.errorCountUntilSkip)
+}
+
+func (k *AuthNKeys) OnSuccess() error {
+	return spooler.HandleSuccess(k.view.UpdateAuthNKeySpoolerRunTimestamp)
+}

internal/auth/repository/eventsourcing/handler/handler.go

@@ -71,7 +71,7 @@ func Register(configs Configs, bulkLimit, errorCount uint64, view *view.View, es
 			repos.OrgEvents,
 			repos.IamEvents,
 			systemDefaults.IamID),
-		newMachineKeys(
+		newAuthNKeys(
 			handler{view, bulkLimit, configs.cycleDuration("MachineKey"), errorCount, es}),
 		newLoginPolicy(
 			handler{view, bulkLimit, configs.cycleDuration("LoginPolicy"), errorCount, es}),

internal/auth/repository/eventsourcing/handler/machine_keys.go

@@ -1,110 +0,0 @@
-package handler
-
-import (
-	"time"
-
-	"github.com/caos/logging"
-
-	"github.com/caos/zitadel/internal/eventstore"
-	es_models "github.com/caos/zitadel/internal/eventstore/models"
-	"github.com/caos/zitadel/internal/eventstore/query"
-	"github.com/caos/zitadel/internal/eventstore/spooler"
-	"github.com/caos/zitadel/internal/user/repository/eventsourcing/model"
-	usr_model "github.com/caos/zitadel/internal/user/repository/view/model"
-)
-
-const (
-	machineKeysTable = "auth.machine_keys"
-)
-
-type MachineKeys struct {
-	handler
-	subscription *eventstore.Subscription
-}
-
-func newMachineKeys(handler handler) *MachineKeys {
-	h := &MachineKeys{
-		handler: handler,
-	}
-
-	h.subscribe()
-
-	return h
-}
-
-func (k *MachineKeys) subscribe() {
-	k.subscription = k.es.Subscribe(k.AggregateTypes()...)
-	go func() {
-		for event := range k.subscription.Events {
-			query.ReduceEvent(k, event)
-		}
-	}()
-}
-
-func (k *MachineKeys) ViewModel() string {
-	return machineKeysTable
-}
-
-func (_ *MachineKeys) AggregateTypes() []es_models.AggregateType {
-	return []es_models.AggregateType{model.UserAggregate}
-}
-
-func (k *MachineKeys) CurrentSequence() (uint64, error) {
-	sequence, err := k.view.GetLatestMachineKeySequence()
-	if err != nil {
-		return 0, err
-	}
-	return sequence.CurrentSequence, nil
-}
-
-func (k *MachineKeys) EventQuery() (*es_models.SearchQuery, error) {
-	sequence, err := k.view.GetLatestMachineKeySequence()
-	if err != nil {
-		return nil, err
-	}
-	return es_models.NewSearchQuery().
-		AggregateTypeFilter(k.AggregateTypes()...).
-		LatestSequenceFilter(sequence.CurrentSequence), nil
-}
-
-func (k *MachineKeys) Reduce(event *es_models.Event) (err error) {
-	switch event.AggregateType {
-	case model.UserAggregate:
-		err = k.processMachineKeys(event)
-	}
-	return err
-}
-
-func (k *MachineKeys) processMachineKeys(event *es_models.Event) (err error) {
-	key := new(usr_model.MachineKeyView)
-	switch event.Type {
-	case model.MachineKeyAdded:
-		err = key.AppendEvent(event)
-		if key.ExpirationDate.Before(time.Now()) {
-			return k.view.ProcessedMachineKeySequence(event)
-		}
-	case model.MachineKeyRemoved:
-		err = key.SetData(event)
-		if err != nil {
-			return err
-		}
-		return k.view.DeleteMachineKey(key.ID, event)
-	case model.UserRemoved:
-		return k.view.DeleteMachineKeysByUserID(event.AggregateID, event)
-	default:
-		return k.view.ProcessedMachineKeySequence(event)
-	}
-	if err != nil {
-		return err
-	}
-	return k.view.PutMachineKey(key, event)
-}
-
-func (k *MachineKeys) OnError(event *es_models.Event, err error) error {
-	logging.LogWithFields("SPOOL-S9fe", "id", event.AggregateID).WithError(err).Warn("something went wrong in machine key handler")
-	return spooler.HandleError(event, err, k.view.GetLatestMachineKeyFailedEvent, k.view.ProcessedMachineKeyFailedEvent, k.view.ProcessedMachineKeySequence, k.errorCountUntilSkip)
-}
-
-func (k *MachineKeys) OnSuccess() error {
-	return spooler.HandleSuccess(k.view.UpdateMachineKeySpoolerRunTimestamp)
-}

internal/auth/repository/eventsourcing/handler/token.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 
 	"github.com/caos/logging"
+
 	caos_errs "github.com/caos/zitadel/internal/errors"
 	"github.com/caos/zitadel/internal/eventstore"
 	"github.com/caos/zitadel/internal/eventstore/models"
@@ -71,9 +72,6 @@ func (t *Token) EventQuery() (*models.SearchQuery, error) {
 	if err != nil {
 		return nil, err
 	}
-	if err != nil {
-		return nil, err
-	}
 	return es_models.NewSearchQuery().
 		AggregateTypeFilter(user_es_model.UserAggregate, project_es_model.ProjectAggregate).
 		LatestSequenceFilter(sequence.CurrentSequence), nil

internal/auth/repository/eventsourcing/repository.go

@@ -2,6 +2,7 @@ package eventsourcing
 
 import (
 	"context"
+
 	"github.com/caos/zitadel/internal/api/authz"
 	"github.com/caos/zitadel/internal/auth/repository/eventsourcing/eventstore"
 	"github.com/caos/zitadel/internal/auth/repository/eventsourcing/handler"
@@ -144,8 +145,9 @@ func Start(conf Config, authZ authz.Config, systemDefaults sd.SystemDefaults, au
 			IAMID:                      systemDefaults.IamID,
 		},
 		eventstore.TokenRepo{
-			UserEvents: user,
-			View:       view,
+			UserEvents:    user,
+			ProjectEvents: project,
+			View:          view,
 		},
 		eventstore.KeyRepository{
 			KeyEvents:          key,

internal/auth/repository/eventsourcing/view/authn_keys.go

@@ -0,0 +1,74 @@
+package view
+
+import (
+	"github.com/caos/zitadel/internal/errors"
+	"github.com/caos/zitadel/internal/eventstore/models"
+	key_model "github.com/caos/zitadel/internal/key/model"
+	"github.com/caos/zitadel/internal/key/repository/view"
+	"github.com/caos/zitadel/internal/key/repository/view/model"
+	"github.com/caos/zitadel/internal/view/repository"
+)
+
+const (
+	authNKeyTable = "auth.authn_keys"
+)
+
+func (v *View) AuthNKeyByIDs(userID, keyID string) (*model.AuthNKeyView, error) {
+	return view.AuthNKeyByIDs(v.Db, authNKeyTable, userID, keyID)
+}
+
+func (v *View) AuthNKeysByObjectID(objectID string) ([]*model.AuthNKeyView, error) {
+	return view.AuthNKeysByObjectID(v.Db, authNKeyTable, objectID)
+}
+
+func (v *View) AuthNKeyByID(keyID string) (*model.AuthNKeyView, error) {
+	return view.AuthNKeyByID(v.Db, authNKeyTable, keyID)
+}
+
+func (v *View) SearchAuthNKeys(request *key_model.AuthNKeySearchRequest) ([]*model.AuthNKeyView, uint64, error) {
+	return view.SearchAuthNKeys(v.Db, authNKeyTable, request)
+}
+
+func (v *View) PutAuthNKey(key *model.AuthNKeyView, event *models.Event) error {
+	err := view.PutAuthNKey(v.Db, authNKeyTable, key)
+	if err != nil {
+		return err
+	}
+	return v.ProcessedAuthNKeySequence(event)
+}
+
+func (v *View) DeleteAuthNKey(keyID string, event *models.Event) error {
+	err := view.DeleteAuthNKey(v.Db, authNKeyTable, keyID)
+	if err != nil && !errors.IsNotFound(err) {
+		return err
+	}
+	return v.ProcessedAuthNKeySequence(event)
+}
+
+func (v *View) DeleteAuthNKeysByObjectID(objectID string, event *models.Event) error {
+	err := view.DeleteAuthNKey(v.Db, authNKeyTable, objectID)
+	if err != nil && !errors.IsNotFound(err) {
+		return err
+	}
+	return v.ProcessedAuthNKeySequence(event)
+}
+
+func (v *View) GetLatestAuthNKeySequence() (*repository.CurrentSequence, error) {
+	return v.latestSequence(authNKeyTable)
+}
+
+func (v *View) ProcessedAuthNKeySequence(event *models.Event) error {
+	return v.saveCurrentSequence(authNKeyTable, event)
+}
+
+func (v *View) UpdateAuthNKeySpoolerRunTimestamp() error {
+	return v.updateSpoolerRunSequence(authNKeyTable)
+}
+
+func (v *View) GetLatestAuthNKeyFailedEvent(sequence uint64) (*repository.FailedEvent, error) {
+	return v.latestFailedEvent(authNKeyTable, sequence)
+}
+
+func (v *View) ProcessedAuthNKeyFailedEvent(failedEvent *repository.FailedEvent) error {
+	return v.saveFailedEvent(failedEvent)
+}

internal/auth/repository/eventsourcing/view/machine_keys.go

@@ -1,74 +0,0 @@
-package view
-
-import (
-	"github.com/caos/zitadel/internal/errors"
-	"github.com/caos/zitadel/internal/eventstore/models"
-	usr_model "github.com/caos/zitadel/internal/user/model"
-	"github.com/caos/zitadel/internal/user/repository/view"
-	"github.com/caos/zitadel/internal/user/repository/view/model"
-	"github.com/caos/zitadel/internal/view/repository"
-)
-
-const (
-	machineKeyTable = "auth.machine_keys"
-)
-
-func (v *View) MachineKeyByIDs(userID, keyID string) (*model.MachineKeyView, error) {
-	return view.MachineKeyByIDs(v.Db, machineKeyTable, userID, keyID)
-}
-
-func (v *View) MachineKeysByUserID(userID string) ([]*model.MachineKeyView, error) {
-	return view.MachineKeysByUserID(v.Db, machineKeyTable, userID)
-}
-
-func (v *View) MachineKeyByID(keyID string) (*model.MachineKeyView, error) {
-	return view.MachineKeyByID(v.Db, machineKeyTable, keyID)
-}
-
-func (v *View) SearchMachineKeys(request *usr_model.MachineKeySearchRequest) ([]*model.MachineKeyView, uint64, error) {
-	return view.SearchMachineKeys(v.Db, machineKeyTable, request)
-}
-
-func (v *View) PutMachineKey(key *model.MachineKeyView, event *models.Event) error {
-	err := view.PutMachineKey(v.Db, machineKeyTable, key)
-	if err != nil {
-		return err
-	}
-	return v.ProcessedMachineKeySequence(event)
-}
-
-func (v *View) DeleteMachineKey(keyID string, event *models.Event) error {
-	err := view.DeleteMachineKey(v.Db, machineKeyTable, keyID)
-	if err != nil && !errors.IsNotFound(err) {
-		return err
-	}
-	return v.ProcessedMachineKeySequence(event)
-}
-
-func (v *View) DeleteMachineKeysByUserID(userID string, event *models.Event) error {
-	err := view.DeleteMachineKey(v.Db, machineKeyTable, userID)
-	if err != nil && !errors.IsNotFound(err) {
-		return err
-	}
-	return v.ProcessedMachineKeySequence(event)
-}
-
-func (v *View) GetLatestMachineKeySequence() (*repository.CurrentSequence, error) {
-	return v.latestSequence(machineKeyTable)
-}
-
-func (v *View) ProcessedMachineKeySequence(event *models.Event) error {
-	return v.saveCurrentSequence(machineKeyTable, event)
-}
-
-func (v *View) UpdateMachineKeySpoolerRunTimestamp() error {
-	return v.updateSpoolerRunSequence(machineKeyTable)
-}
-
-func (v *View) GetLatestMachineKeyFailedEvent(sequence uint64) (*repository.FailedEvent, error) {
-	return v.latestFailedEvent(machineKeyTable, sequence)
-}
-
-func (v *View) ProcessedMachineKeyFailedEvent(failedEvent *repository.FailedEvent) error {
-	return v.saveFailedEvent(failedEvent)
-}

internal/auth/repository/org.go

@@ -4,10 +4,12 @@ import (
 	"context"
 	auth_model "github.com/caos/zitadel/internal/auth/model"
 	iam_model "github.com/caos/zitadel/internal/iam/model"
+	org_model "github.com/caos/zitadel/internal/org/model"
 )
 
 type OrgRepository interface {
 	RegisterOrg(context.Context, *auth_model.RegisterOrg) (*auth_model.RegisterOrg, error)
+	OrgByPrimaryDomain(primaryDomain string) (*org_model.OrgView, error)
 	GetOrgIAMPolicy(ctx context.Context, orgID string) (*iam_model.OrgIAMPolicyView, error)
 	GetDefaultOrgIAMPolicy(ctx context.Context) (*iam_model.OrgIAMPolicyView, error)
 	GetIDPConfigByID(ctx context.Context, idpConfigID string) (*iam_model.IDPConfigView, error)

internal/auth/repository/user.go

@@ -3,6 +3,7 @@ package repository
 import (
 	"context"
 
+	key_model "github.com/caos/zitadel/internal/key/model"
 	org_model "github.com/caos/zitadel/internal/org/model"
 
 	"github.com/caos/zitadel/internal/user/model"
@@ -43,7 +44,7 @@ type UserRepository interface {
 
 	UserByID(ctx context.Context, userID string) (*model.UserView, error)
 
-	MachineKeyByID(ctx context.Context, keyID string) (*model.MachineKeyView, error)
+	MachineKeyByID(ctx context.Context, keyID string) (*key_model.AuthNKeyView, error)
 }
 
 type myUserRepo interface {