feat: token introspection, api clients and auth method private_key_jwt (#1276)

* introspect

* testingapplication key

* date

* client keys

* fix client keys

* fix client keys

* access tokens only for users

* AuthMethodPrivateKeyJWT

* client keys

* set introspection info correctly

* managae apis

* update oidc pkg

* cleanup

* merge msater

* set current sequence in migration

* set current sequence in migration

* set current sequence in migration

* Apply suggestions from code review

Co-authored-by: Fabi <38692350+fgerschwiler@users.noreply.github.com>

* DeleteAuthNKeysByObjectID

* ensure authn keys uptodate

* update oidc version

* merge master

* merge master

Co-authored-by: Fabi <38692350+fgerschwiler@users.noreply.github.com>

internal/management/repository/eventsourcing/eventstore/project.go

@@ -13,6 +13,8 @@ import (
 	es_models "github.com/caos/zitadel/internal/eventstore/models"
 	es_sdk "github.com/caos/zitadel/internal/eventstore/sdk"
 	iam_event "github.com/caos/zitadel/internal/iam/repository/eventsourcing"
+	key_model "github.com/caos/zitadel/internal/key/model"
+	key_view_model "github.com/caos/zitadel/internal/key/repository/view/model"
 	"github.com/caos/zitadel/internal/management/repository/eventsourcing/view"
 	global_model "github.com/caos/zitadel/internal/model"
 	proj_model "github.com/caos/zitadel/internal/project/model"
@@ -405,14 +407,80 @@ func (repo *ProjectRepo) ApplicationChanges(ctx context.Context, id string, appI
 	return changes, nil
 }
 
+func (repo *ProjectRepo) SearchClientKeys(ctx context.Context, request *key_model.AuthNKeySearchRequest) (*key_model.AuthNKeySearchResponse, error) {
+	request.EnsureLimit(repo.SearchLimit)
+	sequence, sequenceErr := repo.View.GetLatestAuthNKeySequence()
+	logging.Log("EVENT-ADwgw").OnError(sequenceErr).Warn("could not read latest authn key sequence")
+	keys, count, err := repo.View.SearchAuthNKeys(request)
+	if err != nil {
+		return nil, err
+	}
+	result := &key_model.AuthNKeySearchResponse{
+		Offset:      request.Offset,
+		Limit:       request.Limit,
+		TotalResult: count,
+		Result:      key_view_model.AuthNKeysToModel(keys),
+	}
+	if sequenceErr == nil {
+		result.Sequence = sequence.CurrentSequence
+		result.Timestamp = sequence.LastSuccessfulSpoolerRun
+	}
+	return result, nil
+}
+
+func (repo *ProjectRepo) GetClientKey(ctx context.Context, projectID, applicationID, keyID string) (*key_model.AuthNKeyView, error) {
+	key, viewErr := repo.View.AuthNKeyByIDs(applicationID, keyID)
+	if viewErr != nil {
+		return nil, viewErr
+	}
+
+	events, esErr := repo.ProjectEvents.ProjectEventsByID(ctx, projectID, key.Sequence)
+	if caos_errs.IsNotFound(viewErr) && len(events) == 0 {
+		return nil, caos_errs.ThrowNotFound(nil, "EVENT-SFf2g", "Errors.User.KeyNotFound")
+	}
+
+	if esErr != nil {
+		logging.Log("EVENT-ADbf2").WithError(viewErr).Debug("error retrieving new events")
+		return key_view_model.AuthNKeyToModel(key), nil
+	}
+
+	viewKey := *key
+	for _, event := range events {
+		err := key.AppendEventIfMyClientKey(event)
+		if err != nil {
+			return key_view_model.AuthNKeyToModel(&viewKey), nil
+		}
+		if key.State != int32(proj_model.AppStateActive) {
+			return nil, caos_errs.ThrowNotFound(nil, "EVENT-Adfg3", "Errors.User.KeyNotFound")
+		}
+	}
+	return key_view_model.AuthNKeyToModel(key), nil
+}
+
+func (repo *ProjectRepo) AddClientKey(ctx context.Context, key *proj_model.ClientKey) (*proj_model.ClientKey, error) {
+	return repo.ProjectEvents.AddClientKey(ctx, key)
+}
+
+func (repo *ProjectRepo) RemoveClientKey(ctx context.Context, projectID, applicationID, keyID string) error {
+	return repo.ProjectEvents.RemoveApplicationKey(ctx, projectID, applicationID, keyID)
+}
+
 func (repo *ProjectRepo) ChangeOIDCConfig(ctx context.Context, config *proj_model.OIDCConfig) (*proj_model.OIDCConfig, error) {
 	return repo.ProjectEvents.ChangeOIDCConfig(ctx, config)
 }
 
+func (repo *ProjectRepo) ChangeAPIConfig(ctx context.Context, config *proj_model.APIConfig) (*proj_model.APIConfig, error) {
+	return repo.ProjectEvents.ChangeAPIConfig(ctx, config)
+}
+
 func (repo *ProjectRepo) ChangeOIDConfigSecret(ctx context.Context, projectID, appID string) (*proj_model.OIDCConfig, error) {
 	return repo.ProjectEvents.ChangeOIDCConfigSecret(ctx, projectID, appID)
 }
 
+func (repo *ProjectRepo) ChangeAPIConfigSecret(ctx context.Context, projectID, appID string) (*proj_model.APIConfig, error) {
+	return repo.ProjectEvents.ChangeAPIConfigSecret(ctx, projectID, appID)
+}
+
 func (repo *ProjectRepo) ProjectGrantByID(ctx context.Context, grantID string) (*proj_model.ProjectGrantView, error) {
 	grant, err := repo.View.ProjectGrantByID(grantID)
 	if err != nil {

internal/management/repository/eventsourcing/eventstore/user.go

@@ -12,6 +12,8 @@ import (
 	es_models "github.com/caos/zitadel/internal/eventstore/models"
 	es_sdk "github.com/caos/zitadel/internal/eventstore/sdk"
 	iam_es_model "github.com/caos/zitadel/internal/iam/repository/view/model"
+	key_model "github.com/caos/zitadel/internal/key/model"
+	key_view_model "github.com/caos/zitadel/internal/key/repository/view/model"
 	"github.com/caos/zitadel/internal/management/repository/eventsourcing/view"
 	global_model "github.com/caos/zitadel/internal/model"
 	org_event "github.com/caos/zitadel/internal/org/repository/eventsourcing"
@@ -301,27 +303,27 @@ func (repo *UserRepo) ChangeMachine(ctx context.Context, machine *usr_model.Mach
 	return repo.UserEvents.ChangeMachine(ctx, machine)
 }
 
-func (repo *UserRepo) GetMachineKey(ctx context.Context, userID, keyID string) (*usr_model.MachineKeyView, error) {
-	key, err := repo.View.MachineKeyByIDs(userID, keyID)
+func (repo *UserRepo) GetMachineKey(ctx context.Context, userID, keyID string) (*key_model.AuthNKeyView, error) {
+	key, err := repo.View.AuthNKeyByIDs(userID, keyID)
 	if err != nil {
 		return nil, err
 	}
-	return model.MachineKeyToModel(key), nil
+	return key_view_model.AuthNKeyToModel(key), nil
 }
 
-func (repo *UserRepo) SearchMachineKeys(ctx context.Context, request *usr_model.MachineKeySearchRequest) (*usr_model.MachineKeySearchResponse, error) {
+func (repo *UserRepo) SearchMachineKeys(ctx context.Context, request *key_model.AuthNKeySearchRequest) (*key_model.AuthNKeySearchResponse, error) {
 	request.EnsureLimit(repo.SearchLimit)
-	sequence, seqErr := repo.View.GetLatestMachineKeySequence()
-	logging.Log("EVENT-Sk8fs").OnError(seqErr).Warn("could not read latest user sequence")
-	keys, count, err := repo.View.SearchMachineKeys(request)
+	sequence, seqErr := repo.View.GetLatestAuthNKeySequence()
+	logging.Log("EVENT-Sk8fs").OnError(seqErr).Warn("could not read latest authn key sequence")
+	keys, count, err := repo.View.SearchAuthNKeys(request)
 	if err != nil {
 		return nil, err
 	}
-	result := &usr_model.MachineKeySearchResponse{
+	result := &key_model.AuthNKeySearchResponse{
 		Offset:      request.Offset,
 		Limit:       request.Limit,
 		TotalResult: count,
-		Result:      model.MachineKeysToModel(keys),
+		Result:      key_view_model.AuthNKeysToModel(keys),
 	}
 	if seqErr == nil {
 		result.Sequence = sequence.CurrentSequence

internal/management/repository/eventsourcing/handler/application.go

@@ -87,6 +87,8 @@ func (a *Application) Reduce(event *models.Event) (err error) {
 	case es_model.ApplicationChanged,
 		es_model.OIDCConfigAdded,
 		es_model.OIDCConfigChanged,
+		es_model.APIConfigAdded,
+		es_model.APIConfigChanged,
 		es_model.ApplicationDeactivated,
 		es_model.ApplicationReactivated:
 		err = app.SetData(event)

internal/management/repository/eventsourcing/handler/authn_keys.go

@@ -0,0 +1,120 @@
+package handler
+
+import (
+	"time"
+
+	"github.com/caos/logging"
+
+	"github.com/caos/zitadel/internal/eventstore"
+	es_models "github.com/caos/zitadel/internal/eventstore/models"
+	"github.com/caos/zitadel/internal/eventstore/query"
+	"github.com/caos/zitadel/internal/eventstore/spooler"
+	key_model "github.com/caos/zitadel/internal/key/repository/view/model"
+	proj_model "github.com/caos/zitadel/internal/project/repository/eventsourcing/model"
+	user_model "github.com/caos/zitadel/internal/user/repository/eventsourcing/model"
+)
+
+const (
+	authnKeysTable = "management.authn_keys"
+)
+
+type AuthNKeys struct {
+	handler
+	subscription *eventstore.Subscription
+}
+
+func newAuthNKeys(handler handler) *AuthNKeys {
+	h := &AuthNKeys{
+		handler: handler,
+	}
+
+	h.subscribe()
+
+	return h
+}
+
+func (k *AuthNKeys) subscribe() {
+	k.subscription = k.es.Subscribe(k.AggregateTypes()...)
+	go func() {
+		for event := range k.subscription.Events {
+			query.ReduceEvent(k, event)
+		}
+	}()
+}
+
+func (k *AuthNKeys) ViewModel() string {
+	return authnKeysTable
+}
+
+func (_ *AuthNKeys) AggregateTypes() []es_models.AggregateType {
+	return []es_models.AggregateType{user_model.UserAggregate, proj_model.ProjectAggregate}
+}
+
+func (k *AuthNKeys) CurrentSequence() (uint64, error) {
+	sequence, err := k.view.GetLatestAuthNKeySequence()
+	if err != nil {
+		return 0, err
+	}
+	return sequence.CurrentSequence, nil
+}
+
+func (k *AuthNKeys) EventQuery() (*es_models.SearchQuery, error) {
+	sequence, err := k.view.GetLatestAuthNKeySequence()
+	if err != nil {
+		return nil, err
+	}
+	return es_models.NewSearchQuery().
+		AggregateTypeFilter(k.AggregateTypes()...).
+		LatestSequenceFilter(sequence.CurrentSequence), nil
+}
+
+func (k *AuthNKeys) Reduce(event *es_models.Event) (err error) {
+	switch event.AggregateType {
+	case user_model.UserAggregate,
+		proj_model.ProjectAggregate:
+		err = k.processAuthNKeys(event)
+	}
+	return err
+}
+
+func (k *AuthNKeys) processAuthNKeys(event *es_models.Event) (err error) {
+	key := new(key_model.AuthNKeyView)
+	switch event.Type {
+	case user_model.MachineKeyAdded,
+		proj_model.ClientKeyAdded:
+		err = key.AppendEvent(event)
+		if key.ExpirationDate.Before(time.Now()) {
+			return k.view.ProcessedAuthNKeySequence(event)
+		}
+	case user_model.MachineKeyRemoved:
+		err = key.SetUserData(event)
+		if err != nil {
+			return err
+		}
+		return k.view.DeleteAuthNKey(key.ID, event)
+	case proj_model.ClientKeyRemoved:
+		err = key.SetClientData(event)
+		if err != nil {
+			return err
+		}
+		return k.view.DeleteAuthNKey(key.ID, event)
+	case user_model.UserRemoved,
+		proj_model.ApplicationRemoved:
+		return k.view.DeleteAuthNKeysByObjectID(event.AggregateID, event)
+	default:
+		return k.view.ProcessedAuthNKeySequence(event)
+	}
+	if err != nil {
+		return err
+	}
+	return k.view.PutAuthNKey(key, event)
+}
+
+func (d *AuthNKeys) OnError(event *es_models.Event, err error) error {
+	logging.LogWithFields("SPOOL-S9fe", "id", event.AggregateID).WithError(err).Warn("something went wrong in machine key handler")
+	return spooler.HandleError(event, err, d.view.GetLatestAuthNKeyFailedEvent, d.view.ProcessedAuthNKeyFailedEvent, d.view.ProcessedAuthNKeySequence, d.errorCountUntilSkip)
+}
+
+func (d *AuthNKeys) OnSuccess() error {
+	return spooler.HandleSuccess(d.view.UpdateAuthNKeySpoolerRunTimestamp)
+}

internal/management/repository/eventsourcing/handler/handler.go

@@ -75,7 +75,7 @@ func Register(configs Configs, bulkLimit, errorCount uint64, view *view.View, es
 			handler{view, bulkLimit, configs.cycleDuration("UserMembership"), errorCount, es},
 			repos.OrgEvents,
 			repos.ProjectEvents),
-		newMachineKeys(
+		newAuthNKeys(
 			handler{view, bulkLimit, configs.cycleDuration("MachineKeys"), errorCount, es}),
 		newIDPConfig(
 			handler{view, bulkLimit, configs.cycleDuration("IDPConfig"), errorCount, es}),

internal/management/repository/eventsourcing/handler/machine_keys.go

@@ -1,110 +0,0 @@
-package handler
-
-import (
-	"time"
-
-	"github.com/caos/logging"
-	"github.com/caos/zitadel/internal/eventstore"
-	"github.com/caos/zitadel/internal/eventstore/models"
-	es_models "github.com/caos/zitadel/internal/eventstore/models"
-	"github.com/caos/zitadel/internal/eventstore/query"
-	"github.com/caos/zitadel/internal/eventstore/spooler"
-	"github.com/caos/zitadel/internal/user/repository/eventsourcing/model"
-	usr_model "github.com/caos/zitadel/internal/user/repository/view/model"
-)
-
-const (
-	machineKeysTable = "management.machine_keys"
-)
-
-type MachineKeys struct {
-	handler
-	subscription *eventstore.Subscription
-}
-
-func newMachineKeys(handler handler) *MachineKeys {
-	h := &MachineKeys{
-		handler: handler,
-	}
-
-	h.subscribe()
-
-	return h
-}
-
-func (m *MachineKeys) subscribe() {
-	m.subscription = m.es.Subscribe(m.AggregateTypes()...)
-	go func() {
-		for event := range m.subscription.Events {
-			query.ReduceEvent(m, event)
-		}
-	}()
-}
-
-func (d *MachineKeys) ViewModel() string {
-	return machineKeysTable
-}
-
-func (_ *MachineKeys) AggregateTypes() []es_models.AggregateType {
-	return []es_models.AggregateType{model.UserAggregate}
-}
-
-func (k *MachineKeys) CurrentSequence() (uint64, error) {
-	sequence, err := k.view.GetLatestMachineKeySequence()
-	if err != nil {
-		return 0, err
-	}
-	return sequence.CurrentSequence, nil
-}
-
-func (d *MachineKeys) EventQuery() (*models.SearchQuery, error) {
-	sequence, err := d.view.GetLatestMachineKeySequence()
-	if err != nil {
-		return nil, err
-	}
-	return es_models.NewSearchQuery().
-		AggregateTypeFilter(d.AggregateTypes()...).
-		LatestSequenceFilter(sequence.CurrentSequence), nil
-}
-
-func (d *MachineKeys) Reduce(event *models.Event) (err error) {
-	switch event.AggregateType {
-	case model.UserAggregate:
-		err = d.processMachineKeys(event)
-	}
-	return err
-}
-
-func (d *MachineKeys) processMachineKeys(event *models.Event) (err error) {
-	key := new(usr_model.MachineKeyView)
-	switch event.Type {
-	case model.MachineKeyAdded:
-		err = key.AppendEvent(event)
-		if key.ExpirationDate.Before(time.Now()) {
-			return d.view.ProcessedMachineKeySequence(event)
-		}
-	case model.MachineKeyRemoved:
-		err = key.SetData(event)
-		if err != nil {
-			return err
-		}
-		return d.view.DeleteMachineKey(key.ID, event)
-	case model.UserRemoved:
-		return d.view.DeleteMachineKeysByUserID(event.AggregateID, event)
-	default:
-		return d.view.ProcessedMachineKeySequence(event)
-	}
-	if err != nil {
-		return err
-	}
-	return d.view.PutMachineKey(key, event)
-}
-
-func (d *MachineKeys) OnError(event *models.Event, err error) error {
-	logging.LogWithFields("SPOOL-S9fe", "id", event.AggregateID).WithError(err).Warn("something went wrong in machine key handler")
-	return spooler.HandleError(event, err, d.view.GetLatestMachineKeyFailedEvent, d.view.ProcessedMachineKeyFailedEvent, d.view.ProcessedMachineKeySequence, d.errorCountUntilSkip)
-}
-
-func (d *MachineKeys) OnSuccess() error {
-	return spooler.HandleSuccess(d.view.UpdateMachineKeySpoolerRunTimestamp)
-}

internal/management/repository/eventsourcing/view/authn_keys.go

@@ -0,0 +1,74 @@
+package view
+
+import (
+	"github.com/caos/zitadel/internal/errors"
+	"github.com/caos/zitadel/internal/eventstore/models"
+	key_model "github.com/caos/zitadel/internal/key/model"
+	"github.com/caos/zitadel/internal/key/repository/view"
+	"github.com/caos/zitadel/internal/key/repository/view/model"
+	"github.com/caos/zitadel/internal/view/repository"
+)
+
+const (
+	authNKeyTable = "management.authn_keys"
+)
+
+func (v *View) AuthNKeyByIDs(objectID, keyID string) (*model.AuthNKeyView, error) {
+	return view.AuthNKeyByIDs(v.Db, authNKeyTable, objectID, keyID)
+}
+
+func (v *View) AuthNKeysByObjectID(objectID string) ([]*model.AuthNKeyView, error) {
+	return view.AuthNKeysByObjectID(v.Db, authNKeyTable, objectID)
+}
+
+func (v *View) AuthNKeyByID(keyID string) (*model.AuthNKeyView, error) {
+	return view.AuthNKeyByID(v.Db, authNKeyTable, keyID)
+}
+
+func (v *View) SearchAuthNKeys(request *key_model.AuthNKeySearchRequest) ([]*model.AuthNKeyView, uint64, error) {
+	return view.SearchAuthNKeys(v.Db, authNKeyTable, request)
+}
+
+func (v *View) PutAuthNKey(key *model.AuthNKeyView, event *models.Event) error {
+	err := view.PutAuthNKey(v.Db, authNKeyTable, key)
+	if err != nil {
+		return err
+	}
+	return v.ProcessedAuthNKeySequence(event)
+}
+
+func (v *View) DeleteAuthNKey(keyID string, event *models.Event) error {
+	err := view.DeleteAuthNKey(v.Db, authNKeyTable, keyID)
+	if err != nil && !errors.IsNotFound(err) {
+		return err
+	}
+	return v.ProcessedAuthNKeySequence(event)
+}
+
+func (v *View) DeleteAuthNKeysByObjectID(objectID string, event *models.Event) error {
+	err := view.DeleteAuthNKey(v.Db, authNKeyTable, objectID)
+	if err != nil && !errors.IsNotFound(err) {
+		return err
+	}
+	return v.ProcessedAuthNKeySequence(event)
+}
+
+func (v *View) GetLatestAuthNKeySequence() (*repository.CurrentSequence, error) {
+	return v.latestSequence(authNKeyTable)
+}
+
+func (v *View) ProcessedAuthNKeySequence(event *models.Event) error {
+	return v.saveCurrentSequence(authNKeyTable, event)
+}
+
+func (v *View) UpdateAuthNKeySpoolerRunTimestamp() error {
+	return v.updateSpoolerRunSequence(authNKeyTable)
+}
+
+func (v *View) GetLatestAuthNKeyFailedEvent(sequence uint64) (*repository.FailedEvent, error) {
+	return v.latestFailedEvent(authNKeyTable, sequence)
+}
+
+func (v *View) ProcessedAuthNKeyFailedEvent(failedEvent *repository.FailedEvent) error {
+	return v.saveFailedEvent(failedEvent)
+}

internal/management/repository/eventsourcing/view/machine_keys.go

@@ -1,70 +0,0 @@
-package view
-
-import (
-	"github.com/caos/zitadel/internal/errors"
-	"github.com/caos/zitadel/internal/eventstore/models"
-	usr_model "github.com/caos/zitadel/internal/user/model"
-	"github.com/caos/zitadel/internal/user/repository/view"
-	"github.com/caos/zitadel/internal/user/repository/view/model"
-	"github.com/caos/zitadel/internal/view/repository"
-)
-
-const (
-	machineKeyTable = "management.machine_keys"
-)
-
-func (v *View) MachineKeyByIDs(userID, keyID string) (*model.MachineKeyView, error) {
-	return view.MachineKeyByIDs(v.Db, machineKeyTable, userID, keyID)
-}
-
-func (v *View) MachineKeysByUserID(userID string) ([]*model.MachineKeyView, error) {
-	return view.MachineKeysByUserID(v.Db, machineKeyTable, userID)
-}
-
-func (v *View) SearchMachineKeys(request *usr_model.MachineKeySearchRequest) ([]*model.MachineKeyView, uint64, error) {
-	return view.SearchMachineKeys(v.Db, machineKeyTable, request)
-}
-
-func (v *View) PutMachineKey(org *model.MachineKeyView, event *models.Event) error {
-	err := view.PutMachineKey(v.Db, machineKeyTable, org)
-	if err != nil {
-		return err
-	}
-	return v.ProcessedMachineKeySequence(event)
-}
-
-func (v *View) DeleteMachineKey(keyID string, event *models.Event) error {
-	err := view.DeleteMachineKey(v.Db, machineKeyTable, keyID)
-	if err != nil && !errors.IsNotFound(err) {
-		return err
-	}
-	return v.ProcessedMachineKeySequence(event)
-}
-
-func (v *View) DeleteMachineKeysByUserID(userID string, event *models.Event) error {
-	err := view.DeleteMachineKey(v.Db, machineKeyTable, userID)
-	if err != nil && !errors.IsNotFound(err) {
-		return err
-	}
-	return v.ProcessedMachineKeySequence(event)
-}
-
-func (v *View) GetLatestMachineKeySequence() (*repository.CurrentSequence, error) {
-	return v.latestSequence(machineKeyTable)
-}
-
-func (v *View) ProcessedMachineKeySequence(event *models.Event) error {
-	return v.saveCurrentSequence(machineKeyTable, event)
-}
-
-func (v *View) UpdateMachineKeySpoolerRunTimestamp() error {
-	return v.updateSpoolerRunSequence(machineKeyTable)
-}
-
-func (v *View) GetLatestMachineKeyFailedEvent(sequence uint64) (*repository.FailedEvent, error) {
-	return v.latestFailedEvent(machineKeyTable, sequence)
-}
-
-func (v *View) ProcessedMachineKeyFailedEvent(failedEvent *repository.FailedEvent) error {
-	return v.saveFailedEvent(failedEvent)
-}