perf(actionsv2): execution target router (#10564)

# Which Problems Are Solved

The event execution system currently uses a projection handler that
subscribes to and processes all events for all instances. This creates a
high static cost because the system over-fetches event data, handling
many events that are not needed by most instances. This inefficiency is
also reflected in high "rows returned" metrics in the database.

# How the Problems Are Solved

Eliminate the use of a project handler. Instead, events for which
"execution targets" are defined, are directly pushed to the queue by the
eventstore. A Router is populated in the Instance object in the authz
middleware.

- By joining the execution targets to the instance, no additional
queries are needed anymore.
- As part of the instance object, execution targets are now cached as
well.
- Events are queued within the same transaction, giving transactional
guarantees on delivery.
- Uses the "insert many fast` variant of River. Multiple jobs are queued
in a single round-trip to the database.
- Fix compatibility with PostgreSQL 15

# Additional Changes

- The signing key was stored as plain-text in the river job payload in
the DB. This violated our [Secrets
Storage](https://zitadel.com/docs/concepts/architecture/secrets#secrets-storage)
principle. This change removed the field and only uses the encrypted
version of the signing key.
- Fixed the target ordering from descending to ascending.
- Some minor linter warnings on the use of `io.WriteString()`.

# Additional Context

- Introduced in https://github.com/zitadel/zitadel/pull/9249
- Closes https://github.com/zitadel/zitadel/issues/10553
- Closes https://github.com/zitadel/zitadel/issues/9832
- Closes https://github.com/zitadel/zitadel/issues/10372
- Closes https://github.com/zitadel/zitadel/issues/10492

---------

Co-authored-by: Stefan Benz <46600784+stebenz@users.noreply.github.com>
(cherry picked from commit a9ebc06c77)

internal/execution/execution.go

@@ -6,14 +6,15 @@ import (
 	"encoding/json"
 	"io"
 	"net/http"
-	"strings"
 	"time"
 
 	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/internal/api/authz"
 	zhttp "github.com/zitadel/zitadel/internal/api/http"
+	"github.com/zitadel/zitadel/internal/crypto"
 	"github.com/zitadel/zitadel/internal/domain"
-	"github.com/zitadel/zitadel/internal/query"
+	target_domain "github.com/zitadel/zitadel/internal/execution/target"
 	"github.com/zitadel/zitadel/internal/repository/execution"
 	"github.com/zitadel/zitadel/internal/telemetry/tracing"
 	"github.com/zitadel/zitadel/internal/zerrors"
@@ -26,27 +27,19 @@ type ContextInfo interface {
 	SetHTTPResponseBody([]byte) error
 }
 
-type Target interface {
-	GetTargetID() string
-	IsInterruptOnError() bool
-	GetEndpoint() string
-	GetTargetType() domain.TargetType
-	GetTimeout() time.Duration
-	GetSigningKey() string
-}
-
 // CallTargets call a list of targets in order with handling of error and responses
 func CallTargets(
 	ctx context.Context,
-	targets []Target,
+	targets []target_domain.Target,
 	info ContextInfo,
+	alg crypto.EncryptionAlgorithm,
 ) (_ interface{}, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() { span.EndWithError(err) }()
 
 	for _, target := range targets {
 		// call the type of target
-		resp, err := CallTarget(ctx, target, info)
+		resp, err := CallTarget(ctx, target, info, alg)
 		// handle error if interrupt is set
 		if err != nil && target.IsInterruptOnError() {
 			return nil, err
@@ -68,22 +61,28 @@ type ContextInfoRequest interface {
 // CallTarget call the desired type of target with handling of responses
 func CallTarget(
 	ctx context.Context,
-	target Target,
+	target target_domain.Target,
 	info ContextInfoRequest,
+	alg crypto.EncryptionAlgorithm,
 ) (res []byte, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() { span.EndWithError(err) }()
 
+	signingKey, err := target.GetSigningKey(alg)
+	if err != nil {
+		return nil, zerrors.ThrowInternal(err, "EXEC-thiiCh5b", "Errors.Internal")
+	}
+
 	switch target.GetTargetType() {
 	// get request, ignore response and return request and error for handling in list of targets
-	case domain.TargetTypeWebhook:
-		return nil, webhook(ctx, target.GetEndpoint(), target.GetTimeout(), info.GetHTTPRequestBody(), target.GetSigningKey())
+	case target_domain.TargetTypeWebhook:
+		return nil, webhook(ctx, target.GetEndpoint(), target.GetTimeout(), info.GetHTTPRequestBody(), signingKey)
 	// get request, return response and error
-	case domain.TargetTypeCall:
-		return Call(ctx, target.GetEndpoint(), target.GetTimeout(), info.GetHTTPRequestBody(), target.GetSigningKey())
-	case domain.TargetTypeAsync:
-		go func(ctx context.Context, target Target, info []byte) {
-			if _, err := Call(ctx, target.GetEndpoint(), target.GetTimeout(), info, target.GetSigningKey()); err != nil {
+	case target_domain.TargetTypeCall:
+		return Call(ctx, target.GetEndpoint(), target.GetTimeout(), info.GetHTTPRequestBody(), signingKey)
+	case target_domain.TargetTypeAsync:
+		go func(ctx context.Context, target target_domain.Target, info []byte) {
+			if _, err := Call(ctx, target.GetEndpoint(), target.GetTimeout(), info, signingKey); err != nil {
 				logging.WithFields("target", target.GetTargetID()).OnError(err).Info(err)
 			}
 		}(context.WithoutCancel(ctx), target, info.GetHTTPRequestBody())
@@ -157,58 +156,29 @@ type ErrorBody struct {
 	ForwardedErrorMessage string `json:"forwardedErrorMessage,omitempty"`
 }
 
-type ExecutionTargetsQueries interface {
-	TargetsByExecutionID(ctx context.Context, ids []string) (execution []*query.ExecutionTarget, err error)
-	TargetsByExecutionIDs(ctx context.Context, ids1, ids2 []string) (execution []*query.ExecutionTarget, err error)
-}
-
-func QueryExecutionTargetsForRequestAndResponse(
+func QueryExecutionTargetsForRequest(
 	ctx context.Context,
-	queries ExecutionTargetsQueries,
 	fullMethod string,
-) ([]Target, []Target) {
+) []target_domain.Target {
 	ctx, span := tracing.NewSpan(ctx)
 	defer span.End()
 
-	targets, err := queries.TargetsByExecutionIDs(ctx,
-		idsForFullMethod(fullMethod, domain.ExecutionTypeRequest),
-		idsForFullMethod(fullMethod, domain.ExecutionTypeResponse),
-	)
-	requestTargets := make([]Target, 0, len(targets))
-	responseTargets := make([]Target, 0, len(targets))
-	if err != nil {
-		logging.WithFields("fullMethod", fullMethod).WithError(err).Info("unable to query targets")
-		return requestTargets, responseTargets
-	}
-
-	for _, target := range targets {
-		if strings.HasPrefix(target.GetExecutionID(), execution.IDAll(domain.ExecutionTypeRequest)) {
-			requestTargets = append(requestTargets, target)
-		} else if strings.HasPrefix(target.GetExecutionID(), execution.IDAll(domain.ExecutionTypeResponse)) {
-			responseTargets = append(responseTargets, target)
-		}
-	}
-
-	return requestTargets, responseTargets
+	requestTargets, _ := authz.GetInstance(ctx).ExecutionRouter().GetEventBestMatch(execution.ID(domain.ExecutionTypeRequest, fullMethod))
+	return requestTargets
 }
 
-func idsForFullMethod(fullMethod string, executionType domain.ExecutionType) []string {
-	return []string{execution.ID(executionType, fullMethod), execution.ID(executionType, serviceFromFullMethod(fullMethod)), execution.IDAll(executionType)}
+func QueryExecutionTargetsForResponse(
+	ctx context.Context,
+	fullMethod string,
+) []target_domain.Target {
+	ctx, span := tracing.NewSpan(ctx)
+	defer span.End()
+
+	responseTargets, _ := authz.GetInstance(ctx).ExecutionRouter().GetEventBestMatch(execution.ID(domain.ExecutionTypeResponse, fullMethod))
+	return responseTargets
 }
 
-func serviceFromFullMethod(s string) string {
-	parts := strings.Split(s, "/")
-	return parts[1]
-}
-
-func QueryExecutionTargetsForFunction(ctx context.Context, query ExecutionTargetsQueries, function string) ([]Target, error) {
-	queriedActionsV2, err := query.TargetsByExecutionID(ctx, []string{function})
-	if err != nil {
-		return nil, err
-	}
-	executionTargets := make([]Target, len(queriedActionsV2))
-	for i, action := range queriedActionsV2 {
-		executionTargets[i] = action
-	}
-	return executionTargets, nil
+func QueryExecutionTargetsForFunction(ctx context.Context, function string) []target_domain.Target {
+	executionTargets, _ := authz.GetInstance(ctx).ExecutionRouter().GetEventBestMatch(function)
+	return executionTargets
 }