refactor(eventstore): move push logic to sql (#8816)

# Which Problems Are Solved

If many events are written to the same aggregate id it can happen that
zitadel [starts to retry the push
transaction](48ffc902cc/internal/eventstore/eventstore.go (L101))
because [the locking
behaviour](48ffc902cc/internal/eventstore/v3/sequence.go (L25))
during push does compute the wrong sequence because newly committed
events are not visible to the transaction. These events impact the
current sequence.

In cases with high command traffic on a single aggregate id this can
have severe impact on general performance of zitadel. Because many
connections of the `eventstore pusher` database pool are blocked by each
other.

# How the Problems Are Solved

To improve the performance this locking mechanism was removed and the
business logic of push is moved to sql functions which reduce network
traffic and can be analyzed by the database before the actual push. For
clients of the eventstore framework nothing changed.

# Additional Changes

- after a connection is established prefetches the newly added database
types
- `eventstore.BaseEvent` now returns the correct revision of the event

# Additional Context

- part of https://github.com/zitadel/zitadel/issues/8931

---------

Co-authored-by: Tim Möhlmann <tim+github@zitadel.com>
Co-authored-by: Livio Spring <livio.a@gmail.com>
Co-authored-by: Max Peintner <max@caos.ch>
Co-authored-by: Elio Bischof <elio@zitadel.com>
Co-authored-by: Stefan Benz <46600784+stebenz@users.noreply.github.com>
Co-authored-by: Miguel Cabrerizo <30386061+doncicuto@users.noreply.github.com>
Co-authored-by: Joakim Lodén <Loddan@users.noreply.github.com>
Co-authored-by: Yxnt <Yxnt@users.noreply.github.com>
Co-authored-by: Stefan Benz <stefan@caos.ch>
Co-authored-by: Harsha Reddy <harsha.reddy@klaviyo.com>
Co-authored-by: Zach H <zhirschtritt@gmail.com>

internal/database/cockroach/crdb.go

@@ -3,15 +3,18 @@ package cockroach
 import (
 	"context"
 	"database/sql"
+	"fmt"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/jackc/pgx/v5"
 	"github.com/jackc/pgx/v5/pgxpool"
 	"github.com/jackc/pgx/v5/stdlib"
 	"github.com/mitchellh/mapstructure"
 	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/internal/api/authz"
 	"github.com/zitadel/zitadel/internal/database/dialect"
 )
 
@@ -72,6 +75,12 @@ func (_ *Config) Decode(configs []interface{}) (dialect.Connector, error) {
 }
 
 func (c *Config) Connect(useAdmin bool, pusherRatio, spoolerRatio float64, purpose dialect.DBPurpose) (*sql.DB, *pgxpool.Pool, error) {
+	dialect.RegisterAfterConnect(func(ctx context.Context, c *pgx.Conn) error {
+		// CockroachDB by default does not allow multiple modifications of the same table using ON CONFLICT
+		// This is needed to fill the fields table of the eventstore during eventstore.Push.
+		_, err := c.Exec(ctx, "SET enable_multiple_modifications_of_table = on")
+		return err
+	})
 	connConfig, err := dialect.NewConnectionConfig(c.MaxOpenConns, c.MaxIdleConns, pusherRatio, spoolerRatio, purpose)
 	if err != nil {
 		return nil, nil, err
@@ -82,6 +91,29 @@ func (c *Config) Connect(useAdmin bool, pusherRatio, spoolerRatio float64, purpo
 		return nil, nil, err
 	}
 
+	if len(connConfig.AfterConnect) > 0 {
+		config.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error {
+			for _, f := range connConfig.AfterConnect {
+				if err := f(ctx, conn); err != nil {
+					return err
+				}
+			}
+			return nil
+		}
+	}
+
+	// For the pusher we set the app name with the instance ID
+	if purpose == dialect.DBPurposeEventPusher {
+		config.BeforeAcquire = func(ctx context.Context, conn *pgx.Conn) bool {
+			return setAppNameWithID(ctx, conn, purpose, authz.GetInstance(ctx).InstanceID())
+		}
+		config.AfterRelease = func(conn *pgx.Conn) bool {
+			ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+			defer cancel()
+			return setAppNameWithID(ctx, conn, purpose, "IDLE")
+		}
+	}
+
 	if connConfig.MaxOpenConns != 0 {
 		config.MaxConns = int32(connConfig.MaxOpenConns)
 	}
@@ -200,3 +232,11 @@ func (c Config) String(useAdmin bool, appName string) string {
 
 	return strings.Join(fields, " ")
 }
+
+func setAppNameWithID(ctx context.Context, conn *pgx.Conn, purpose dialect.DBPurpose, id string) bool {
+	// needs to be set like this because psql complains about parameters in the SET statement
+	query := fmt.Sprintf("SET application_name = '%s_%s'", purpose.AppName(), id)
+	_, err := conn.Exec(ctx, query)
+	logging.OnError(err).Warn("failed to set application name")
+	return err == nil
+}