refactor(eventstore): move push logic to sql (#8816)

# Which Problems Are Solved

If many events are written to the same aggregate id it can happen that
zitadel [starts to retry the push
transaction](48ffc902cc/internal/eventstore/eventstore.go (L101))
because [the locking
behaviour](48ffc902cc/internal/eventstore/v3/sequence.go (L25))
during push does compute the wrong sequence because newly committed
events are not visible to the transaction. These events impact the
current sequence.

In cases with high command traffic on a single aggregate id this can
have severe impact on general performance of zitadel. Because many
connections of the `eventstore pusher` database pool are blocked by each
other.

# How the Problems Are Solved

To improve the performance this locking mechanism was removed and the
business logic of push is moved to sql functions which reduce network
traffic and can be analyzed by the database before the actual push. For
clients of the eventstore framework nothing changed.

# Additional Changes

- after a connection is established prefetches the newly added database
types
- `eventstore.BaseEvent` now returns the correct revision of the event

# Additional Context

- part of https://github.com/zitadel/zitadel/issues/8931

---------

Co-authored-by: Tim Möhlmann <tim+github@zitadel.com>
Co-authored-by: Livio Spring <livio.a@gmail.com>
Co-authored-by: Max Peintner <max@caos.ch>
Co-authored-by: Elio Bischof <elio@zitadel.com>
Co-authored-by: Stefan Benz <46600784+stebenz@users.noreply.github.com>
Co-authored-by: Miguel Cabrerizo <30386061+doncicuto@users.noreply.github.com>
Co-authored-by: Joakim Lodén <Loddan@users.noreply.github.com>
Co-authored-by: Yxnt <Yxnt@users.noreply.github.com>
Co-authored-by: Stefan Benz <stefan@caos.ch>
Co-authored-by: Harsha Reddy <harsha.reddy@klaviyo.com>
Co-authored-by: Zach H <zhirschtritt@gmail.com>

internal/eventstore/v3/event.go

@@ -1,11 +1,13 @@
 package eventstore
 
 import (
+	"context"
 	"encoding/json"
+	"strconv"
 	"time"
 
 	"github.com/zitadel/logging"
-
+	"github.com/zitadel/zitadel/internal/api/authz"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
@@ -14,33 +16,98 @@ var (
 	_ eventstore.Event = (*event)(nil)
 )
 
+type command struct {
+	InstanceID    string
+	AggregateType string
+	AggregateID   string
+	CommandType   string
+	Revision      uint16
+	Payload       Payload
+	Creator       string
+	Owner         string
+}
+
+func (c *command) Aggregate() *eventstore.Aggregate {
+	return &eventstore.Aggregate{
+		ID:            c.AggregateID,
+		Type:          eventstore.AggregateType(c.AggregateType),
+		ResourceOwner: c.Owner,
+		InstanceID:    c.InstanceID,
+		Version:       eventstore.Version("v" + strconv.Itoa(int(c.Revision))),
+	}
+}
+
 type event struct {
-	aggregate *eventstore.Aggregate
-	creator   string
-	revision  uint16
-	typ       eventstore.EventType
+	command   *command
 	createdAt time.Time
 	sequence  uint64
 	position  float64
-	payload   Payload
 }
 
-func commandToEvent(sequence *latestSequence, command eventstore.Command) (_ *event, err error) {
+// TODO: remove on v3
+func commandToEventOld(sequence *latestSequence, cmd eventstore.Command) (_ *event, err error) {
 	var payload Payload
-	if command.Payload() != nil {
-		payload, err = json.Marshal(command.Payload())
+	if cmd.Payload() != nil {
+		payload, err = json.Marshal(cmd.Payload())
 		if err != nil {
 			logging.WithError(err).Warn("marshal payload failed")
 			return nil, zerrors.ThrowInternal(err, "V3-MInPK", "Errors.Internal")
 		}
 	}
 	return &event{
-		aggregate: sequence.aggregate,
-		creator:   command.Creator(),
-		revision:  command.Revision(),
-		typ:       command.Type(),
-		payload:   payload,
-		sequence:  sequence.sequence,
+		command: &command{
+			InstanceID:    sequence.aggregate.InstanceID,
+			AggregateType: string(sequence.aggregate.Type),
+			AggregateID:   sequence.aggregate.ID,
+			CommandType:   string(cmd.Type()),
+			Revision:      cmd.Revision(),
+			Payload:       payload,
+			Creator:       cmd.Creator(),
+			Owner:         sequence.aggregate.ResourceOwner,
+		},
+		sequence: sequence.sequence,
+	}, nil
+}
+
+func commandsToEvents(ctx context.Context, cmds []eventstore.Command) (_ []eventstore.Event, _ []*command, err error) {
+	events := make([]eventstore.Event, len(cmds))
+	commands := make([]*command, len(cmds))
+	for i, cmd := range cmds {
+		if cmd.Aggregate().InstanceID == "" {
+			cmd.Aggregate().InstanceID = authz.GetInstance(ctx).InstanceID()
+		}
+		events[i], err = commandToEvent(cmd)
+		if err != nil {
+			return nil, nil, err
+		}
+		commands[i] = events[i].(*event).command
+	}
+	return events, commands, nil
+}
+
+func commandToEvent(cmd eventstore.Command) (_ eventstore.Event, err error) {
+	var payload Payload
+	if cmd.Payload() != nil {
+		payload, err = json.Marshal(cmd.Payload())
+		if err != nil {
+			logging.WithError(err).Warn("marshal payload failed")
+			return nil, zerrors.ThrowInternal(err, "V3-MInPK", "Errors.Internal")
+		}
+	}
+
+	command := &command{
+		InstanceID:    cmd.Aggregate().InstanceID,
+		AggregateType: string(cmd.Aggregate().Type),
+		AggregateID:   cmd.Aggregate().ID,
+		CommandType:   string(cmd.Type()),
+		Revision:      cmd.Revision(),
+		Payload:       payload,
+		Creator:       cmd.Creator(),
+		Owner:         cmd.Aggregate().ResourceOwner,
+	}
+
+	return &event{
+		command: command,
 	}, nil
 }
 
@@ -56,22 +123,22 @@ func (e *event) EditorUser() string {
 
 // Aggregate implements [eventstore.Event]
 func (e *event) Aggregate() *eventstore.Aggregate {
-	return e.aggregate
+	return e.command.Aggregate()
 }
 
 // Creator implements [eventstore.Event]
 func (e *event) Creator() string {
-	return e.creator
+	return e.command.Creator
 }
 
 // Revision implements [eventstore.Event]
 func (e *event) Revision() uint16 {
-	return e.revision
+	return e.command.Revision
 }
 
 // Type implements [eventstore.Event]
 func (e *event) Type() eventstore.EventType {
-	return e.typ
+	return eventstore.EventType(e.command.CommandType)
 }
 
 // CreatedAt implements [eventstore.Event]
@@ -91,10 +158,10 @@ func (e *event) Position() float64 {
 
 // Unmarshal implements [eventstore.Event]
 func (e *event) Unmarshal(ptr any) error {
-	if len(e.payload) == 0 {
+	if len(e.command.Payload) == 0 {
 		return nil
 	}
-	if err := json.Unmarshal(e.payload, ptr); err != nil {
+	if err := json.Unmarshal(e.command.Payload, ptr); err != nil {
 		return zerrors.ThrowInternal(err, "V3-u8qVo", "Errors.Internal")
 	}
 
@@ -103,5 +170,5 @@ func (e *event) Unmarshal(ptr any) error {
 
 // DataAsBytes implements [eventstore.Event]
 func (e *event) DataAsBytes() []byte {
-	return e.payload
+	return e.command.Payload
 }