fix(eventstore): use decimal, correct mirror (#9914)

# Eventstore fixes

- `event.Position` used float64 before which can lead to [precision
loss](https://github.com/golang/go/issues/47300). The type got replaced
by [a type without precision
loss](https://github.com/jackc/pgx-shopspring-decimal)
- the handler reported the wrong error if the current state was updated
and therefore took longer to retry failed events.

# Mirror fixes

- max age of auth requests can be configured to speed up copying data
from `auth.auth_requests` table. Auth requests last updated before the
set age will be ignored. Default is 1 month
- notification projections are skipped because notifications should be
sent by the source system. The projections are set to the latest
position
- ensure that mirror can be executed multiple times

---------

Co-authored-by: Livio Spring <livio.a@gmail.com>
This commit is contained in:
Silvan
2025-05-28 23:54:18 +02:00
committed by GitHub
parent 046b165db8
commit 131f70db34
51 changed files with 436 additions and 236 deletions

View File

@@ -8,7 +8,9 @@ import (
"io"
"time"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/stdlib"
"github.com/shopspring/decimal"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/zitadel/logging"
@@ -89,7 +91,7 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
previousMigration, err := queryLastSuccessfulMigration(ctx, destinationES, source.DatabaseName())
logging.OnError(err).Fatal("unable to query latest successful migration")
var maxPosition float64
var maxPosition decimal.Decimal
err = source.QueryRowContext(ctx,
func(row *sql.Row) error {
return row.Scan(&maxPosition)
@@ -101,7 +103,7 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
logging.WithFields("from", previousMigration.Position, "to", maxPosition).Info("start event migration")
nextPos := make(chan bool, 1)
pos := make(chan float64, 1)
pos := make(chan decimal.Decimal, 1)
errs := make(chan error, 3)
go func() {
@@ -152,7 +154,7 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
go func() {
defer close(pos)
for range nextPos {
var position float64
var position decimal.Decimal
err := dest.QueryRowContext(
ctx,
func(row *sql.Row) error {
@@ -175,6 +177,10 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
tag, err := conn.PgConn().CopyFrom(ctx, reader, "COPY eventstore.events2 FROM STDIN")
eventCount = tag.RowsAffected()
if err != nil {
pgErr := new(pgconn.PgError)
errors.As(err, &pgErr)
logging.WithError(err).WithField("pg_err_details", pgErr.Detail).Error("unable to copy events into destination")
return zerrors.ThrowUnknown(err, "MIGRA-DTHi7", "unable to copy events into destination")
}
@@ -187,7 +193,7 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
logging.WithFields("took", time.Since(start), "count", eventCount).Info("events migrated")
}
func writeCopyEventsDone(ctx context.Context, es *eventstore.EventStore, id, source string, position float64, errs <-chan error) {
func writeCopyEventsDone(ctx context.Context, es *eventstore.EventStore, id, source string, position decimal.Decimal, errs <-chan error) {
joinedErrs := make([]error, 0, len(errs))
for err := range errs {
joinedErrs = append(joinedErrs, err)