fix(eventstore): use decimal, correct mirror (#9914)

# Eventstore fixes - `event.Position` used float64 before which can lead to [precision loss](https://github.com/golang/go/issues/47300). The type got replaced by [a type without precision loss](https://github.com/jackc/pgx-shopspring-decimal) - the handler reported the wrong error if the current state was updated and therefore took longer to retry failed events. # Mirror fixes - max age of auth requests can be configured to speed up copying data from `auth.auth_requests` table. Auth requests last updated before the set age will be ignored. Default is 1 month - notification projections are skipped because notifications should be sent by the source system. The projections are set to the latest position - ensure that mirror can be executed multiple times --------- Co-authored-by: Livio Spring <livio.a@gmail.com>
2025-08-11 19:17:32 +00:00 · 2025-05-28 23:54:18 +02:00
parent 046b165db8
commit 131f70db34
51 changed files with 436 additions and 236 deletions
--- a/internal/v2/eventstore/event_store.go
+++ b/internal/v2/eventstore/event_store.go
@@ -2,6 +2,8 @@ package eventstore

 import (
 	"context"
+
+	"github.com/shopspring/decimal"
 )

 func NewEventstore(querier Querier, pusher Pusher) *EventStore {
@@ -30,12 +32,12 @@ type healthier interface {
 }

 type GlobalPosition struct {
-	Position        float64
+	Position        decimal.Decimal
 	InPositionOrder uint32
 }

 func (gp GlobalPosition) IsLess(other GlobalPosition) bool {
-	return gp.Position < other.Position || (gp.Position == other.Position && gp.InPositionOrder < other.InPositionOrder)
+	return gp.Position.LessThan(other.Position) || (gp.Position.Equal(other.Position) && gp.InPositionOrder < other.InPositionOrder)
 }

 type Reducer interface {
--- a/internal/v2/eventstore/postgres/push_test.go
+++ b/internal/v2/eventstore/postgres/push_test.go
@@ -8,6 +8,8 @@ import (
 	"testing"
 	"time"

+	"github.com/shopspring/decimal"
+
 	"github.com/zitadel/zitadel/internal/v2/database/mock"
 	"github.com/zitadel/zitadel/internal/v2/eventstore"
 	"github.com/zitadel/zitadel/internal/zerrors"
@@ -818,7 +820,7 @@ func Test_push(t *testing.T) {
 							[][]driver.Value{
 								{
 									time.Now(),
-									float64(123),
+									decimal.NewFromFloat(123).String(),
 								},
 							},
 						),
@@ -899,11 +901,11 @@ func Test_push(t *testing.T) {
 							[][]driver.Value{
 								{
 									time.Now(),
-									float64(123),
+									decimal.NewFromFloat(123).String(),
 								},
 								{
 									time.Now(),
-									float64(123.1),
+									decimal.NewFromFloat(123.1).String(),
 								},
 							},
 						),
@@ -984,11 +986,11 @@ func Test_push(t *testing.T) {
 							[][]driver.Value{
 								{
 									time.Now(),
-									float64(123),
+									decimal.NewFromFloat(123).String(),
 								},
 								{
 									time.Now(),
-									float64(123.1),
+									decimal.NewFromFloat(123.1).String(),
 								},
 							},
 						),
@@ -1044,7 +1046,7 @@ func Test_push(t *testing.T) {
 							[][]driver.Value{
 								{
 									time.Now(),
-									float64(123),
+									decimal.NewFromFloat(123).String(),
 								},
 							},
 						),
@@ -1099,7 +1101,7 @@ func Test_push(t *testing.T) {
 							[][]driver.Value{
 								{
 									time.Now(),
-									float64(123),
+									decimal.NewFromFloat(123).String(),
 								},
 							},
 						),
@@ -1181,11 +1183,11 @@ func Test_push(t *testing.T) {
 							[][]driver.Value{
 								{
 									time.Now(),
-									float64(123),
+									decimal.NewFromFloat(123).String(),
 								},
 								{
 									time.Now(),
-									float64(123.1),
+									decimal.NewFromFloat(123.1).String(),
 								},
 							},
 						),
@@ -1272,11 +1274,11 @@ func Test_push(t *testing.T) {
 							[][]driver.Value{
 								{
 									time.Now(),
-									float64(123),
+									decimal.NewFromFloat(123).String(),
 								},
 								{
 									time.Now(),
-									float64(123.1),
+									decimal.NewFromFloat(123.1).String(),
 								},
 							},
 						),
--- a/internal/v2/eventstore/postgres/query_test.go
+++ b/internal/v2/eventstore/postgres/query_test.go
@@ -8,6 +8,8 @@ import (
 	"testing"
 	"time"

+	"github.com/shopspring/decimal"
+
 	"github.com/zitadel/zitadel/internal/v2/database"
 	"github.com/zitadel/zitadel/internal/v2/database/mock"
 	"github.com/zitadel/zitadel/internal/v2/eventstore"
@@ -541,13 +543,13 @@ func Test_writeFilter(t *testing.T) {
 			args: args{
 				filter: eventstore.NewFilter(
 					eventstore.FilterPagination(
-						eventstore.PositionGreater(123.4, 0),
+						eventstore.PositionGreater(decimal.NewFromFloat(123.4), 0),
 					),
 				),
 			},
 			want: wantQuery{
 				query: " WHERE instance_id = $1 AND position > $2 ORDER BY position, in_tx_order",
-				args:  []any{"i1", 123.4},
+				args:  []any{"i1", decimal.NewFromFloat(123.4)},
 			},
 		},
 		{
@@ -555,18 +557,18 @@ func Test_writeFilter(t *testing.T) {
 			args: args{
 				filter: eventstore.NewFilter(
 					eventstore.FilterPagination(
-						// 	eventstore.PositionGreater(123.4, 0),
+						// 	eventstore.PositionGreater(decimal.NewFromFloat(123.4), 0),
 						// 	eventstore.PositionLess(125.4, 10),
 						eventstore.PositionBetween(
-							&eventstore.GlobalPosition{Position: 123.4},
-							&eventstore.GlobalPosition{Position: 125.4, InPositionOrder: 10},
+							&eventstore.GlobalPosition{Position: decimal.NewFromFloat(123.4)},
+							&eventstore.GlobalPosition{Position: decimal.NewFromFloat(125.4), InPositionOrder: 10},
 						),
 					),
 				),
 			},
 			want: wantQuery{
 				query: " WHERE instance_id = $1 AND ((position = $2 AND in_tx_order < $3) OR position < $4) AND position > $5 ORDER BY position, in_tx_order",
-				args:  []any{"i1", 125.4, uint32(10), 125.4, 123.4},
+				args:  []any{"i1", decimal.NewFromFloat(125.4), uint32(10), decimal.NewFromFloat(125.4), decimal.NewFromFloat(123.4)},
 				// TODO: (adlerhurst) would require some refactoring to reuse existing args
 				// query: " WHERE instance_id = $1 AND position > $2 AND ((position = $3 AND in_tx_order < $4) OR position < $3) ORDER BY position, in_tx_order",
 				// args:  []any{"i1", 123.4, 125.4, uint32(10)},
@@ -577,13 +579,13 @@ func Test_writeFilter(t *testing.T) {
 			args: args{
 				filter: eventstore.NewFilter(
 					eventstore.FilterPagination(
-						eventstore.PositionGreater(123.4, 12),
+						eventstore.PositionGreater(decimal.NewFromFloat(123.4), 12),
 					),
 				),
 			},
 			want: wantQuery{
 				query: " WHERE instance_id = $1 AND ((position = $2 AND in_tx_order > $3) OR position > $4) ORDER BY position, in_tx_order",
-				args:  []any{"i1", 123.4, uint32(12), 123.4},
+				args:  []any{"i1", decimal.NewFromFloat(123.4), uint32(12), decimal.NewFromFloat(123.4)},
 			},
 		},
 		{
@@ -593,13 +595,13 @@ func Test_writeFilter(t *testing.T) {
 					eventstore.FilterPagination(
 						eventstore.Limit(10),
 						eventstore.Offset(3),
-						eventstore.PositionGreater(123.4, 12),
+						eventstore.PositionGreater(decimal.NewFromFloat(123.4), 12),
 					),
 				),
 			},
 			want: wantQuery{
 				query: " WHERE instance_id = $1 AND ((position = $2 AND in_tx_order > $3) OR position > $4) ORDER BY position, in_tx_order LIMIT $5 OFFSET $6",
-				args:  []any{"i1", 123.4, uint32(12), 123.4, uint32(10), uint32(3)},
+				args:  []any{"i1", decimal.NewFromFloat(123.4), uint32(12), decimal.NewFromFloat(123.4), uint32(10), uint32(3)},
 			},
 		},
 		{
@@ -609,14 +611,14 @@ func Test_writeFilter(t *testing.T) {
 					eventstore.FilterPagination(
 						eventstore.Limit(10),
 						eventstore.Offset(3),
-						eventstore.PositionGreater(123.4, 12),
+						eventstore.PositionGreater(decimal.NewFromFloat(123.4), 12),
 					),
 					eventstore.AppendAggregateFilter("user"),
 				),
 			},
 			want: wantQuery{
 				query: " WHERE instance_id = $1 AND aggregate_type = $2 AND ((position = $3 AND in_tx_order > $4) OR position > $5) ORDER BY position, in_tx_order LIMIT $6 OFFSET $7",
-				args:  []any{"i1", "user", 123.4, uint32(12), 123.4, uint32(10), uint32(3)},
+				args:  []any{"i1", "user", decimal.NewFromFloat(123.4), uint32(12), decimal.NewFromFloat(123.4), uint32(10), uint32(3)},
 			},
 		},
 		{
@@ -626,7 +628,7 @@ func Test_writeFilter(t *testing.T) {
 					eventstore.FilterPagination(
 						eventstore.Limit(10),
 						eventstore.Offset(3),
-						eventstore.PositionGreater(123.4, 12),
+						eventstore.PositionGreater(decimal.NewFromFloat(123.4), 12),
 					),
 					eventstore.AppendAggregateFilter("user"),
 					eventstore.AppendAggregateFilter(
@@ -637,7 +639,7 @@ func Test_writeFilter(t *testing.T) {
 			},
 			want: wantQuery{
 				query: " WHERE instance_id = $1 AND (aggregate_type = $2 OR (aggregate_type = $3 AND aggregate_id = $4)) AND ((position = $5 AND in_tx_order > $6) OR position > $7) ORDER BY position, in_tx_order LIMIT $8 OFFSET $9",
-				args:  []any{"i1", "user", "org", "o1", 123.4, uint32(12), 123.4, uint32(10), uint32(3)},
+				args:  []any{"i1", "user", "org", "o1", decimal.NewFromFloat(123.4), uint32(12), decimal.NewFromFloat(123.4), uint32(10), uint32(3)},
 			},
 		},
 	}
@@ -956,7 +958,7 @@ func Test_writeQueryUse_examples(t *testing.T) {
 							),
 							eventstore.FilterPagination(
 								// used because we need to check for first login and an app which is not console
-								eventstore.PositionGreater(12, 4),
+								eventstore.PositionGreater(decimal.NewFromInt(12), 4),
 							),
 						),
 						eventstore.NewFilter(
@@ -1065,9 +1067,9 @@ func Test_writeQueryUse_examples(t *testing.T) {
 					"instance",
 					"user",
 					"user.token.added",
-					float64(12),
+					decimal.NewFromInt(12),
 					uint32(4),
-					float64(12),
+					decimal.NewFromInt(12),
 					"instance",
 					"instance",
 					[]string{"instance.idp.config.added", "instance.idp.oauth.added", "instance.idp.oidc.added", "instance.idp.jwt.added", "instance.idp.azure.added", "instance.idp.github.added", "instance.idp.github.enterprise.added", "instance.idp.gitlab.added", "instance.idp.gitlab.selfhosted.added", "instance.idp.google.added", "instance.idp.ldap.added", "instance.idp.config.apple.added", "instance.idp.saml.added"},
@@ -1201,7 +1203,7 @@ func Test_executeQuery(t *testing.T) {
 						time.Now(),
 						"event.type",
 						uint32(23),
-						float64(123),
+						decimal.NewFromInt(123).String(),
 						uint32(0),
 						nil,
 						"gigi",
@@ -1235,7 +1237,7 @@ func Test_executeQuery(t *testing.T) {
 						time.Now(),
 						"event.type",
 						uint32(23),
-						float64(123),
+						decimal.NewFromInt(123).String(),
 						uint32(0),
 						[]byte(`{"name": "gigi"}`),
 						"gigi",
@@ -1269,7 +1271,7 @@ func Test_executeQuery(t *testing.T) {
 						time.Now(),
 						"event.type",
 						uint32(23),
-						float64(123),
+						decimal.NewFromInt(123).String(),
 						uint32(0),
 						nil,
 						"gigi",
@@ -1283,7 +1285,7 @@ func Test_executeQuery(t *testing.T) {
 						time.Now(),
 						"event.type",
 						uint32(24),
-						float64(124),
+						decimal.NewFromInt(124).String(),
 						uint32(0),
 						[]byte(`{"name": "gigi"}`),
 						"gigi",
@@ -1317,7 +1319,7 @@ func Test_executeQuery(t *testing.T) {
 						time.Now(),
 						"event.type",
 						uint32(23),
-						float64(123),
+						decimal.NewFromInt(123).String(),
 						uint32(0),
 						nil,
 						"gigi",
@@ -1331,7 +1333,7 @@ func Test_executeQuery(t *testing.T) {
 						time.Now(),
 						"event.type",
 						uint32(24),
-						float64(124),
+						decimal.NewFromInt(124).String(),
 						uint32(0),
 						[]byte(`{"name": "gigi"}`),
 						"gigi",
--- a/internal/v2/eventstore/query.go
+++ b/internal/v2/eventstore/query.go
@@ -7,6 +7,8 @@ import (
 	"slices"
 	"time"

+	"github.com/shopspring/decimal"
+
 	"github.com/zitadel/zitadel/internal/v2/database"
 )

@@ -723,7 +725,7 @@ func (pc *PositionCondition) Min() *GlobalPosition {
 // PositionGreater prepares the condition as follows
 // if inPositionOrder is set: position = AND in_tx_order > OR or position >
 // if inPositionOrder is NOT set: position >
-func PositionGreater(position float64, inPositionOrder uint32) paginationOpt {
+func PositionGreater(position decimal.Decimal, inPositionOrder uint32) paginationOpt {
 	return func(p *Pagination) {
 		p.ensurePosition()
 		p.position.min = &GlobalPosition{
@@ -743,7 +745,7 @@ func GlobalPositionGreater(position *GlobalPosition) paginationOpt {
 // PositionLess prepares the condition as follows
 // if inPositionOrder is set: position = AND in_tx_order > OR or position >
 // if inPositionOrder is NOT set: position >
-func PositionLess(position float64, inPositionOrder uint32) paginationOpt {
+func PositionLess(position decimal.Decimal, inPositionOrder uint32) paginationOpt {
 	return func(p *Pagination) {
 		p.ensurePosition()
 		p.position.max = &GlobalPosition{
--- a/internal/v2/eventstore/query_test.go
+++ b/internal/v2/eventstore/query_test.go
@@ -6,6 +6,8 @@ import (
 	"testing"
 	"time"

+	"github.com/shopspring/decimal"
+
 	"github.com/zitadel/zitadel/internal/v2/database"
 )

@@ -74,13 +76,13 @@ func TestPaginationOpt(t *testing.T) {
 			name: "global position greater",
 			args: args{
 				opts: []paginationOpt{
-					GlobalPositionGreater(&GlobalPosition{Position: 10}),
+					GlobalPositionGreater(&GlobalPosition{Position: decimal.NewFromInt(10)}),
 				},
 			},
 			want: &Pagination{
 				position: &PositionCondition{
 					min: &GlobalPosition{
-						Position:        10,
+						Position:        decimal.NewFromInt(10),
 						InPositionOrder: 0,
 					},
 				},
@@ -90,13 +92,13 @@ func TestPaginationOpt(t *testing.T) {
 			name: "position greater",
 			args: args{
 				opts: []paginationOpt{
-					PositionGreater(10, 0),
+					PositionGreater(decimal.NewFromInt(10), 0),
 				},
 			},
 			want: &Pagination{
 				position: &PositionCondition{
 					min: &GlobalPosition{
-						Position:        10,
+						Position:        decimal.NewFromInt(10),
 						InPositionOrder: 0,
 					},
 				},
@@ -107,13 +109,13 @@ func TestPaginationOpt(t *testing.T) {
 			name: "position less",
 			args: args{
 				opts: []paginationOpt{
-					PositionLess(10, 12),
+					PositionLess(decimal.NewFromInt(10), 12),
 				},
 			},
 			want: &Pagination{
 				position: &PositionCondition{
 					max: &GlobalPosition{
-						Position:        10,
+						Position:        decimal.NewFromInt(10),
 						InPositionOrder: 12,
 					},
 				},
@@ -123,13 +125,13 @@ func TestPaginationOpt(t *testing.T) {
 			name: "global position less",
 			args: args{
 				opts: []paginationOpt{
-					GlobalPositionLess(&GlobalPosition{Position: 12, InPositionOrder: 24}),
+					GlobalPositionLess(&GlobalPosition{Position: decimal.NewFromInt(12), InPositionOrder: 24}),
 				},
 			},
 			want: &Pagination{
 				position: &PositionCondition{
 					max: &GlobalPosition{
-						Position:        12,
+						Position:        decimal.NewFromInt(12),
 						InPositionOrder: 24,
 					},
 				},
@@ -140,19 +142,19 @@ func TestPaginationOpt(t *testing.T) {
 			args: args{
 				opts: []paginationOpt{
 					PositionBetween(
-						&GlobalPosition{10, 12},
-						&GlobalPosition{20, 0},
+						&GlobalPosition{decimal.NewFromInt(10), 12},
+						&GlobalPosition{decimal.NewFromInt(20), 0},
 					),
 				},
 			},
 			want: &Pagination{
 				position: &PositionCondition{
 					min: &GlobalPosition{
-						Position:        10,
+						Position:        decimal.NewFromInt(10),
 						InPositionOrder: 12,
 					},
 					max: &GlobalPosition{
-						Position:        20,
+						Position:        decimal.NewFromInt(20),
 						InPositionOrder: 0,
 					},
 				},