fix(eventstore): precise decimal (#8527)

# Which Problems Are Solved Float64 which was used for the event.Position field is [not precise in go and gets rounded](https://github.com/golang/go/issues/47300). This can lead to unprecies position tracking of events and therefore projections especially on cockcoachdb as the position used there is a big number. example of a unprecies position: exact: 1725257931223002628 float64: 1725257931223002624.000000 # How the Problems Are Solved The float64 was replaced by [github.com/jackc/pgx-shopspring-decimal](https://github.com/jackc/pgx-shopspring-decimal). # Additional Changes Correct behaviour of makefile for load tests. Rename `latestSequence`-queries to `latestPosition`
2025-08-11 18:07:31 +00:00 · 2024-09-06 11:19:19 +02:00
parent 2981ff04da
commit b522588d98
47 changed files with 319 additions and 215 deletions
--- a/internal/v2/database/number_filter.go
+++ b/internal/v2/database/number_filter.go
@@ -3,6 +3,7 @@ package database
 import (
 	"time"

+	"github.com/shopspring/decimal"
 	"github.com/zitadel/logging"
 	"golang.org/x/exp/constraints"
 )
@@ -94,7 +95,7 @@ func (c numberCompare) String() string {
 }

 type number interface {
-	constraints.Integer | constraints.Float | time.Time
+	constraints.Integer | constraints.Float | time.Time | decimal.Decimal
 	// TODO: condition must know if it's args are named parameters or not
 	// constraints.Integer | constraints.Float | time.Time | placeholder
 }
--- a/internal/v2/eventstore/event_store.go
+++ b/internal/v2/eventstore/event_store.go
@@ -2,6 +2,8 @@ package eventstore

 import (
 	"context"
+
+	"github.com/shopspring/decimal"
 )

 func NewEventstore(querier Querier, pusher Pusher) *EventStore {
@@ -30,12 +32,12 @@ type healthier interface {
 }

 type GlobalPosition struct {
-	Position        float64
+	Position        decimal.Decimal
 	InPositionOrder uint32
 }

 func (gp GlobalPosition) IsLess(other GlobalPosition) bool {
-	return gp.Position < other.Position || (gp.Position == other.Position && gp.InPositionOrder < other.InPositionOrder)
+	return gp.Position.LessThan(other.Position) || (gp.Position.Equal(other.Position) && gp.InPositionOrder < other.InPositionOrder)
 }

 type Reducer interface {
--- a/internal/v2/eventstore/postgres/push_test.go
+++ b/internal/v2/eventstore/postgres/push_test.go
@@ -8,6 +8,8 @@ import (
 	"testing"
 	"time"

+	"github.com/shopspring/decimal"
+
 	"github.com/zitadel/zitadel/internal/v2/database/mock"
 	"github.com/zitadel/zitadel/internal/v2/eventstore"
 	"github.com/zitadel/zitadel/internal/zerrors"
@@ -818,7 +820,7 @@ func Test_push(t *testing.T) {
 							[][]driver.Value{
 								{
 									time.Now(),
-									float64(123),
+									decimal.NewFromFloat(123).String(),
 								},
 							},
 						),
@@ -899,11 +901,11 @@ func Test_push(t *testing.T) {
 							[][]driver.Value{
 								{
 									time.Now(),
-									float64(123),
+									decimal.NewFromFloat(123).String(),
 								},
 								{
 									time.Now(),
-									float64(123.1),
+									decimal.NewFromFloat(123.1).String(),
 								},
 							},
 						),
@@ -984,11 +986,11 @@ func Test_push(t *testing.T) {
 							[][]driver.Value{
 								{
 									time.Now(),
-									float64(123),
+									decimal.NewFromFloat(123).String(),
 								},
 								{
 									time.Now(),
-									float64(123.1),
+									decimal.NewFromFloat(123.1).String(),
 								},
 							},
 						),
@@ -1044,7 +1046,7 @@ func Test_push(t *testing.T) {
 							[][]driver.Value{
 								{
 									time.Now(),
-									float64(123),
+									decimal.NewFromFloat(123).String(),
 								},
 							},
 						),
@@ -1099,7 +1101,7 @@ func Test_push(t *testing.T) {
 							[][]driver.Value{
 								{
 									time.Now(),
-									float64(123),
+									decimal.NewFromFloat(123).String(),
 								},
 							},
 						),
@@ -1181,11 +1183,11 @@ func Test_push(t *testing.T) {
 							[][]driver.Value{
 								{
 									time.Now(),
-									float64(123),
+									decimal.NewFromFloat(123).String(),
 								},
 								{
 									time.Now(),
-									float64(123.1),
+									decimal.NewFromFloat(123.1).String(),
 								},
 							},
 						),
@@ -1272,11 +1274,11 @@ func Test_push(t *testing.T) {
 							[][]driver.Value{
 								{
 									time.Now(),
-									float64(123),
+									decimal.NewFromFloat(123).String(),
 								},
 								{
 									time.Now(),
-									float64(123.1),
+									decimal.NewFromFloat(123.1).String(),
 								},
 							},
 						),
--- a/internal/v2/eventstore/postgres/query_test.go
+++ b/internal/v2/eventstore/postgres/query_test.go
@@ -8,6 +8,8 @@ import (
 	"testing"
 	"time"

+	"github.com/shopspring/decimal"
+
 	"github.com/zitadel/zitadel/internal/v2/database"
 	"github.com/zitadel/zitadel/internal/v2/database/mock"
 	"github.com/zitadel/zitadel/internal/v2/eventstore"
@@ -541,13 +543,13 @@ func Test_writeFilter(t *testing.T) {
 			args: args{
 				filter: eventstore.NewFilter(
 					eventstore.FilterPagination(
-						eventstore.PositionGreater(123.4, 0),
+						eventstore.PositionGreater(decimal.NewFromFloat(123.4), 0),
 					),
 				),
 			},
 			want: wantQuery{
 				query: " WHERE instance_id = $1 AND position > $2 ORDER BY position, in_tx_order",
-				args:  []any{"i1", 123.4},
+				args:  []any{"i1", decimal.NewFromFloat(123.4)},
 			},
 		},
 		{
@@ -555,18 +557,18 @@ func Test_writeFilter(t *testing.T) {
 			args: args{
 				filter: eventstore.NewFilter(
 					eventstore.FilterPagination(
-						// 	eventstore.PositionGreater(123.4, 0),
+						// 	eventstore.PositionGreater(decimal.NewFromFloat(123.4), 0),
 						// 	eventstore.PositionLess(125.4, 10),
 						eventstore.PositionBetween(
-							&eventstore.GlobalPosition{Position: 123.4},
-							&eventstore.GlobalPosition{Position: 125.4, InPositionOrder: 10},
+							&eventstore.GlobalPosition{Position: decimal.NewFromFloat(123.4)},
+							&eventstore.GlobalPosition{Position: decimal.NewFromFloat(125.4), InPositionOrder: 10},
 						),
 					),
 				),
 			},
 			want: wantQuery{
 				query: " WHERE instance_id = $1 AND ((position = $2 AND in_tx_order < $3) OR position < $4) AND position > $5 ORDER BY position, in_tx_order",
-				args:  []any{"i1", 125.4, uint32(10), 125.4, 123.4},
+				args:  []any{"i1", decimal.NewFromFloat(125.4), uint32(10), decimal.NewFromFloat(125.4), decimal.NewFromFloat(123.4)},
 				// TODO: (adlerhurst) would require some refactoring to reuse existing args
 				// query: " WHERE instance_id = $1 AND position > $2 AND ((position = $3 AND in_tx_order < $4) OR position < $3) ORDER BY position, in_tx_order",
 				// args:  []any{"i1", 123.4, 125.4, uint32(10)},
@@ -577,13 +579,13 @@ func Test_writeFilter(t *testing.T) {
 			args: args{
 				filter: eventstore.NewFilter(
 					eventstore.FilterPagination(
-						eventstore.PositionGreater(123.4, 12),
+						eventstore.PositionGreater(decimal.NewFromFloat(123.4), 12),
 					),
 				),
 			},
 			want: wantQuery{
 				query: " WHERE instance_id = $1 AND ((position = $2 AND in_tx_order > $3) OR position > $4) ORDER BY position, in_tx_order",
-				args:  []any{"i1", 123.4, uint32(12), 123.4},
+				args:  []any{"i1", decimal.NewFromFloat(123.4), uint32(12), decimal.NewFromFloat(123.4)},
 			},
 		},
 		{
@@ -593,13 +595,13 @@ func Test_writeFilter(t *testing.T) {
 					eventstore.FilterPagination(
 						eventstore.Limit(10),
 						eventstore.Offset(3),
-						eventstore.PositionGreater(123.4, 12),
+						eventstore.PositionGreater(decimal.NewFromFloat(123.4), 12),
 					),
 				),
 			},
 			want: wantQuery{
 				query: " WHERE instance_id = $1 AND ((position = $2 AND in_tx_order > $3) OR position > $4) ORDER BY position, in_tx_order LIMIT $5 OFFSET $6",
-				args:  []any{"i1", 123.4, uint32(12), 123.4, uint32(10), uint32(3)},
+				args:  []any{"i1", decimal.NewFromFloat(123.4), uint32(12), decimal.NewFromFloat(123.4), uint32(10), uint32(3)},
 			},
 		},
 		{
@@ -609,14 +611,14 @@ func Test_writeFilter(t *testing.T) {
 					eventstore.FilterPagination(
 						eventstore.Limit(10),
 						eventstore.Offset(3),
-						eventstore.PositionGreater(123.4, 12),
+						eventstore.PositionGreater(decimal.NewFromFloat(123.4), 12),
 					),
 					eventstore.AppendAggregateFilter("user"),
 				),
 			},
 			want: wantQuery{
 				query: " WHERE instance_id = $1 AND aggregate_type = $2 AND ((position = $3 AND in_tx_order > $4) OR position > $5) ORDER BY position, in_tx_order LIMIT $6 OFFSET $7",
-				args:  []any{"i1", "user", 123.4, uint32(12), 123.4, uint32(10), uint32(3)},
+				args:  []any{"i1", "user", decimal.NewFromFloat(123.4), uint32(12), decimal.NewFromFloat(123.4), uint32(10), uint32(3)},
 			},
 		},
 		{
@@ -626,7 +628,7 @@ func Test_writeFilter(t *testing.T) {
 					eventstore.FilterPagination(
 						eventstore.Limit(10),
 						eventstore.Offset(3),
-						eventstore.PositionGreater(123.4, 12),
+						eventstore.PositionGreater(decimal.NewFromFloat(123.4), 12),
 					),
 					eventstore.AppendAggregateFilter("user"),
 					eventstore.AppendAggregateFilter(
@@ -637,7 +639,7 @@ func Test_writeFilter(t *testing.T) {
 			},
 			want: wantQuery{
 				query: " WHERE instance_id = $1 AND (aggregate_type = $2 OR (aggregate_type = $3 AND aggregate_id = $4)) AND ((position = $5 AND in_tx_order > $6) OR position > $7) ORDER BY position, in_tx_order LIMIT $8 OFFSET $9",
-				args:  []any{"i1", "user", "org", "o1", 123.4, uint32(12), 123.4, uint32(10), uint32(3)},
+				args:  []any{"i1", "user", "org", "o1", decimal.NewFromFloat(123.4), uint32(12), decimal.NewFromFloat(123.4), uint32(10), uint32(3)},
 			},
 		},
 	}
@@ -956,7 +958,7 @@ func Test_writeQueryUse_examples(t *testing.T) {
 							),
 							eventstore.FilterPagination(
 								// used because we need to check for first login and an app which is not console
-								eventstore.PositionGreater(12, 4),
+								eventstore.PositionGreater(decimal.NewFromInt(12), 4),
 							),
 						),
 						eventstore.NewFilter(
@@ -1065,9 +1067,9 @@ func Test_writeQueryUse_examples(t *testing.T) {
 					"instance",
 					"user",
 					"user.token.added",
-					float64(12),
+					decimal.NewFromInt(12),
 					uint32(4),
-					float64(12),
+					decimal.NewFromInt(12),
 					"instance",
 					"instance",
 					[]string{"instance.idp.config.added", "instance.idp.oauth.added", "instance.idp.oidc.added", "instance.idp.jwt.added", "instance.idp.azure.added", "instance.idp.github.added", "instance.idp.github.enterprise.added", "instance.idp.gitlab.added", "instance.idp.gitlab.selfhosted.added", "instance.idp.google.added", "instance.idp.ldap.added", "instance.idp.config.apple.added", "instance.idp.saml.added"},
@@ -1201,7 +1203,7 @@ func Test_executeQuery(t *testing.T) {
 						time.Now(),
 						"event.type",
 						uint32(23),
-						float64(123),
+						decimal.NewFromInt(123).String(),
 						uint32(0),
 						nil,
 						"gigi",
@@ -1235,7 +1237,7 @@ func Test_executeQuery(t *testing.T) {
 						time.Now(),
 						"event.type",
 						uint32(23),
-						float64(123),
+						decimal.NewFromInt(123).String(),
 						uint32(0),
 						[]byte(`{"name": "gigi"}`),
 						"gigi",
@@ -1269,7 +1271,7 @@ func Test_executeQuery(t *testing.T) {
 						time.Now(),
 						"event.type",
 						uint32(23),
-						float64(123),
+						decimal.NewFromInt(123).String(),
 						uint32(0),
 						nil,
 						"gigi",
@@ -1283,7 +1285,7 @@ func Test_executeQuery(t *testing.T) {
 						time.Now(),
 						"event.type",
 						uint32(24),
-						float64(124),
+						decimal.NewFromInt(124).String(),
 						uint32(0),
 						[]byte(`{"name": "gigi"}`),
 						"gigi",
@@ -1317,7 +1319,7 @@ func Test_executeQuery(t *testing.T) {
 						time.Now(),
 						"event.type",
 						uint32(23),
-						float64(123),
+						decimal.NewFromInt(123).String(),
 						uint32(0),
 						nil,
 						"gigi",
@@ -1331,7 +1333,7 @@ func Test_executeQuery(t *testing.T) {
 						time.Now(),
 						"event.type",
 						uint32(24),
-						float64(124),
+						decimal.NewFromInt(124).String(),
 						uint32(0),
 						[]byte(`{"name": "gigi"}`),
 						"gigi",
--- a/internal/v2/eventstore/query.go
+++ b/internal/v2/eventstore/query.go
@@ -7,6 +7,8 @@ import (
 	"slices"
 	"time"

+	"github.com/shopspring/decimal"
+
 	"github.com/zitadel/zitadel/internal/v2/database"
 )

@@ -723,7 +725,7 @@ func (pc *PositionCondition) Min() *GlobalPosition {
 // PositionGreater prepares the condition as follows
 // if inPositionOrder is set: position = AND in_tx_order > OR or position >
 // if inPositionOrder is NOT set: position >
-func PositionGreater(position float64, inPositionOrder uint32) paginationOpt {
+func PositionGreater(position decimal.Decimal, inPositionOrder uint32) paginationOpt {
 	return func(p *Pagination) {
 		p.ensurePosition()
 		p.position.min = &GlobalPosition{
@@ -743,7 +745,7 @@ func GlobalPositionGreater(position *GlobalPosition) paginationOpt {
 // PositionLess prepares the condition as follows
 // if inPositionOrder is set: position = AND in_tx_order > OR or position >
 // if inPositionOrder is NOT set: position >
-func PositionLess(position float64, inPositionOrder uint32) paginationOpt {
+func PositionLess(position decimal.Decimal, inPositionOrder uint32) paginationOpt {
 	return func(p *Pagination) {
 		p.ensurePosition()
 		p.position.max = &GlobalPosition{
--- a/internal/v2/eventstore/query_test.go
+++ b/internal/v2/eventstore/query_test.go
@@ -6,6 +6,8 @@ import (
 	"testing"
 	"time"

+	"github.com/shopspring/decimal"
+
 	"github.com/zitadel/zitadel/internal/v2/database"
 )

@@ -74,13 +76,13 @@ func TestPaginationOpt(t *testing.T) {
 			name: "global position greater",
 			args: args{
 				opts: []paginationOpt{
-					GlobalPositionGreater(&GlobalPosition{Position: 10}),
+					GlobalPositionGreater(&GlobalPosition{Position: decimal.NewFromInt(10)}),
 				},
 			},
 			want: &Pagination{
 				position: &PositionCondition{
 					min: &GlobalPosition{
-						Position:        10,
+						Position:        decimal.NewFromInt(10),
 						InPositionOrder: 0,
 					},
 				},
@@ -90,13 +92,13 @@ func TestPaginationOpt(t *testing.T) {
 			name: "position greater",
 			args: args{
 				opts: []paginationOpt{
-					PositionGreater(10, 0),
+					PositionGreater(decimal.NewFromInt(10), 0),
 				},
 			},
 			want: &Pagination{
 				position: &PositionCondition{
 					min: &GlobalPosition{
-						Position:        10,
+						Position:        decimal.NewFromInt(10),
 						InPositionOrder: 0,
 					},
 				},
@@ -107,13 +109,13 @@ func TestPaginationOpt(t *testing.T) {
 			name: "position less",
 			args: args{
 				opts: []paginationOpt{
-					PositionLess(10, 12),
+					PositionLess(decimal.NewFromInt(10), 12),
 				},
 			},
 			want: &Pagination{
 				position: &PositionCondition{
 					max: &GlobalPosition{
-						Position:        10,
+						Position:        decimal.NewFromInt(10),
 						InPositionOrder: 12,
 					},
 				},
@@ -123,13 +125,13 @@ func TestPaginationOpt(t *testing.T) {
 			name: "global position less",
 			args: args{
 				opts: []paginationOpt{
-					GlobalPositionLess(&GlobalPosition{Position: 12, InPositionOrder: 24}),
+					GlobalPositionLess(&GlobalPosition{Position: decimal.NewFromInt(12), InPositionOrder: 24}),
 				},
 			},
 			want: &Pagination{
 				position: &PositionCondition{
 					max: &GlobalPosition{
-						Position:        12,
+						Position:        decimal.NewFromInt(12),
 						InPositionOrder: 24,
 					},
 				},
@@ -140,19 +142,19 @@ func TestPaginationOpt(t *testing.T) {
 			args: args{
 				opts: []paginationOpt{
 					PositionBetween(
-						&GlobalPosition{10, 12},
-						&GlobalPosition{20, 0},
+						&GlobalPosition{decimal.NewFromInt(10), 12},
+						&GlobalPosition{decimal.NewFromInt(20), 0},
 					),
 				},
 			},
 			want: &Pagination{
 				position: &PositionCondition{
 					min: &GlobalPosition{
-						Position:        10,
+						Position:        decimal.NewFromInt(10),
 						InPositionOrder: 12,
 					},
 					max: &GlobalPosition{
-						Position:        20,
+						Position:        decimal.NewFromInt(20),
 						InPositionOrder: 0,
 					},
 				},
--- a/internal/v2/readmodel/last_successful_mirror.go
+++ b/internal/v2/readmodel/last_successful_mirror.go
@@ -1,6 +1,8 @@
 package readmodel

 import (
+	"github.com/shopspring/decimal"
+
 	"github.com/zitadel/zitadel/internal/v2/eventstore"
 	"github.com/zitadel/zitadel/internal/v2/system"
 	"github.com/zitadel/zitadel/internal/v2/system/mirror"
@@ -8,7 +10,7 @@ import (

 type LastSuccessfulMirror struct {
 	ID       string
-	Position float64
+	Position decimal.Decimal
 	source   string
 }

@@ -53,7 +55,7 @@ func (h *LastSuccessfulMirror) Reduce(events ...*eventstore.StorageEvent) (err e

 func (h *LastSuccessfulMirror) reduceSucceeded(event *eventstore.StorageEvent) error {
 	// if position is set we skip all older events
-	if h.Position > 0 {
+	if h.Position.GreaterThan(decimal.NewFromInt(0)) {
 		return nil

 	}
--- a/internal/v2/system/mirror/succeeded.go
+++ b/internal/v2/system/mirror/succeeded.go
@@ -1,6 +1,8 @@
 package mirror

 import (
+	"github.com/shopspring/decimal"
+
 	"github.com/zitadel/zitadel/internal/v2/eventstore"
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
@@ -9,7 +11,7 @@ type succeededPayload struct {
 	// Source is the name of the database data are mirrored from
 	Source string `json:"source"`
 	// Position until data will be mirrored
-	Position float64 `json:"position"`
+	Position decimal.Decimal `json:"position"`
 }

 const SucceededType = eventTypePrefix + "succeeded"
@@ -38,7 +40,7 @@ func SucceededEventFromStorage(event *eventstore.StorageEvent) (e *SucceededEven
 	}, nil
 }

-func NewSucceededCommand(source string, position float64) *eventstore.Command {
+func NewSucceededCommand(source string, position decimal.Decimal) *eventstore.Command {
 	return &eventstore.Command{
 		Action: eventstore.Action[any]{
 			Creator:  Creator,