TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-14 06:08:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(eventstore): use decimal for position (#9881)

Browse Source 
Float64 which was used for the event.Position field is [not precise in
go and gets rounded](https://github.com/golang/go/issues/47300). This
can lead to unprecies position tracking of events and therefore
projections especially on cockcoachdb as the position used there is a
big number.

example of a unprecies position:
exact: 1725257931223002628
float64: 1725257931223002624.000000

The float64 was replaced by
[github.com/jackc/pgx-shopspring-decimal](https://github.com/jackc/pgx-shopspring-decimal).

Rename `latestSequence`-queries to `latestPosition`

closes https://github.com/zitadel/zitadel/issues/8863
This commit is contained in:
Silvan
2025-05-14 12:14:08 +02:00
committed by adlerhurst
parent ed7eee8a77
commit e14639c0ad
49 changed files with 366 additions and 236 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
internal/eventstore/handler/v2/field_handler.go
View File

@@ -8,6 +8,7 @@ import (
"time"
"github.com/jackc/pgx/v5/pgconn"
"github.com/shopspring/decimal"
"github.com/zitadel/zitadel/internal/eventstore"
)
@@ -126,10 +127,15 @@ func (h *FieldHandler) processEvents(ctx context.Context, config *triggerConfig)
return additionalIteration, err
}
// stop execution if currentState.eventTimestamp >= config.maxCreatedAt
if config.maxPosition != 0 && currentState.position >= config.maxPosition {
if !config.maxPosition.IsZero() && currentState.position.GreaterThanOrEqual(config.maxPosition) {
return false, nil
}
if config.minPosition.GreaterThan(decimal.NewFromInt(0)) {
currentState.position = config.minPosition
currentState.offset = 0
}
events, additionalIteration, err := h.fetchEvents(ctx, tx, currentState)
if err != nil {
return additionalIteration, err
@@ -159,7 +165,7 @@ func (h *FieldHandler) fetchEvents(ctx context.Context, tx *sql.Tx, currentState
idx, offset := skipPreviouslyReducedEvents(events, currentState)
if currentState.position == events[len(events)-1].Position() {
if currentState.position.Equal(events[len(events)-1].Position()) {
offset += currentState.offset
}
currentState.position = events[len(events)-1].Position()
@@ -189,9 +195,9 @@ func (h *FieldHandler) fetchEvents(ctx context.Context, tx *sql.Tx, currentState
}
func skipPreviouslyReducedEvents(events []eventstore.Event, currentState *state) (index int, offset uint32) {
var position float64
var position decimal.Decimal
for i, event := range events {
if event.Position() != position {
if !event.Position().Equal(position) {
offset = 0
position = event.Position()
}

27
internal/eventstore/handler/v2/handler.go
View File

@@ -4,13 +4,13 @@ import (
"context"
"database/sql"
"errors"
"math"
"math/rand"
"slices"
"sync"
"time"
"github.com/jackc/pgx/v5/pgconn"
"github.com/shopspring/decimal"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/api/authz"
@@ -381,7 +381,8 @@ func (h *Handler) existingInstances(ctx context.Context) ([]string, error) {
type triggerConfig struct {
awaitRunning bool
maxPosition float64
maxPosition decimal.Decimal
minPosition decimal.Decimal
}
type TriggerOpt func(conf *triggerConfig)
@@ -392,12 +393,18 @@ func WithAwaitRunning() TriggerOpt {
}
}
func WithMaxPosition(position float64) TriggerOpt {
func WithMaxPosition(position decimal.Decimal) TriggerOpt {
return func(conf *triggerConfig) {
conf.maxPosition = position
}
}
func WithMinPosition(position decimal.Decimal) TriggerOpt {
return func(conf *triggerConfig) {
conf.minPosition = position
}
}
func (h *Handler) Trigger(ctx context.Context, opts ...TriggerOpt) (_ context.Context, err error) {
config := new(triggerConfig)
for _, opt := range opts {
@@ -506,10 +513,15 @@ func (h *Handler) processEvents(ctx context.Context, config *triggerConfig) (add
return additionalIteration, err
}
// stop execution if currentState.position >= config.maxPosition
if config.maxPosition != 0 && currentState.position >= config.maxPosition {
if !config.maxPosition.Equal(decimal.Decimal{}) && currentState.position.GreaterThanOrEqual(config.maxPosition) {
return false, nil
}
if config.minPosition.GreaterThan(decimal.NewFromInt(0)) {
currentState.position = config.minPosition
currentState.offset = 0
}
var statements []*Statement
statements, additionalIteration, err = h.generateStatements(ctx, tx, currentState)
if err != nil {
@@ -601,7 +613,7 @@ func (h *Handler) generateStatements(ctx context.Context, tx *sql.Tx, currentSta
func skipPreviouslyReducedStatements(statements []*Statement, currentState *state) int {
for i, statement := range statements {
if statement.Position == currentState.position &&
if statement.Position.Equal(currentState.position) &&
statement.Aggregate.ID == currentState.aggregateID &&
statement.Aggregate.Type == currentState.aggregateType &&
statement.Sequence == currentState.sequence {
@@ -665,9 +677,8 @@ func (h *Handler) eventQuery(currentState *state) *eventstore.SearchQueryBuilder
OrderAsc().
InstanceID(currentState.instanceID)
if currentState.position > 0 {
// decrease position by 10 because builder.PositionAfter filters for position > and we need position >=
builder = builder.PositionAfter(math.Float64frombits(math.Float64bits(currentState.position) - 10))
if currentState.position.GreaterThan(decimal.Decimal{}) {
builder = builder.PositionAtLeast(currentState.position)
if currentState.offset > 0 {
builder = builder.Offset(currentState.offset)
}

8
internal/eventstore/handler/v2/state.go
View File

@@ -7,6 +7,8 @@ import (
"errors"
"time"
"github.com/shopspring/decimal"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/zerrors"
@@ -14,7 +16,7 @@ import (
type state struct {
instanceID string
position float64
position decimal.Decimal
eventTimestamp time.Time
aggregateType eventstore.AggregateType
aggregateID string
@@ -45,7 +47,7 @@ func (h *Handler) currentState(ctx context.Context, tx *sql.Tx, config *triggerC
aggregateType = new(sql.NullString)
sequence = new(sql.NullInt64)
timestamp = new(sql.NullTime)
position = new(sql.NullFloat64)
position = new(decimal.NullDecimal)
offset = new(sql.NullInt64)
)
@@ -75,7 +77,7 @@ func (h *Handler) currentState(ctx context.Context, tx *sql.Tx, config *triggerC
currentState.aggregateType = eventstore.AggregateType(aggregateType.String)
currentState.sequence = uint64(sequence.Int64)
currentState.eventTimestamp = timestamp.Time
currentState.position = position.Float64
currentState.position = position.Decimal
// psql does not provide unsigned numbers so we work around it
currentState.offset = uint32(offset.Int64)
return currentState, nil

13
internal/eventstore/handler/v2/state_test.go
View File

@@ -11,6 +11,7 @@ import (
"time"
"github.com/jackc/pgx/v5/pgconn"
"github.com/shopspring/decimal"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/database/mock"
@@ -166,7 +167,7 @@ func TestHandler_updateLastUpdated(t *testing.T) {
updatedState: &state{
instanceID: "instance",
eventTimestamp: time.Now(),
position: 42,
position: decimal.NewFromInt(42),
},
},
isErr: func(t *testing.T, err error) {
@@ -192,7 +193,7 @@ func TestHandler_updateLastUpdated(t *testing.T) {
updatedState: &state{
instanceID: "instance",
eventTimestamp: time.Now(),
position: 42,
position: decimal.NewFromInt(42),
},
},
isErr: func(t *testing.T, err error) {
@@ -217,7 +218,7 @@ func TestHandler_updateLastUpdated(t *testing.T) {
eventstore.AggregateType("aggregate type"),
uint64(42),
mock.AnyType[time.Time]{},
float64(42),
decimal.NewFromInt(42),
uint32(0),
),
mock.WithExecRowsAffected(1),
@@ -228,7 +229,7 @@ func TestHandler_updateLastUpdated(t *testing.T) {
updatedState: &state{
instanceID: "instance",
eventTimestamp: time.Now(),
position: 42,
position: decimal.NewFromInt(42),
aggregateType: "aggregate type",
aggregateID: "aggregate id",
sequence: 42,
@@ -397,7 +398,7 @@ func TestHandler_currentState(t *testing.T) {
"aggregate type",
int64(42),
testTime,
float64(42),
decimal.NewFromInt(42).String(),
uint16(10),
},
},
@@ -412,7 +413,7 @@ func TestHandler_currentState(t *testing.T) {
currentState: &state{
instanceID: "instance",
eventTimestamp: testTime,
position: 42,
position: decimal.NewFromInt(42),
aggregateType: "aggregate type",
aggregateID: "aggregate id",
sequence: 42,

3
internal/eventstore/handler/v2/statement.go
View File

@@ -9,6 +9,7 @@ import (
"strings"
"time"
"github.com/shopspring/decimal"
"github.com/zitadel/logging"
"golang.org/x/exp/constraints"
@@ -82,7 +83,7 @@ func (h *Handler) reduce(event eventstore.Event) (*Statement, error) {
type Statement struct {
Aggregate *eventstore.Aggregate
Sequence uint64
Position float64
Position decimal.Decimal
CreationDate time.Time
offset uint32