TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-12 00:27:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(eventstore): use decimal, correct mirror (#9916)

Browse Source 
# Eventstore fixes

- `event.Position` used float64 before which can lead to [precision
loss](https://github.com/golang/go/issues/47300). The type got replaced
by [a type without precision
loss](https://github.com/jackc/pgx-shopspring-decimal)
- the handler reported the wrong error if the current state was updated
and therefore took longer to retry failed events.

# Mirror fixes

- max age of auth requests can be configured to speed up copying data
from `auth.auth_requests` table. Auth requests last updated before the
set age will be ignored. Default is 1 month
- notification projections are skipped because notifications should be
sent by the source system. The projections are set to the latest
position
- ensure that mirror can be executed multiple times
This commit is contained in:
Silvan
2025-05-27 17:13:17 +02:00
committed by GitHub
parent b979923928
commit 7c5480f94e
51 changed files with 436 additions and 236 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
internal/query/access_token.go
View File

@@ -5,6 +5,7 @@ import (
"strings"
"time"
"github.com/shopspring/decimal"
"golang.org/x/text/language"
"github.com/zitadel/zitadel/internal/domain"
@@ -140,7 +141,7 @@ func (q *Queries) accessTokenByOIDCSessionAndTokenID(ctx context.Context, oidcSe
// checkSessionNotTerminatedAfter checks if a [session.TerminateType] event (or user events leading to a session termination)
// occurred after a certain time and will return an error if so.
func (q *Queries) checkSessionNotTerminatedAfter(ctx context.Context, sessionID, userID string, position float64, fingerprintID string) (err error) {
func (q *Queries) checkSessionNotTerminatedAfter(ctx context.Context, sessionID, userID string, position decimal.Decimal, fingerprintID string) (err error) {
ctx, span := tracing.NewSpan(ctx)
defer func() { span.EndWithError(err) }()
@@ -165,7 +166,7 @@ func (q *Queries) checkSessionNotTerminatedAfter(ctx context.Context, sessionID,
}
type sessionTerminatedModel struct {
position float64
position decimal.Decimal
sessionID string
userID string
fingerPrintID string

11
internal/query/current_state.go
View File

@@ -10,6 +10,7 @@ import (
"time"
sq "github.com/Masterminds/squirrel"
"github.com/shopspring/decimal"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/eventstore"
@@ -25,7 +26,7 @@ type Stateful interface {
type State struct {
LastRun time.Time
Position float64
Position decimal.Decimal
EventCreatedAt time.Time
AggregateID string
AggregateType eventstore.AggregateType
@@ -220,7 +221,7 @@ func prepareLatestState() (sq.SelectBuilder, func(*sql.Row) (*State, error)) {
var (
creationDate sql.NullTime
lastUpdated sql.NullTime
position sql.NullFloat64
position decimal.NullDecimal
)
err := row.Scan(
&creationDate,
@@ -233,7 +234,7 @@ func prepareLatestState() (sq.SelectBuilder, func(*sql.Row) (*State, error)) {
return &State{
EventCreatedAt: creationDate.Time,
LastRun: lastUpdated.Time,
Position: position.Float64,
Position: position.Decimal,
}, nil
}
}
@@ -258,7 +259,7 @@ func prepareCurrentStateQuery() (sq.SelectBuilder, func(*sql.Rows) (*CurrentStat
var (
lastRun sql.NullTime
eventDate sql.NullTime
currentPosition sql.NullFloat64
currentPosition decimal.NullDecimal
aggregateType sql.NullString
aggregateID sql.NullString
sequence sql.NullInt64
@@ -279,7 +280,7 @@ func prepareCurrentStateQuery() (sq.SelectBuilder, func(*sql.Rows) (*CurrentStat
}
currentState.State.EventCreatedAt = eventDate.Time
currentState.State.LastRun = lastRun.Time
currentState.Position = currentPosition.Float64
currentState.Position = currentPosition.Decimal
currentState.AggregateType = eventstore.AggregateType(aggregateType.String)
currentState.AggregateID = aggregateID.String
currentState.Sequence = uint64(sequence.Int64)

8
internal/query/current_state_test.go
View File

@@ -7,6 +7,8 @@ import (
"fmt"
"regexp"
"testing"
"github.com/shopspring/decimal"
)
var (
@@ -86,7 +88,7 @@ func Test_CurrentSequencesPrepares(t *testing.T) {
State: State{
EventCreatedAt: testNow,
LastRun: testNow,
Position: 20211108,
Position: decimal.NewFromInt(20211108),
AggregateID: "agg-id",
AggregateType: "agg-type",
Sequence: 20211108,
@@ -133,7 +135,7 @@ func Test_CurrentSequencesPrepares(t *testing.T) {
ProjectionName: "projection-name",
State: State{
EventCreatedAt: testNow,
Position: 20211108,
Position: decimal.NewFromInt(20211108),
LastRun: testNow,
AggregateID: "agg-id",
AggregateType: "agg-type",
@@ -144,7 +146,7 @@ func Test_CurrentSequencesPrepares(t *testing.T) {
ProjectionName: "projection-name2",
State: State{
EventCreatedAt: testNow,
Position: 20211108,
Position: decimal.NewFromInt(20211108),
LastRun: testNow,
AggregateID: "agg-id",
AggregateType: "agg-type",

38
internal/query/projection/projection.go
View File

@@ -2,8 +2,10 @@ package projection
import (
"context"
"errors"
"fmt"
"github.com/jackc/pgx/v5/pgconn"
"github.com/zitadel/logging"
internal_authz "github.com/zitadel/zitadel/internal/api/authz"
@@ -212,11 +214,19 @@ func Start(ctx context.Context) {
func ProjectInstance(ctx context.Context) error {
for i, projection := range projections {
logging.WithFields("name", projection.ProjectionName(), "instance", internal_authz.GetInstance(ctx).InstanceID(), "index", fmt.Sprintf("%d/%d", i, len(projections))).Info("starting projection")
_, err := projection.Trigger(ctx)
if err != nil {
return err
for {
_, err := projection.Trigger(ctx)
if err == nil {
break
}
var pgErr *pgconn.PgError
errors.As(err, &pgErr)
if pgErr.Code != database.PgUniqueConstraintErrorCode {
return err
}
logging.WithFields("name", projection.ProjectionName(), "instance", internal_authz.GetInstance(ctx).InstanceID()).WithError(err).Debug("projection failed because of unique constraint, retrying")
}
logging.WithFields("name", projection.ProjectionName(), "instance", internal_authz.GetInstance(ctx).InstanceID()).Info("projection done")
logging.WithFields("name", projection.ProjectionName(), "instance", internal_authz.GetInstance(ctx).InstanceID(), "index", fmt.Sprintf("%d/%d", i, len(projections))).Info("projection done")
}
return nil
}
@@ -224,11 +234,19 @@ func ProjectInstance(ctx context.Context) error {
func ProjectInstanceFields(ctx context.Context) error {
for i, fieldProjection := range fields {
logging.WithFields("name", fieldProjection.ProjectionName(), "instance", internal_authz.GetInstance(ctx).InstanceID(), "index", fmt.Sprintf("%d/%d", i, len(fields))).Info("starting fields projection")
err := fieldProjection.Trigger(ctx)
if err != nil {
return err
for {
err := fieldProjection.Trigger(ctx)
if err == nil {
break
}
var pgErr *pgconn.PgError
errors.As(err, &pgErr)
if pgErr.Code != database.PgUniqueConstraintErrorCode {
return err
}
logging.WithFields("name", fieldProjection.ProjectionName(), "instance", internal_authz.GetInstance(ctx).InstanceID()).WithError(err).Debug("fields projection failed because of unique constraint, retrying")
}
logging.WithFields("name", fieldProjection.ProjectionName(), "instance", internal_authz.GetInstance(ctx).InstanceID()).Info("fields projection done")
logging.WithFields("name", fieldProjection.ProjectionName(), "instance", internal_authz.GetInstance(ctx).InstanceID(), "index", fmt.Sprintf("%d/%d", i, len(fields))).Info("fields projection done")
}
return nil
}
@@ -257,6 +275,10 @@ func applyCustomConfig(config handler.Config, customConfig CustomConfig) handler
return config
}
// we know this is ugly, but we need to have a singleton slice of all projections
// and are only able to initialize it after all projections are created
// as setup and start currently create them individually, we make sure we get the right one
// will be refactored when changing to new id based projections
func newFieldsList() {
fields = []*handler.FieldHandler{
ProjectGrantFields,

4
internal/query/user_grant.go
View File

@@ -280,7 +280,7 @@ func (q *Queries) UserGrants(ctx context.Context, queries *UserGrantsQueries, sh
return nil, zerrors.ThrowInternal(err, "QUERY-wXnQR", "Errors.Query.SQLStatement")
}
latestSequence, err := q.latestState(ctx, userGrantTable)
latestState, err := q.latestState(ctx, userGrantTable)
if err != nil {
return nil, err
}
@@ -293,7 +293,7 @@ func (q *Queries) UserGrants(ctx context.Context, queries *UserGrantsQueries, sh
return nil, err
}
grants.State = latestSequence
grants.State = latestState
return grants, nil
}

4
internal/query/user_membership.go
View File

@@ -143,7 +143,7 @@ func (q *Queries) Memberships(ctx context.Context, queries *MembershipSearchQuer
if err != nil {
return nil, zerrors.ThrowInvalidArgument(err, "QUERY-T84X9", "Errors.Query.InvalidRequest")
}
latestSequence, err := q.latestState(ctx, orgMemberTable, instanceMemberTable, projectMemberTable, projectGrantMemberTable)
latestState, err := q.latestState(ctx, orgMemberTable, instanceMemberTable, projectMemberTable, projectGrantMemberTable)
if err != nil {
return nil, err
}
@@ -156,7 +156,7 @@ func (q *Queries) Memberships(ctx context.Context, queries *MembershipSearchQuer
if err != nil {
return nil, err
}
memberships.State = latestSequence
memberships.State = latestState
return memberships, nil
}