TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-11 20:17:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(eventstore): use decimal, correct mirror (#9905)

Browse Source 
back port #9812, #9878, #9881, #9884

---------

Co-authored-by: Livio Spring <livio.a@gmail.com>
Co-authored-by: Stefan Benz <46600784+stebenz@users.noreply.github.com>
This commit is contained in:
Silvan
2025-05-20 12:58:12 +02:00
committed by GitHub
parent 1df6638847
commit f3338cdb9a
58 changed files with 545 additions and 331 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
.golangci.yaml
View File

@@ -4,12 +4,7 @@ issues:
max-issues-per-linter: 0 max-issues-per-linter: 0
# Set to 0 to disable. # Set to 0 to disable.
max-same-issues: 0 max-same-issues: 0
exclude-dirs:
run:
concurrency: 4
timeout: 10m
go: '1.22'
skip-dirs:
- .artifacts - .artifacts
- .backups - .backups
- .codecov - .codecov
@@ -25,6 +20,11 @@ run:
- openapi - openapi
- proto - proto
- tools - tools
run:
concurrency: 4
timeout: 10m
go: '1.22'
linters: linters:
enable: enable:
# Simple linter to check that your code does not contain non-ASCII identifiers [fast: true, auto-fix: false] # Simple linter to check that your code does not contain non-ASCII identifiers [fast: true, auto-fix: false]

15
cmd/mirror/auth.go
View File

@@ -4,6 +4,7 @@ import (
"context" "context"
_ "embed" _ "embed"
"io" "io"
"strconv"
"time" "time"
"github.com/jackc/pgx/v5/stdlib" "github.com/jackc/pgx/v5/stdlib"
@@ -41,12 +42,16 @@ func copyAuth(ctx context.Context, config *Migration) {
logging.OnError(err).Fatal("unable to connect to destination database") logging.OnError(err).Fatal("unable to connect to destination database")
defer destClient.Close() defer destClient.Close()
copyAuthRequests(ctx, sourceClient, destClient) copyAuthRequests(ctx, sourceClient, destClient, config.MaxAuthRequestAge)
} }
func copyAuthRequests(ctx context.Context, source, dest *database.DB) { func copyAuthRequests(ctx context.Context, source, dest *database.DB, maxAuthRequestAge time.Duration) {
start := time.Now() start := time.Now()
logging.Info("creating index on auth.auth_requests.change_date to speed up copy in source database")
_, err := source.ExecContext(ctx, "CREATE INDEX CONCURRENTLY IF NOT EXISTS auth_requests_change_date ON auth.auth_requests (change_date)")
logging.OnError(err).Fatal("unable to create index on auth.auth_requests.change_date")
sourceConn, err := source.Conn(ctx) sourceConn, err := source.Conn(ctx)
logging.OnError(err).Fatal("unable to acquire connection") logging.OnError(err).Fatal("unable to acquire connection")
defer sourceConn.Close() defer sourceConn.Close()
@@ -55,9 +60,9 @@ func copyAuthRequests(ctx context.Context, source, dest *database.DB) {
errs := make(chan error, 1) errs := make(chan error, 1)
go func() { go func() {
err = sourceConn.Raw(func(driverConn interface{}) error { err = sourceConn.Raw(func(driverConn any) error {
conn := driverConn.(*stdlib.Conn).Conn() conn := driverConn.(*stdlib.Conn).Conn()
_, err := conn.PgConn().CopyTo(ctx, w, "COPY (SELECT id, regexp_replace(request::TEXT, '\\\\u0000', '', 'g')::JSON request, code, request_type, creation_date, change_date, instance_id FROM auth.auth_requests "+instanceClause()+") TO STDOUT") _, err := conn.PgConn().CopyTo(ctx, w, "COPY (SELECT id, regexp_replace(request::TEXT, '\\\\u0000', '', 'g')::JSON request, code, request_type, creation_date, change_date, instance_id FROM auth.auth_requests "+instanceClause()+" AND change_date > NOW() - INTERVAL '"+strconv.FormatFloat(maxAuthRequestAge.Seconds(), 'f', -1, 64)+" seconds') TO STDOUT")
w.Close() w.Close()
return err return err
}) })
@@ -69,7 +74,7 @@ func copyAuthRequests(ctx context.Context, source, dest *database.DB) {
defer destConn.Close() defer destConn.Close()
var affected int64 var affected int64
err = destConn.Raw(func(driverConn interface{}) error { err = destConn.Raw(func(driverConn any) error {
conn := driverConn.(*stdlib.Conn).Conn() conn := driverConn.(*stdlib.Conn).Conn()
if shouldReplace { if shouldReplace {

3
cmd/mirror/config.go
View File

@@ -23,7 +23,8 @@ type Migration struct {
Source database.Config Source database.Config
Destination database.Config Destination database.Config
EventBulkSize uint32 EventBulkSize uint32
MaxAuthRequestAge time.Duration
Log *logging.Config Log *logging.Config
Machine *id.Config Machine *id.Config

132
cmd/mirror/defaults.yaml
View File

@@ -1,84 +1,88 @@
Source: Source:
cockroach: cockroach:
Host: localhost # ZITADEL_DATABASE_COCKROACH_HOST Host: localhost # ZITADEL_SOURCE_COCKROACH_HOST
Port: 26257 # ZITADEL_DATABASE_COCKROACH_PORT Port: 26257 # ZITADEL_SOURCE_COCKROACH_PORT
Database: zitadel # ZITADEL_DATABASE_COCKROACH_DATABASE Database: zitadel # ZITADEL_SOURCE_COCKROACH_DATABASE
MaxOpenConns: 6 # ZITADEL_DATABASE_COCKROACH_MAXOPENCONNS MaxOpenConns: 6 # ZITADEL_SOURCE_COCKROACH_MAXOPENCONNS
MaxIdleConns: 6 # ZITADEL_DATABASE_COCKROACH_MAXIDLECONNS MaxIdleConns: 6 # ZITADEL_SOURCE_COCKROACH_MAXIDLECONNS
EventPushConnRatio: 0.33 # ZITADEL_DATABASE_COCKROACH_EVENTPUSHCONNRATIO EventPushConnRatio: 0.33 # ZITADEL_SOURCE_COCKROACH_EVENTPUSHCONNRATIO
ProjectionSpoolerConnRatio: 0.33 # ZITADEL_DATABASE_COCKROACH_PROJECTIONSPOOLERCONNRATIO ProjectionSpoolerConnRatio: 0.33 # ZITADEL_SOURCE_COCKROACH_PROJECTIONSPOOLERCONNRATIO
MaxConnLifetime: 30m # ZITADEL_DATABASE_COCKROACH_MAXCONNLIFETIME MaxConnLifetime: 30m # ZITADEL_SOURCE_COCKROACH_MAXCONNLIFETIME
MaxConnIdleTime: 5m # ZITADEL_DATABASE_COCKROACH_MAXCONNIDLETIME MaxConnIdleTime: 5m # ZITADEL_SOURCE_COCKROACH_MAXCONNIDLETIME
Options: "" # ZITADEL_DATABASE_COCKROACH_OPTIONS Options: "" # ZITADEL_SOURCE_COCKROACH_OPTIONS
User: User:
Username: zitadel # ZITADEL_DATABASE_COCKROACH_USER_USERNAME Username: zitadel # ZITADEL_SOURCE_COCKROACH_USER_USERNAME
Password: "" # ZITADEL_DATABASE_COCKROACH_USER_PASSWORD Password: "" # ZITADEL_SOURCE_COCKROACH_USER_PASSWORD
SSL: SSL:
Mode: disable # ZITADEL_DATABASE_COCKROACH_USER_SSL_MODE Mode: disable # ZITADEL_SOURCE_COCKROACH_USER_SSL_MODE
RootCert: "" # ZITADEL_DATABASE_COCKROACH_USER_SSL_ROOTCERT RootCert: "" # ZITADEL_SOURCE_COCKROACH_USER_SSL_ROOTCERT
Cert: "" # ZITADEL_DATABASE_COCKROACH_USER_SSL_CERT Cert: "" # ZITADEL_SOURCE_COCKROACH_USER_SSL_CERT
Key: "" # ZITADEL_DATABASE_COCKROACH_USER_SSL_KEY Key: "" # ZITADEL_SOURCE_COCKROACH_USER_SSL_KEY
# Postgres is used as soon as a value is set # Postgres is used as soon as a value is set
# The values describe the possible fields to set values # The values describe the possible fields to set values
postgres: postgres:
Host: # ZITADEL_DATABASE_POSTGRES_HOST Host: # ZITADEL_SOURCE_POSTGRES_HOST
Port: # ZITADEL_DATABASE_POSTGRES_PORT Port: # ZITADEL_SOURCE_POSTGRES_PORT
Database: # ZITADEL_DATABASE_POSTGRES_DATABASE Database: # ZITADEL_SOURCE_POSTGRES_DATABASE
MaxOpenConns: # ZITADEL_DATABASE_POSTGRES_MAXOPENCONNS MaxOpenConns: # ZITADEL_SOURCE_POSTGRES_MAXOPENCONNS
MaxIdleConns: # ZITADEL_DATABASE_POSTGRES_MAXIDLECONNS MaxIdleConns: # ZITADEL_SOURCE_POSTGRES_MAXIDLECONNS
MaxConnLifetime: # ZITADEL_DATABASE_POSTGRES_MAXCONNLIFETIME MaxConnLifetime: # ZITADEL_SOURCE_POSTGRES_MAXCONNLIFETIME
MaxConnIdleTime: # ZITADEL_DATABASE_POSTGRES_MAXCONNIDLETIME MaxConnIdleTime: # ZITADEL_SOURCE_POSTGRES_MAXCONNIDLETIME
Options: # ZITADEL_DATABASE_POSTGRES_OPTIONS Options: # ZITADEL_SOURCE_POSTGRES_OPTIONS
User: User:
Username: # ZITADEL_DATABASE_POSTGRES_USER_USERNAME Username: # ZITADEL_SOURCE_POSTGRES_USER_USERNAME
Password: # ZITADEL_DATABASE_POSTGRES_USER_PASSWORD Password: # ZITADEL_SOURCE_POSTGRES_USER_PASSWORD
SSL: SSL:
Mode: # ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE Mode: # ZITADEL_SOURCE_POSTGRES_USER_SSL_MODE
RootCert: # ZITADEL_DATABASE_POSTGRES_USER_SSL_ROOTCERT RootCert: # ZITADEL_SOURCE_POSTGRES_USER_SSL_ROOTCERT
Cert: # ZITADEL_DATABASE_POSTGRES_USER_SSL_CERT Cert: # ZITADEL_SOURCE_POSTGRES_USER_SSL_CERT
Key: # ZITADEL_DATABASE_POSTGRES_USER_SSL_KEY Key: # ZITADEL_SOURCE_POSTGRES_USER_SSL_KEY
Destination: Destination:
cockroach: cockroach:
Host: localhost # ZITADEL_DATABASE_COCKROACH_HOST Host: localhost # ZITADEL_DESTINATION_COCKROACH_HOST
Port: 26257 # ZITADEL_DATABASE_COCKROACH_PORT Port: 26257 # ZITADEL_DESTINATION_COCKROACH_PORT
Database: zitadel # ZITADEL_DATABASE_COCKROACH_DATABASE Database: zitadel # ZITADEL_DESTINATION_COCKROACH_DATABASE
MaxOpenConns: 0 # ZITADEL_DATABASE_COCKROACH_MAXOPENCONNS MaxOpenConns: 0 # ZITADEL_DESTINATION_COCKROACH_MAXOPENCONNS
MaxIdleConns: 0 # ZITADEL_DATABASE_COCKROACH_MAXIDLECONNS MaxIdleConns: 0 # ZITADEL_DESTINATION_COCKROACH_MAXIDLECONNS
MaxConnLifetime: 30m # ZITADEL_DATABASE_COCKROACH_MAXCONNLIFETIME MaxConnLifetime: 30m # ZITADEL_DESTINATION_COCKROACH_MAXCONNLIFETIME
MaxConnIdleTime: 5m # ZITADEL_DATABASE_COCKROACH_MAXCONNIDLETIME MaxConnIdleTime: 5m # ZITADEL_DESTINATION_COCKROACH_MAXCONNIDLETIME
EventPushConnRatio: 0.01 # ZITADEL_DATABASE_COCKROACH_EVENTPUSHCONNRATIO EventPushConnRatio: 0.01 # ZITADEL_DESTINATION_COCKROACH_EVENTPUSHCONNRATIO
ProjectionSpoolerConnRatio: 0.5 # ZITADEL_DATABASE_COCKROACH_PROJECTIONSPOOLERCONNRATIO ProjectionSpoolerConnRatio: 0.5 # ZITADEL_DESTINATION_COCKROACH_PROJECTIONSPOOLERCONNRATIO
Options: "" # ZITADEL_DATABASE_COCKROACH_OPTIONS Options: "" # ZITADEL_DESTINATION_COCKROACH_OPTIONS
User: User:
Username: zitadel # ZITADEL_DATABASE_COCKROACH_USER_USERNAME Username: zitadel # ZITADEL_DESTINATION_COCKROACH_USER_USERNAME
Password: "" # ZITADEL_DATABASE_COCKROACH_USER_PASSWORD Password: "" # ZITADEL_DESTINATION_COCKROACH_USER_PASSWORD
SSL: SSL:
Mode: disable # ZITADEL_DATABASE_COCKROACH_USER_SSL_MODE Mode: disable # ZITADEL_DESTINATION_COCKROACH_USER_SSL_MODE
RootCert: "" # ZITADEL_DATABASE_COCKROACH_USER_SSL_ROOTCERT RootCert: "" # ZITADEL_DESTINATION_COCKROACH_USER_SSL_ROOTCERT
Cert: "" # ZITADEL_DATABASE_COCKROACH_USER_SSL_CERT Cert: "" # ZITADEL_DESTINATION_COCKROACH_USER_SSL_CERT
Key: "" # ZITADEL_DATABASE_COCKROACH_USER_SSL_KEY Key: "" # ZITADEL_DESTINATION_COCKROACH_USER_SSL_KEY
# Postgres is used as soon as a value is set # Postgres is used as soon as a value is set
# The values describe the possible fields to set values # The values describe the possible fields to set values
postgres: postgres:
Host: # ZITADEL_DATABASE_POSTGRES_HOST Host: # ZITADEL_DESTINATION_POSTGRES_HOST
Port: # ZITADEL_DATABASE_POSTGRES_PORT Port: # ZITADEL_DESTINATION_POSTGRES_PORT
Database: # ZITADEL_DATABASE_POSTGRES_DATABASE Database: # ZITADEL_DESTINATION_POSTGRES_DATABASE
MaxOpenConns: # ZITADEL_DATABASE_POSTGRES_MAXOPENCONNS MaxOpenConns: # ZITADEL_DESTINATION_POSTGRES_MAXOPENCONNS
MaxIdleConns: # ZITADEL_DATABASE_POSTGRES_MAXIDLECONNS MaxIdleConns: # ZITADEL_DESTINATION_POSTGRES_MAXIDLECONNS
MaxConnLifetime: # ZITADEL_DATABASE_POSTGRES_MAXCONNLIFETIME MaxConnLifetime: # ZITADEL_DESTINATION_POSTGRES_MAXCONNLIFETIME
MaxConnIdleTime: # ZITADEL_DATABASE_POSTGRES_MAXCONNIDLETIME MaxConnIdleTime: # ZITADEL_DESTINATION_POSTGRES_MAXCONNIDLETIME
Options: # ZITADEL_DATABASE_POSTGRES_OPTIONS Options: # ZITADEL_DESTINATION_POSTGRES_OPTIONS
User: User:
Username: # ZITADEL_DATABASE_POSTGRES_USER_USERNAME Username: # ZITADEL_DESTINATION_POSTGRES_USER_USERNAME
Password: # ZITADEL_DATABASE_POSTGRES_USER_PASSWORD Password: # ZITADEL_DESTINATION_POSTGRES_USER_PASSWORD
SSL: SSL:
Mode: # ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE Mode: # ZITADEL_DESTINATION_POSTGRES_USER_SSL_MODE
RootCert: # ZITADEL_DATABASE_POSTGRES_USER_SSL_ROOTCERT RootCert: # ZITADEL_DESTINATION_POSTGRES_USER_SSL_ROOTCERT
Cert: # ZITADEL_DATABASE_POSTGRES_USER_SSL_CERT Cert: # ZITADEL_DESTINATION_POSTGRES_USER_SSL_CERT
Key: # ZITADEL_DATABASE_POSTGRES_USER_SSL_KEY Key: # ZITADEL_DESTINATION_POSTGRES_USER_SSL_KEY
EventBulkSize: 10000
EventBulkSize: 10000 # ZITADEL_EVENTBULKSIZE
# The maximum duration an auth request was last updated before it gets ignored.
# Default is 30 days
MaxAuthRequestAge: 720h # ZITADEL_MAXAUTHREQUESTAGE
Projections: Projections:
# The maximum duration a transaction remains open # The maximum duration a transaction remains open
@@ -87,14 +91,14 @@ Projections:
TransactionDuration: 0s # ZITADEL_PROJECTIONS_TRANSACTIONDURATION TransactionDuration: 0s # ZITADEL_PROJECTIONS_TRANSACTIONDURATION
# turn off scheduler during operation # turn off scheduler during operation
RequeueEvery: 0s RequeueEvery: 0s
ConcurrentInstances: 7 ConcurrentInstances: 7 # ZITADEL_PROJECTIONS_CONCURRENTINSTANCES
EventBulkLimit: 1000 EventBulkLimit: 1000 # ZITADEL_PROJECTIONS_EVENTBULKLIMIT
Customizations: Customizations:
notifications: notifications:
MaxFailureCount: 1 MaxFailureCount: 1
Eventstore: Eventstore:
MaxRetries: 3 MaxRetries: 3 # ZITADEL_EVENTSTORE_MAXRETRIES
Auth: Auth:
Spooler: Spooler:

10
cmd/mirror/event.go
View File

@@ -3,6 +3,8 @@ package mirror
import ( import (
"context" "context"
"github.com/shopspring/decimal"
"github.com/zitadel/zitadel/internal/v2/eventstore" "github.com/zitadel/zitadel/internal/v2/eventstore"
"github.com/zitadel/zitadel/internal/v2/projection" "github.com/zitadel/zitadel/internal/v2/projection"
"github.com/zitadel/zitadel/internal/v2/readmodel" "github.com/zitadel/zitadel/internal/v2/readmodel"
@@ -30,12 +32,12 @@ func queryLastSuccessfulMigration(ctx context.Context, destinationES *eventstore
return lastSuccess, nil return lastSuccess, nil
} }
func writeMigrationStart(ctx context.Context, sourceES *eventstore.EventStore, id string, destination string) (_ float64, err error) { func writeMigrationStart(ctx context.Context, sourceES *eventstore.EventStore, id string, destination string) (_ decimal.Decimal, err error) {
var cmd *eventstore.Command var cmd *eventstore.Command
if len(instanceIDs) > 0 { if len(instanceIDs) > 0 {
cmd, err = mirror_event.NewStartedInstancesCommand(destination, instanceIDs) cmd, err = mirror_event.NewStartedInstancesCommand(destination, instanceIDs)
if err != nil { if err != nil {
return 0, err return decimal.Decimal{}, err
} }
} else { } else {
cmd = mirror_event.NewStartedSystemCommand(destination) cmd = mirror_event.NewStartedSystemCommand(destination)
@@ -58,12 +60,12 @@ func writeMigrationStart(ctx context.Context, sourceES *eventstore.EventStore, i
), ),
) )
if err != nil { if err != nil {
return 0, err return decimal.Decimal{}, err
} }
return position.Position, nil return position.Position, nil
} }
func writeMigrationSucceeded(ctx context.Context, destinationES *eventstore.EventStore, id, source string, position float64) error { func writeMigrationSucceeded(ctx context.Context, destinationES *eventstore.EventStore, id, source string, position decimal.Decimal) error {
return destinationES.Push( return destinationES.Push(
ctx, ctx,
eventstore.NewPushIntent( eventstore.NewPushIntent(

17
cmd/mirror/event_store.go
View File

@@ -8,7 +8,9 @@ import (
"io" "io"
"time" "time"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/stdlib" "github.com/jackc/pgx/v5/stdlib"
"github.com/shopspring/decimal"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/zitadel/logging" "github.com/zitadel/logging"
@@ -68,6 +70,7 @@ func positionQuery(db *db.DB) string {
} }
func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) { func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
logging.Info("starting to copy events")
start := time.Now() start := time.Now()
reader, writer := io.Pipe() reader, writer := io.Pipe()
@@ -96,7 +99,7 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
logging.WithFields("from", previousMigration.Position, "to", maxPosition).Info("start event migration") logging.WithFields("from", previousMigration.Position, "to", maxPosition).Info("start event migration")
nextPos := make(chan bool, 1) nextPos := make(chan bool, 1)
pos := make(chan float64, 1) pos := make(chan decimal.Decimal, 1)
errs := make(chan error, 3) errs := make(chan error, 3)
go func() { go func() {
@@ -126,7 +129,10 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
if err != nil { if err != nil {
return zerrors.ThrowUnknownf(err, "MIGRA-KTuSq", "unable to copy events from source during iteration %d", i) return zerrors.ThrowUnknownf(err, "MIGRA-KTuSq", "unable to copy events from source during iteration %d", i)
} }
logging.WithFields("batch_count", i).Info("batch of events copied")
if tag.RowsAffected() < int64(bulkSize) { if tag.RowsAffected() < int64(bulkSize) {
logging.WithFields("batch_count", i).Info("last batch of events copied")
return nil return nil
} }
@@ -144,7 +150,7 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
go func() { go func() {
defer close(pos) defer close(pos)
for range nextPos { for range nextPos {
var position float64 var position decimal.Decimal
err := dest.QueryRowContext( err := dest.QueryRowContext(
ctx, ctx,
func(row *sql.Row) error { func(row *sql.Row) error {
@@ -167,6 +173,10 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
tag, err := conn.PgConn().CopyFrom(ctx, reader, "COPY eventstore.events2 FROM STDIN") tag, err := conn.PgConn().CopyFrom(ctx, reader, "COPY eventstore.events2 FROM STDIN")
eventCount = tag.RowsAffected() eventCount = tag.RowsAffected()
if err != nil { if err != nil {
pgErr := new(pgconn.PgError)
errors.As(err, &pgErr)
logging.WithError(err).WithField("pg_err_details", pgErr.Detail).Error("unable to copy events into destination")
return zerrors.ThrowUnknown(err, "MIGRA-DTHi7", "unable to copy events into destination") return zerrors.ThrowUnknown(err, "MIGRA-DTHi7", "unable to copy events into destination")
} }
@@ -179,7 +189,7 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
logging.WithFields("took", time.Since(start), "count", eventCount).Info("events migrated") logging.WithFields("took", time.Since(start), "count", eventCount).Info("events migrated")
} }
func writeCopyEventsDone(ctx context.Context, es *eventstore.EventStore, id, source string, position float64, errs <-chan error) { func writeCopyEventsDone(ctx context.Context, es *eventstore.EventStore, id, source string, position decimal.Decimal, errs <-chan error) {
joinedErrs := make([]error, 0, len(errs)) joinedErrs := make([]error, 0, len(errs))
for err := range errs { for err := range errs {
joinedErrs = append(joinedErrs, err) joinedErrs = append(joinedErrs, err)
@@ -198,6 +208,7 @@ func writeCopyEventsDone(ctx context.Context, es *eventstore.EventStore, id, sou
} }
func copyUniqueConstraints(ctx context.Context, source, dest *db.DB) { func copyUniqueConstraints(ctx context.Context, source, dest *db.DB) {
logging.Info("starting to copy unique constraints")
start := time.Now() start := time.Now()
reader, writer := io.Pipe() reader, writer := io.Pipe()
errs := make(chan error, 1) errs := make(chan error, 1)

23
cmd/mirror/projections.go
View File

@@ -3,6 +3,7 @@ package mirror
import ( import (
"context" "context"
"database/sql" "database/sql"
"fmt"
"net/http" "net/http"
"sync" "sync"
"time" "time"
@@ -103,6 +104,7 @@ func projections(
config *ProjectionsConfig, config *ProjectionsConfig,
masterKey string, masterKey string,
) { ) {
logging.Info("starting to fill projections")
start := time.Now() start := time.Now()
client, err := database.Connect(config.Destination, false) client, err := database.Connect(config.Destination, false)
@@ -118,7 +120,11 @@ func projections(
logging.OnError(err).Fatal("unable create static storage") logging.OnError(err).Fatal("unable create static storage")
config.Eventstore.Querier = old_es.NewCRDB(client) config.Eventstore.Querier = old_es.NewCRDB(client)
config.Eventstore.Pusher = new_es.NewEventstore(client)
newES := new_es.NewEventstore(client)
config.Eventstore.Pusher = newES
config.Eventstore.Searcher = newES
es := eventstore.NewEventstore(config.Eventstore) es := eventstore.NewEventstore(config.Eventstore)
esV4 := es_v4.NewEventstoreFromOne(es_v4_pg.New(client, &es_v4_pg.Config{ esV4 := es_v4.NewEventstoreFromOne(es_v4_pg.New(client, &es_v4_pg.Config{
MaxRetries: config.Eventstore.MaxRetries, MaxRetries: config.Eventstore.MaxRetries,
@@ -251,8 +257,10 @@ func projections(
go execProjections(ctx, instances, failedInstances, &wg) go execProjections(ctx, instances, failedInstances, &wg)
} }
for _, instance := range queryInstanceIDs(ctx, client) { existingInstances := queryInstanceIDs(ctx, client)
for i, instance := range existingInstances {
instances <- instance instances <- instance
logging.WithFields("id", instance, "index", fmt.Sprintf("%d/%d", i, len(existingInstances))).Info("instance queued for projection")
} }
close(instances) close(instances)
wg.Wait() wg.Wait()
@@ -264,7 +272,7 @@ func projections(
func execProjections(ctx context.Context, instances <-chan string, failedInstances chan<- string, wg *sync.WaitGroup) { func execProjections(ctx context.Context, instances <-chan string, failedInstances chan<- string, wg *sync.WaitGroup) {
for instance := range instances { for instance := range instances {
logging.WithFields("instance", instance).Info("start projections") logging.WithFields("instance", instance).Info("starting projections")
ctx = internal_authz.WithInstanceID(ctx, instance) ctx = internal_authz.WithInstanceID(ctx, instance)
err := projection.ProjectInstance(ctx) err := projection.ProjectInstance(ctx)
@@ -281,6 +289,13 @@ func execProjections(ctx context.Context, instances <-chan string, failedInstanc
continue continue
} }
err = projection.ProjectInstanceFields(ctx)
if err != nil {
logging.WithFields("instance", instance).WithError(err).Info("trigger fields failed")
failedInstances <- instance
continue
}
err = auth_handler.ProjectInstance(ctx) err = auth_handler.ProjectInstance(ctx)
if err != nil { if err != nil {
logging.WithFields("instance", instance).OnError(err).Info("trigger auth handler failed") logging.WithFields("instance", instance).OnError(err).Info("trigger auth handler failed")
@@ -299,7 +314,7 @@ func execProjections(ctx context.Context, instances <-chan string, failedInstanc
wg.Done() wg.Done()
} }
// returns the instance configured by flag // queryInstanceIDs returns the instance configured by flag
// or all instances which are not removed // or all instances which are not removed
func queryInstanceIDs(ctx context.Context, source *database.DB) []string { func queryInstanceIDs(ctx context.Context, source *database.DB) []string {
if len(instanceIDs) > 0 { if len(instanceIDs) > 0 {

14
cmd/mirror/system.go
View File

@@ -46,6 +46,7 @@ func copySystem(ctx context.Context, config *Migration) {
} }
func copyAssets(ctx context.Context, source, dest *database.DB) { func copyAssets(ctx context.Context, source, dest *database.DB) {
logging.Info("starting to copy assets")
start := time.Now() start := time.Now()
sourceConn, err := source.Conn(ctx) sourceConn, err := source.Conn(ctx)
@@ -70,7 +71,7 @@ func copyAssets(ctx context.Context, source, dest *database.DB) {
logging.OnError(err).Fatal("unable to acquire dest connection") logging.OnError(err).Fatal("unable to acquire dest connection")
defer destConn.Close() defer destConn.Close()
var eventCount int64 var assetCount int64
err = destConn.Raw(func(driverConn interface{}) error { err = destConn.Raw(func(driverConn interface{}) error {
conn := driverConn.(*stdlib.Conn).Conn() conn := driverConn.(*stdlib.Conn).Conn()
@@ -82,16 +83,17 @@ func copyAssets(ctx context.Context, source, dest *database.DB) {
} }
tag, err := conn.PgConn().CopyFrom(ctx, r, "COPY system.assets (instance_id, asset_type, resource_owner, name, content_type, data, updated_at) FROM stdin") tag, err := conn.PgConn().CopyFrom(ctx, r, "COPY system.assets (instance_id, asset_type, resource_owner, name, content_type, data, updated_at) FROM stdin")
eventCount = tag.RowsAffected() assetCount = tag.RowsAffected()
return err return err
}) })
logging.OnError(err).Fatal("unable to copy assets to destination") logging.OnError(err).Fatal("unable to copy assets to destination")
logging.OnError(<-errs).Fatal("unable to copy assets from source") logging.OnError(<-errs).Fatal("unable to copy assets from source")
logging.WithFields("took", time.Since(start), "count", eventCount).Info("assets migrated") logging.WithFields("took", time.Since(start), "count", assetCount).Info("assets migrated")
} }
func copyEncryptionKeys(ctx context.Context, source, dest *database.DB) { func copyEncryptionKeys(ctx context.Context, source, dest *database.DB) {
logging.Info("starting to copy encryption keys")
start := time.Now() start := time.Now()
sourceConn, err := source.Conn(ctx) sourceConn, err := source.Conn(ctx)
@@ -116,7 +118,7 @@ func copyEncryptionKeys(ctx context.Context, source, dest *database.DB) {
logging.OnError(err).Fatal("unable to acquire dest connection") logging.OnError(err).Fatal("unable to acquire dest connection")
defer destConn.Close() defer destConn.Close()
var eventCount int64 var keyCount int64
err = destConn.Raw(func(driverConn interface{}) error { err = destConn.Raw(func(driverConn interface{}) error {
conn := driverConn.(*stdlib.Conn).Conn() conn := driverConn.(*stdlib.Conn).Conn()
@@ -128,11 +130,11 @@ func copyEncryptionKeys(ctx context.Context, source, dest *database.DB) {
} }
tag, err := conn.PgConn().CopyFrom(ctx, r, "COPY system.encryption_keys FROM stdin") tag, err := conn.PgConn().CopyFrom(ctx, r, "COPY system.encryption_keys FROM stdin")
eventCount = tag.RowsAffected() keyCount = tag.RowsAffected()
return err return err
}) })
logging.OnError(err).Fatal("unable to copy encryption keys to destination") logging.OnError(err).Fatal("unable to copy encryption keys to destination")
logging.OnError(<-errs).Fatal("unable to copy encryption keys from source") logging.OnError(<-errs).Fatal("unable to copy encryption keys from source")
logging.WithFields("took", time.Since(start), "count", eventCount).Info("encryption keys migrated") logging.WithFields("took", time.Since(start), "count", keyCount).Info("encryption keys migrated")
} }

3
docs/docs/self-hosting/manage/cli/mirror.mdx
View File

@@ -163,6 +163,9 @@ Destination:
# As cockroachdb first copies the data into memory this parameter is used to iterate through the events table and fetch only the given amount of events per iteration # As cockroachdb first copies the data into memory this parameter is used to iterate through the events table and fetch only the given amount of events per iteration
EventBulkSize: 10000 # ZITADEL_EVENTBULKSIZE EventBulkSize: 10000 # ZITADEL_EVENTBULKSIZE
# The maximum duration an auth request was last updated before it gets ignored.
# Default is 30 days
MaxAuthRequestAge: 720h # ZITADEL_MAXAUTHREQUESTAGE
Projections: Projections:
# Defines how many projections are allowed to run in parallel # Defines how many projections are allowed to run in parallel

9
go.mod
View File

@@ -42,7 +42,8 @@ require (
github.com/h2non/gock v1.2.0 github.com/h2non/gock v1.2.0
github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/improbable-eng/grpc-web v0.15.0 github.com/improbable-eng/grpc-web v0.15.0
github.com/jackc/pgx/v5 v5.7.2 github.com/jackc/pgx-shopspring-decimal v0.0.0-20220624020537-1d36b5a1853e
github.com/jackc/pgx/v5 v5.7.4
github.com/jarcoal/jpath v0.0.0-20140328210829-f76b8b2dbf52 github.com/jarcoal/jpath v0.0.0-20140328210829-f76b8b2dbf52
github.com/jinzhu/gorm v1.9.16 github.com/jinzhu/gorm v1.9.16
github.com/k3a/html2text v1.2.1 github.com/k3a/html2text v1.2.1
@@ -57,8 +58,11 @@ require (
github.com/pquerna/otp v1.4.0 github.com/pquerna/otp v1.4.0
github.com/rakyll/statik v0.1.7 github.com/rakyll/statik v0.1.7
github.com/redis/go-redis/v9 v9.7.0 github.com/redis/go-redis/v9 v9.7.0
github.com/riverqueue/river v0.16.0
github.com/riverqueue/river/riverdriver v0.16.0
github.com/rs/cors v1.11.1 github.com/rs/cors v1.11.1
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
github.com/shopspring/decimal v1.4.0
github.com/sony/gobreaker/v2 v2.0.0 github.com/sony/gobreaker/v2 v2.0.0
github.com/sony/sonyflake v1.2.0 github.com/sony/sonyflake v1.2.0
github.com/spf13/cobra v1.8.1 github.com/spf13/cobra v1.8.1
@@ -124,8 +128,6 @@ require (
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect
github.com/pkg/errors v0.9.1 // indirect github.com/pkg/errors v0.9.1 // indirect
github.com/riverqueue/river v0.16.0 // indirect
github.com/riverqueue/river/riverdriver v0.16.0 // indirect
github.com/riverqueue/river/rivershared v0.16.0 // indirect github.com/riverqueue/river/rivershared v0.16.0 // indirect
github.com/riverqueue/river/rivertype v0.16.0 // indirect github.com/riverqueue/river/rivertype v0.16.0 // indirect
github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect
@@ -137,7 +139,6 @@ require (
github.com/tidwall/sjson v1.2.5 // indirect github.com/tidwall/sjson v1.2.5 // indirect
github.com/yuin/gopher-lua v1.1.1 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect
github.com/zenazn/goji v1.0.1 // indirect github.com/zenazn/goji v1.0.1 // indirect
go.uber.org/goleak v1.3.0 // indirect
go.uber.org/multierr v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect
golang.org/x/time v0.5.0 // indirect golang.org/x/time v0.5.0 // indirect
google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d // indirect google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d // indirect

19
go.sum
View File

@@ -416,16 +416,16 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa h1:s+4MhCQ6YrzisK6hFJUX53drDT4UsSW3DEhKn0ifuHw=
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.7.0 h1:FG6VLIdzvAPhnYqP14sQ2xhFLkiUQHCs6ySqO91kF4g= github.com/jackc/pgx-shopspring-decimal v0.0.0-20220624020537-1d36b5a1853e h1:i3gQ/Zo7sk4LUVbsAjTNeC4gIjoPNIZVzs4EXstssV4=
github.com/jackc/pgx/v5 v5.7.0/go.mod h1:awP1KNnjylvpxHuHP63gzjhnGkI1iw+PMoIwvoleN/8= github.com/jackc/pgx-shopspring-decimal v0.0.0-20220624020537-1d36b5a1853e/go.mod h1:zUHglCZ4mpDUPgIwqEKoba6+tcUQzRdb1+DPTuYe9pI=
github.com/jackc/pgx/v5 v5.7.2 h1:mLoDLV6sonKlvjIEsV56SkWNCnuNv531l94GaIzO+XI= github.com/jackc/pgx/v5 v5.7.4 h1:9wKznZrhWa2QiHL+NjTSPP6yjl3451BX3imWDnokYlg=
github.com/jackc/pgx/v5 v5.7.2/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ= github.com/jackc/pgx/v5 v5.7.4/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ=
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jarcoal/jpath v0.0.0-20140328210829-f76b8b2dbf52 h1:jny9eqYPwkG8IVy7foUoRjQmFLcArCSz+uPsL6KS0HQ= github.com/jarcoal/jpath v0.0.0-20140328210829-f76b8b2dbf52 h1:jny9eqYPwkG8IVy7foUoRjQmFLcArCSz+uPsL6KS0HQ=
@@ -648,6 +648,8 @@ github.com/riverqueue/river v0.16.0 h1:YyQrs0kGgjuABwgat02DPUYS0TMyG2ZFlzvf6+fSF
github.com/riverqueue/river v0.16.0/go.mod h1:pEZ8Gc15XyFjVY89nJeL256ub5z18XF7ukYn8ktqQrs= github.com/riverqueue/river v0.16.0/go.mod h1:pEZ8Gc15XyFjVY89nJeL256ub5z18XF7ukYn8ktqQrs=
github.com/riverqueue/river/riverdriver v0.16.0 h1:y4Df4e1Xk3Id0nnu1VxHJn9118OzmRHcmvOxM/i1Q30= github.com/riverqueue/river/riverdriver v0.16.0 h1:y4Df4e1Xk3Id0nnu1VxHJn9118OzmRHcmvOxM/i1Q30=
github.com/riverqueue/river/riverdriver v0.16.0/go.mod h1:7Kdf5HQDrLyLUUqPqXobaK+7zbcMctWeAl7yhg4nHes= github.com/riverqueue/river/riverdriver v0.16.0/go.mod h1:7Kdf5HQDrLyLUUqPqXobaK+7zbcMctWeAl7yhg4nHes=
github.com/riverqueue/river/riverdriver/riverdatabasesql v0.16.0 h1:T/DcMmZXiJAyLN3CSyAoNcf3U4oAD9Ht/8Vd5SXv5YU=
github.com/riverqueue/river/riverdriver/riverdatabasesql v0.16.0/go.mod h1:a9EUhD2yGsAeM9eWo+QrGGbL8LVWoGj2m8KEzm0xUxE=
github.com/riverqueue/river/riverdriver/riverpgxv5 v0.16.0 h1:6HP296OPN+3ORL9qG1f561pldB5eovkLzfkNIQmaTXI= github.com/riverqueue/river/riverdriver/riverpgxv5 v0.16.0 h1:6HP296OPN+3ORL9qG1f561pldB5eovkLzfkNIQmaTXI=
github.com/riverqueue/river/riverdriver/riverpgxv5 v0.16.0/go.mod h1:MAeBNoTQ+CD3nRvV9mF6iCBfsGJTxYHZeZSP4MYoeUE= github.com/riverqueue/river/riverdriver/riverpgxv5 v0.16.0/go.mod h1:MAeBNoTQ+CD3nRvV9mF6iCBfsGJTxYHZeZSP4MYoeUE=
github.com/riverqueue/river/rivershared v0.16.0 h1:L1lQ3gMwdIsxA6yF0/PwAdsFP0T82yBD1V03q2GuJDU= github.com/riverqueue/river/rivershared v0.16.0 h1:L1lQ3gMwdIsxA6yF0/PwAdsFP0T82yBD1V03q2GuJDU=
@@ -679,6 +681,8 @@ github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4=
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -727,7 +731,6 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
@@ -924,8 +927,6 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

8
internal/admin/repository/eventsourcing/handler/handler.go
View File

@@ -2,9 +2,13 @@ package handler
import ( import (
"context" "context"
"fmt"
"time" "time"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/view" "github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/view"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/database" "github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore" "github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/eventstore/handler/v2" "github.com/zitadel/zitadel/internal/eventstore/handler/v2"
@@ -57,11 +61,13 @@ func Start(ctx context.Context) {
} }
func ProjectInstance(ctx context.Context) error { func ProjectInstance(ctx context.Context) error {
for _, projection := range projections { for i, projection := range projections {
logging.WithFields("name", projection.ProjectionName(), "instance", authz.GetInstance(ctx).InstanceID(), "index", fmt.Sprintf("%d/%d", i, len(projections))).Info("starting admin projection")
_, err := projection.Trigger(ctx) _, err := projection.Trigger(ctx)
if err != nil { if err != nil {
return err return err
} }
logging.WithFields("name", projection.ProjectionName(), "instance", authz.GetInstance(ctx).InstanceID(), "index", fmt.Sprintf("%d/%d", i, len(projections))).Info("admin projection done")
} }
return nil return nil
} }

17
internal/api/oidc/key.go
View File

@@ -11,6 +11,7 @@ import (
"github.com/go-jose/go-jose/v4" "github.com/go-jose/go-jose/v4"
"github.com/jonboulle/clockwork" "github.com/jonboulle/clockwork"
"github.com/muhlemmer/gu" "github.com/muhlemmer/gu"
"github.com/shopspring/decimal"
"github.com/zitadel/logging" "github.com/zitadel/logging"
"github.com/zitadel/oidc/v3/pkg/op" "github.com/zitadel/oidc/v3/pkg/op"
@@ -350,14 +351,14 @@ func (o *OPStorage) getSigningKey(ctx context.Context) (op.SigningKey, error) {
if len(keys.Keys) > 0 { if len(keys.Keys) > 0 {
return PrivateKeyToSigningKey(SelectSigningKey(keys.Keys), o.encAlg) return PrivateKeyToSigningKey(SelectSigningKey(keys.Keys), o.encAlg)
} }
var position float64 var position decimal.Decimal
if keys.State != nil { if keys.State != nil {
position = keys.State.Position position = keys.State.Position
} }
return nil, o.refreshSigningKey(ctx, position) return nil, o.refreshSigningKey(ctx, position)
} }
func (o *OPStorage) refreshSigningKey(ctx context.Context, position float64) error { func (o *OPStorage) refreshSigningKey(ctx context.Context, position decimal.Decimal) error {
ok, err := o.ensureIsLatestKey(ctx, position) ok, err := o.ensureIsLatestKey(ctx, position)
if err != nil || !ok { if err != nil || !ok {
return zerrors.ThrowInternal(err, "OIDC-ASfh3", "cannot ensure that projection is up to date") return zerrors.ThrowInternal(err, "OIDC-ASfh3", "cannot ensure that projection is up to date")
@@ -369,12 +370,12 @@ func (o *OPStorage) refreshSigningKey(ctx context.Context, position float64) err
return zerrors.ThrowInternal(nil, "OIDC-Df1bh", "") return zerrors.ThrowInternal(nil, "OIDC-Df1bh", "")
} }
func (o *OPStorage) ensureIsLatestKey(ctx context.Context, position float64) (bool, error) { func (o *OPStorage) ensureIsLatestKey(ctx context.Context, position decimal.Decimal) (bool, error) {
maxSequence, err := o.getMaxKeySequence(ctx) maxSequence, err := o.getMaxKeyPosition(ctx)
if err != nil { if err != nil {
return false, fmt.Errorf("error retrieving new events: %w", err) return false, fmt.Errorf("error retrieving new events: %w", err)
} }
return position >= maxSequence, nil return position.GreaterThanOrEqual(maxSequence), nil
} }
func PrivateKeyToSigningKey(key query.PrivateKey, algorithm crypto.EncryptionAlgorithm) (_ op.SigningKey, err error) { func PrivateKeyToSigningKey(key query.PrivateKey, algorithm crypto.EncryptionAlgorithm) (_ op.SigningKey, err error) {
@@ -412,9 +413,9 @@ func (o *OPStorage) lockAndGenerateSigningKeyPair(ctx context.Context) error {
return o.command.GenerateSigningKeyPair(setOIDCCtx(ctx), "RS256") return o.command.GenerateSigningKeyPair(setOIDCCtx(ctx), "RS256")
} }
func (o *OPStorage) getMaxKeySequence(ctx context.Context) (float64, error) { func (o *OPStorage) getMaxKeyPosition(ctx context.Context) (decimal.Decimal, error) {
return o.eventstore.LatestSequence(ctx, return o.eventstore.LatestPosition(ctx,
eventstore.NewSearchQueryBuilder(eventstore.ColumnsMaxSequence). eventstore.NewSearchQueryBuilder(eventstore.ColumnsMaxPosition).
ResourceOwner(authz.GetInstance(ctx).InstanceID()). ResourceOwner(authz.GetInstance(ctx).InstanceID()).
AwaitOpenTransactions(). AwaitOpenTransactions().
AllowTimeTravel(). AllowTimeTravel().

17
internal/api/saml/certificate.go
View File

@@ -6,6 +6,7 @@ import (
"time" "time"
"github.com/go-jose/go-jose/v4" "github.com/go-jose/go-jose/v4"
"github.com/shopspring/decimal"
"github.com/zitadel/logging" "github.com/zitadel/logging"
"github.com/zitadel/saml/pkg/provider/key" "github.com/zitadel/saml/pkg/provider/key"
@@ -76,7 +77,7 @@ func (p *Storage) getCertificateAndKey(ctx context.Context, usage crypto.KeyUsag
return p.certificateToCertificateAndKey(selectCertificate(certs.Certificates)) return p.certificateToCertificateAndKey(selectCertificate(certs.Certificates))
} }
var position float64 var position decimal.Decimal
if certs.State != nil { if certs.State != nil {
position = certs.State.Position position = certs.State.Position
} }
@@ -87,7 +88,7 @@ func (p *Storage) getCertificateAndKey(ctx context.Context, usage crypto.KeyUsag
func (p *Storage) refreshCertificate( func (p *Storage) refreshCertificate(
ctx context.Context, ctx context.Context,
usage crypto.KeyUsage, usage crypto.KeyUsage,
position float64, position decimal.Decimal,
) error { ) error {
ok, err := p.ensureIsLatestCertificate(ctx, position) ok, err := p.ensureIsLatestCertificate(ctx, position)
if err != nil { if err != nil {
@@ -103,12 +104,12 @@ func (p *Storage) refreshCertificate(
return nil return nil
} }
func (p *Storage) ensureIsLatestCertificate(ctx context.Context, position float64) (bool, error) { func (p *Storage) ensureIsLatestCertificate(ctx context.Context, position decimal.Decimal) (bool, error) {
maxSequence, err := p.getMaxKeySequence(ctx) maxSequence, err := p.getMaxKeyPosition(ctx)
if err != nil { if err != nil {
return false, fmt.Errorf("error retrieving new events: %w", err) return false, fmt.Errorf("error retrieving new events: %w", err)
} }
return position >= maxSequence, nil return position.GreaterThanOrEqual(maxSequence), nil
} }
func (p *Storage) lockAndGenerateCertificateAndKey(ctx context.Context, usage crypto.KeyUsage) error { func (p *Storage) lockAndGenerateCertificateAndKey(ctx context.Context, usage crypto.KeyUsage) error {
@@ -151,9 +152,9 @@ func (p *Storage) lockAndGenerateCertificateAndKey(ctx context.Context, usage cr
} }
} }
func (p *Storage) getMaxKeySequence(ctx context.Context) (float64, error) { func (p *Storage) getMaxKeyPosition(ctx context.Context) (decimal.Decimal, error) {
return p.eventstore.LatestSequence(ctx, return p.eventstore.LatestPosition(ctx,
eventstore.NewSearchQueryBuilder(eventstore.ColumnsMaxSequence). eventstore.NewSearchQueryBuilder(eventstore.ColumnsMaxPosition).
ResourceOwner(authz.GetInstance(ctx).InstanceID()). ResourceOwner(authz.GetInstance(ctx).InstanceID()).
AwaitOpenTransactions(). AwaitOpenTransactions().
AddQuery(). AddQuery().

8
internal/auth/repository/eventsourcing/handler/handler.go
View File

@@ -2,8 +2,12 @@ package handler
import ( import (
"context" "context"
"fmt"
"time" "time"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/auth/repository/eventsourcing/view" "github.com/zitadel/zitadel/internal/auth/repository/eventsourcing/view"
"github.com/zitadel/zitadel/internal/database" "github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore" "github.com/zitadel/zitadel/internal/eventstore"
@@ -72,11 +76,13 @@ func Projections() []*handler2.Handler {
} }
func ProjectInstance(ctx context.Context) error { func ProjectInstance(ctx context.Context) error {
for _, projection := range projections { for i, projection := range projections {
logging.WithFields("name", projection.ProjectionName(), "instance", authz.GetInstance(ctx).InstanceID(), "index", fmt.Sprintf("%d/%d", i, len(projections))).Info("starting auth projection")
_, err := projection.Trigger(ctx) _, err := projection.Trigger(ctx)
if err != nil { if err != nil {
return err return err
} }
logging.WithFields("name", projection.ProjectionName(), "instance", authz.GetInstance(ctx).InstanceID(), "index", fmt.Sprintf("%d/%d", i, len(projections))).Info("auth projection done")
} }
return nil return nil
} }

10
internal/database/cockroach/crdb.go
View File

@@ -73,10 +73,16 @@ func (_ *Config) Decode(configs []interface{}) (dialect.Connector, error) {
} }
func (c *Config) Connect(useAdmin bool) (*sql.DB, *pgxpool.Pool, error) { func (c *Config) Connect(useAdmin bool) (*sql.DB, *pgxpool.Pool, error) {
dialect.RegisterAfterConnect(func(ctx context.Context, c *pgx.Conn) error { dialect.RegisterAfterConnect(func(ctx context.Context, conn *pgx.Conn) error {
// CockroachDB by default does not allow multiple modifications of the same table using ON CONFLICT // CockroachDB by default does not allow multiple modifications of the same table using ON CONFLICT
// This is needed to fill the fields table of the eventstore during eventstore.Push. // This is needed to fill the fields table of the eventstore during eventstore.Push.
_, err := c.Exec(ctx, "SET enable_multiple_modifications_of_table = on")
// This modification is only needed on crdb so we check if the connection is crdb
// postgres doesn't have parameter so we check if the parameter is empty
if conn.PgConn().ParameterStatus("crdb_version") == "" {
return nil
}
_, err := conn.Exec(ctx, "SET enable_multiple_modifications_of_table = on")
return err return err
}) })
connConfig := dialect.NewConnectionConfig(c.MaxOpenConns, c.MaxIdleConns) connConfig := dialect.NewConnectionConfig(c.MaxOpenConns, c.MaxIdleConns)

8
internal/database/dialect/connections.go
View File

@@ -5,6 +5,7 @@ import (
"errors" "errors"
"reflect" "reflect"
pgxdecimal "github.com/jackc/pgx-shopspring-decimal"
"github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype" "github.com/jackc/pgx/v5/pgtype"
) )
@@ -23,7 +24,12 @@ type ConnectionConfig struct {
AfterRelease []func(c *pgx.Conn) error AfterRelease []func(c *pgx.Conn) error
} }
var afterConnectFuncs []func(ctx context.Context, c *pgx.Conn) error var afterConnectFuncs = []func(ctx context.Context, c *pgx.Conn) error{
func(ctx context.Context, c *pgx.Conn) error {
pgxdecimal.Register(c.TypeMap())
return nil
},
}
func RegisterAfterConnect(f func(ctx context.Context, c *pgx.Conn) error) { func RegisterAfterConnect(f func(ctx context.Context, c *pgx.Conn) error) {
afterConnectFuncs = append(afterConnectFuncs, f) afterConnectFuncs = append(afterConnectFuncs, f)

4
internal/eventstore/event.go
View File

@@ -5,6 +5,8 @@ import (
"reflect" "reflect"
"time" "time"
"github.com/shopspring/decimal"
"github.com/zitadel/zitadel/internal/zerrors" "github.com/zitadel/zitadel/internal/zerrors"
) )
@@ -44,7 +46,7 @@ type Event interface {
// CreatedAt is the time the event was created at // CreatedAt is the time the event was created at
CreatedAt() time.Time CreatedAt() time.Time
// Position is the global position of the event // Position is the global position of the event
Position() float64 Position() decimal.Decimal
// Unmarshal parses the payload and stores the result // Unmarshal parses the payload and stores the result
// in the value pointed to by ptr. If ptr is nil or not a pointer, // in the value pointed to by ptr. If ptr is nil or not a pointer,

5
internal/eventstore/event_base.go
View File

@@ -7,6 +7,7 @@ import (
"strings" "strings"
"time" "time"
"github.com/shopspring/decimal"
"github.com/zitadel/logging" "github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/api/authz" "github.com/zitadel/zitadel/internal/api/authz"
@@ -25,7 +26,7 @@ type BaseEvent struct {
Agg *Aggregate Agg *Aggregate
Seq uint64 Seq uint64
Pos float64 Pos decimal.Decimal
Creation time.Time Creation time.Time
previousAggregateSequence uint64 previousAggregateSequence uint64
previousAggregateTypeSequence uint64 previousAggregateTypeSequence uint64
@@ -38,7 +39,7 @@ type BaseEvent struct {
} }
// Position implements Event. // Position implements Event.
func (e *BaseEvent) Position() float64 { func (e *BaseEvent) Position() decimal.Decimal {
return e.Pos return e.Pos
} }

11
internal/eventstore/eventstore.go
View File

@@ -7,6 +7,7 @@ import (
"time" "time"
"github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgconn"
"github.com/shopspring/decimal"
"github.com/zitadel/logging" "github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/api/authz" "github.com/zitadel/zitadel/internal/api/authz"
@@ -229,11 +230,11 @@ func (es *Eventstore) FilterToReducer(ctx context.Context, searchQuery *SearchQu
}) })
} }
// LatestSequence filters the latest sequence for the given search query // LatestPosition filters the latest position for the given search query
func (es *Eventstore) LatestSequence(ctx context.Context, queryFactory *SearchQueryBuilder) (float64, error) { func (es *Eventstore) LatestPosition(ctx context.Context, queryFactory *SearchQueryBuilder) (decimal.Decimal, error) {
queryFactory.InstanceID(authz.GetInstance(ctx).InstanceID()) queryFactory.InstanceID(authz.GetInstance(ctx).InstanceID())
return es.querier.LatestSequence(ctx, queryFactory) return es.querier.LatestPosition(ctx, queryFactory)
} }
// InstanceIDs returns the distinct instance ids found by the search query // InstanceIDs returns the distinct instance ids found by the search query
@@ -265,8 +266,8 @@ type Querier interface {
Health(ctx context.Context) error Health(ctx context.Context) error
// FilterToReducer calls r for every event returned from the storage // FilterToReducer calls r for every event returned from the storage
FilterToReducer(ctx context.Context, searchQuery *SearchQueryBuilder, r Reducer) error FilterToReducer(ctx context.Context, searchQuery *SearchQueryBuilder, r Reducer) error
// LatestSequence returns the latest sequence found by the search query // LatestPosition returns the latest position found by the search query
LatestSequence(ctx context.Context, queryFactory *SearchQueryBuilder) (float64, error) LatestPosition(ctx context.Context, queryFactory *SearchQueryBuilder) (decimal.Decimal, error)
// InstanceIDs returns the instance ids found by the search query // InstanceIDs returns the instance ids found by the search query
InstanceIDs(ctx context.Context, queryFactory *SearchQueryBuilder) ([]string, error) InstanceIDs(ctx context.Context, queryFactory *SearchQueryBuilder) ([]string, error)
// Client returns the underlying database connection // Client returns the underlying database connection

16
internal/eventstore/eventstore_querier_test.go
View File

@@ -4,6 +4,8 @@ import (
"context" "context"
"testing" "testing"
"github.com/shopspring/decimal"
"github.com/zitadel/zitadel/internal/eventstore" "github.com/zitadel/zitadel/internal/eventstore"
) )
@@ -131,7 +133,7 @@ func TestCRDB_Filter(t *testing.T) {
} }
} }
func TestCRDB_LatestSequence(t *testing.T) { func TestCRDB_LatestPosition(t *testing.T) {
type args struct { type args struct {
searchQuery *eventstore.SearchQueryBuilder searchQuery *eventstore.SearchQueryBuilder
} }
@@ -139,7 +141,7 @@ func TestCRDB_LatestSequence(t *testing.T) {
existingEvents []eventstore.Command existingEvents []eventstore.Command
} }
type res struct { type res struct {
sequence float64 position decimal.Decimal
} }
tests := []struct { tests := []struct {
name string name string
@@ -151,7 +153,7 @@ func TestCRDB_LatestSequence(t *testing.T) {
{ {
name: "aggregate type filter no sequence", name: "aggregate type filter no sequence",
args: args{ args: args{
searchQuery: eventstore.NewSearchQueryBuilder(eventstore.ColumnsMaxSequence). searchQuery: eventstore.NewSearchQueryBuilder(eventstore.ColumnsMaxPosition).
AddQuery(). AddQuery().
AggregateTypes("not found"). AggregateTypes("not found").
Builder(), Builder(),
@@ -168,7 +170,7 @@ func TestCRDB_LatestSequence(t *testing.T) {
{ {
name: "aggregate type filter sequence", name: "aggregate type filter sequence",
args: args{ args: args{
searchQuery: eventstore.NewSearchQueryBuilder(eventstore.ColumnsMaxSequence). searchQuery: eventstore.NewSearchQueryBuilder(eventstore.ColumnsMaxPosition).
AddQuery(). AddQuery().
AggregateTypes(eventstore.AggregateType(t.Name())). AggregateTypes(eventstore.AggregateType(t.Name())).
Builder(), Builder(),
@@ -202,12 +204,12 @@ func TestCRDB_LatestSequence(t *testing.T) {
return return
} }
sequence, err := db.LatestSequence(context.Background(), tt.args.searchQuery) position, err := db.LatestPosition(context.Background(), tt.args.searchQuery)
if (err != nil) != tt.wantErr { if (err != nil) != tt.wantErr {
t.Errorf("CRDB.query() error = %v, wantErr %v", err, tt.wantErr) t.Errorf("CRDB.query() error = %v, wantErr %v", err, tt.wantErr)
} }
if tt.res.sequence > sequence { if tt.res.position.GreaterThan(position) {
t.Errorf("CRDB.query() expected sequence: %v got %v", tt.res.sequence, sequence) t.Errorf("CRDB.query() expected sequence: %v got %v", tt.res.position, position)
} }
}) })
} }

17
internal/eventstore/eventstore_test.go
View File

@@ -9,6 +9,7 @@ import (
"time" "time"
"github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgconn"
"github.com/shopspring/decimal"
"github.com/zitadel/zitadel/internal/api/authz" "github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/api/service" "github.com/zitadel/zitadel/internal/api/service"
@@ -397,7 +398,7 @@ func (repo *testPusher) Push(_ context.Context, _ database.ContextQueryExecuter,
type testQuerier struct { type testQuerier struct {
events []Event events []Event
sequence float64 sequence decimal.Decimal
instances []string instances []string
err error err error
t *testing.T t *testing.T
@@ -430,9 +431,9 @@ func (repo *testQuerier) FilterToReducer(ctx context.Context, searchQuery *Searc
return nil return nil
} }
func (repo *testQuerier) LatestSequence(ctx context.Context, queryFactory *SearchQueryBuilder) (float64, error) { func (repo *testQuerier) LatestPosition(ctx context.Context, queryFactory *SearchQueryBuilder) (decimal.Decimal, error) {
if repo.err != nil { if repo.err != nil {
return 0, repo.err return decimal.Decimal{}, repo.err
} }
return repo.sequence, nil return repo.sequence, nil
} }
@@ -1076,7 +1077,7 @@ func TestEventstore_FilterEvents(t *testing.T) {
} }
} }
func TestEventstore_LatestSequence(t *testing.T) { func TestEventstore_LatestPosition(t *testing.T) {
type args struct { type args struct {
query *SearchQueryBuilder query *SearchQueryBuilder
} }
@@ -1096,7 +1097,7 @@ func TestEventstore_LatestSequence(t *testing.T) {
name: "no events", name: "no events",
args: args{ args: args{
query: &SearchQueryBuilder{ query: &SearchQueryBuilder{
columns: ColumnsMaxSequence, columns: ColumnsMaxPosition,
queries: []*SearchQuery{ queries: []*SearchQuery{
{ {
builder: &SearchQueryBuilder{}, builder: &SearchQueryBuilder{},
@@ -1119,7 +1120,7 @@ func TestEventstore_LatestSequence(t *testing.T) {
name: "repo error", name: "repo error",
args: args{ args: args{
query: &SearchQueryBuilder{ query: &SearchQueryBuilder{
columns: ColumnsMaxSequence, columns: ColumnsMaxPosition,
queries: []*SearchQuery{ queries: []*SearchQuery{
{ {
builder: &SearchQueryBuilder{}, builder: &SearchQueryBuilder{},
@@ -1142,7 +1143,7 @@ func TestEventstore_LatestSequence(t *testing.T) {
name: "found events", name: "found events",
args: args{ args: args{
query: &SearchQueryBuilder{ query: &SearchQueryBuilder{
columns: ColumnsMaxSequence, columns: ColumnsMaxPosition,
queries: []*SearchQuery{ queries: []*SearchQuery{
{ {
builder: &SearchQueryBuilder{}, builder: &SearchQueryBuilder{},
@@ -1168,7 +1169,7 @@ func TestEventstore_LatestSequence(t *testing.T) {
querier: tt.fields.repo, querier: tt.fields.repo,
} }
_, err := es.LatestSequence(context.Background(), tt.args.query) _, err := es.LatestPosition(context.Background(), tt.args.query)
if (err != nil) != tt.res.wantErr { if (err != nil) != tt.res.wantErr {
t.Errorf("Eventstore.aggregatesToEvents() error = %v, wantErr %v", err, tt.res.wantErr) t.Errorf("Eventstore.aggregatesToEvents() error = %v, wantErr %v", err, tt.res.wantErr)
} }

14
internal/eventstore/handler/v2/field_handler.go
View File

@@ -8,6 +8,7 @@ import (
"time" "time"
"github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgconn"
"github.com/shopspring/decimal"
"github.com/zitadel/zitadel/internal/eventstore" "github.com/zitadel/zitadel/internal/eventstore"
) )
@@ -126,10 +127,15 @@ func (h *FieldHandler) processEvents(ctx context.Context, config *triggerConfig)
return additionalIteration, err return additionalIteration, err
} }
// stop execution if currentState.eventTimestamp >= config.maxCreatedAt // stop execution if currentState.eventTimestamp >= config.maxCreatedAt
if config.maxPosition != 0 && currentState.position >= config.maxPosition { if !config.maxPosition.IsZero() && currentState.position.GreaterThanOrEqual(config.maxPosition) {
return false, nil return false, nil
} }
if config.minPosition.GreaterThan(decimal.NewFromInt(0)) {
currentState.position = config.minPosition
currentState.offset = 0
}
events, additionalIteration, err := h.fetchEvents(ctx, tx, currentState) events, additionalIteration, err := h.fetchEvents(ctx, tx, currentState)
if err != nil { if err != nil {
return additionalIteration, err return additionalIteration, err
@@ -159,7 +165,7 @@ func (h *FieldHandler) fetchEvents(ctx context.Context, tx *sql.Tx, currentState
idx, offset := skipPreviouslyReducedEvents(events, currentState) idx, offset := skipPreviouslyReducedEvents(events, currentState)
if currentState.position == events[len(events)-1].Position() { if currentState.position.Equal(events[len(events)-1].Position()) {
offset += currentState.offset offset += currentState.offset
} }
currentState.position = events[len(events)-1].Position() currentState.position = events[len(events)-1].Position()
@@ -189,9 +195,9 @@ func (h *FieldHandler) fetchEvents(ctx context.Context, tx *sql.Tx, currentState
} }
func skipPreviouslyReducedEvents(events []eventstore.Event, currentState *state) (index int, offset uint32) { func skipPreviouslyReducedEvents(events []eventstore.Event, currentState *state) (index int, offset uint32) {
var position float64 var position decimal.Decimal
for i, event := range events { for i, event := range events {
if event.Position() != position { if !event.Position().Equal(position) {
offset = 0 offset = 0
position = event.Position() position = event.Position()
} }

27
internal/eventstore/handler/v2/handler.go
View File

@@ -4,13 +4,13 @@ import (
"context" "context"
"database/sql" "database/sql"
"errors" "errors"
"math"
"math/rand" "math/rand"
"slices" "slices"
"sync" "sync"
"time" "time"
"github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgconn"
"github.com/shopspring/decimal"
"github.com/zitadel/logging" "github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/api/authz" "github.com/zitadel/zitadel/internal/api/authz"
@@ -381,7 +381,8 @@ func (h *Handler) existingInstances(ctx context.Context) ([]string, error) {
type triggerConfig struct { type triggerConfig struct {
awaitRunning bool awaitRunning bool
maxPosition float64 maxPosition decimal.Decimal
minPosition decimal.Decimal
} }
type TriggerOpt func(conf *triggerConfig) type TriggerOpt func(conf *triggerConfig)
@@ -392,12 +393,18 @@ func WithAwaitRunning() TriggerOpt {
} }
} }
func WithMaxPosition(position float64) TriggerOpt { func WithMaxPosition(position decimal.Decimal) TriggerOpt {
return func(conf *triggerConfig) { return func(conf *triggerConfig) {
conf.maxPosition = position conf.maxPosition = position
} }
} }
func WithMinPosition(position decimal.Decimal) TriggerOpt {
return func(conf *triggerConfig) {
conf.minPosition = position
}
}
func (h *Handler) Trigger(ctx context.Context, opts ...TriggerOpt) (_ context.Context, err error) { func (h *Handler) Trigger(ctx context.Context, opts ...TriggerOpt) (_ context.Context, err error) {
config := new(triggerConfig) config := new(triggerConfig)
for _, opt := range opts { for _, opt := range opts {
@@ -506,10 +513,15 @@ func (h *Handler) processEvents(ctx context.Context, config *triggerConfig) (add
return additionalIteration, err return additionalIteration, err
} }
// stop execution if currentState.position >= config.maxPosition // stop execution if currentState.position >= config.maxPosition
if config.maxPosition != 0 && currentState.position >= config.maxPosition { if !config.maxPosition.Equal(decimal.Decimal{}) && currentState.position.GreaterThanOrEqual(config.maxPosition) {
return false, nil return false, nil
} }
if config.minPosition.GreaterThan(decimal.NewFromInt(0)) {
currentState.position = config.minPosition
currentState.offset = 0
}
var statements []*Statement var statements []*Statement
statements, additionalIteration, err = h.generateStatements(ctx, tx, currentState) statements, additionalIteration, err = h.generateStatements(ctx, tx, currentState)
if err != nil { if err != nil {
@@ -601,7 +613,7 @@ func (h *Handler) generateStatements(ctx context.Context, tx *sql.Tx, currentSta
func skipPreviouslyReducedStatements(statements []*Statement, currentState *state) int { func skipPreviouslyReducedStatements(statements []*Statement, currentState *state) int {
for i, statement := range statements { for i, statement := range statements {
if statement.Position == currentState.position && if statement.Position.Equal(currentState.position) &&
statement.Aggregate.ID == currentState.aggregateID && statement.Aggregate.ID == currentState.aggregateID &&
statement.Aggregate.Type == currentState.aggregateType && statement.Aggregate.Type == currentState.aggregateType &&
statement.Sequence == currentState.sequence { statement.Sequence == currentState.sequence {
@@ -665,9 +677,8 @@ func (h *Handler) eventQuery(currentState *state) *eventstore.SearchQueryBuilder
OrderAsc(). OrderAsc().
InstanceID(currentState.instanceID) InstanceID(currentState.instanceID)
if currentState.position > 0 { if currentState.position.GreaterThan(decimal.Decimal{}) {
// decrease position by 10 because builder.PositionAfter filters for position > and we need position >= builder = builder.PositionAtLeast(currentState.position)
builder = builder.PositionAfter(math.Float64frombits(math.Float64bits(currentState.position) - 10))
if currentState.offset > 0 { if currentState.offset > 0 {
builder = builder.Offset(currentState.offset) builder = builder.Offset(currentState.offset)
} }

8
internal/eventstore/handler/v2/state.go
View File

@@ -7,6 +7,8 @@ import (
"errors" "errors"
"time" "time"
"github.com/shopspring/decimal"
"github.com/zitadel/zitadel/internal/api/authz" "github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/eventstore" "github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/zerrors" "github.com/zitadel/zitadel/internal/zerrors"
@@ -14,7 +16,7 @@ import (
type state struct { type state struct {
instanceID string instanceID string
position float64 position decimal.Decimal
eventTimestamp time.Time eventTimestamp time.Time
aggregateType eventstore.AggregateType aggregateType eventstore.AggregateType
aggregateID string aggregateID string
@@ -45,7 +47,7 @@ func (h *Handler) currentState(ctx context.Context, tx *sql.Tx, config *triggerC
aggregateType = new(sql.NullString) aggregateType = new(sql.NullString)
sequence = new(sql.NullInt64) sequence = new(sql.NullInt64)
timestamp = new(sql.NullTime) timestamp = new(sql.NullTime)
position = new(sql.NullFloat64) position = new(decimal.NullDecimal)
offset = new(sql.NullInt64) offset = new(sql.NullInt64)
) )
@@ -75,7 +77,7 @@ func (h *Handler) currentState(ctx context.Context, tx *sql.Tx, config *triggerC
currentState.aggregateType = eventstore.AggregateType(aggregateType.String) currentState.aggregateType = eventstore.AggregateType(aggregateType.String)
currentState.sequence = uint64(sequence.Int64) currentState.sequence = uint64(sequence.Int64)
currentState.eventTimestamp = timestamp.Time currentState.eventTimestamp = timestamp.Time
currentState.position = position.Float64 currentState.position = position.Decimal
// psql does not provide unsigned numbers so we work around it // psql does not provide unsigned numbers so we work around it
currentState.offset = uint32(offset.Int64) currentState.offset = uint32(offset.Int64)
return currentState, nil return currentState, nil

13
internal/eventstore/handler/v2/state_test.go
View File

@@ -11,6 +11,7 @@ import (
"time" "time"
"github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgconn"
"github.com/shopspring/decimal"
"github.com/zitadel/zitadel/internal/api/authz" "github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/database/mock" "github.com/zitadel/zitadel/internal/database/mock"
@@ -166,7 +167,7 @@ func TestHandler_updateLastUpdated(t *testing.T) {
updatedState: &state{ updatedState: &state{
instanceID: "instance", instanceID: "instance",
eventTimestamp: time.Now(), eventTimestamp: time.Now(),
position: 42, position: decimal.NewFromInt(42),
}, },
}, },
isErr: func(t *testing.T, err error) { isErr: func(t *testing.T, err error) {
@@ -192,7 +193,7 @@ func TestHandler_updateLastUpdated(t *testing.T) {
updatedState: &state{ updatedState: &state{
instanceID: "instance", instanceID: "instance",
eventTimestamp: time.Now(), eventTimestamp: time.Now(),
position: 42, position: decimal.NewFromInt(42),
}, },
}, },
isErr: func(t *testing.T, err error) { isErr: func(t *testing.T, err error) {
@@ -217,7 +218,7 @@ func TestHandler_updateLastUpdated(t *testing.T) {
eventstore.AggregateType("aggregate type"), eventstore.AggregateType("aggregate type"),
uint64(42), uint64(42),
mock.AnyType[time.Time]{}, mock.AnyType[time.Time]{},
float64(42), decimal.NewFromInt(42),
uint32(0), uint32(0),
), ),
mock.WithExecRowsAffected(1), mock.WithExecRowsAffected(1),
@@ -228,7 +229,7 @@ func TestHandler_updateLastUpdated(t *testing.T) {
updatedState: &state{ updatedState: &state{
instanceID: "instance", instanceID: "instance",
eventTimestamp: time.Now(), eventTimestamp: time.Now(),
position: 42, position: decimal.NewFromInt(42),
aggregateType: "aggregate type", aggregateType: "aggregate type",
aggregateID: "aggregate id", aggregateID: "aggregate id",
sequence: 42, sequence: 42,
@@ -397,7 +398,7 @@ func TestHandler_currentState(t *testing.T) {
"aggregate type", "aggregate type",
int64(42), int64(42),
testTime, testTime,
float64(42), decimal.NewFromInt(42).String(),
uint16(10), uint16(10),
}, },
}, },
@@ -412,7 +413,7 @@ func TestHandler_currentState(t *testing.T) {
currentState: &state{ currentState: &state{
instanceID: "instance", instanceID: "instance",
eventTimestamp: testTime, eventTimestamp: testTime,
position: 42, position: decimal.NewFromInt(42),
aggregateType: "aggregate type", aggregateType: "aggregate type",
aggregateID: "aggregate id", aggregateID: "aggregate id",
sequence: 42, sequence: 42,

3
internal/eventstore/handler/v2/statement.go
View File

@@ -9,6 +9,7 @@ import (
"strings" "strings"
"time" "time"
"github.com/shopspring/decimal"
"github.com/zitadel/logging" "github.com/zitadel/logging"
"golang.org/x/exp/constraints" "golang.org/x/exp/constraints"
@@ -82,7 +83,7 @@ func (h *Handler) reduce(event eventstore.Event) (*Statement, error) {
type Statement struct { type Statement struct {
Aggregate *eventstore.Aggregate Aggregate *eventstore.Aggregate
Sequence uint64 Sequence uint64
Position float64 Position decimal.Decimal
CreationDate time.Time CreationDate time.Time
offset uint32 offset uint32

22
internal/eventstore/local_crdb_test.go
View File

@@ -2,13 +2,14 @@ package eventstore_test
import ( import (
"context" "context"
"database/sql"
"encoding/json" "encoding/json"
"os" "os"
"testing" "testing"
"time" "time"
"github.com/cockroachdb/cockroach-go/v2/testserver" "github.com/cockroachdb/cockroach-go/v2/testserver"
pgxdecimal "github.com/jackc/pgx-shopspring-decimal"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool" "github.com/jackc/pgx/v5/pgxpool"
"github.com/jackc/pgx/v5/stdlib" "github.com/jackc/pgx/v5/stdlib"
"github.com/zitadel/logging" "github.com/zitadel/logging"
@@ -41,13 +42,15 @@ func TestMain(m *testing.M) {
testCRDBClient = &database.DB{ testCRDBClient = &database.DB{
Database: new(testDB), Database: new(testDB),
} }
config, err := pgxpool.ParseConfig(ts.PGURL().String())
connConfig, err := pgxpool.ParseConfig(ts.PGURL().String())
if err != nil { if err != nil {
logging.WithFields("error", err).Fatal("unable to parse db url") logging.WithFields("error", err).Fatal("unable to parse db config")
} }
connConfig.AfterConnect = new_es.RegisterEventstoreTypes config.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error {
pool, err := pgxpool.NewWithConfig(context.Background(), connConfig) pgxdecimal.Register(conn.TypeMap())
return new_es.RegisterEventstoreTypes(ctx, conn)
}
pool, err := pgxpool.NewWithConfig(context.Background(), config)
if err != nil { if err != nil {
logging.WithFields("error", err).Fatal("unable to create db pool") logging.WithFields("error", err).Fatal("unable to create db pool")
} }
@@ -112,10 +115,15 @@ func initDB(ctx context.Context, db *database.DB) error {
} }
func connectLocalhost() (*database.DB, error) { func connectLocalhost() (*database.DB, error) {
client, err := sql.Open("pgx", "postgresql://root@localhost:26257/defaultdb?sslmode=disable") config, err := pgxpool.ParseConfig("postgresql://root@localhost:26257/defaultdb?sslmode=disable")
if err != nil { if err != nil {
return nil, err return nil, err
} }
pool, err := pgxpool.NewWithConfig(context.Background(), config)
if err != nil {
return nil, err
}
client := stdlib.OpenDBFromPool(pool)
if err = client.Ping(); err != nil { if err = client.Ping(); err != nil {
return nil, err return nil, err
} }

22
internal/eventstore/read_model.go
View File

@@ -1,19 +1,23 @@
package eventstore package eventstore
import "time" import (
"time"
"github.com/shopspring/decimal"
)
// ReadModel is the minimum representation of a read model. // ReadModel is the minimum representation of a read model.
// It implements a basic reducer // It implements a basic reducer
// it might be saved in a database or in memory // it might be saved in a database or in memory
type ReadModel struct { type ReadModel struct {
AggregateID string `json:"-"` AggregateID string `json:"-"`
ProcessedSequence uint64 `json:"-"` ProcessedSequence uint64 `json:"-"`
CreationDate time.Time `json:"-"` CreationDate time.Time `json:"-"`
ChangeDate time.Time `json:"-"` ChangeDate time.Time `json:"-"`
Events []Event `json:"-"` Events []Event `json:"-"`
ResourceOwner string `json:"-"` ResourceOwner string `json:"-"`
InstanceID string `json:"-"` InstanceID string `json:"-"`
Position float64 `json:"-"` Position decimal.Decimal `json:"-"`
} }
// AppendEvents adds all the events to the read model. // AppendEvents adds all the events to the read model.

5
internal/eventstore/repository/event.go
View File

@@ -7,6 +7,7 @@ import (
"strings" "strings"
"time" "time"
"github.com/shopspring/decimal"
"github.com/zitadel/logging" "github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/eventstore" "github.com/zitadel/zitadel/internal/eventstore"
@@ -22,7 +23,7 @@ type Event struct {
// Seq is the sequence of the event // Seq is the sequence of the event
Seq uint64 Seq uint64
// Pos is the global sequence of the event multiple events can have the same sequence // Pos is the global sequence of the event multiple events can have the same sequence
Pos float64 Pos decimal.Decimal
//CreationDate is the time the event is created //CreationDate is the time the event is created
// it's used for human readability. // it's used for human readability.
@@ -97,7 +98,7 @@ func (e *Event) Sequence() uint64 {
} }
// Position implements [eventstore.Event] // Position implements [eventstore.Event]
func (e *Event) Position() float64 { func (e *Event) Position() decimal.Decimal {
return e.Pos return e.Pos
} }

15
internal/eventstore/repository/mock/repository.mock.go
View File

@@ -13,6 +13,7 @@ import (
context "context" context "context"
reflect "reflect" reflect "reflect"
decimal "github.com/shopspring/decimal"
database "github.com/zitadel/zitadel/internal/database" database "github.com/zitadel/zitadel/internal/database"
eventstore "github.com/zitadel/zitadel/internal/eventstore" eventstore "github.com/zitadel/zitadel/internal/eventstore"
gomock "go.uber.org/mock/gomock" gomock "go.uber.org/mock/gomock"
@@ -98,19 +99,19 @@ func (mr *MockQuerierMockRecorder) InstanceIDs(arg0, arg1 any) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstanceIDs", reflect.TypeOf((*MockQuerier)(nil).InstanceIDs), arg0, arg1) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstanceIDs", reflect.TypeOf((*MockQuerier)(nil).InstanceIDs), arg0, arg1)
} }
// LatestSequence mocks base method. // LatestPosition mocks base method.
func (m *MockQuerier) LatestSequence(arg0 context.Context, arg1 *eventstore.SearchQueryBuilder) (float64, error) { func (m *MockQuerier) LatestPosition(arg0 context.Context, arg1 *eventstore.SearchQueryBuilder) (decimal.Decimal, error) {
m.ctrl.T.Helper() m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "LatestSequence", arg0, arg1) ret := m.ctrl.Call(m, "LatestPosition", arg0, arg1)
ret0, _ := ret[0].(float64) ret0, _ := ret[0].(decimal.Decimal)
ret1, _ := ret[1].(error) ret1, _ := ret[1].(error)
return ret0, ret1 return ret0, ret1
} }
// LatestSequence indicates an expected call of LatestSequence. // LatestPosition indicates an expected call of LatestPosition.
func (mr *MockQuerierMockRecorder) LatestSequence(arg0, arg1 any) *gomock.Call { func (mr *MockQuerierMockRecorder) LatestPosition(arg0, arg1 any) *gomock.Call {
mr.mock.ctrl.T.Helper() mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LatestSequence", reflect.TypeOf((*MockQuerier)(nil).LatestSequence), arg0, arg1) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LatestPosition", reflect.TypeOf((*MockQuerier)(nil).LatestPosition), arg0, arg1)
} }
// MockPusher is a mock of Pusher interface. // MockPusher is a mock of Pusher interface.

5
internal/eventstore/repository/mock/repository.mock.impl.go
View File

@@ -7,6 +7,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/shopspring/decimal"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"go.uber.org/mock/gomock" "go.uber.org/mock/gomock"
@@ -187,8 +188,8 @@ func (e *mockEvent) Sequence() uint64 {
return e.sequence return e.sequence
} }
func (e *mockEvent) Position() float64 { func (e *mockEvent) Position() decimal.Decimal {
return 0 return decimal.Decimal{}
} }
func (e *mockEvent) CreatedAt() time.Time { func (e *mockEvent) CreatedAt() time.Time {

10
internal/eventstore/repository/search_query.go
View File

@@ -3,6 +3,8 @@ package repository
import ( import (
"database/sql" "database/sql"
"github.com/shopspring/decimal"
"github.com/zitadel/zitadel/internal/database" "github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore" "github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/zerrors" "github.com/zitadel/zitadel/internal/zerrors"
@@ -58,6 +60,8 @@ const (
//OperationNotIn checks if a stored value does not match one of the passed value list //OperationNotIn checks if a stored value does not match one of the passed value list
OperationNotIn OperationNotIn
OperationGreaterOrEquals
operationCount operationCount
) )
@@ -252,10 +256,10 @@ func instanceIDsFilter(builder *eventstore.SearchQueryBuilder, query *SearchQuer
} }
func positionAfterFilter(builder *eventstore.SearchQueryBuilder, query *SearchQuery) *Filter { func positionAfterFilter(builder *eventstore.SearchQueryBuilder, query *SearchQuery) *Filter {
if builder.GetPositionAfter() == 0 { if builder.GetPositionAtLeast().IsZero() {
return nil return nil
} }
query.Position = NewFilter(FieldPosition, builder.GetPositionAfter(), OperationGreater) query.Position = NewFilter(FieldPosition, builder.GetPositionAtLeast(), OperationGreaterOrEquals)
return query.Position return query.Position
} }
@@ -297,7 +301,7 @@ func eventDataFilter(query *eventstore.SearchQuery) *Filter {
} }
func eventPositionAfterFilter(query *eventstore.SearchQuery) *Filter { func eventPositionAfterFilter(query *eventstore.SearchQuery) *Filter {
if pos := query.GetPositionAfter(); pos != 0 { if pos := query.GetPositionAfter(); !pos.Equal(decimal.Decimal{}) {
return NewFilter(FieldPosition, pos, OperationGreater) return NewFilter(FieldPosition, pos, OperationGreater)
} }
return nil return nil

13
internal/eventstore/repository/sql/crdb.go
View File

@@ -11,6 +11,7 @@ import (
"github.com/cockroachdb/cockroach-go/v2/crdb" "github.com/cockroachdb/cockroach-go/v2/crdb"
"github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgconn"
"github.com/shopspring/decimal"
"github.com/zitadel/logging" "github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/api/authz" "github.com/zitadel/zitadel/internal/api/authz"
@@ -264,11 +265,11 @@ func (crdb *CRDB) FilterToReducer(ctx context.Context, searchQuery *eventstore.S
return err return err
} }
// LatestSequence returns the latest sequence found by the search query // LatestPosition returns the latest position found by the search query
func (db *CRDB) LatestSequence(ctx context.Context, searchQuery *eventstore.SearchQueryBuilder) (float64, error) { func (db *CRDB) LatestPosition(ctx context.Context, searchQuery *eventstore.SearchQueryBuilder) (decimal.Decimal, error) {
var position sql.NullFloat64 var position decimal.Decimal
err := query(ctx, db, searchQuery, &position, false) err := query(ctx, db, searchQuery, &position, false)
return position.Float64, err return position, err
} }
// InstanceIDs returns the instance ids found by the search query // InstanceIDs returns the instance ids found by the search query
@@ -335,7 +336,7 @@ func (db *CRDB) eventQuery(useV1 bool) string {
" FROM eventstore.events2" " FROM eventstore.events2"
} }
func (db *CRDB) maxSequenceQuery(useV1 bool) string { func (db *CRDB) maxPositionQuery(useV1 bool) string {
if useV1 { if useV1 {
return `SELECT event_sequence FROM eventstore.events` return `SELECT event_sequence FROM eventstore.events`
} }
@@ -413,6 +414,8 @@ func (db *CRDB) operation(operation repository.Operation) string {
return "=" return "="
case repository.OperationGreater: case repository.OperationGreater:
return ">" return ">"
case repository.OperationGreaterOrEquals:
return ">="
case repository.OperationLess: case repository.OperationLess:
return "<" return "<"
case repository.OperationJSONContains: case repository.OperationJSONContains:

4
internal/eventstore/repository/sql/crdb_test.go
View File

@@ -4,6 +4,8 @@ import (
"database/sql" "database/sql"
"testing" "testing"
"github.com/shopspring/decimal"
"github.com/zitadel/zitadel/internal/eventstore" "github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/eventstore/repository" "github.com/zitadel/zitadel/internal/eventstore/repository"
) )
@@ -312,7 +314,7 @@ func generateEvent(t *testing.T, aggregateID string, opts ...func(*repository.Ev
ResourceOwner: sql.NullString{String: "ro", Valid: true}, ResourceOwner: sql.NullString{String: "ro", Valid: true},
Typ: "test.created", Typ: "test.created",
Version: "v1", Version: "v1",
Pos: 42, Pos: decimal.NewFromInt(42),
} }
for _, opt := range opts { for _, opt := range opts {

8
internal/eventstore/repository/sql/local_crdb_test.go
View File

@@ -8,6 +8,8 @@ import (
"time" "time"
"github.com/cockroachdb/cockroach-go/v2/testserver" "github.com/cockroachdb/cockroach-go/v2/testserver"
pgxdecimal "github.com/jackc/pgx-shopspring-decimal"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool" "github.com/jackc/pgx/v5/pgxpool"
"github.com/jackc/pgx/v5/stdlib" "github.com/jackc/pgx/v5/stdlib"
"github.com/zitadel/logging" "github.com/zitadel/logging"
@@ -15,7 +17,6 @@ import (
"github.com/zitadel/zitadel/cmd/initialise" "github.com/zitadel/zitadel/cmd/initialise"
"github.com/zitadel/zitadel/internal/database" "github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/database/cockroach" "github.com/zitadel/zitadel/internal/database/cockroach"
new_es "github.com/zitadel/zitadel/internal/eventstore/v3"
) )
var ( var (
@@ -36,7 +37,10 @@ func TestMain(m *testing.M) {
if err != nil { if err != nil {
logging.WithFields("error", err).Fatal("unable to parse db url") logging.WithFields("error", err).Fatal("unable to parse db url")
} }
connConfig.AfterConnect = new_es.RegisterEventstoreTypes connConfig.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error {
pgxdecimal.Register(conn.TypeMap())
return nil
}
pool, err := pgxpool.NewWithConfig(context.Background(), connConfig) pool, err := pgxpool.NewWithConfig(context.Background(), connConfig)
if err != nil { if err != nil {
logging.WithFields("error", err).Fatal("unable to create db pool") logging.WithFields("error", err).Fatal("unable to create db pool")

25
internal/eventstore/repository/sql/query.go
View File

@@ -9,6 +9,7 @@ import (
"strconv" "strconv"
"strings" "strings"
"github.com/shopspring/decimal"
"github.com/zitadel/logging" "github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/api/call" "github.com/zitadel/zitadel/internal/api/call"
@@ -25,7 +26,7 @@ type querier interface {
conditionFormat(repository.Operation) string conditionFormat(repository.Operation) string
placeholder(query string) string placeholder(query string) string
eventQuery(useV1 bool) string eventQuery(useV1 bool) string
maxSequenceQuery(useV1 bool) string maxPositionQuery(useV1 bool) string
instanceIDsQuery(useV1 bool) string instanceIDsQuery(useV1 bool) string
Client() *database.DB Client() *database.DB
orderByEventSequence(desc, shouldOrderBySequence, useV1 bool) string orderByEventSequence(desc, shouldOrderBySequence, useV1 bool) string
@@ -74,7 +75,7 @@ func query(ctx context.Context, criteria querier, searchQuery *eventstore.Search
// instead of using the max function of the database (which doesn't work for postgres) // instead of using the max function of the database (which doesn't work for postgres)
// we select the most recent row // we select the most recent row
if q.Columns == eventstore.ColumnsMaxSequence { if q.Columns == eventstore.ColumnsMaxPosition {
q.Limit = 1 q.Limit = 1
q.Desc = true q.Desc = true
} }
@@ -91,7 +92,7 @@ func query(ctx context.Context, criteria querier, searchQuery *eventstore.Search
switch q.Columns { switch q.Columns {
case eventstore.ColumnsEvent, case eventstore.ColumnsEvent,
eventstore.ColumnsMaxSequence: eventstore.ColumnsMaxPosition:
query += criteria.orderByEventSequence(q.Desc, shouldOrderBySequence, useV1) query += criteria.orderByEventSequence(q.Desc, shouldOrderBySequence, useV1)
} }
@@ -147,8 +148,8 @@ func query(ctx context.Context, criteria querier, searchQuery *eventstore.Search
func prepareColumns(criteria querier, columns eventstore.Columns, useV1 bool) (string, func(s scan, dest interface{}) error) { func prepareColumns(criteria querier, columns eventstore.Columns, useV1 bool) (string, func(s scan, dest interface{}) error) {
switch columns { switch columns {
case eventstore.ColumnsMaxSequence: case eventstore.ColumnsMaxPosition:
return criteria.maxSequenceQuery(useV1), maxSequenceScanner return criteria.maxPositionQuery(useV1), maxPositionScanner
case eventstore.ColumnsInstanceIDs: case eventstore.ColumnsInstanceIDs:
return criteria.instanceIDsQuery(useV1), instanceIDsScanner return criteria.instanceIDsQuery(useV1), instanceIDsScanner
case eventstore.ColumnsEvent: case eventstore.ColumnsEvent:
@@ -166,13 +167,15 @@ func prepareTimeTravel(ctx context.Context, criteria querier, allow bool) string
return criteria.Timetravel(took) return criteria.Timetravel(took)
} }
func maxSequenceScanner(row scan, dest interface{}) (err error) { func maxPositionScanner(row scan, dest interface{}) (err error) {
position, ok := dest.(*sql.NullFloat64) position, ok := dest.(*decimal.Decimal)
if !ok { if !ok {
return zerrors.ThrowInvalidArgumentf(nil, "SQL-NBjA9", "type must be sql.NullInt64 got: %T", dest) return zerrors.ThrowInvalidArgumentf(nil, "SQL-NBjA9", "type must be decimal.Decimal got: %T", dest)
} }
err = row(position) var res decimal.NullDecimal
err = row(&res)
if err == nil || errors.Is(err, sql.ErrNoRows) { if err == nil || errors.Is(err, sql.ErrNoRows) {
*position = res.Decimal
return nil return nil
} }
return zerrors.ThrowInternal(err, "SQL-bN5xg", "something went wrong") return zerrors.ThrowInternal(err, "SQL-bN5xg", "something went wrong")
@@ -201,7 +204,7 @@ func eventsScanner(useV1 bool) func(scanner scan, dest interface{}) (err error)
return zerrors.ThrowInvalidArgumentf(nil, "SQL-4GP6F", "events scanner: invalid type %T", dest) return zerrors.ThrowInvalidArgumentf(nil, "SQL-4GP6F", "events scanner: invalid type %T", dest)
} }
event := new(repository.Event) event := new(repository.Event)
position := new(sql.NullFloat64) position := new(decimal.NullDecimal)
if useV1 { if useV1 {
err = scanner( err = scanner(
@@ -238,7 +241,7 @@ func eventsScanner(useV1 bool) func(scanner scan, dest interface{}) (err error)
logging.New().WithError(err).Warn("unable to scan row") logging.New().WithError(err).Warn("unable to scan row")
return zerrors.ThrowInternal(err, "SQL-M0dsf", "unable to scan row") return zerrors.ThrowInternal(err, "SQL-M0dsf", "unable to scan row")
} }
event.Pos = position.Float64 event.Pos = position.Decimal
return reduce(event) return reduce(event)
} }
} }

39
internal/eventstore/repository/sql/query_test.go
View File

@@ -11,6 +11,7 @@ import (
"time" "time"
"github.com/DATA-DOG/go-sqlmock" "github.com/DATA-DOG/go-sqlmock"
"github.com/shopspring/decimal"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/zitadel/zitadel/internal/database" "github.com/zitadel/zitadel/internal/database"
@@ -110,36 +111,36 @@ func Test_prepareColumns(t *testing.T) {
{ {
name: "max column", name: "max column",
args: args{ args: args{
columns: eventstore.ColumnsMaxSequence, columns: eventstore.ColumnsMaxPosition,
dest: new(sql.NullFloat64), dest: new(decimal.Decimal),
useV1: true, useV1: true,
}, },
res: res{ res: res{
query: `SELECT event_sequence FROM eventstore.events`, query: `SELECT event_sequence FROM eventstore.events`,
expected: sql.NullFloat64{Float64: 43, Valid: true}, expected: decimal.NewFromInt(42),
}, },
fields: fields{ fields: fields{
dbRow: []interface{}{sql.NullFloat64{Float64: 43, Valid: true}}, dbRow: []interface{}{decimal.NewNullDecimal(decimal.NewFromInt(42))},
}, },
}, },
{ {
name: "max column v2", name: "max column v2",
args: args{ args: args{
columns: eventstore.ColumnsMaxSequence, columns: eventstore.ColumnsMaxPosition,
dest: new(sql.NullFloat64), dest: new(decimal.Decimal),
}, },
res: res{ res: res{
query: `SELECT "position" FROM eventstore.events2`, query: `SELECT "position" FROM eventstore.events2`,
expected: sql.NullFloat64{Float64: 43, Valid: true}, expected: decimal.NewFromInt(42),
}, },
fields: fields{ fields: fields{
dbRow: []interface{}{sql.NullFloat64{Float64: 43, Valid: true}}, dbRow: []interface{}{decimal.NewNullDecimal(decimal.NewFromInt(42))},
}, },
}, },
{ {
name: "max sequence wrong dest type", name: "max sequence wrong dest type",
args: args{ args: args{
columns: eventstore.ColumnsMaxSequence, columns: eventstore.ColumnsMaxPosition,
dest: new(uint64), dest: new(uint64),
}, },
res: res{ res: res{
@@ -179,11 +180,11 @@ func Test_prepareColumns(t *testing.T) {
res: res{ res: res{
query: `SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision FROM eventstore.events2`, query: `SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision FROM eventstore.events2`,
expected: []eventstore.Event{ expected: []eventstore.Event{
&repository.Event{AggregateID: "hodor", AggregateType: "user", Seq: 5, Pos: 42, Data: nil, Version: "v1"}, &repository.Event{AggregateID: "hodor", AggregateType: "user", Seq: 5, Pos: decimal.NewFromInt(42), Data: nil, Version: "v1"},
}, },
}, },
fields: fields{ fields: fields{
dbRow: []interface{}{time.Time{}, eventstore.EventType(""), uint64(5), sql.NullFloat64{Float64: 42, Valid: true}, sql.RawBytes(nil), "", sql.NullString{}, "", eventstore.AggregateType("user"), "hodor", uint8(1)}, dbRow: []interface{}{time.Time{}, eventstore.EventType(""), uint64(5), decimal.NewNullDecimal(decimal.NewFromInt(42)), sql.RawBytes(nil), "", sql.NullString{}, "", eventstore.AggregateType("user"), "hodor", uint8(1)},
}, },
}, },
{ {
@@ -198,11 +199,11 @@ func Test_prepareColumns(t *testing.T) {
res: res{ res: res{
query: `SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision FROM eventstore.events2`, query: `SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision FROM eventstore.events2`,
expected: []eventstore.Event{ expected: []eventstore.Event{
&repository.Event{AggregateID: "hodor", AggregateType: "user", Seq: 5, Pos: 0, Data: nil, Version: "v1"}, &repository.Event{AggregateID: "hodor", AggregateType: "user", Seq: 5, Pos: decimal.Decimal{}, Data: nil, Version: "v1"},
}, },
}, },
fields: fields{ fields: fields{
dbRow: []interface{}{time.Time{}, eventstore.EventType(""), uint64(5), sql.NullFloat64{Float64: 0, Valid: false}, sql.RawBytes(nil), "", sql.NullString{}, "", eventstore.AggregateType("user"), "hodor", uint8(1)}, dbRow: []interface{}{time.Time{}, eventstore.EventType(""), uint64(5), decimal.NullDecimal{}, sql.RawBytes(nil), "", sql.NullString{}, "", eventstore.AggregateType("user"), "hodor", uint8(1)},
}, },
}, },
{ {
@@ -1006,7 +1007,7 @@ func Test_query_events_mocked(t *testing.T) {
InstanceID("instanceID"). InstanceID("instanceID").
OrderDesc(). OrderDesc().
Limit(5). Limit(5).
PositionAfter(123.456). PositionAtLeast(decimal.NewFromFloat(123.456)).
AddQuery(). AddQuery().
AggregateTypes("notify"). AggregateTypes("notify").
EventTypes("notify.foo.bar"). EventTypes("notify.foo.bar").
@@ -1020,9 +1021,9 @@ func Test_query_events_mocked(t *testing.T) {
fields: fields{ fields: fields{
mock: newMockClient(t).expectQuery(t, mock: newMockClient(t).expectQuery(t,
regexp.QuoteMeta( regexp.QuoteMeta(
`SELECT creation_date, event_type, event_sequence, event_data, editor_user, resource_owner, instance_id, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE instance_id = $1 AND aggregate_type = $2 AND event_type = $3 AND "position" > $4 AND aggregate_id NOT IN (SELECT aggregate_id FROM eventstore.events WHERE aggregate_type = $5 AND event_type = ANY($6) AND instance_id = $7 AND "position" > $8) ORDER BY event_sequence DESC LIMIT $9`, `SELECT creation_date, event_type, event_sequence, event_data, editor_user, resource_owner, instance_id, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE instance_id = $1 AND aggregate_type = $2 AND event_type = $3 AND "position" >= $4 AND aggregate_id NOT IN (SELECT aggregate_id FROM eventstore.events WHERE aggregate_type = $5 AND event_type = ANY($6) AND instance_id = $7 AND "position" >= $8) ORDER BY event_sequence DESC LIMIT $9`,
), ),
[]driver.Value{"instanceID", eventstore.AggregateType("notify"), eventstore.EventType("notify.foo.bar"), 123.456, eventstore.AggregateType("notify"), []eventstore.EventType{"notification.failed", "notification.success"}, "instanceID", 123.456, uint64(5)}, []driver.Value{"instanceID", eventstore.AggregateType("notify"), eventstore.EventType("notify.foo.bar"), decimal.NewFromFloat(123.456), eventstore.AggregateType("notify"), []eventstore.EventType{"notification.failed", "notification.success"}, "instanceID", decimal.NewFromFloat(123.456), uint64(5)},
), ),
}, },
res: res{ res: res{
@@ -1037,7 +1038,7 @@ func Test_query_events_mocked(t *testing.T) {
InstanceID("instanceID"). InstanceID("instanceID").
OrderDesc(). OrderDesc().
Limit(5). Limit(5).
PositionAfter(123.456). PositionAtLeast(decimal.NewFromFloat(123.456)).
AddQuery(). AddQuery().
AggregateTypes("notify"). AggregateTypes("notify").
EventTypes("notify.foo.bar"). EventTypes("notify.foo.bar").
@@ -1051,9 +1052,9 @@ func Test_query_events_mocked(t *testing.T) {
fields: fields{ fields: fields{
mock: newMockClient(t).expectQuery(t, mock: newMockClient(t).expectQuery(t,
regexp.QuoteMeta( regexp.QuoteMeta(
`SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision FROM eventstore.events2 WHERE instance_id = $1 AND aggregate_type = $2 AND event_type = $3 AND "position" > $4 AND aggregate_id NOT IN (SELECT aggregate_id FROM eventstore.events2 WHERE aggregate_type = $5 AND event_type = ANY($6) AND instance_id = $7 AND "position" > $8) ORDER BY "position" DESC, in_tx_order DESC LIMIT $9`, `SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision FROM eventstore.events2 WHERE instance_id = $1 AND aggregate_type = $2 AND event_type = $3 AND "position" >= $4 AND aggregate_id NOT IN (SELECT aggregate_id FROM eventstore.events2 WHERE aggregate_type = $5 AND event_type = ANY($6) AND instance_id = $7 AND "position" >= $8) ORDER BY "position" DESC, in_tx_order DESC LIMIT $9`,
), ),
[]driver.Value{"instanceID", eventstore.AggregateType("notify"), eventstore.EventType("notify.foo.bar"), 123.456, eventstore.AggregateType("notify"), []eventstore.EventType{"notification.failed", "notification.success"}, "instanceID", 123.456, uint64(5)}, []driver.Value{"instanceID", eventstore.AggregateType("notify"), eventstore.EventType("notify.foo.bar"), decimal.NewFromFloat(123.456), eventstore.AggregateType("notify"), []eventstore.EventType{"notification.failed", "notification.success"}, "instanceID", decimal.NewFromFloat(123.456), uint64(5)},
), ),
}, },
res: res{ res: res{

24
internal/eventstore/search_query.go
View File

@@ -5,6 +5,8 @@ import (
"database/sql" "database/sql"
"time" "time"
"github.com/shopspring/decimal"
"github.com/zitadel/zitadel/internal/api/authz" "github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/zerrors" "github.com/zitadel/zitadel/internal/zerrors"
) )
@@ -26,7 +28,7 @@ type SearchQueryBuilder struct {
lockRows bool lockRows bool
lockOption LockOption lockOption LockOption
allowTimeTravel bool allowTimeTravel bool
positionAfter float64 positionAtLeast decimal.Decimal
awaitOpenTransactions bool awaitOpenTransactions bool
creationDateAfter time.Time creationDateAfter time.Time
creationDateBefore time.Time creationDateBefore time.Time
@@ -81,8 +83,8 @@ func (b *SearchQueryBuilder) GetAllowTimeTravel() bool {
return b.allowTimeTravel return b.allowTimeTravel
} }
func (b SearchQueryBuilder) GetPositionAfter() float64 { func (b SearchQueryBuilder) GetPositionAtLeast() decimal.Decimal {
return b.positionAfter return b.positionAtLeast
} }
func (b SearchQueryBuilder) GetAwaitOpenTransactions() bool { func (b SearchQueryBuilder) GetAwaitOpenTransactions() bool {
@@ -118,7 +120,7 @@ type SearchQuery struct {
aggregateIDs []string aggregateIDs []string
eventTypes []EventType eventTypes []EventType
eventData map[string]interface{} eventData map[string]interface{}
positionAfter float64 positionAfter decimal.Decimal
} }
func (q SearchQuery) GetAggregateTypes() []AggregateType { func (q SearchQuery) GetAggregateTypes() []AggregateType {
@@ -137,7 +139,7 @@ func (q SearchQuery) GetEventData() map[string]interface{} {
return q.eventData return q.eventData
} }
func (q SearchQuery) GetPositionAfter() float64 { func (q SearchQuery) GetPositionAfter() decimal.Decimal {
return q.positionAfter return q.positionAfter
} }
@@ -161,8 +163,8 @@ type Columns int8
const ( const (
//ColumnsEvent represents all fields of an event //ColumnsEvent represents all fields of an event
ColumnsEvent = iota + 1 ColumnsEvent = iota + 1
// ColumnsMaxSequence represents the latest sequence of the filtered events // ColumnsMaxPosition represents the latest sequence of the filtered events
ColumnsMaxSequence ColumnsMaxPosition
// ColumnsInstanceIDs represents the instance ids of the filtered events // ColumnsInstanceIDs represents the instance ids of the filtered events
ColumnsInstanceIDs ColumnsInstanceIDs
@@ -296,9 +298,9 @@ func (builder *SearchQueryBuilder) AllowTimeTravel() *SearchQueryBuilder {
return builder return builder
} }
// PositionAfter filters for events which happened after the specified time // PositionAtLeast filters for events which happened after the specified time
func (builder *SearchQueryBuilder) PositionAfter(position float64) *SearchQueryBuilder { func (builder *SearchQueryBuilder) PositionAtLeast(position decimal.Decimal) *SearchQueryBuilder {
builder.positionAfter = position builder.positionAtLeast = position
return builder return builder
} }
@@ -405,7 +407,7 @@ func (query *SearchQuery) EventData(data map[string]interface{}) *SearchQuery {
return query return query
} }
func (query *SearchQuery) PositionAfter(position float64) *SearchQuery { func (query *SearchQuery) PositionAfter(position decimal.Decimal) *SearchQuery {
query.positionAfter = position query.positionAfter = position
return query return query
} }

4
internal/eventstore/search_query_test.go
View File

@@ -116,10 +116,10 @@ func TestSearchQuerybuilderSetters(t *testing.T) {
{ {
name: "set columns", name: "set columns",
args: args{ args: args{
setters: []func(*SearchQueryBuilder) *SearchQueryBuilder{testSetColumns(ColumnsMaxSequence)}, setters: []func(*SearchQueryBuilder) *SearchQueryBuilder{testSetColumns(ColumnsMaxPosition)},
}, },
res: &SearchQueryBuilder{ res: &SearchQueryBuilder{
columns: ColumnsMaxSequence, columns: ColumnsMaxPosition,
}, },
}, },
{ {

6
internal/eventstore/v1/models/event.go
View File

@@ -5,6 +5,8 @@ import (
"reflect" "reflect"
"time" "time"
"github.com/shopspring/decimal"
"github.com/zitadel/zitadel/internal/eventstore" "github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/zerrors" "github.com/zitadel/zitadel/internal/zerrors"
) )
@@ -20,7 +22,7 @@ var _ eventstore.Event = (*Event)(nil)
type Event struct { type Event struct {
ID string ID string
Seq uint64 Seq uint64
Pos float64 Pos decimal.Decimal
CreationDate time.Time CreationDate time.Time
Typ eventstore.EventType Typ eventstore.EventType
PreviousSequence uint64 PreviousSequence uint64
@@ -80,7 +82,7 @@ func (e *Event) Sequence() uint64 {
} }
// Position implements [eventstore.Event] // Position implements [eventstore.Event]
func (e *Event) Position() float64 { func (e *Event) Position() decimal.Decimal {
return e.Pos return e.Pos
} }

7
internal/eventstore/v3/event.go
View File

@@ -6,6 +6,7 @@ import (
"strconv" "strconv"
"time" "time"
"github.com/shopspring/decimal"
"github.com/zitadel/logging" "github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/api/authz" "github.com/zitadel/zitadel/internal/api/authz"
@@ -42,7 +43,7 @@ type event struct {
command *command command *command
createdAt time.Time createdAt time.Time
sequence uint64 sequence uint64
position float64 position decimal.Decimal
} }
// TODO: remove on v3 // TODO: remove on v3
@@ -152,8 +153,8 @@ func (e *event) Sequence() uint64 {
return e.sequence return e.sequence
} }
// Sequence implements [eventstore.Event] // Position implements [eventstore.Event]
func (e *event) Position() float64 { func (e *event) Position() decimal.Decimal {
return e.position return e.position
} }

2
internal/execution/execution_test.go
View File

@@ -61,7 +61,7 @@ func Test_Call(t *testing.T) {
args{ args{
ctx: context.Background(), ctx: context.Background(),
timeout: time.Second, timeout: time.Second,
sleep: time.Second, sleep: 2 * time.Second,
method: http.MethodPost, method: http.MethodPost,
body: []byte("{\"request\": \"values\"}"), body: []byte("{\"request\": \"values\"}"),
respBody: []byte("{\"response\": \"values\"}"), respBody: []byte("{\"response\": \"values\"}"),

28
internal/notification/projections.go
View File

@@ -2,8 +2,12 @@ package notification
import ( import (
"context" "context"
"fmt"
"time" "time"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/command" "github.com/zitadel/zitadel/internal/command"
"github.com/zitadel/zitadel/internal/crypto" "github.com/zitadel/zitadel/internal/crypto"
"github.com/zitadel/zitadel/internal/database" "github.com/zitadel/zitadel/internal/database"
@@ -63,12 +67,34 @@ func Start(ctx context.Context) {
worker.Start(ctx) worker.Start(ctx)
} }
func SetCurrentState(ctx context.Context, es *eventstore.Eventstore) error {
if len(projections) == 0 {
return nil
}
position, err := es.LatestPosition(ctx, eventstore.NewSearchQueryBuilder(eventstore.ColumnsMaxPosition).InstanceID(authz.GetInstance(ctx).InstanceID()).OrderDesc().Limit(1))
if err != nil {
return err
}
for i, projection := range projections {
logging.WithFields("name", projection.ProjectionName(), "instance", authz.GetInstance(ctx).InstanceID(), "index", fmt.Sprintf("%d/%d", i, len(projections))).Info("set current state of notification projection")
_, err = projection.Trigger(ctx, handler.WithMinPosition(position))
if err != nil {
return err
}
logging.WithFields("name", projection.ProjectionName(), "instance", authz.GetInstance(ctx).InstanceID(), "index", fmt.Sprintf("%d/%d", i, len(projections))).Info("current state of notification projection set")
}
return nil
}
func ProjectInstance(ctx context.Context) error { func ProjectInstance(ctx context.Context) error {
for _, projection := range projections { for i, projection := range projections {
logging.WithFields("name", projection.ProjectionName(), "instance", authz.GetInstance(ctx).InstanceID(), "index", fmt.Sprintf("%d/%d", i, len(projections))).Info("starting notification projection")
_, err := projection.Trigger(ctx) _, err := projection.Trigger(ctx)
if err != nil { if err != nil {
return err return err
} }
logging.WithFields("name", projection.ProjectionName(), "instance", authz.GetInstance(ctx).InstanceID(), "index", fmt.Sprintf("%d/%d", i, len(projections))).Info("notification projection done")
} }
return nil return nil
} }

5
internal/query/access_token.go
View File

@@ -5,6 +5,7 @@ import (
"strings" "strings"
"time" "time"
"github.com/shopspring/decimal"
"golang.org/x/text/language" "golang.org/x/text/language"
"github.com/zitadel/zitadel/internal/domain" "github.com/zitadel/zitadel/internal/domain"
@@ -140,7 +141,7 @@ func (q *Queries) accessTokenByOIDCSessionAndTokenID(ctx context.Context, oidcSe
// checkSessionNotTerminatedAfter checks if a [session.TerminateType] event (or user events leading to a session termination) // checkSessionNotTerminatedAfter checks if a [session.TerminateType] event (or user events leading to a session termination)
// occurred after a certain time and will return an error if so. // occurred after a certain time and will return an error if so.
func (q *Queries) checkSessionNotTerminatedAfter(ctx context.Context, sessionID, userID string, position float64, fingerprintID string) (err error) { func (q *Queries) checkSessionNotTerminatedAfter(ctx context.Context, sessionID, userID string, position decimal.Decimal, fingerprintID string) (err error) {
ctx, span := tracing.NewSpan(ctx) ctx, span := tracing.NewSpan(ctx)
defer func() { span.EndWithError(err) }() defer func() { span.EndWithError(err) }()
@@ -165,7 +166,7 @@ func (q *Queries) checkSessionNotTerminatedAfter(ctx context.Context, sessionID,
} }
type sessionTerminatedModel struct { type sessionTerminatedModel struct {
position float64 position decimal.Decimal
sessionID string sessionID string
userID string userID string
fingerPrintID string fingerPrintID string

11
internal/query/current_state.go
View File

@@ -10,6 +10,7 @@ import (
"time" "time"
sq "github.com/Masterminds/squirrel" sq "github.com/Masterminds/squirrel"
"github.com/shopspring/decimal"
"github.com/zitadel/logging" "github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/api/call" "github.com/zitadel/zitadel/internal/api/call"
@@ -26,7 +27,7 @@ type Stateful interface {
type State struct { type State struct {
LastRun time.Time LastRun time.Time
Position float64 Position decimal.Decimal
EventCreatedAt time.Time EventCreatedAt time.Time
AggregateID string AggregateID string
AggregateType eventstore.AggregateType AggregateType eventstore.AggregateType
@@ -221,7 +222,7 @@ func prepareLatestState(ctx context.Context, db prepareDatabase) (sq.SelectBuild
var ( var (
creationDate sql.NullTime creationDate sql.NullTime
lastUpdated sql.NullTime lastUpdated sql.NullTime
position sql.NullFloat64 position decimal.NullDecimal
) )
err := row.Scan( err := row.Scan(
&creationDate, &creationDate,
@@ -234,7 +235,7 @@ func prepareLatestState(ctx context.Context, db prepareDatabase) (sq.SelectBuild
return &State{ return &State{
EventCreatedAt: creationDate.Time, EventCreatedAt: creationDate.Time,
LastRun: lastUpdated.Time, LastRun: lastUpdated.Time,
Position: position.Float64, Position: position.Decimal,
}, nil }, nil
} }
} }
@@ -259,7 +260,7 @@ func prepareCurrentStateQuery(ctx context.Context, db prepareDatabase) (sq.Selec
var ( var (
lastRun sql.NullTime lastRun sql.NullTime
eventDate sql.NullTime eventDate sql.NullTime
currentPosition sql.NullFloat64 currentPosition decimal.NullDecimal
aggregateType sql.NullString aggregateType sql.NullString
aggregateID sql.NullString aggregateID sql.NullString
sequence sql.NullInt64 sequence sql.NullInt64
@@ -280,7 +281,7 @@ func prepareCurrentStateQuery(ctx context.Context, db prepareDatabase) (sq.Selec
} }
currentState.State.EventCreatedAt = eventDate.Time currentState.State.EventCreatedAt = eventDate.Time
currentState.State.LastRun = lastRun.Time currentState.State.LastRun = lastRun.Time
currentState.Position = currentPosition.Float64 currentState.Position = currentPosition.Decimal
currentState.AggregateType = eventstore.AggregateType(aggregateType.String) currentState.AggregateType = eventstore.AggregateType(aggregateType.String)
currentState.AggregateID = aggregateID.String currentState.AggregateID = aggregateID.String
currentState.Sequence = uint64(sequence.Int64) currentState.Sequence = uint64(sequence.Int64)

8
internal/query/current_state_test.go
View File

@@ -7,6 +7,8 @@ import (
"fmt" "fmt"
"regexp" "regexp"
"testing" "testing"
"github.com/shopspring/decimal"
) )
var ( var (
@@ -87,7 +89,7 @@ func Test_CurrentSequencesPrepares(t *testing.T) {
State: State{ State: State{
EventCreatedAt: testNow, EventCreatedAt: testNow,
LastRun: testNow, LastRun: testNow,
Position: 20211108, Position: decimal.NewFromInt(20211108),
AggregateID: "agg-id", AggregateID: "agg-id",
AggregateType: "agg-type", AggregateType: "agg-type",
Sequence: 20211108, Sequence: 20211108,
@@ -134,7 +136,7 @@ func Test_CurrentSequencesPrepares(t *testing.T) {
ProjectionName: "projection-name", ProjectionName: "projection-name",
State: State{ State: State{
EventCreatedAt: testNow, EventCreatedAt: testNow,
Position: 20211108, Position: decimal.NewFromInt(20211108),
LastRun: testNow, LastRun: testNow,
AggregateID: "agg-id", AggregateID: "agg-id",
AggregateType: "agg-type", AggregateType: "agg-type",
@@ -145,7 +147,7 @@ func Test_CurrentSequencesPrepares(t *testing.T) {
ProjectionName: "projection-name2", ProjectionName: "projection-name2",
State: State{ State: State{
EventCreatedAt: testNow, EventCreatedAt: testNow,
Position: 20211108, Position: decimal.NewFromInt(20211108),
LastRun: testNow, LastRun: testNow,
AggregateID: "agg-id", AggregateID: "agg-id",
AggregateType: "agg-type", AggregateType: "agg-type",

34
internal/query/projection/projection.go
View File

@@ -2,6 +2,9 @@ package projection
import ( import (
"context" "context"
"fmt"
"github.com/zitadel/logging"
internal_authz "github.com/zitadel/zitadel/internal/api/authz" internal_authz "github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/crypto" "github.com/zitadel/zitadel/internal/crypto"
@@ -89,6 +92,7 @@ var (
) )
type projection interface { type projection interface {
ProjectionName() string
Start(ctx context.Context) Start(ctx context.Context)
Init(ctx context.Context) error Init(ctx context.Context) error
Trigger(ctx context.Context, opts ...handler.TriggerOpt) (_ context.Context, err error) Trigger(ctx context.Context, opts ...handler.TriggerOpt) (_ context.Context, err error)
@@ -97,6 +101,7 @@ type projection interface {
var ( var (
projections []projection projections []projection
fields []*handler.FieldHandler
) )
func Create(ctx context.Context, sqlClient *database.DB, es handler.EventStore, config Config, keyEncryptionAlgorithm crypto.EncryptionAlgorithm, certEncryptionAlgorithm crypto.EncryptionAlgorithm, systemUsers map[string]*internal_authz.SystemAPIUser) error { func Create(ctx context.Context, sqlClient *database.DB, es handler.EventStore, config Config, keyEncryptionAlgorithm crypto.EncryptionAlgorithm, certEncryptionAlgorithm crypto.EncryptionAlgorithm, systemUsers map[string]*internal_authz.SystemAPIUser) error {
@@ -178,6 +183,7 @@ func Create(ctx context.Context, sqlClient *database.DB, es handler.EventStore,
MembershipFields = newFillMembershipFields(applyCustomConfig(projectionConfig, config.Customizations[fieldsMemberships])) MembershipFields = newFillMembershipFields(applyCustomConfig(projectionConfig, config.Customizations[fieldsMemberships]))
newProjectionsList() newProjectionsList()
newFieldsList()
return nil return nil
} }
@@ -201,7 +207,8 @@ func Start(ctx context.Context) {
} }
func ProjectInstance(ctx context.Context) error { func ProjectInstance(ctx context.Context) error {
for _, projection := range projections { for i, projection := range projections {
logging.WithFields("name", projection.ProjectionName(), "instance", internal_authz.GetInstance(ctx).InstanceID(), "index", fmt.Sprintf("%d/%d", i, len(projections))).Info("starting projection")
_, err := projection.Trigger(ctx) _, err := projection.Trigger(ctx)
if err != nil { if err != nil {
return err return err
@@ -210,6 +217,18 @@ func ProjectInstance(ctx context.Context) error {
return nil return nil
} }
func ProjectInstanceFields(ctx context.Context) error {
for i, fieldProjection := range fields {
logging.WithFields("name", fieldProjection.ProjectionName(), "instance", internal_authz.GetInstance(ctx).InstanceID(), "index", fmt.Sprintf("%d/%d", i, len(fields))).Info("starting fields projection")
err := fieldProjection.Trigger(ctx)
if err != nil {
return err
}
logging.WithFields("name", fieldProjection.ProjectionName(), "instance", internal_authz.GetInstance(ctx).InstanceID()).Info("fields projection done")
}
return nil
}
func ApplyCustomConfig(customConfig CustomConfig) handler.Config { func ApplyCustomConfig(customConfig CustomConfig) handler.Config {
return applyCustomConfig(projectionConfig, customConfig) return applyCustomConfig(projectionConfig, customConfig)
} }
@@ -234,6 +253,19 @@ func applyCustomConfig(config handler.Config, customConfig CustomConfig) handler
return config return config
} }
// we know this is ugly, but we need to have a singleton slice of all projections
// and are only able to initialize it after all projections are created
// as setup and start currently create them individually, we make sure we get the right one
// will be refactored when changing to new id based projections
func newFieldsList() {
fields = []*handler.FieldHandler{
ProjectGrantFields,
OrgDomainVerifiedFields,
InstanceDomainFields,
MembershipFields,
}
}
// we know this is ugly, but we need to have a singleton slice of all projections // we know this is ugly, but we need to have a singleton slice of all projections
// and are only able to initialize it after all projections are created // and are only able to initialize it after all projections are created
// as setup and start currently create them individually, we make sure we get the right one // as setup and start currently create them individually, we make sure we get the right one

4
internal/query/user_grant.go
View File

@@ -281,7 +281,7 @@ func (q *Queries) UserGrants(ctx context.Context, queries *UserGrantsQueries, sh
return nil, zerrors.ThrowInternal(err, "QUERY-wXnQR", "Errors.Query.SQLStatement") return nil, zerrors.ThrowInternal(err, "QUERY-wXnQR", "Errors.Query.SQLStatement")
} }
latestSequence, err := q.latestState(ctx, userGrantTable) latestState, err := q.latestState(ctx, userGrantTable)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -294,7 +294,7 @@ func (q *Queries) UserGrants(ctx context.Context, queries *UserGrantsQueries, sh
return nil, err return nil, err
} }
grants.State = latestSequence grants.State = latestState
return grants, nil return grants, nil
} }

4
internal/query/user_membership.go
View File

@@ -144,7 +144,7 @@ func (q *Queries) Memberships(ctx context.Context, queries *MembershipSearchQuer
if err != nil { if err != nil {
return nil, zerrors.ThrowInvalidArgument(err, "QUERY-T84X9", "Errors.Query.InvalidRequest") return nil, zerrors.ThrowInvalidArgument(err, "QUERY-T84X9", "Errors.Query.InvalidRequest")
} }
latestSequence, err := q.latestState(ctx, orgMemberTable, instanceMemberTable, projectMemberTable, projectGrantMemberTable) latestState, err := q.latestState(ctx, orgMemberTable, instanceMemberTable, projectMemberTable, projectGrantMemberTable)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -157,7 +157,7 @@ func (q *Queries) Memberships(ctx context.Context, queries *MembershipSearchQuer
if err != nil { if err != nil {
return nil, err return nil, err
} }
memberships.State = latestSequence memberships.State = latestState
return memberships, nil return memberships, nil
} }

3
internal/v2/database/number_filter.go
View File

@@ -3,6 +3,7 @@ package database
import ( import (
"time" "time"
"github.com/shopspring/decimal"
"github.com/zitadel/logging" "github.com/zitadel/logging"
"golang.org/x/exp/constraints" "golang.org/x/exp/constraints"
) )
@@ -94,7 +95,7 @@ func (c numberCompare) String() string {
} }
type number interface { type number interface {
constraints.Integer | constraints.Float | time.Time constraints.Integer | constraints.Float | time.Time | decimal.Decimal
// TODO: condition must know if it's args are named parameters or not // TODO: condition must know if it's args are named parameters or not
// constraints.Integer | constraints.Float | time.Time | placeholder // constraints.Integer | constraints.Float | time.Time | placeholder
} }

6
internal/v2/eventstore/event_store.go
View File

@@ -2,6 +2,8 @@ package eventstore
import ( import (
"context" "context"
"github.com/shopspring/decimal"
) )
func NewEventstore(querier Querier, pusher Pusher) *EventStore { func NewEventstore(querier Querier, pusher Pusher) *EventStore {
@@ -30,12 +32,12 @@ type healthier interface {
} }
type GlobalPosition struct { type GlobalPosition struct {
Position float64 Position decimal.Decimal
InPositionOrder uint32 InPositionOrder uint32
} }
func (gp GlobalPosition) IsLess(other GlobalPosition) bool { func (gp GlobalPosition) IsLess(other GlobalPosition) bool {
return gp.Position < other.Position || (gp.Position == other.Position && gp.InPositionOrder < other.InPositionOrder) return gp.Position.LessThan(other.Position) || (gp.Position.Equal(other.Position) && gp.InPositionOrder < other.InPositionOrder)
} }
type Reducer interface { type Reducer interface {

24
internal/v2/eventstore/postgres/push_test.go
View File

@@ -8,6 +8,8 @@ import (
"testing" "testing"
"time" "time"
"github.com/shopspring/decimal"
"github.com/zitadel/zitadel/internal/v2/database/mock" "github.com/zitadel/zitadel/internal/v2/database/mock"
"github.com/zitadel/zitadel/internal/v2/eventstore" "github.com/zitadel/zitadel/internal/v2/eventstore"
"github.com/zitadel/zitadel/internal/zerrors" "github.com/zitadel/zitadel/internal/zerrors"
@@ -818,7 +820,7 @@ func Test_push(t *testing.T) {
[][]driver.Value{ [][]driver.Value{
{ {
time.Now(), time.Now(),
float64(123), decimal.NewFromFloat(123).String(),
}, },
}, },
), ),
@@ -899,11 +901,11 @@ func Test_push(t *testing.T) {
[][]driver.Value{ [][]driver.Value{
{ {
time.Now(), time.Now(),
float64(123), decimal.NewFromFloat(123).String(),
}, },
{ {
time.Now(), time.Now(),
float64(123.1), decimal.NewFromFloat(123.1).String(),
}, },
}, },
), ),
@@ -984,11 +986,11 @@ func Test_push(t *testing.T) {
[][]driver.Value{ [][]driver.Value{
{ {
time.Now(), time.Now(),
float64(123), decimal.NewFromFloat(123).String(),
}, },
{ {
time.Now(), time.Now(),
float64(123.1), decimal.NewFromFloat(123.1).String(),
}, },
}, },
), ),
@@ -1044,7 +1046,7 @@ func Test_push(t *testing.T) {
[][]driver.Value{ [][]driver.Value{
{ {
time.Now(), time.Now(),
float64(123), decimal.NewFromFloat(123).String(),
}, },
}, },
), ),
@@ -1099,7 +1101,7 @@ func Test_push(t *testing.T) {
[][]driver.Value{ [][]driver.Value{
{ {
time.Now(), time.Now(),
float64(123), decimal.NewFromFloat(123).String(),
}, },
}, },
), ),
@@ -1181,11 +1183,11 @@ func Test_push(t *testing.T) {
[][]driver.Value{ [][]driver.Value{
{ {
time.Now(), time.Now(),
float64(123), decimal.NewFromFloat(123).String(),
}, },
{ {
time.Now(), time.Now(),
float64(123.1), decimal.NewFromFloat(123.1).String(),
}, },
}, },
), ),
@@ -1272,11 +1274,11 @@ func Test_push(t *testing.T) {
[][]driver.Value{ [][]driver.Value{
{ {
time.Now(), time.Now(),
float64(123), decimal.NewFromFloat(123).String(),
}, },
{ {
time.Now(), time.Now(),
float64(123.1), decimal.NewFromFloat(123.1).String(),
}, },
}, },
), ),

48
internal/v2/eventstore/postgres/query_test.go
View File

@@ -8,6 +8,8 @@ import (
"testing" "testing"
"time" "time"
"github.com/shopspring/decimal"
"github.com/zitadel/zitadel/internal/v2/database" "github.com/zitadel/zitadel/internal/v2/database"
"github.com/zitadel/zitadel/internal/v2/database/mock" "github.com/zitadel/zitadel/internal/v2/database/mock"
"github.com/zitadel/zitadel/internal/v2/eventstore" "github.com/zitadel/zitadel/internal/v2/eventstore"
@@ -541,13 +543,13 @@ func Test_writeFilter(t *testing.T) {
args: args{ args: args{
filter: eventstore.NewFilter( filter: eventstore.NewFilter(
eventstore.FilterPagination( eventstore.FilterPagination(
eventstore.PositionGreater(123.4, 0), eventstore.PositionGreater(decimal.NewFromFloat(123.4), 0),
), ),
), ),
}, },
want: wantQuery{ want: wantQuery{
query: " WHERE instance_id = $1 AND position > $2 ORDER BY position, in_tx_order", query: " WHERE instance_id = $1 AND position > $2 ORDER BY position, in_tx_order",
args: []any{"i1", 123.4}, args: []any{"i1", decimal.NewFromFloat(123.4)},
}, },
}, },
{ {
@@ -555,18 +557,18 @@ func Test_writeFilter(t *testing.T) {
args: args{ args: args{
filter: eventstore.NewFilter( filter: eventstore.NewFilter(
eventstore.FilterPagination( eventstore.FilterPagination(
// eventstore.PositionGreater(123.4, 0), // eventstore.PositionGreater(decimal.NewFromFloat(123.4), 0),
// eventstore.PositionLess(125.4, 10), // eventstore.PositionLess(125.4, 10),
eventstore.PositionBetween( eventstore.PositionBetween(
&eventstore.GlobalPosition{Position: 123.4}, &eventstore.GlobalPosition{Position: decimal.NewFromFloat(123.4)},
&eventstore.GlobalPosition{Position: 125.4, InPositionOrder: 10}, &eventstore.GlobalPosition{Position: decimal.NewFromFloat(125.4), InPositionOrder: 10},
), ),
), ),
), ),
}, },
want: wantQuery{ want: wantQuery{
query: " WHERE instance_id = $1 AND ((position = $2 AND in_tx_order < $3) OR position < $4) AND position > $5 ORDER BY position, in_tx_order", query: " WHERE instance_id = $1 AND ((position = $2 AND in_tx_order < $3) OR position < $4) AND position > $5 ORDER BY position, in_tx_order",
args: []any{"i1", 125.4, uint32(10), 125.4, 123.4}, args: []any{"i1", decimal.NewFromFloat(125.4), uint32(10), decimal.NewFromFloat(125.4), decimal.NewFromFloat(123.4)},
// TODO: (adlerhurst) would require some refactoring to reuse existing args // TODO: (adlerhurst) would require some refactoring to reuse existing args
// query: " WHERE instance_id = $1 AND position > $2 AND ((position = $3 AND in_tx_order < $4) OR position < $3) ORDER BY position, in_tx_order", // query: " WHERE instance_id = $1 AND position > $2 AND ((position = $3 AND in_tx_order < $4) OR position < $3) ORDER BY position, in_tx_order",
// args: []any{"i1", 123.4, 125.4, uint32(10)}, // args: []any{"i1", 123.4, 125.4, uint32(10)},
@@ -577,13 +579,13 @@ func Test_writeFilter(t *testing.T) {
args: args{ args: args{
filter: eventstore.NewFilter( filter: eventstore.NewFilter(
eventstore.FilterPagination( eventstore.FilterPagination(
eventstore.PositionGreater(123.4, 12), eventstore.PositionGreater(decimal.NewFromFloat(123.4), 12),
), ),
), ),
}, },
want: wantQuery{ want: wantQuery{
query: " WHERE instance_id = $1 AND ((position = $2 AND in_tx_order > $3) OR position > $4) ORDER BY position, in_tx_order", query: " WHERE instance_id = $1 AND ((position = $2 AND in_tx_order > $3) OR position > $4) ORDER BY position, in_tx_order",
args: []any{"i1", 123.4, uint32(12), 123.4}, args: []any{"i1", decimal.NewFromFloat(123.4), uint32(12), decimal.NewFromFloat(123.4)},
}, },
}, },
{ {
@@ -593,13 +595,13 @@ func Test_writeFilter(t *testing.T) {
eventstore.FilterPagination( eventstore.FilterPagination(
eventstore.Limit(10), eventstore.Limit(10),
eventstore.Offset(3), eventstore.Offset(3),
eventstore.PositionGreater(123.4, 12), eventstore.PositionGreater(decimal.NewFromFloat(123.4), 12),
), ),
), ),
}, },
want: wantQuery{ want: wantQuery{
query: " WHERE instance_id = $1 AND ((position = $2 AND in_tx_order > $3) OR position > $4) ORDER BY position, in_tx_order LIMIT $5 OFFSET $6", query: " WHERE instance_id = $1 AND ((position = $2 AND in_tx_order > $3) OR position > $4) ORDER BY position, in_tx_order LIMIT $5 OFFSET $6",
args: []any{"i1", 123.4, uint32(12), 123.4, uint32(10), uint32(3)}, args: []any{"i1", decimal.NewFromFloat(123.4), uint32(12), decimal.NewFromFloat(123.4), uint32(10), uint32(3)},
}, },
}, },
{ {
@@ -609,14 +611,14 @@ func Test_writeFilter(t *testing.T) {
eventstore.FilterPagination( eventstore.FilterPagination(
eventstore.Limit(10), eventstore.Limit(10),
eventstore.Offset(3), eventstore.Offset(3),
eventstore.PositionGreater(123.4, 12), eventstore.PositionGreater(decimal.NewFromFloat(123.4), 12),
), ),
eventstore.AppendAggregateFilter("user"), eventstore.AppendAggregateFilter("user"),
), ),
}, },
want: wantQuery{ want: wantQuery{
query: " WHERE instance_id = $1 AND aggregate_type = $2 AND ((position = $3 AND in_tx_order > $4) OR position > $5) ORDER BY position, in_tx_order LIMIT $6 OFFSET $7", query: " WHERE instance_id = $1 AND aggregate_type = $2 AND ((position = $3 AND in_tx_order > $4) OR position > $5) ORDER BY position, in_tx_order LIMIT $6 OFFSET $7",
args: []any{"i1", "user", 123.4, uint32(12), 123.4, uint32(10), uint32(3)}, args: []any{"i1", "user", decimal.NewFromFloat(123.4), uint32(12), decimal.NewFromFloat(123.4), uint32(10), uint32(3)},
}, },
}, },
{ {
@@ -626,7 +628,7 @@ func Test_writeFilter(t *testing.T) {
eventstore.FilterPagination( eventstore.FilterPagination(
eventstore.Limit(10), eventstore.Limit(10),
eventstore.Offset(3), eventstore.Offset(3),
eventstore.PositionGreater(123.4, 12), eventstore.PositionGreater(decimal.NewFromFloat(123.4), 12),
), ),
eventstore.AppendAggregateFilter("user"), eventstore.AppendAggregateFilter("user"),
eventstore.AppendAggregateFilter( eventstore.AppendAggregateFilter(
@@ -637,7 +639,7 @@ func Test_writeFilter(t *testing.T) {
}, },
want: wantQuery{ want: wantQuery{
query: " WHERE instance_id = $1 AND (aggregate_type = $2 OR (aggregate_type = $3 AND aggregate_id = $4)) AND ((position = $5 AND in_tx_order > $6) OR position > $7) ORDER BY position, in_tx_order LIMIT $8 OFFSET $9", query: " WHERE instance_id = $1 AND (aggregate_type = $2 OR (aggregate_type = $3 AND aggregate_id = $4)) AND ((position = $5 AND in_tx_order > $6) OR position > $7) ORDER BY position, in_tx_order LIMIT $8 OFFSET $9",
args: []any{"i1", "user", "org", "o1", 123.4, uint32(12), 123.4, uint32(10), uint32(3)}, args: []any{"i1", "user", "org", "o1", decimal.NewFromFloat(123.4), uint32(12), decimal.NewFromFloat(123.4), uint32(10), uint32(3)},
}, },
}, },
} }
@@ -956,7 +958,7 @@ func Test_writeQueryUse_examples(t *testing.T) {
), ),
eventstore.FilterPagination( eventstore.FilterPagination(
// used because we need to check for first login and an app which is not console // used because we need to check for first login and an app which is not console
eventstore.PositionGreater(12, 4), eventstore.PositionGreater(decimal.NewFromInt(12), 4),
), ),
), ),
eventstore.NewFilter( eventstore.NewFilter(
@@ -1065,9 +1067,9 @@ func Test_writeQueryUse_examples(t *testing.T) {
"instance", "instance",
"user", "user",
"user.token.added", "user.token.added",
float64(12), decimal.NewFromInt(12),
uint32(4), uint32(4),
float64(12), decimal.NewFromInt(12),
"instance", "instance",
"instance", "instance",
[]string{"instance.idp.config.added", "instance.idp.oauth.added", "instance.idp.oidc.added", "instance.idp.jwt.added", "instance.idp.azure.added", "instance.idp.github.added", "instance.idp.github.enterprise.added", "instance.idp.gitlab.added", "instance.idp.gitlab.selfhosted.added", "instance.idp.google.added", "instance.idp.ldap.added", "instance.idp.config.apple.added", "instance.idp.saml.added"}, []string{"instance.idp.config.added", "instance.idp.oauth.added", "instance.idp.oidc.added", "instance.idp.jwt.added", "instance.idp.azure.added", "instance.idp.github.added", "instance.idp.github.enterprise.added", "instance.idp.gitlab.added", "instance.idp.gitlab.selfhosted.added", "instance.idp.google.added", "instance.idp.ldap.added", "instance.idp.config.apple.added", "instance.idp.saml.added"},
@@ -1201,7 +1203,7 @@ func Test_executeQuery(t *testing.T) {
time.Now(), time.Now(),
"event.type", "event.type",
uint32(23), uint32(23),
float64(123), decimal.NewFromInt(123).String(),
uint32(0), uint32(0),
nil, nil,
"gigi", "gigi",
@@ -1235,7 +1237,7 @@ func Test_executeQuery(t *testing.T) {
time.Now(), time.Now(),
"event.type", "event.type",
uint32(23), uint32(23),
float64(123), decimal.NewFromInt(123).String(),
uint32(0), uint32(0),
[]byte(`{"name": "gigi"}`), []byte(`{"name": "gigi"}`),
"gigi", "gigi",
@@ -1269,7 +1271,7 @@ func Test_executeQuery(t *testing.T) {
time.Now(), time.Now(),
"event.type", "event.type",
uint32(23), uint32(23),
float64(123), decimal.NewFromInt(123).String(),
uint32(0), uint32(0),
nil, nil,
"gigi", "gigi",
@@ -1283,7 +1285,7 @@ func Test_executeQuery(t *testing.T) {
time.Now(), time.Now(),
"event.type", "event.type",
uint32(24), uint32(24),
float64(124), decimal.NewFromInt(124).String(),
uint32(0), uint32(0),
[]byte(`{"name": "gigi"}`), []byte(`{"name": "gigi"}`),
"gigi", "gigi",
@@ -1317,7 +1319,7 @@ func Test_executeQuery(t *testing.T) {
time.Now(), time.Now(),
"event.type", "event.type",
uint32(23), uint32(23),
float64(123), decimal.NewFromInt(123).String(),
uint32(0), uint32(0),
nil, nil,
"gigi", "gigi",
@@ -1331,7 +1333,7 @@ func Test_executeQuery(t *testing.T) {
time.Now(), time.Now(),
"event.type", "event.type",
uint32(24), uint32(24),
float64(124), decimal.NewFromInt(124).String(),
uint32(0), uint32(0),
[]byte(`{"name": "gigi"}`), []byte(`{"name": "gigi"}`),
"gigi", "gigi",

6
internal/v2/eventstore/query.go
View File

@@ -7,6 +7,8 @@ import (
"slices" "slices"
"time" "time"
"github.com/shopspring/decimal"
"github.com/zitadel/zitadel/internal/v2/database" "github.com/zitadel/zitadel/internal/v2/database"
) )
@@ -723,7 +725,7 @@ func (pc *PositionCondition) Min() *GlobalPosition {
// PositionGreater prepares the condition as follows // PositionGreater prepares the condition as follows
// if inPositionOrder is set: position = AND in_tx_order > OR or position > // if inPositionOrder is set: position = AND in_tx_order > OR or position >
// if inPositionOrder is NOT set: position > // if inPositionOrder is NOT set: position >
func PositionGreater(position float64, inPositionOrder uint32) paginationOpt { func PositionGreater(position decimal.Decimal, inPositionOrder uint32) paginationOpt {
return func(p *Pagination) { return func(p *Pagination) {
p.ensurePosition() p.ensurePosition()
p.position.min = &GlobalPosition{ p.position.min = &GlobalPosition{
@@ -743,7 +745,7 @@ func GlobalPositionGreater(position *GlobalPosition) paginationOpt {
// PositionLess prepares the condition as follows // PositionLess prepares the condition as follows
// if inPositionOrder is set: position = AND in_tx_order > OR or position > // if inPositionOrder is set: position = AND in_tx_order > OR or position >
// if inPositionOrder is NOT set: position > // if inPositionOrder is NOT set: position >
func PositionLess(position float64, inPositionOrder uint32) paginationOpt { func PositionLess(position decimal.Decimal, inPositionOrder uint32) paginationOpt {
return func(p *Pagination) { return func(p *Pagination) {
p.ensurePosition() p.ensurePosition()
p.position.max = &GlobalPosition{ p.position.max = &GlobalPosition{

26
internal/v2/eventstore/query_test.go
View File

@@ -6,6 +6,8 @@ import (
"testing" "testing"
"time" "time"
"github.com/shopspring/decimal"
"github.com/zitadel/zitadel/internal/v2/database" "github.com/zitadel/zitadel/internal/v2/database"
) )
@@ -74,13 +76,13 @@ func TestPaginationOpt(t *testing.T) {
name: "global position greater", name: "global position greater",
args: args{ args: args{
opts: []paginationOpt{ opts: []paginationOpt{
GlobalPositionGreater(&GlobalPosition{Position: 10}), GlobalPositionGreater(&GlobalPosition{Position: decimal.NewFromInt(10)}),
}, },
}, },
want: &Pagination{ want: &Pagination{
position: &PositionCondition{ position: &PositionCondition{
min: &GlobalPosition{ min: &GlobalPosition{
Position: 10, Position: decimal.NewFromInt(10),
InPositionOrder: 0, InPositionOrder: 0,
}, },
}, },
@@ -90,13 +92,13 @@ func TestPaginationOpt(t *testing.T) {
name: "position greater", name: "position greater",
args: args{ args: args{
opts: []paginationOpt{ opts: []paginationOpt{
PositionGreater(10, 0), PositionGreater(decimal.NewFromInt(10), 0),
}, },
}, },
want: &Pagination{ want: &Pagination{
position: &PositionCondition{ position: &PositionCondition{
min: &GlobalPosition{ min: &GlobalPosition{
Position: 10, Position: decimal.NewFromInt(10),
InPositionOrder: 0, InPositionOrder: 0,
}, },
}, },
@@ -107,13 +109,13 @@ func TestPaginationOpt(t *testing.T) {
name: "position less", name: "position less",
args: args{ args: args{
opts: []paginationOpt{ opts: []paginationOpt{
PositionLess(10, 12), PositionLess(decimal.NewFromInt(10), 12),
}, },
}, },
want: &Pagination{ want: &Pagination{
position: &PositionCondition{ position: &PositionCondition{
max: &GlobalPosition{ max: &GlobalPosition{
Position: 10, Position: decimal.NewFromInt(10),
InPositionOrder: 12, InPositionOrder: 12,
}, },
}, },
@@ -123,13 +125,13 @@ func TestPaginationOpt(t *testing.T) {
name: "global position less", name: "global position less",
args: args{ args: args{
opts: []paginationOpt{ opts: []paginationOpt{
GlobalPositionLess(&GlobalPosition{Position: 12, InPositionOrder: 24}), GlobalPositionLess(&GlobalPosition{Position: decimal.NewFromInt(12), InPositionOrder: 24}),
}, },
}, },
want: &Pagination{ want: &Pagination{
position: &PositionCondition{ position: &PositionCondition{
max: &GlobalPosition{ max: &GlobalPosition{
Position: 12, Position: decimal.NewFromInt(12),
InPositionOrder: 24, InPositionOrder: 24,
}, },
}, },
@@ -140,19 +142,19 @@ func TestPaginationOpt(t *testing.T) {
args: args{ args: args{
opts: []paginationOpt{ opts: []paginationOpt{
PositionBetween( PositionBetween(
&GlobalPosition{10, 12}, &GlobalPosition{decimal.NewFromInt(10), 12},
&GlobalPosition{20, 0}, &GlobalPosition{decimal.NewFromInt(20), 0},
), ),
}, },
}, },
want: &Pagination{ want: &Pagination{
position: &PositionCondition{ position: &PositionCondition{
min: &GlobalPosition{ min: &GlobalPosition{
Position: 10, Position: decimal.NewFromInt(10),
InPositionOrder: 12, InPositionOrder: 12,
}, },
max: &GlobalPosition{ max: &GlobalPosition{
Position: 20, Position: decimal.NewFromInt(20),
InPositionOrder: 0, InPositionOrder: 0,
}, },
}, },

7
internal/v2/readmodel/last_successful_mirror.go
View File

@@ -1,6 +1,8 @@
package readmodel package readmodel
import ( import (
"github.com/shopspring/decimal"
"github.com/zitadel/zitadel/internal/v2/eventstore" "github.com/zitadel/zitadel/internal/v2/eventstore"
"github.com/zitadel/zitadel/internal/v2/system" "github.com/zitadel/zitadel/internal/v2/system"
"github.com/zitadel/zitadel/internal/v2/system/mirror" "github.com/zitadel/zitadel/internal/v2/system/mirror"
@@ -8,7 +10,7 @@ import (
type LastSuccessfulMirror struct { type LastSuccessfulMirror struct {
ID string ID string
Position float64 Position decimal.Decimal
source string source string
} }
@@ -34,6 +36,7 @@ func (p *LastSuccessfulMirror) Filter() *eventstore.Filter {
), ),
eventstore.FilterPagination( eventstore.FilterPagination(
eventstore.Descending(), eventstore.Descending(),
eventstore.Limit(1),
), ),
) )
} }
@@ -53,7 +56,7 @@ func (h *LastSuccessfulMirror) Reduce(events ...*eventstore.StorageEvent) (err e
func (h *LastSuccessfulMirror) reduceSucceeded(event *eventstore.StorageEvent) error { func (h *LastSuccessfulMirror) reduceSucceeded(event *eventstore.StorageEvent) error {
// if position is set we skip all older events // if position is set we skip all older events
if h.Position > 0 { if h.Position.GreaterThan(decimal.NewFromInt(0)) {
return nil return nil
} }

6
internal/v2/system/mirror/succeeded.go
View File

@@ -1,6 +1,8 @@
package mirror package mirror
import ( import (
"github.com/shopspring/decimal"
"github.com/zitadel/zitadel/internal/v2/eventstore" "github.com/zitadel/zitadel/internal/v2/eventstore"
"github.com/zitadel/zitadel/internal/zerrors" "github.com/zitadel/zitadel/internal/zerrors"
) )
@@ -9,7 +11,7 @@ type succeededPayload struct {
// Source is the name of the database data are mirrored from // Source is the name of the database data are mirrored from
Source string `json:"source"` Source string `json:"source"`
// Position until data will be mirrored // Position until data will be mirrored
Position float64 `json:"position"` Position decimal.Decimal `json:"position"`
} }
const SucceededType = eventTypePrefix + "succeeded" const SucceededType = eventTypePrefix + "succeeded"
@@ -38,7 +40,7 @@ func SucceededEventFromStorage(event *eventstore.StorageEvent) (e *SucceededEven
}, nil }, nil
} }
func NewSucceededCommand(source string, position float64) *eventstore.Command { func NewSucceededCommand(source string, position decimal.Decimal) *eventstore.Command {
return &eventstore.Command{ return &eventstore.Command{
Action: eventstore.Action[any]{ Action: eventstore.Action[any]{
Creator: Creator, Creator: Creator,