TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2024-12-04 23:45:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf(query): remove transactions for queries (#8614)

Browse Source 
# Which Problems Are Solved

Queries currently execute 3 statements, begin, query, commit

# How the Problems Are Solved

remove transaction handling from query methods in database package

# Additional Changes

- Bump versions of `core_grpc_dependencies`-receipt in Makefile

# Additional info

During load tests we saw a lot of idle transactions of `zitadel_queries`
application name which is the connection pool used to query data in
zitadel. Executed query:

`select query_start - xact_start, pid, application_name, backend_start,
xact_start, query_start, state_change, wait_event_type,
wait_event,substring(query, 1, 200) query from pg_stat_activity where
datname = 'zitadel' and state <> 'idle';`

Mostly the last query executed was `begin isolation level read committed
read only`.

example: 

```
    ?column?     |  pid  |      application_name      |         backend_start         |          xact_start           |          query_start          |         state_change          | wait_event_type |  wait_event  |                                                                                                  query                                                                                                   
-----------------+-------+----------------------------+-------------------------------+-------------------------------+-------------------------------+-------------------------------+-----------------+--------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 00:00:00        | 33030 | zitadel_queries            | 2024-10-16 16:25:53.906036+00 | 2024-10-16 16:30:19.191661+00 | 2024-10-16 16:30:19.191661+00 | 2024-10-16 16:30:19.19169+00  | Client          | ClientRead   | begin isolation level read committed read only
 00:00:00        | 33035 | zitadel_queries            | 2024-10-16 16:25:53.909629+00 | 2024-10-16 16:30:19.19179+00  | 2024-10-16 16:30:19.19179+00  | 2024-10-16 16:30:19.191805+00 | Client          | ClientRead   | begin isolation level read committed read only
 00:00:00.00412  | 33028 | zitadel_queries            | 2024-10-16 16:25:53.904247+00 | 2024-10-16 16:30:19.187734+00 | 2024-10-16 16:30:19.191854+00 | 2024-10-16 16:30:19.191964+00 | Client          | ClientRead   | SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision FROM eventstore.events2 WHERE instance_id = $1 AND aggregate_type 
 00:00:00.084662 | 33134 | zitadel_es_pusher          | 2024-10-16 16:29:54.979692+00 | 2024-10-16 16:30:19.178578+00 | 2024-10-16 16:30:19.26324+00  | 2024-10-16 16:30:19.263267+00 | Client          | ClientRead   | RELEASE SAVEPOINT cockroach_restart
 00:00:00.084768 | 33139 | zitadel_es_pusher          | 2024-10-16 16:29:54.979585+00 | 2024-10-16 16:30:19.180762+00 | 2024-10-16 16:30:19.26553+00  | 2024-10-16 16:30:19.265531+00 | LWLock          | WALWriteLock | commit
 00:00:00.077377 | 33136 | zitadel_es_pusher          | 2024-10-16 16:29:54.978582+00 | 2024-10-16 16:30:19.187883+00 | 2024-10-16 16:30:19.26526+00  | 2024-10-16 16:30:19.265431+00 | Client          | ClientRead   | WITH existing AS (                                                                                                                                                                                      +
                 |       |                            |                               |                               |                               |                               |                 |              |     (SELECT instance_id, aggregate_type, aggregate_id, "sequence" FROM eventstore.events2 WHERE instance_id = $1 AND aggregate_type = $2 AND aggregate_id = $3 ORDER BY "sequence" DE
 00:00:00.012309 | 33123 | zitadel_es_pusher          | 2024-10-16 16:29:54.963484+00 | 2024-10-16 16:30:19.175066+00 | 2024-10-16 16:30:19.187375+00 | 2024-10-16 16:30:19.187376+00 | IO              | WalSync      | commit
 00:00:00        | 33034 | zitadel_queries            | 2024-10-16 16:25:53.90791+00  | 2024-10-16 16:30:19.262921+00 | 2024-10-16 16:30:19.262921+00 | 2024-10-16 16:30:19.263133+00 | Client          | ClientRead   | begin isolation level read committed read only
 00:00:00        | 33039 | zitadel_queries            | 2024-10-16 16:25:53.914106+00 | 2024-10-16 16:30:19.191676+00 | 2024-10-16 16:30:19.191676+00 | 2024-10-16 16:30:19.191687+00 | Client          | ClientRead   | begin isolation level read committed read only
 00:00:00.24539  | 33083 | zitadel_projection_spooler | 2024-10-16 16:27:49.895548+00 | 2024-10-16 16:30:19.020058+00 | 2024-10-16 16:30:19.265448+00 | 2024-10-16 16:30:19.26546+00  | Client          | ClientRead   | SAVEPOINT exec_stmt
 00:00:00        | 33125 | zitadel_es_pusher          | 2024-10-16 16:29:54.963859+00 | 2024-10-16 16:30:19.191715+00 | 2024-10-16 16:30:19.191715+00 | 2024-10-16 16:30:19.191729+00 | Client          | ClientRead   | begin
 00:00:00.004292 | 33032 | zitadel_queries            | 2024-10-16 16:25:53.906624+00 | 2024-10-16 16:30:19.187713+00 | 2024-10-16 16:30:19.192005+00 | 2024-10-16 16:30:19.192062+00 | Client          | ClientRead   | SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision FROM eventstore.events2 WHERE instance_id = $1 AND aggregate_type 
 00:00:00        | 33031 | zitadel_queries            | 2024-10-16 16:25:53.906422+00 | 2024-10-16 16:30:19.191625+00 | 2024-10-16 16:30:19.191625+00 | 2024-10-16 16:30:19.191645+00 | Client          | ClientRead   | begin isolation level read committed read only

```

The amount of idle transactions is significantly less if the query
transactions are removed:

example: 

```
    ?column?     |  pid  |      application_name      |         backend_start         |          xact_start           |          query_start          |         state_change          | wait_event_type | wait_event |                                                                                                  query                                                                                                   
-----------------+-------+----------------------------+-------------------------------+-------------------------------+-------------------------------+-------------------------------+-----------------+------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 00:00:00.000094 | 32741 | zitadel_queries            | 2024-10-16 16:23:49.73935+00  | 2024-10-16 16:24:59.785589+00 | 2024-10-16 16:24:59.785683+00 | 2024-10-16 16:24:59.785684+00 |                 |            | SELECT created_at, event_type, "sequence", "position", payload, creator, "owner", instance_id, aggregate_type, aggregate_id, revision FROM eventstore.events2 WHERE instance_id = $1 AND aggregate_type 
 00:00:00        | 32762 | zitadel_es_pusher          | 2024-10-16 16:24:02.275136+00 | 2024-10-16 16:24:59.784586+00 | 2024-10-16 16:24:59.784586+00 | 2024-10-16 16:24:59.784607+00 | Client          | ClientRead | begin
 00:00:00.000167 | 32742 | zitadel_queries            | 2024-10-16 16:23:49.740489+00 | 2024-10-16 16:24:59.784274+00 | 2024-10-16 16:24:59.784441+00 | 2024-10-16 16:24:59.784442+00 |                 |            | with usr as (                                                                                                                                                                                           +
                 |       |                            |                               |                               |                               |                               |                 |            |         select u.id, u.creation_date, u.change_date, u.sequence, u.state, u.resource_owner, u.username, n.login_name as preferred_login_name                                                            +
                 |       |                            |                               |                               |                               |                               |                 |            |         from projections.users13 u                                                                                                                                                                      +
                 |       |                            |                               |                               |                               |                               |                 |            |         left join projections.l
 00:00:00.256014 | 32759 | zitadel_projection_spooler | 2024-10-16 16:24:01.418429+00 | 2024-10-16 16:24:59.52959+00  | 2024-10-16 16:24:59.785604+00 | 2024-10-16 16:24:59.785649+00 | Client          | ClientRead | UPDATE projections.milestones SET reached_date = $1 WHERE (instance_id = $2) AND (type = $3) AND (reached_date IS NULL)
 00:00:00.014199 | 32773 | zitadel_es_pusher          | 2024-10-16 16:24:02.320404+00 | 2024-10-16 16:24:59.769509+00 | 2024-10-16 16:24:59.783708+00 | 2024-10-16 16:24:59.783709+00 | IO              | WalSync    | commit
 00:00:00        | 32765 | zitadel_es_pusher          | 2024-10-16 16:24:02.28173+00  | 2024-10-16 16:24:59.780413+00 | 2024-10-16 16:24:59.780413+00 | 2024-10-16 16:24:59.780426+00 | Client          | ClientRead | begin
 00:00:00.012729 | 32777 | zitadel_es_pusher          | 2024-10-16 16:24:02.339737+00 | 2024-10-16 16:24:59.767432+00 | 2024-10-16 16:24:59.780161+00 | 2024-10-16 16:24:59.780195+00 | Client          | ClientRead | RELEASE SAVEPOINT cockroach_restart
```

---------

Co-authored-by: Tim Möhlmann <tim+github@zitadel.com>
Co-authored-by: Livio Spring <livio.a@gmail.com>
Co-authored-by: Max Peintner <max@caos.ch>
Co-authored-by: Elio Bischof <elio@zitadel.com>
Co-authored-by: Stefan Benz <46600784+stebenz@users.noreply.github.com>
Co-authored-by: Miguel Cabrerizo <30386061+doncicuto@users.noreply.github.com>
Co-authored-by: Joakim Lodén <Loddan@users.noreply.github.com>
Co-authored-by: Yxnt <Yxnt@users.noreply.github.com>
Co-authored-by: Stefan Benz <stefan@caos.ch>
Co-authored-by: Harsha Reddy <harsha.reddy@klaviyo.com>
Co-authored-by: Zach H <zhirschtritt@gmail.com>
This commit is contained in:
Silvan 2024-11-04 10:06:14 +01:00 committed by GitHub
parent 9422766e17
commit 9c3e5e467b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 44 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.releaserc.js
View File

@ -10,6 +10,7 @@ module.exports = {
"@semantic-release/github", "@semantic-release/github",
{ {
draftRelease: true, draftRelease: true,
successComment: false,
assets: [ assets: [
{ {
path: ".artifacts/zitadel-linux-amd64/zitadel-linux-amd64.tar.gz", path: ".artifacts/zitadel-linux-amd64/zitadel-linux-amd64.tar.gz",

12
Makefile
View File

@ -63,12 +63,12 @@ endif
.PHONY: core_grpc_dependencies .PHONY: core_grpc_dependencies
core_grpc_dependencies: core_grpc_dependencies:
go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2 # https://pkg.go.dev/google.golang.org/protobuf/cmd/protoc-gen-go?tab=versions go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.35.1 # https://pkg.go.dev/google.golang.org/protobuf/cmd/protoc-gen-go?tab=versions
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.4 # https://pkg.go.dev/google.golang.org/grpc/cmd/protoc-gen-go-grpc?tab=versions go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.5.1 # https://pkg.go.dev/google.golang.org/grpc/cmd/protoc-gen-go-grpc?tab=versions
go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway@v2.20.0 # https://pkg.go.dev/github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway?tab=versions go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway@v2.22.0 # https://pkg.go.dev/github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway?tab=versions
go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2@v2.20.0 # https://pkg.go.dev/github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2?tab=versions go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2@v2.22.0 # https://pkg.go.dev/github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2?tab=versions
go install github.com/envoyproxy/protoc-gen-validate@v1.0.4 # https://pkg.go.dev/github.com/envoyproxy/protoc-gen-validate?tab=versions go install github.com/envoyproxy/protoc-gen-validate@v1.1.0 # https://pkg.go.dev/github.com/envoyproxy/protoc-gen-validate?tab=versions
go install github.com/bufbuild/buf/cmd/buf@v1.34.0 # https://pkg.go.dev/github.com/bufbuild/buf/cmd/buf?tab=versions go install github.com/bufbuild/buf/cmd/buf@v1.45.0 # https://pkg.go.dev/github.com/bufbuild/buf/cmd/buf?tab=versions
.PHONY: core_api .PHONY: core_api
core_api: core_api_generator core_grpc_dependencies core_api: core_api_generator core_grpc_dependencies

6
internal/crypto/database/database_test.go
View File

@ -468,17 +468,13 @@ func dbMock(t *testing.T, expectations ...func(m sqlmock.Sqlmock)) db {
func expectQueryErr(query string, err error, args ...driver.Value) func(m sqlmock.Sqlmock) { func expectQueryErr(query string, err error, args ...driver.Value) func(m sqlmock.Sqlmock) {
return func(m sqlmock.Sqlmock) { return func(m sqlmock.Sqlmock) {
m.ExpectBegin()
m.ExpectQuery(regexp.QuoteMeta(query)).WithArgs(args...).WillReturnError(err) m.ExpectQuery(regexp.QuoteMeta(query)).WithArgs(args...).WillReturnError(err)
m.ExpectRollback()
} }
} }
func expectQueryScanErr(stmt string, cols []string, rows [][]driver.Value, args ...driver.Value) func(m sqlmock.Sqlmock) { func expectQueryScanErr(stmt string, cols []string, rows [][]driver.Value, args ...driver.Value) func(m sqlmock.Sqlmock) {
return func(m sqlmock.Sqlmock) { return func(m sqlmock.Sqlmock) {
m.ExpectBegin()
q := m.ExpectQuery(regexp.QuoteMeta(stmt)).WithArgs(args...) q := m.ExpectQuery(regexp.QuoteMeta(stmt)).WithArgs(args...)
m.ExpectRollback()
result := m.NewRows(cols) result := m.NewRows(cols)
count := uint64(len(rows)) count := uint64(len(rows))
for _, row := range rows { for _, row := range rows {
@ -494,9 +490,7 @@ func expectQueryScanErr(stmt string, cols []string, rows [][]driver.Value, args
func expectQuery(stmt string, cols []string, rows [][]driver.Value, args ...driver.Value) func(m sqlmock.Sqlmock) { func expectQuery(stmt string, cols []string, rows [][]driver.Value, args ...driver.Value) func(m sqlmock.Sqlmock) {
return func(m sqlmock.Sqlmock) { return func(m sqlmock.Sqlmock) {
m.ExpectBegin()
q := m.ExpectQuery(regexp.QuoteMeta(stmt)).WithArgs(args...) q := m.ExpectQuery(regexp.QuoteMeta(stmt)).WithArgs(args...)
m.ExpectCommit()
result := m.NewRows(cols) result := m.NewRows(cols)
count := uint64(len(rows)) count := uint64(len(rows))
for _, row := range rows { for _, row := range rows {

30
internal/database/database.go
View File

@ -40,20 +40,7 @@ func (db *DB) Query(scan func(*sql.Rows) error, query string, args ...any) error
} }
func (db *DB) QueryContext(ctx context.Context, scan func(rows *sql.Rows) error, query string, args ...any) (err error) { func (db *DB) QueryContext(ctx context.Context, scan func(rows *sql.Rows) error, query string, args ...any) (err error) {
tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true, Isolation: sql.LevelReadCommitted}) rows, err := db.DB.QueryContext(ctx, query, args...)
if err != nil {
return err
}
defer func() {
if err != nil {
rollbackErr := tx.Rollback()
logging.OnError(rollbackErr).Info("commit of read only transaction failed")
return
}
err = tx.Commit()
}()
rows, err := tx.QueryContext(ctx, query, args...)
if err != nil { if err != nil {
return err return err
} }
@ -73,20 +60,7 @@ func (db *DB) QueryRow(scan func(*sql.Row) error, query string, args ...any) (er
} }
func (db *DB) QueryRowContext(ctx context.Context, scan func(row *sql.Row) error, query string, args ...any) (err error) { func (db *DB) QueryRowContext(ctx context.Context, scan func(row *sql.Row) error, query string, args ...any) (err error) {
tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true, Isolation: sql.LevelReadCommitted}) row := db.DB.QueryRowContext(ctx, query, args...)
if err != nil {
return err
}
defer func() {
if err != nil {
rollbackErr := tx.Rollback()
logging.OnError(rollbackErr).Info("commit of read only transaction failed")
return
}
err = tx.Commit()
}()
row := tx.QueryRowContext(ctx, query, args...)
logging.OnError(row.Err()).Error("unexpected query error") logging.OnError(row.Err()).Error("unexpected query error")
err = scan(row) err = scan(row)

7
internal/database/database_test.go
View File

@ -31,7 +31,7 @@ func TestQueryJSONObject(t *testing.T) {
{ {
name: "tx error", name: "tx error",
mock: func(t *testing.T) *mock.SQLMock { mock: func(t *testing.T) *mock.SQLMock {
return mock.NewSQLMock(t, mock.ExpectBegin(sql.ErrConnDone)) return mock.NewSQLMock(t, mock.ExpectQuery("select $1;", mock.WithQueryErr(sql.ErrConnDone)))
}, },
wantErr: zerrors.ThrowInternal(sql.ErrConnDone, "DATAB-Oath6", "Errors.Internal"), wantErr: zerrors.ThrowInternal(sql.ErrConnDone, "DATAB-Oath6", "Errors.Internal"),
}, },
@ -39,7 +39,6 @@ func TestQueryJSONObject(t *testing.T) {
name: "no rows", name: "no rows",
mock: func(t *testing.T) *mock.SQLMock { mock: func(t *testing.T) *mock.SQLMock {
return mock.NewSQLMock(t, return mock.NewSQLMock(t,
mock.ExpectBegin(nil),
mock.ExpectQuery(query, mock.ExpectQuery(query,
mock.WithQueryArgs(arg), mock.WithQueryArgs(arg),
mock.WithQueryResult([]string{"json"}, [][]driver.Value{}), mock.WithQueryResult([]string{"json"}, [][]driver.Value{}),
@ -52,12 +51,10 @@ func TestQueryJSONObject(t *testing.T) {
name: "unmarshal error", name: "unmarshal error",
mock: func(t *testing.T) *mock.SQLMock { mock: func(t *testing.T) *mock.SQLMock {
return mock.NewSQLMock(t, return mock.NewSQLMock(t,
mock.ExpectBegin(nil),
mock.ExpectQuery(query, mock.ExpectQuery(query,
mock.WithQueryArgs(arg), mock.WithQueryArgs(arg),
mock.WithQueryResult([]string{"json"}, [][]driver.Value{{`~~~`}}), mock.WithQueryResult([]string{"json"}, [][]driver.Value{{`~~~`}}),
), ),
mock.ExpectCommit(nil),
) )
}, },
wantErr: zerrors.ThrowInternal(nil, "DATAB-Vohs6", "Errors.Internal"), wantErr: zerrors.ThrowInternal(nil, "DATAB-Vohs6", "Errors.Internal"),
@ -66,12 +63,10 @@ func TestQueryJSONObject(t *testing.T) {
name: "success", name: "success",
mock: func(t *testing.T) *mock.SQLMock { mock: func(t *testing.T) *mock.SQLMock {
return mock.NewSQLMock(t, return mock.NewSQLMock(t,
mock.ExpectBegin(nil),
mock.ExpectQuery(query, mock.ExpectQuery(query,
mock.WithQueryArgs(arg), mock.WithQueryArgs(arg),
mock.WithQueryResult([]string{"json"}, [][]driver.Value{{`{"a":1}`}}), mock.WithQueryResult([]string{"json"}, [][]driver.Value{{`{"a":1}`}}),
), ),
mock.ExpectCommit(nil),
) )
}, },
want: &dst{A: 1}, want: &dst{A: 1},

5
internal/eventstore/repository/sql/query_test.go
View File

@ -870,9 +870,7 @@ type dbMock struct {
} }
func (m *dbMock) expectQuery(t *testing.T, expectedQuery string, args []driver.Value, events ...*repository.Event) *dbMock { func (m *dbMock) expectQuery(t *testing.T, expectedQuery string, args []driver.Value, events ...*repository.Event) *dbMock {
m.mock.ExpectBegin()
query := m.mock.ExpectQuery(expectedQuery).WithArgs(args...) query := m.mock.ExpectQuery(expectedQuery).WithArgs(args...)
m.mock.ExpectCommit()
rows := m.mock.NewRows([]string{"sequence"}) rows := m.mock.NewRows([]string{"sequence"})
for _, event := range events { for _, event := range events {
rows = rows.AddRow(event.Seq) rows = rows.AddRow(event.Seq)
@ -882,9 +880,7 @@ func (m *dbMock) expectQuery(t *testing.T, expectedQuery string, args []driver.V
} }
func (m *dbMock) expectQueryScanErr(t *testing.T, expectedQuery string, args []driver.Value, events ...*repository.Event) *dbMock { func (m *dbMock) expectQueryScanErr(t *testing.T, expectedQuery string, args []driver.Value, events ...*repository.Event) *dbMock {
m.mock.ExpectBegin()
query := m.mock.ExpectQuery(expectedQuery).WithArgs(args...) query := m.mock.ExpectQuery(expectedQuery).WithArgs(args...)
m.mock.ExpectRollback()
rows := m.mock.NewRows([]string{"sequence"}) rows := m.mock.NewRows([]string{"sequence"})
for _, event := range events { for _, event := range events {
rows = rows.AddRow(event.Seq) rows = rows.AddRow(event.Seq)
@ -894,7 +890,6 @@ func (m *dbMock) expectQueryScanErr(t *testing.T, expectedQuery string, args []d
} }
func (m *dbMock) expectQueryErr(t *testing.T, expectedQuery string, args []driver.Value, err error) *dbMock { func (m *dbMock) expectQueryErr(t *testing.T, expectedQuery string, args []driver.Value, err error) *dbMock {
m.mock.ExpectBegin()
m.mock.ExpectQuery(expectedQuery).WithArgs(args...).WillReturnError(err) m.mock.ExpectQuery(expectedQuery).WithArgs(args...).WillReturnError(err)
return m return m
} }

17
internal/eventstore/v3/push.go
View File

@ -14,10 +14,14 @@ import (
"github.com/zitadel/logging" "github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/eventstore" "github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/telemetry/tracing"
"github.com/zitadel/zitadel/internal/zerrors" "github.com/zitadel/zitadel/internal/zerrors"
) )
func (es *Eventstore) Push(ctx context.Context, commands ...eventstore.Command) (events []eventstore.Event, err error) { func (es *Eventstore) Push(ctx context.Context, commands ...eventstore.Command) (events []eventstore.Event, err error) {
ctx, span := tracing.NewSpan(ctx)
defer func() { span.EndWithError(err) }()
tx, err := es.client.BeginTx(ctx, nil) tx, err := es.client.BeginTx(ctx, nil)
if err != nil { if err != nil {
return nil, err return nil, err
@ -27,18 +31,21 @@ func (es *Eventstore) Push(ctx context.Context, commands ...eventstore.Command)
sequences []*latestSequence sequences []*latestSequence
) )
err = crdb.ExecuteInTx(ctx, &transaction{tx}, func() error { err = crdb.ExecuteInTx(ctx, &transaction{tx}, func() (err error) {
sequences, err = latestSequences(ctx, tx, commands) inTxCtx, span := tracing.NewSpan(ctx)
defer func() { span.EndWithError(err) }()
sequences, err = latestSequences(inTxCtx, tx, commands)
if err != nil { if err != nil {
return err return err
} }
events, err = insertEvents(ctx, tx, sequences, commands) events, err = insertEvents(inTxCtx, tx, sequences, commands)
if err != nil { if err != nil {
return err return err
} }
if err = handleUniqueConstraints(ctx, tx, commands); err != nil { if err = handleUniqueConstraints(inTxCtx, tx, commands); err != nil {
return err return err
} }
@ -51,7 +58,7 @@ func (es *Eventstore) Push(ctx context.Context, commands ...eventstore.Command)
} }
} }
return handleFieldCommands(ctx, tx, commands) return handleFieldCommands(inTxCtx, tx, commands)
}) })
if err != nil { if err != nil {

8
internal/integration/oidc.go
View File

@ -411,5 +411,13 @@ func (i *Instance) CreateOIDCJWTProfileClient(ctx context.Context) (machine *man
if err != nil { if err != nil {
return nil, "", nil, err return nil, "", nil, err
} }
mustAwait(func() error {
_, err := i.Client.Mgmt.GetMachineKeyByIDs(ctx, &management.GetMachineKeyByIDsRequest{
UserId: machine.GetUserId(),
KeyId: keyResp.GetKeyId(),
})
return err
})
return machine, name, keyResp.GetKeyDetails(), nil return machine, name, keyResp.GetKeyDetails(), nil
} }

2
internal/query/device_auth_test.go
View File

@ -57,11 +57,9 @@ func TestQueries_DeviceAuthRequestByUserCode(t *testing.T) {
} }
defer client.Close() defer client.Close()
mock.ExpectBegin()
mock.ExpectQuery(expectedDeviceAuthWhereUserCodeQuery).WillReturnRows( mock.ExpectQuery(expectedDeviceAuthWhereUserCodeQuery).WillReturnRows(
mock.NewRows(deviceAuthSelectColumns).AddRow(expectedDeviceAuthValues...), mock.NewRows(deviceAuthSelectColumns).AddRow(expectedDeviceAuthValues...),
) )
mock.ExpectCommit()
q := Queries{ q := Queries{
client: &database.DB{DB: client}, client: &database.DB{DB: client},
} }

10
internal/query/prepare_test.go
View File

@ -82,9 +82,7 @@ type sqlExpectation func(sqlmock.Sqlmock) sqlmock.Sqlmock
func mockQuery(stmt string, cols []string, row []driver.Value, args ...driver.Value) func(m sqlmock.Sqlmock) sqlmock.Sqlmock { func mockQuery(stmt string, cols []string, row []driver.Value, args ...driver.Value) func(m sqlmock.Sqlmock) sqlmock.Sqlmock {
return func(m sqlmock.Sqlmock) sqlmock.Sqlmock { return func(m sqlmock.Sqlmock) sqlmock.Sqlmock {
m.ExpectBegin()
q := m.ExpectQuery(stmt).WithArgs(args...) q := m.ExpectQuery(stmt).WithArgs(args...)
m.ExpectCommit()
result := m.NewRows(cols) result := m.NewRows(cols)
if len(row) > 0 { if len(row) > 0 {
result.AddRow(row...) result.AddRow(row...)
@ -96,9 +94,7 @@ func mockQuery(stmt string, cols []string, row []driver.Value, args ...driver.Va
func mockQueryScanErr(stmt string, cols []string, row []driver.Value, args ...driver.Value) func(m sqlmock.Sqlmock) sqlmock.Sqlmock { func mockQueryScanErr(stmt string, cols []string, row []driver.Value, args ...driver.Value) func(m sqlmock.Sqlmock) sqlmock.Sqlmock {
return func(m sqlmock.Sqlmock) sqlmock.Sqlmock { return func(m sqlmock.Sqlmock) sqlmock.Sqlmock {
m.ExpectBegin()
q := m.ExpectQuery(stmt).WithArgs(args...) q := m.ExpectQuery(stmt).WithArgs(args...)
m.ExpectRollback()
result := m.NewRows(cols) result := m.NewRows(cols)
if len(row) > 0 { if len(row) > 0 {
result.AddRow(row...) result.AddRow(row...)
@ -110,9 +106,7 @@ func mockQueryScanErr(stmt string, cols []string, row []driver.Value, args ...dr
func mockQueries(stmt string, cols []string, rows [][]driver.Value, args ...driver.Value) func(m sqlmock.Sqlmock) sqlmock.Sqlmock { func mockQueries(stmt string, cols []string, rows [][]driver.Value, args ...driver.Value) func(m sqlmock.Sqlmock) sqlmock.Sqlmock {
return func(m sqlmock.Sqlmock) sqlmock.Sqlmock { return func(m sqlmock.Sqlmock) sqlmock.Sqlmock {
m.ExpectBegin()
q := m.ExpectQuery(stmt).WithArgs(args...) q := m.ExpectQuery(stmt).WithArgs(args...)
m.ExpectCommit()
result := m.NewRows(cols) result := m.NewRows(cols)
count := uint64(len(rows)) count := uint64(len(rows))
for _, row := range rows { for _, row := range rows {
@ -129,9 +123,7 @@ func mockQueries(stmt string, cols []string, rows [][]driver.Value, args ...driv
func mockQueriesScanErr(stmt string, cols []string, rows [][]driver.Value, args ...driver.Value) func(m sqlmock.Sqlmock) sqlmock.Sqlmock { func mockQueriesScanErr(stmt string, cols []string, rows [][]driver.Value, args ...driver.Value) func(m sqlmock.Sqlmock) sqlmock.Sqlmock {
return func(m sqlmock.Sqlmock) sqlmock.Sqlmock { return func(m sqlmock.Sqlmock) sqlmock.Sqlmock {
m.ExpectBegin()
q := m.ExpectQuery(stmt).WithArgs(args...) q := m.ExpectQuery(stmt).WithArgs(args...)
m.ExpectRollback()
result := m.NewRows(cols) result := m.NewRows(cols)
count := uint64(len(rows)) count := uint64(len(rows))
for _, row := range rows { for _, row := range rows {
@ -148,10 +140,8 @@ func mockQueriesScanErr(stmt string, cols []string, rows [][]driver.Value, args
func mockQueryErr(stmt string, err error, args ...driver.Value) func(m sqlmock.Sqlmock) sqlmock.Sqlmock { func mockQueryErr(stmt string, err error, args ...driver.Value) func(m sqlmock.Sqlmock) sqlmock.Sqlmock {
return func(m sqlmock.Sqlmock) sqlmock.Sqlmock { return func(m sqlmock.Sqlmock) sqlmock.Sqlmock {
m.ExpectBegin()
q := m.ExpectQuery(stmt).WithArgs(args...) q := m.ExpectQuery(stmt).WithArgs(args...)
q.WillReturnError(err) q.WillReturnError(err)
m.ExpectRollback()
return m return m
} }
} }

25
internal/view/repository/db_mock_test.go
View File

@ -176,58 +176,48 @@ func (db *dbMock) expectRollback(err error) *dbMock {
func (db *dbMock) expectGetByID(table, key, value string) *dbMock { func (db *dbMock) expectGetByID(table, key, value string) *dbMock {
query := fmt.Sprintf(expectedGetByID, table, key) query := fmt.Sprintf(expectedGetByID, table, key)
db.mock.ExpectBegin()
db.mock.ExpectQuery(query). db.mock.ExpectQuery(query).
WithArgs(value). WithArgs(value).
WillReturnRows(db.mock.NewRows([]string{key}). WillReturnRows(db.mock.NewRows([]string{key}).
AddRow(key)) AddRow(key))
db.mock.ExpectCommit()
return db return db
} }
func (db *dbMock) expectGetByIDErr(table, key, value string, err error) *dbMock { func (db *dbMock) expectGetByIDErr(table, key, value string, err error) *dbMock {
query := fmt.Sprintf(expectedGetByID, table, key) query := fmt.Sprintf(expectedGetByID, table, key)
db.mock.ExpectBegin()
db.mock.ExpectQuery(query). db.mock.ExpectQuery(query).
WithArgs(value). WithArgs(value).
WillReturnError(err) WillReturnError(err)
db.mock.ExpectCommit()
return db return db
} }
func (db *dbMock) expectGetByQuery(table, key, method, value string) *dbMock { func (db *dbMock) expectGetByQuery(table, key, method, value string) *dbMock {
query := fmt.Sprintf(expectedGetByQuery, table, key, method) query := fmt.Sprintf(expectedGetByQuery, table, key, method)
db.mock.ExpectBegin()
db.mock.ExpectQuery(query). db.mock.ExpectQuery(query).
WithArgs(value). WithArgs(value).
WillReturnRows(db.mock.NewRows([]string{key}). WillReturnRows(db.mock.NewRows([]string{key}).
AddRow(key)) AddRow(key))
db.mock.ExpectCommit()
return db return db
} }
func (db *dbMock) expectGetByQueryCaseSensitive(table, key, method, value string) *dbMock { func (db *dbMock) expectGetByQueryCaseSensitive(table, key, method, value string) *dbMock {
query := fmt.Sprintf(expectedGetByQueryCaseSensitive, table, key, method) query := fmt.Sprintf(expectedGetByQueryCaseSensitive, table, key, method)
db.mock.ExpectBegin()
db.mock.ExpectQuery(query). db.mock.ExpectQuery(query).
WithArgs(value). WithArgs(value).
WillReturnRows(db.mock.NewRows([]string{key}). WillReturnRows(db.mock.NewRows([]string{key}).
AddRow(key)) AddRow(key))
db.mock.ExpectCommit()
return db return db
} }
func (db *dbMock) expectGetByQueryErr(table, key, method, value string, err error) *dbMock { func (db *dbMock) expectGetByQueryErr(table, key, method, value string, err error) *dbMock {
query := fmt.Sprintf(expectedGetByQuery, table, key, method) query := fmt.Sprintf(expectedGetByQuery, table, key, method)
db.mock.ExpectBegin()
db.mock.ExpectQuery(query). db.mock.ExpectQuery(query).
WithArgs(value). WithArgs(value).
WillReturnError(err) WillReturnError(err)
db.mock.ExpectCommit()
return db return db
} }
@ -324,14 +314,11 @@ func (db *dbMock) expectGetSearchRequestNoParams(table string, resultAmount, tot
rows.AddRow(fmt.Sprintf("hodor-%d", i)) rows.AddRow(fmt.Sprintf("hodor-%d", i))
} }
db.mock.ExpectBegin()
db.mock.ExpectQuery(queryCount). db.mock.ExpectQuery(queryCount).
WillReturnRows(db.mock.NewRows([]string{"count"}).AddRow(total)) WillReturnRows(db.mock.NewRows([]string{"count"}).AddRow(total))
db.mock.ExpectQuery(query). db.mock.ExpectQuery(query).
WillReturnRows(rows) WillReturnRows(rows)
db.mock.ExpectCommit()
return db return db
} }
@ -344,12 +331,10 @@ func (db *dbMock) expectGetSearchRequestWithLimit(table string, limit, resultAmo
rows.AddRow(fmt.Sprintf("hodor-%d", i)) rows.AddRow(fmt.Sprintf("hodor-%d", i))
} }
db.mock.ExpectBegin()
db.mock.ExpectQuery(queryCount). db.mock.ExpectQuery(queryCount).
WillReturnRows(db.mock.NewRows([]string{"count"}).AddRow(total)) WillReturnRows(db.mock.NewRows([]string{"count"}).AddRow(total))
db.mock.ExpectQuery(query). db.mock.ExpectQuery(query).
WillReturnRows(rows) WillReturnRows(rows)
db.mock.ExpectCommit()
return db return db
} }
@ -362,12 +347,10 @@ func (db *dbMock) expectGetSearchRequestWithOffset(table string, offset, resultA
rows.AddRow(fmt.Sprintf("hodor-%d", i)) rows.AddRow(fmt.Sprintf("hodor-%d", i))
} }
db.mock.ExpectBegin()
db.mock.ExpectQuery(queryCount). db.mock.ExpectQuery(queryCount).
WillReturnRows(db.mock.NewRows([]string{"count"}).AddRow(total)) WillReturnRows(db.mock.NewRows([]string{"count"}).AddRow(total))
db.mock.ExpectQuery(query). db.mock.ExpectQuery(query).
WillReturnRows(rows) WillReturnRows(rows)
db.mock.ExpectCommit()
return db return db
} }
@ -380,12 +363,10 @@ func (db *dbMock) expectGetSearchRequestWithSorting(table, sorting string, sorti
rows.AddRow(fmt.Sprintf("hodor-%d", i)) rows.AddRow(fmt.Sprintf("hodor-%d", i))
} }
db.mock.ExpectBegin()
db.mock.ExpectQuery(queryCount). db.mock.ExpectQuery(queryCount).
WillReturnRows(db.mock.NewRows([]string{"count"}).AddRow(total)) WillReturnRows(db.mock.NewRows([]string{"count"}).AddRow(total))
db.mock.ExpectQuery(query). db.mock.ExpectQuery(query).
WillReturnRows(rows) WillReturnRows(rows)
db.mock.ExpectCommit()
return db return db
} }
@ -398,14 +379,12 @@ func (db *dbMock) expectGetSearchRequestWithSearchQuery(table, key, method, valu
rows.AddRow(fmt.Sprintf("hodor-%d", i)) rows.AddRow(fmt.Sprintf("hodor-%d", i))
} }
db.mock.ExpectBegin()
db.mock.ExpectQuery(queryCount). db.mock.ExpectQuery(queryCount).
WithArgs(value). WithArgs(value).
WillReturnRows(db.mock.NewRows([]string{"count"}).AddRow(total)) WillReturnRows(db.mock.NewRows([]string{"count"}).AddRow(total))
db.mock.ExpectQuery(query). db.mock.ExpectQuery(query).
WithArgs(value). WithArgs(value).
WillReturnRows(rows) WillReturnRows(rows)
db.mock.ExpectCommit()
return db return db
} }
@ -418,14 +397,12 @@ func (db *dbMock) expectGetSearchRequestWithAllParams(table, key, method, value,
rows.AddRow(fmt.Sprintf("hodor-%d", i)) rows.AddRow(fmt.Sprintf("hodor-%d", i))
} }
db.mock.ExpectBegin()
db.mock.ExpectQuery(queryCount). db.mock.ExpectQuery(queryCount).
WithArgs(value). WithArgs(value).
WillReturnRows(db.mock.NewRows([]string{"count"}).AddRow(total)) WillReturnRows(db.mock.NewRows([]string{"count"}).AddRow(total))
db.mock.ExpectQuery(query). db.mock.ExpectQuery(query).
WithArgs(value). WithArgs(value).
WillReturnRows(rows) WillReturnRows(rows)
db.mock.ExpectCommit()
return db return db
} }
@ -438,11 +415,9 @@ func (db *dbMock) expectGetSearchRequestErr(table string, resultAmount, total in
rows.AddRow(fmt.Sprintf("hodor-%d", i)) rows.AddRow(fmt.Sprintf("hodor-%d", i))
} }
db.mock.ExpectBegin()
db.mock.ExpectQuery(queryCount). db.mock.ExpectQuery(queryCount).
WillReturnRows(db.mock.NewRows([]string{"count"}).AddRow(total)) WillReturnRows(db.mock.NewRows([]string{"count"}).AddRow(total))
db.mock.ExpectQuery(query). db.mock.ExpectQuery(query).
WillReturnError(err) WillReturnError(err)
db.mock.ExpectCommit()
return db return db
} }

11
internal/view/repository/query.go
View File

@ -1,12 +1,9 @@
package repository package repository
import ( import (
"context"
"database/sql"
"fmt" "fmt"
"github.com/jinzhu/gorm" "github.com/jinzhu/gorm"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/database" "github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/domain" "github.com/zitadel/zitadel/internal/domain"
@ -51,14 +48,6 @@ func PrepareSearchQuery(table string, request SearchRequest) func(db *gorm.DB, r
} }
} }
query = query.BeginTx(context.Background(), &sql.TxOptions{ReadOnly: true})
defer func() {
if err := query.Commit().Error; err != nil {
logging.OnError(err).Info("commit failed")
}
query.RollbackUnlessCommitted()
}()
query = query.Count(&count) query = query.Count(&count)
if res == nil { if res == nil {
return count, nil return count, nil

12
internal/view/repository/requests.go
View File

@ -1,8 +1,6 @@
package repository package repository
import ( import (
"context"
"database/sql"
"errors" "errors"
"fmt" "fmt"
"strings" "strings"
@ -24,15 +22,7 @@ func PrepareGetByQuery(table string, queries ...SearchQuery) func(db *gorm.DB, r
} }
} }
tx := query.BeginTx(context.Background(), &sql.TxOptions{ReadOnly: true}) err := query.Take(res).Error
defer func() {
if err := tx.Commit().Error; err != nil {
logging.OnError(err).Info("commit failed")
}
tx.RollbackUnlessCommitted()
}()
err := tx.Take(res).Error
if err == nil { if err == nil {
return nil return nil
} }

70
pkg/grpc/protoc/v2/options.pb.go
View File

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT. // Code generated by protoc-gen-go. DO NOT EDIT.
// versions: // versions:
// protoc-gen-go v1.34.2 // protoc-gen-go v1.35.1
// protoc (unknown) // protoc (unknown)
// source: zitadel/protoc_gen_zitadel/v2/options.proto // source: zitadel/protoc_gen_zitadel/v2/options.proto
@ -32,11 +32,9 @@ type Options struct {
func (x *Options) Reset() { func (x *Options) Reset() {
*x = Options{} *x = Options{}
if protoimpl.UnsafeEnabled { mi := &file_zitadel_protoc_gen_zitadel_v2_options_proto_msgTypes[0]
mi := &file_zitadel_protoc_gen_zitadel_v2_options_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi)
ms.StoreMessageInfo(mi)
}
} }
func (x *Options) String() string { func (x *Options) String() string {
@ -47,7 +45,7 @@ func (*Options) ProtoMessage() {}
func (x *Options) ProtoReflect() protoreflect.Message { func (x *Options) ProtoReflect() protoreflect.Message {
mi := &file_zitadel_protoc_gen_zitadel_v2_options_proto_msgTypes[0] mi := &file_zitadel_protoc_gen_zitadel_v2_options_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil { if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil { if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi) ms.StoreMessageInfo(mi)
@ -87,11 +85,9 @@ type AuthOption struct {
func (x *AuthOption) Reset() { func (x *AuthOption) Reset() {
*x = AuthOption{} *x = AuthOption{}
if protoimpl.UnsafeEnabled { mi := &file_zitadel_protoc_gen_zitadel_v2_options_proto_msgTypes[1]
mi := &file_zitadel_protoc_gen_zitadel_v2_options_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi)
ms.StoreMessageInfo(mi)
}
} }
func (x *AuthOption) String() string { func (x *AuthOption) String() string {
@ -102,7 +98,7 @@ func (*AuthOption) ProtoMessage() {}
func (x *AuthOption) ProtoReflect() protoreflect.Message { func (x *AuthOption) ProtoReflect() protoreflect.Message {
mi := &file_zitadel_protoc_gen_zitadel_v2_options_proto_msgTypes[1] mi := &file_zitadel_protoc_gen_zitadel_v2_options_proto_msgTypes[1]
if protoimpl.UnsafeEnabled && x != nil { if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil { if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi) ms.StoreMessageInfo(mi)
@ -141,11 +137,9 @@ type CustomHTTPResponse struct {
func (x *CustomHTTPResponse) Reset() { func (x *CustomHTTPResponse) Reset() {
*x = CustomHTTPResponse{} *x = CustomHTTPResponse{}
if protoimpl.UnsafeEnabled { mi := &file_zitadel_protoc_gen_zitadel_v2_options_proto_msgTypes[2]
mi := &file_zitadel_protoc_gen_zitadel_v2_options_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi)
ms.StoreMessageInfo(mi)
}
} }
func (x *CustomHTTPResponse) String() string { func (x *CustomHTTPResponse) String() string {
@ -156,7 +150,7 @@ func (*CustomHTTPResponse) ProtoMessage() {}
func (x *CustomHTTPResponse) ProtoReflect() protoreflect.Message { func (x *CustomHTTPResponse) ProtoReflect() protoreflect.Message {
mi := &file_zitadel_protoc_gen_zitadel_v2_options_proto_msgTypes[2] mi := &file_zitadel_protoc_gen_zitadel_v2_options_proto_msgTypes[2]
if protoimpl.UnsafeEnabled && x != nil { if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil { if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi) ms.StoreMessageInfo(mi)
@ -273,44 +267,6 @@ func file_zitadel_protoc_gen_zitadel_v2_options_proto_init() {
if File_zitadel_protoc_gen_zitadel_v2_options_proto != nil { if File_zitadel_protoc_gen_zitadel_v2_options_proto != nil {
return return
} }
if !protoimpl.UnsafeEnabled {
file_zitadel_protoc_gen_zitadel_v2_options_proto_msgTypes[0].Exporter = func(v any, i int) any {
switch v := v.(*Options); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_zitadel_protoc_gen_zitadel_v2_options_proto_msgTypes[1].Exporter = func(v any, i int) any {
switch v := v.(*AuthOption); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_zitadel_protoc_gen_zitadel_v2_options_proto_msgTypes[2].Exporter = func(v any, i int) any {
switch v := v.(*CustomHTTPResponse); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
type x struct{} type x struct{}
out := protoimpl.TypeBuilder{ out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{ File: protoimpl.DescBuilder{