TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-11 21:37:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(database): support for postgres (#3998)

Browse Source 
* beginning with postgres statements

* try pgx

* use pgx

* database

* init works for postgres

* arrays working

* init for cockroach

* init

* start tests

* tests

* TESTS

* ch

* ch

* chore: use go 1.18

* read stmts

* fix typo

* tests

* connection string

* add missing error handler

* cleanup

* start all apis

* go mod tidy

* old update

* switch back to minute

* on conflict

* replace string slice with `database.StringArray` in db models

* fix tests and start

* update go version in dockerfile

* setup go

* clean up

* remove notification migration

* update

* docs: add deploy guide for postgres

* fix: revert sonyflake

* use `database.StringArray` for daos

* use `database.StringArray` every where

* new tables

* index naming,
metadata primary key,
project grant role key type

* docs(postgres): change to beta

* chore: correct compose

* fix(defaults): add empty postgres config

* refactor: remove unused code

* docs: add postgres to self hosted

* fix broken link

* so?

* change title

* add mdx to link

* fix stmt

* update goreleaser in test-code

* docs: improve postgres example

* update more projections

* fix: add beta log for postgres

* revert index name change

* prerelease

* fix: add sequence to v1 "reduce paniced"

* log if nil

* add logging

* fix: log output

* fix(import): check if org exists and user

* refactor: imports

* fix(user): ignore malformed events

* refactor: method naming

* fix: test

* refactor: correct errors.Is call

* ci: don't build dev binaries on main

* fix(go releaser): update version to 1.11.0

* fix(user): projection should not break

* fix(user): handle error properly

* docs: correct config example

* Update .releaserc.js

* Update .releaserc.js

Co-authored-by: Livio Amstutz <livio.a@gmail.com>
Co-authored-by: Elio Bischof <eliobischof@gmail.com>
This commit is contained in:
Silvan
2022-08-31 09:52:43 +02:00
committed by GitHub
parent d6c9815945
commit 77b4fc5487
189 changed files with 3401 additions and 2956 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
internal/eventstore/repository/sql/crdb.go
View File

@@ -9,6 +9,7 @@ import (
"strings"
"github.com/cockroachdb/cockroach-go/v2/crdb"
"github.com/jackc/pgconn"
"github.com/lib/pq"
"github.com/zitadel/logging"
@@ -30,7 +31,7 @@ const (
" SELECT MAX(event_sequence) seq, 1 join_me" +
" FROM eventstore.events" +
" WHERE aggregate_type = $2" +
" AND (CASE WHEN $9::STRING IS NULL THEN instance_id is null else instance_id = $9::STRING END)" +
" AND (CASE WHEN $9::TEXT IS NULL THEN instance_id is null else instance_id = $9::TEXT END)" +
") AS agg_type " +
// combined with
"LEFT JOIN " +
@@ -39,7 +40,7 @@ const (
" SELECT event_sequence seq, resource_owner ro, 1 join_me" +
" FROM eventstore.events" +
" WHERE aggregate_type = $2 AND aggregate_id = $3" +
" AND (CASE WHEN $9::STRING IS NULL THEN instance_id is null else instance_id = $9::STRING END)" +
" AND (CASE WHEN $9::TEXT IS NULL THEN instance_id is null else instance_id = $9::TEXT END)" +
" ORDER BY event_sequence DESC" +
" LIMIT 1" +
") AS agg USING(join_me)" +
@@ -69,9 +70,9 @@ const (
" $5::JSONB AS event_data," +
" $6::VARCHAR AS editor_user," +
" $7::VARCHAR AS editor_service," +
" IFNULL((resource_owner), $8::VARCHAR) AS resource_owner," +
" COALESCE((resource_owner), $8::VARCHAR) AS resource_owner," +
" $9::VARCHAR AS instance_id," +
" NEXTVAL(CONCAT('eventstore.', IF($9 <> '', CONCAT('i_', $9), 'system'), '_seq'))," +
" NEXTVAL(CONCAT('eventstore.', (CASE WHEN $9 <> '' THEN CONCAT('i_', $9) ELSE 'system' END), '_seq'))," +
" aggregate_sequence AS previous_aggregate_sequence," +
" aggregate_type_sequence AS previous_aggregate_type_sequence " +
"FROM previous_data " +
@@ -156,7 +157,7 @@ func (db *CRDB) Push(ctx context.Context, events []*repository.Event, uniqueCons
var instanceRegexp = regexp.MustCompile(`eventstore\.i_[0-9a-zA-Z]{1,}_seq`)
func (db *CRDB) CreateInstance(ctx context.Context, instanceID string) error {
row := db.client.QueryRowContext(ctx, "SELECT CONCAT('eventstore.i_', $1, '_seq')", instanceID)
row := db.client.QueryRowContext(ctx, "SELECT CONCAT('eventstore.i_', $1::TEXT, '_seq')", instanceID)
if row.Err() != nil {
return caos_errs.ThrowInvalidArgument(row.Err(), "SQL-7gtFA", "Errors.InvalidArgument")
}
@@ -218,7 +219,7 @@ func (db *CRDB) Filter(ctx context.Context, searchQuery *repository.SearchQuery)
return events, nil
}
//LatestSequence returns the latest sequence found by the search query
// LatestSequence returns the latest sequence found by the search query
func (db *CRDB) LatestSequence(ctx context.Context, searchQuery *repository.SearchQuery) (uint64, error) {
var seq Sequence
err := query(ctx, db, searchQuery, &seq)
@@ -228,7 +229,7 @@ func (db *CRDB) LatestSequence(ctx context.Context, searchQuery *repository.Sear
return uint64(seq), nil
}
//InstanceIDs returns the instance ids found by the search query
// InstanceIDs returns the instance ids found by the search query
func (db *CRDB) InstanceIDs(ctx context.Context, searchQuery *repository.SearchQuery) ([]string, error) {
var ids []string
err := query(ctx, db, searchQuery, &ids)
@@ -331,7 +332,7 @@ var (
placeholder = regexp.MustCompile(`\?`)
)
//placeholder replaces all "?" with postgres placeholders ($<NUMBER>)
// placeholder replaces all "?" with postgres placeholders ($<NUMBER>)
func (db *CRDB) placeholder(query string) string {
occurances := placeholder.FindAllStringIndex(query, -1)
if len(occurances) == 0 {
@@ -355,5 +356,10 @@ func (db *CRDB) isUniqueViolationError(err error) bool {
return true
}
}
if pgxErr, ok := err.(*pgconn.PgError); ok {
if pgxErr.Code == "23505" {
return true
}
}
return false
}

37
internal/eventstore/repository/sql/crdb_test.go
View File

@@ -6,8 +6,7 @@ import (
"sync"
"testing"
"github.com/lib/pq"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore/repository"
)
@@ -279,7 +278,7 @@ func TestCRDB_Push_OneAggregate(t *testing.T) {
uniqueCount int
assetCount int
aggType repository.AggregateType
aggID []string
aggID database.StringArray
}
type res struct {
wantErr bool
@@ -430,7 +429,7 @@ func TestCRDB_Push_OneAggregate(t *testing.T) {
t.Errorf("CRDB.Push() error = %v, wantErr %v", err, tt.res.wantErr)
}
countEventRow := testCRDBClient.QueryRow("SELECT COUNT(*) FROM eventstore.events where aggregate_type = $1 AND aggregate_id = ANY($2)", tt.res.eventsRes.aggType, pq.Array(tt.res.eventsRes.aggID))
countEventRow := testCRDBClient.QueryRow("SELECT COUNT(*) FROM eventstore.events where aggregate_type = $1 AND aggregate_id = ANY($2)", tt.res.eventsRes.aggType, tt.res.eventsRes.aggID)
var eventCount int
err := countEventRow.Scan(&eventCount)
if err != nil {
@@ -462,8 +461,8 @@ func TestCRDB_Push_MultipleAggregate(t *testing.T) {
}
type eventsRes struct {
pushedEventsCount int
aggType []repository.AggregateType
aggID []string
aggType database.StringArray
aggID database.StringArray
}
type res struct {
wantErr bool
@@ -487,7 +486,7 @@ func TestCRDB_Push_MultipleAggregate(t *testing.T) {
eventsRes: eventsRes{
pushedEventsCount: 2,
aggID: []string{"100", "101"},
aggType: []repository.AggregateType{repository.AggregateType(t.Name())},
aggType: database.StringArray{t.Name()},
},
},
},
@@ -506,7 +505,7 @@ func TestCRDB_Push_MultipleAggregate(t *testing.T) {
eventsRes: eventsRes{
pushedEventsCount: 4,
aggID: []string{"102", "103"},
aggType: []repository.AggregateType{repository.AggregateType(t.Name())},
aggType: database.StringArray{t.Name()},
},
},
},
@@ -533,7 +532,7 @@ func TestCRDB_Push_MultipleAggregate(t *testing.T) {
eventsRes: eventsRes{
pushedEventsCount: 12,
aggID: []string{"106", "107", "108"},
aggType: []repository.AggregateType{repository.AggregateType(t.Name())},
aggType: database.StringArray{t.Name()},
},
},
},
@@ -547,7 +546,7 @@ func TestCRDB_Push_MultipleAggregate(t *testing.T) {
t.Errorf("CRDB.Push() error = %v, wantErr %v", err, tt.res.wantErr)
}
countRow := testCRDBClient.QueryRow("SELECT COUNT(*) FROM eventstore.events where aggregate_type = ANY($1) AND aggregate_id = ANY($2)", pq.Array(tt.res.eventsRes.aggType), pq.Array(tt.res.eventsRes.aggID))
countRow := testCRDBClient.QueryRow("SELECT COUNT(*) FROM eventstore.events where aggregate_type = ANY($1) AND aggregate_id = ANY($2)", tt.res.eventsRes.aggType, tt.res.eventsRes.aggID)
var count int
err := countRow.Scan(&count)
if err != nil {
@@ -645,8 +644,8 @@ func TestCRDB_Push_Parallel(t *testing.T) {
}
type eventsRes struct {
pushedEventsCount int
aggTypes []repository.AggregateType
aggIDs []string
aggTypes database.StringArray
aggIDs database.StringArray
}
type res struct {
errCount int
@@ -681,7 +680,7 @@ func TestCRDB_Push_Parallel(t *testing.T) {
eventsRes: eventsRes{
aggIDs: []string{"200", "201", "202", "203"},
pushedEventsCount: 9,
aggTypes: []repository.AggregateType{repository.AggregateType(t.Name())},
aggTypes: database.StringArray{t.Name()},
},
},
},
@@ -718,7 +717,7 @@ func TestCRDB_Push_Parallel(t *testing.T) {
eventsRes: eventsRes{
aggIDs: []string{"204", "205", "206"},
pushedEventsCount: 14,
aggTypes: []repository.AggregateType{repository.AggregateType(t.Name())},
aggTypes: database.StringArray{t.Name()},
},
},
},
@@ -748,7 +747,7 @@ func TestCRDB_Push_Parallel(t *testing.T) {
eventsRes: eventsRes{
aggIDs: []string{"207", "208"},
pushedEventsCount: 11,
aggTypes: []repository.AggregateType{repository.AggregateType(t.Name())},
aggTypes: database.StringArray{t.Name()},
},
},
},
@@ -781,7 +780,7 @@ func TestCRDB_Push_Parallel(t *testing.T) {
t.Errorf("CRDB.Push() error count = %d, wanted err count %d, errs: %v", len(errs), tt.res.errCount, errs)
}
rows, err := testCRDBClient.Query("SELECT event_data FROM eventstore.events where aggregate_type = ANY($1) AND aggregate_id = ANY($2) order by event_sequence", pq.Array(tt.res.eventsRes.aggTypes), pq.Array(tt.res.eventsRes.aggIDs))
rows, err := testCRDBClient.Query("SELECT event_data FROM eventstore.events where aggregate_type = ANY($1) AND aggregate_id = ANY($2) order by event_sequence", tt.res.eventsRes.aggTypes, tt.res.eventsRes.aggIDs)
if err != nil {
t.Error("unable to query inserted rows: ", err)
return
@@ -993,10 +992,10 @@ func TestCRDB_Push_ResourceOwner(t *testing.T) {
events []*repository.Event
}
type res struct {
resourceOwners []string
resourceOwners database.StringArray
}
type fields struct {
aggregateIDs []string
aggregateIDs database.StringArray
aggregateType string
}
tests := []struct {
@@ -1128,7 +1127,7 @@ func TestCRDB_Push_ResourceOwner(t *testing.T) {
}
}
rows, err := testCRDBClient.Query("SELECT resource_owner FROM eventstore.events WHERE aggregate_type = $1 AND aggregate_id = ANY($2) ORDER BY event_sequence", tt.fields.aggregateType, pq.Array(tt.fields.aggregateIDs))
rows, err := testCRDBClient.Query("SELECT resource_owner FROM eventstore.events WHERE aggregate_type = $1 AND aggregate_id = ANY($2) ORDER BY event_sequence", tt.fields.aggregateType, tt.fields.aggregateIDs)
if err != nil {
t.Error("unable to query inserted rows: ", err)
return

21
internal/eventstore/repository/sql/local_crdb_test.go
View File

@@ -9,6 +9,8 @@ import (
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/cmd/initialise"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/database/cockroach"
)
var (
@@ -42,15 +44,22 @@ func TestMain(m *testing.M) {
}
func initDB(db *sql.DB) error {
username := "zitadel"
database := "zitadel"
err := initialise.Initialise(db, initialise.VerifyUser(username, ""),
initialise.VerifyDatabase(database),
initialise.VerifyGrant(database, username))
config := new(database.Config)
config.SetConnector(&cockroach.Config{User: cockroach.User{Username: "zitadel"}, Database: "zitadel"})
if err := initialise.ReadStmts("cockroach"); err != nil {
return err
}
err := initialise.Init(db,
initialise.VerifyUser(config.Username(), ""),
initialise.VerifyDatabase(config.Database()),
initialise.VerifyGrant(config.Database(), config.Username()))
if err != nil {
return err
}
return initialise.VerifyZitadel(db)
return initialise.VerifyZitadel(db, *config)
}
func fillUniqueData(unique_type, field, instanceID string) error {

3
internal/eventstore/repository/sql/query.go
View File

@@ -8,7 +8,6 @@ import (
"fmt"
"strings"
"github.com/lib/pq"
"github.com/zitadel/logging"
z_errors "github.com/zitadel/zitadel/internal/errors"
@@ -170,8 +169,6 @@ func prepareCondition(criteria querier, filters [][]*repository.Filter) (clause
for _, f := range filter {
value := f.Value
switch value.(type) {
case []bool, []float64, []int64, []string, []repository.AggregateType, []repository.EventType, *[]bool, *[]float64, *[]int64, *[]string, *[]repository.AggregateType, *[]repository.EventType:
value = pq.Array(value)
case map[string]interface{}:
var err error
value, err = json.Marshal(value)

5
internal/eventstore/repository/sql/query_test.go
View File

@@ -9,7 +9,6 @@ import (
"time"
"github.com/DATA-DOG/go-sqlmock"
"github.com/lib/pq"
"github.com/zitadel/zitadel/internal/errors"
"github.com/zitadel/zitadel/internal/eventstore/repository"
@@ -265,7 +264,7 @@ func Test_prepareCondition(t *testing.T) {
},
res: res{
clause: " WHERE ( aggregate_type = ANY(?) )",
values: []interface{}{pq.Array([]repository.AggregateType{"user", "org"})},
values: []interface{}{[]repository.AggregateType{"user", "org"}},
},
},
{
@@ -281,7 +280,7 @@ func Test_prepareCondition(t *testing.T) {
},
res: res{
clause: " WHERE ( aggregate_type = ANY(?) AND aggregate_id = ? AND event_type = ANY(?) )",
values: []interface{}{pq.Array([]repository.AggregateType{"user", "org"}), "1234", pq.Array([]repository.EventType{"user.created", "org.created"})},
values: []interface{}{[]repository.AggregateType{"user", "org"}, "1234", []repository.EventType{"user.created", "org.created"}},
},
},
}