feat(org): add org metadata functionality (#4234)

* feat(org): add org metadata functionality

* fix(metadata): add unit tests and review for org metadata

* fix(org-metadata): move endpoints to /

Co-authored-by: Fabi <38692350+hifabienne@users.noreply.github.com>
This commit is contained in:
Stefan Benz
2022-09-20 15:32:09 +01:00
committed by GitHub
parent 05cb672cff
commit 2c1f9ac4a8
25 changed files with 2267 additions and 27 deletions

View File

@@ -0,0 +1,224 @@
package query
import (
"context"
"database/sql"
errs "errors"
"time"
sq "github.com/Masterminds/squirrel"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/errors"
"github.com/zitadel/zitadel/internal/query/projection"
)
type OrgMetadataList struct {
SearchResponse
Metadata []*OrgMetadata
}
type OrgMetadata struct {
CreationDate time.Time
ChangeDate time.Time
ResourceOwner string
Sequence uint64
Key string
Value []byte
}
type OrgMetadataSearchQueries struct {
SearchRequest
Queries []SearchQuery
}
var (
orgMetadataTable = table{
name: projection.OrgMetadataProjectionTable,
}
OrgMetadataOrgIDCol = Column{
name: projection.OrgMetadataColumnOrgID,
table: orgMetadataTable,
}
OrgMetadataCreationDateCol = Column{
name: projection.OrgMetadataColumnCreationDate,
table: orgMetadataTable,
}
OrgMetadataChangeDateCol = Column{
name: projection.OrgMetadataColumnChangeDate,
table: orgMetadataTable,
}
OrgMetadataResourceOwnerCol = Column{
name: projection.OrgMetadataColumnResourceOwner,
table: orgMetadataTable,
}
OrgMetadataInstanceIDCol = Column{
name: projection.OrgMetadataColumnInstanceID,
table: orgMetadataTable,
}
OrgMetadataSequenceCol = Column{
name: projection.OrgMetadataColumnSequence,
table: orgMetadataTable,
}
OrgMetadataKeyCol = Column{
name: projection.OrgMetadataColumnKey,
table: orgMetadataTable,
}
OrgMetadataValueCol = Column{
name: projection.OrgMetadataColumnValue,
table: orgMetadataTable,
}
)
func (q *Queries) GetOrgMetadataByKey(ctx context.Context, shouldTriggerBulk bool, orgID string, key string, queries ...SearchQuery) (*OrgMetadata, error) {
if shouldTriggerBulk {
projection.OrgMetadataProjection.Trigger(ctx)
}
query, scan := prepareOrgMetadataQuery()
for _, q := range queries {
query = q.toQuery(query)
}
stmt, args, err := query.Where(
sq.Eq{
OrgMetadataOrgIDCol.identifier(): orgID,
OrgMetadataKeyCol.identifier(): key,
OrgMetadataInstanceIDCol.identifier(): authz.GetInstance(ctx).InstanceID(),
}).ToSql()
if err != nil {
return nil, errors.ThrowInternal(err, "QUERY-aDaG2", "Errors.Query.SQLStatment")
}
row := q.client.QueryRowContext(ctx, stmt, args...)
return scan(row)
}
func (q *Queries) SearchOrgMetadata(ctx context.Context, shouldTriggerBulk bool, orgID string, queries *OrgMetadataSearchQueries) (*OrgMetadataList, error) {
if shouldTriggerBulk {
projection.OrgMetadataProjection.Trigger(ctx)
}
query, scan := prepareOrgMetadataListQuery()
stmt, args, err := queries.toQuery(query).Where(
sq.Eq{
OrgMetadataOrgIDCol.identifier(): orgID,
OrgMetadataInstanceIDCol.identifier(): authz.GetInstance(ctx).InstanceID(),
}).
ToSql()
if err != nil {
return nil, errors.ThrowInternal(err, "QUERY-Egbld", "Errors.Query.SQLStatment")
}
rows, err := q.client.QueryContext(ctx, stmt, args...)
if err != nil {
return nil, errors.ThrowInternal(err, "QUERY-Ho2wf", "Errors.Internal")
}
metadata, err := scan(rows)
if err != nil {
return nil, err
}
metadata.LatestSequence, err = q.latestSequence(ctx, orgMetadataTable)
return metadata, err
}
func (q *OrgMetadataSearchQueries) toQuery(query sq.SelectBuilder) sq.SelectBuilder {
query = q.SearchRequest.toQuery(query)
for _, q := range q.Queries {
query = q.toQuery(query)
}
return query
}
func (r *OrgMetadataSearchQueries) AppendMyResourceOwnerQuery(orgID string) error {
query, err := NewOrgMetadataResourceOwnerSearchQuery(orgID)
if err != nil {
return err
}
r.Queries = append(r.Queries, query)
return nil
}
func NewOrgMetadataResourceOwnerSearchQuery(value string) (SearchQuery, error) {
return NewTextQuery(OrgMetadataResourceOwnerCol, value, TextEquals)
}
func NewOrgMetadataKeySearchQuery(value string, comparison TextComparison) (SearchQuery, error) {
return NewTextQuery(OrgMetadataKeyCol, value, comparison)
}
func prepareOrgMetadataQuery() (sq.SelectBuilder, func(*sql.Row) (*OrgMetadata, error)) {
return sq.Select(
OrgMetadataCreationDateCol.identifier(),
OrgMetadataChangeDateCol.identifier(),
OrgMetadataResourceOwnerCol.identifier(),
OrgMetadataSequenceCol.identifier(),
OrgMetadataKeyCol.identifier(),
OrgMetadataValueCol.identifier(),
).
From(orgMetadataTable.identifier()).
PlaceholderFormat(sq.Dollar),
func(row *sql.Row) (*OrgMetadata, error) {
m := new(OrgMetadata)
err := row.Scan(
&m.CreationDate,
&m.ChangeDate,
&m.ResourceOwner,
&m.Sequence,
&m.Key,
&m.Value,
)
if err != nil {
if errs.Is(err, sql.ErrNoRows) {
return nil, errors.ThrowNotFound(err, "QUERY-Rph32", "Errors.Metadata.NotFound")
}
return nil, errors.ThrowInternal(err, "QUERY-Hajt2", "Errors.Internal")
}
return m, nil
}
}
func prepareOrgMetadataListQuery() (sq.SelectBuilder, func(*sql.Rows) (*OrgMetadataList, error)) {
return sq.Select(
OrgMetadataCreationDateCol.identifier(),
OrgMetadataChangeDateCol.identifier(),
OrgMetadataResourceOwnerCol.identifier(),
OrgMetadataSequenceCol.identifier(),
OrgMetadataKeyCol.identifier(),
OrgMetadataValueCol.identifier(),
countColumn.identifier()).
From(orgMetadataTable.identifier()).
PlaceholderFormat(sq.Dollar),
func(rows *sql.Rows) (*OrgMetadataList, error) {
metadata := make([]*OrgMetadata, 0)
var count uint64
for rows.Next() {
m := new(OrgMetadata)
err := rows.Scan(
&m.CreationDate,
&m.ChangeDate,
&m.ResourceOwner,
&m.Sequence,
&m.Key,
&m.Value,
&count,
)
if err != nil {
return nil, err
}
metadata = append(metadata, m)
}
if err := rows.Close(); err != nil {
return nil, errors.ThrowInternal(err, "QUERY-dd3gh", "Errors.Query.CloseRows")
}
return &OrgMetadataList{
Metadata: metadata,
SearchResponse: SearchResponse{
Count: count,
},
}, nil
}
}

View File

@@ -0,0 +1,248 @@
package query
import (
"database/sql"
"database/sql/driver"
"errors"
"fmt"
"regexp"
"testing"
errs "github.com/zitadel/zitadel/internal/errors"
)
var (
orgMetadataQuery = `SELECT projections.org_metadata.creation_date,` +
` projections.org_metadata.change_date,` +
` projections.org_metadata.resource_owner,` +
` projections.org_metadata.sequence,` +
` projections.org_metadata.key,` +
` projections.org_metadata.value` +
` FROM projections.org_metadata`
orgMetadataCols = []string{
"creation_date",
"change_date",
"resource_owner",
"sequence",
"key",
"value",
}
orgMetadataListQuery = `SELECT projections.org_metadata.creation_date,` +
` projections.org_metadata.change_date,` +
` projections.org_metadata.resource_owner,` +
` projections.org_metadata.sequence,` +
` projections.org_metadata.key,` +
` projections.org_metadata.value,` +
` COUNT(*) OVER ()` +
` FROM projections.org_metadata`
orgMetadataListCols = []string{
"creation_date",
"change_date",
"resource_owner",
"sequence",
"key",
"value",
"count",
}
)
func Test_OrgMetadataPrepares(t *testing.T) {
type want struct {
sqlExpectations sqlExpectation
err checkErr
}
tests := []struct {
name string
prepare interface{}
want want
object interface{}
}{
{
name: "prepareOrgMetadataQuery no result",
prepare: prepareOrgMetadataQuery,
want: want{
sqlExpectations: mockQuery(
regexp.QuoteMeta(orgMetadataQuery),
nil,
nil,
),
err: func(err error) (error, bool) {
if !errs.IsNotFound(err) {
return fmt.Errorf("err should be zitadel.NotFoundError got: %w", err), false
}
return nil, true
},
},
object: (*OrgMetadata)(nil),
},
{
name: "prepareOrgMetadataQuery found",
prepare: prepareOrgMetadataQuery,
want: want{
sqlExpectations: mockQuery(
regexp.QuoteMeta(orgMetadataQuery),
orgMetadataCols,
[]driver.Value{
testNow,
testNow,
"resource_owner",
uint64(20211108),
"key",
[]byte("value"),
},
),
},
object: &OrgMetadata{
CreationDate: testNow,
ChangeDate: testNow,
ResourceOwner: "resource_owner",
Sequence: 20211108,
Key: "key",
Value: []byte("value"),
},
},
{
name: "prepareOrgMetadataQuery sql err",
prepare: prepareOrgMetadataQuery,
want: want{
sqlExpectations: mockQueryErr(
regexp.QuoteMeta(orgMetadataQuery),
sql.ErrConnDone,
),
err: func(err error) (error, bool) {
if !errors.Is(err, sql.ErrConnDone) {
return fmt.Errorf("err should be sql.ErrConnDone got: %w", err), false
}
return nil, true
},
},
object: nil,
},
{
name: "prepareOrgMetadataListQuery no result",
prepare: prepareOrgMetadataListQuery,
want: want{
sqlExpectations: mockQueries(
regexp.QuoteMeta(orgMetadataListQuery),
nil,
nil,
),
err: func(err error) (error, bool) {
if !errs.IsNotFound(err) {
return fmt.Errorf("err should be zitadel.NotFoundError got: %w", err), false
}
return nil, true
},
},
object: &OrgMetadataList{Metadata: []*OrgMetadata{}},
},
{
name: "prepareOrgMetadataListQuery one result",
prepare: prepareOrgMetadataListQuery,
want: want{
sqlExpectations: mockQueries(
regexp.QuoteMeta(orgMetadataListQuery),
orgMetadataListCols,
[][]driver.Value{
{
testNow,
testNow,
"resource_owner",
uint64(20211108),
"key",
[]byte("value"),
},
},
),
},
object: &OrgMetadataList{
SearchResponse: SearchResponse{
Count: 1,
},
Metadata: []*OrgMetadata{
{
CreationDate: testNow,
ChangeDate: testNow,
ResourceOwner: "resource_owner",
Sequence: 20211108,
Key: "key",
Value: []byte("value"),
},
},
},
},
{
name: "prepareOrgMetadataListQuery multiple results",
prepare: prepareOrgMetadataListQuery,
want: want{
sqlExpectations: mockQueries(
regexp.QuoteMeta(orgMetadataListQuery),
orgMetadataListCols,
[][]driver.Value{
{
testNow,
testNow,
"resource_owner",
uint64(20211108),
"key",
[]byte("value"),
},
{
testNow,
testNow,
"resource_owner",
uint64(20211108),
"key2",
[]byte("value2"),
},
},
),
},
object: &OrgMetadataList{
SearchResponse: SearchResponse{
Count: 2,
},
Metadata: []*OrgMetadata{
{
CreationDate: testNow,
ChangeDate: testNow,
ResourceOwner: "resource_owner",
Sequence: 20211108,
Key: "key",
Value: []byte("value"),
},
{
CreationDate: testNow,
ChangeDate: testNow,
ResourceOwner: "resource_owner",
Sequence: 20211108,
Key: "key2",
Value: []byte("value2"),
},
},
},
},
{
name: "prepareOrgMetadataListQuery sql err",
prepare: prepareOrgMetadataListQuery,
want: want{
sqlExpectations: mockQueryErr(
regexp.QuoteMeta(orgMetadataListQuery),
sql.ErrConnDone,
),
err: func(err error) (error, bool) {
if !errors.Is(err, sql.ErrConnDone) {
return fmt.Errorf("err should be sql.ErrConnDone got: %w", err), false
}
return nil, true
},
},
object: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assertPrepare(t, tt.prepare, tt.object, tt.want.sqlExpectations, tt.want.err)
})
}
}

View File

@@ -0,0 +1,132 @@
package projection
import (
"context"
"github.com/zitadel/zitadel/internal/errors"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/eventstore/handler"
"github.com/zitadel/zitadel/internal/eventstore/handler/crdb"
"github.com/zitadel/zitadel/internal/repository/org"
)
const (
OrgMetadataProjectionTable = "projections.org_metadata"
OrgMetadataColumnOrgID = "org_id"
OrgMetadataColumnCreationDate = "creation_date"
OrgMetadataColumnChangeDate = "change_date"
OrgMetadataColumnSequence = "sequence"
OrgMetadataColumnResourceOwner = "resource_owner"
OrgMetadataColumnInstanceID = "instance_id"
OrgMetadataColumnKey = "key"
OrgMetadataColumnValue = "value"
)
type orgMetadataProjection struct {
crdb.StatementHandler
}
func newOrgMetadataProjection(ctx context.Context, config crdb.StatementHandlerConfig) *orgMetadataProjection {
p := new(orgMetadataProjection)
config.ProjectionName = OrgMetadataProjectionTable
config.Reducers = p.reducers()
config.InitCheck = crdb.NewTableCheck(
crdb.NewTable([]*crdb.Column{
crdb.NewColumn(OrgMetadataColumnOrgID, crdb.ColumnTypeText),
crdb.NewColumn(OrgMetadataColumnCreationDate, crdb.ColumnTypeTimestamp),
crdb.NewColumn(OrgMetadataColumnChangeDate, crdb.ColumnTypeTimestamp),
crdb.NewColumn(OrgMetadataColumnSequence, crdb.ColumnTypeInt64),
crdb.NewColumn(OrgMetadataColumnResourceOwner, crdb.ColumnTypeText),
crdb.NewColumn(OrgMetadataColumnInstanceID, crdb.ColumnTypeText),
crdb.NewColumn(OrgMetadataColumnKey, crdb.ColumnTypeText),
crdb.NewColumn(OrgMetadataColumnValue, crdb.ColumnTypeBytes, crdb.Nullable()),
},
crdb.NewPrimaryKey(OrgMetadataColumnInstanceID, OrgMetadataColumnOrgID, OrgMetadataColumnKey),
),
)
p.StatementHandler = crdb.NewStatementHandler(ctx, config)
return p
}
func (p *orgMetadataProjection) reducers() []handler.AggregateReducer {
return []handler.AggregateReducer{
{
Aggregate: org.AggregateType,
EventRedusers: []handler.EventReducer{
{
Event: org.MetadataSetType,
Reduce: p.reduceMetadataSet,
},
{
Event: org.MetadataRemovedType,
Reduce: p.reduceMetadataRemoved,
},
{
Event: org.MetadataRemovedAllType,
Reduce: p.reduceMetadataRemovedAll,
},
{
Event: org.OrgRemovedEventType,
Reduce: p.reduceMetadataRemovedAll,
},
},
},
}
}
func (p *orgMetadataProjection) reduceMetadataSet(event eventstore.Event) (*handler.Statement, error) {
e, ok := event.(*org.MetadataSetEvent)
if !ok {
return nil, errors.ThrowInvalidArgumentf(nil, "HANDL-Ghn53", "reduce.wrong.event.type %s", org.MetadataSetType)
}
return crdb.NewUpsertStatement(
e,
[]handler.Column{
handler.NewCol(OrgMetadataColumnInstanceID, nil),
handler.NewCol(OrgMetadataColumnOrgID, nil),
handler.NewCol(OrgMetadataColumnKey, e.Key),
},
[]handler.Column{
handler.NewCol(OrgMetadataColumnInstanceID, e.Aggregate().InstanceID),
handler.NewCol(OrgMetadataColumnOrgID, e.Aggregate().ID),
handler.NewCol(OrgMetadataColumnKey, e.Key),
handler.NewCol(OrgMetadataColumnResourceOwner, e.Aggregate().ResourceOwner),
handler.NewCol(OrgMetadataColumnCreationDate, e.CreationDate()),
handler.NewCol(OrgMetadataColumnChangeDate, e.CreationDate()),
handler.NewCol(OrgMetadataColumnSequence, e.Sequence()),
handler.NewCol(OrgMetadataColumnValue, e.Value),
},
), nil
}
func (p *orgMetadataProjection) reduceMetadataRemoved(event eventstore.Event) (*handler.Statement, error) {
e, ok := event.(*org.MetadataRemovedEvent)
if !ok {
return nil, errors.ThrowInvalidArgumentf(nil, "HANDL-Bm542", "reduce.wrong.event.type %s", org.MetadataRemovedType)
}
return crdb.NewDeleteStatement(
e,
[]handler.Condition{
handler.NewCond(OrgMetadataColumnOrgID, e.Aggregate().ID),
handler.NewCond(OrgMetadataColumnKey, e.Key),
},
), nil
}
func (p *orgMetadataProjection) reduceMetadataRemovedAll(event eventstore.Event) (*handler.Statement, error) {
switch event.(type) {
case *org.MetadataRemovedAllEvent,
*org.OrgRemovedEvent:
//ok
default:
return nil, errors.ThrowInvalidArgumentf(nil, "HANDL-Bmnf3", "reduce.wrong.event.type %v", []eventstore.EventType{org.MetadataRemovedAllType, org.OrgRemovedEventType})
}
return crdb.NewDeleteStatement(
event,
[]handler.Condition{
handler.NewCond(OrgMetadataColumnOrgID, event.Aggregate().ID),
},
), nil
}

View File

@@ -0,0 +1,158 @@
package projection
import (
"testing"
"github.com/zitadel/zitadel/internal/errors"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/eventstore/handler"
"github.com/zitadel/zitadel/internal/eventstore/repository"
"github.com/zitadel/zitadel/internal/repository/org"
)
func TestOrgMetadataProjection_reduces(t *testing.T) {
type args struct {
event func(t *testing.T) eventstore.Event
}
tests := []struct {
name string
args args
reduce func(event eventstore.Event) (*handler.Statement, error)
want wantReduce
}{
{
name: "reduceMetadataSet",
args: args{
event: getEvent(testEvent(
repository.EventType(org.MetadataSetType),
org.AggregateType,
[]byte(`{
"key": "key",
"value": "dmFsdWU="
}`),
), org.MetadataSetEventMapper),
},
reduce: (&orgMetadataProjection{}).reduceMetadataSet,
want: wantReduce{
aggregateType: org.AggregateType,
sequence: 15,
previousSequence: 10,
projection: OrgMetadataProjectionTable,
executer: &testExecuter{
executions: []execution{
{
expectedStmt: "INSERT INTO projections.org_metadata (instance_id, org_id, key, resource_owner, creation_date, change_date, sequence, value) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (instance_id, org_id, key) DO UPDATE SET (resource_owner, creation_date, change_date, sequence, value) = (EXCLUDED.resource_owner, EXCLUDED.creation_date, EXCLUDED.change_date, EXCLUDED.sequence, EXCLUDED.value)",
expectedArgs: []interface{}{
"instance-id",
"agg-id",
"key",
"ro-id",
anyArg{},
anyArg{},
uint64(15),
[]byte("value"),
},
},
},
},
},
},
{
name: "reduceMetadataRemoved",
args: args{
event: getEvent(testEvent(
repository.EventType(org.MetadataRemovedType),
org.AggregateType,
[]byte(`{
"key": "key"
}`),
), org.MetadataRemovedEventMapper),
},
reduce: (&orgMetadataProjection{}).reduceMetadataRemoved,
want: wantReduce{
aggregateType: org.AggregateType,
sequence: 15,
previousSequence: 10,
projection: OrgMetadataProjectionTable,
executer: &testExecuter{
executions: []execution{
{
expectedStmt: "DELETE FROM projections.org_metadata WHERE (org_id = $1) AND (key = $2)",
expectedArgs: []interface{}{
"agg-id",
"key",
},
},
},
},
},
},
{
name: "reduceMetadataRemovedAll",
args: args{
event: getEvent(testEvent(
repository.EventType(org.MetadataRemovedAllType),
org.AggregateType,
nil,
), org.MetadataRemovedAllEventMapper),
},
reduce: (&orgMetadataProjection{}).reduceMetadataRemovedAll,
want: wantReduce{
aggregateType: org.AggregateType,
sequence: 15,
previousSequence: 10,
projection: OrgMetadataProjectionTable,
executer: &testExecuter{
executions: []execution{
{
expectedStmt: "DELETE FROM projections.org_metadata WHERE (org_id = $1)",
expectedArgs: []interface{}{
"agg-id",
},
},
},
},
},
},
{
name: "reduceMetadataRemovedAll (org removed)",
args: args{
event: getEvent(testEvent(
repository.EventType(org.OrgRemovedEventType),
org.AggregateType,
nil,
), org.OrgRemovedEventMapper),
},
reduce: (&orgMetadataProjection{}).reduceMetadataRemovedAll,
want: wantReduce{
aggregateType: org.AggregateType,
sequence: 15,
previousSequence: 10,
projection: OrgMetadataProjectionTable,
executer: &testExecuter{
executions: []execution{
{
expectedStmt: "DELETE FROM projections.org_metadata WHERE (org_id = $1)",
expectedArgs: []interface{}{
"agg-id",
},
},
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
event := baseEvent(t)
got, err := tt.reduce(event)
if _, ok := err.(errors.InvalidArgument); !ok {
t.Errorf("no wrong event mapping: %v, got: %v", err, got)
}
event = tt.args.event(t)
got, err = tt.reduce(event)
assertReduce(t, got, err, tt.want)
})
}
}

View File

@@ -20,6 +20,7 @@ const (
var (
projectionConfig crdb.StatementHandlerConfig
OrgProjection *orgProjection
OrgMetadataProjection *orgMetadataProjection
ActionProjection *actionProjection
FlowProjection *flowProjection
ProjectProjection *projectProjection
@@ -82,6 +83,7 @@ func Start(ctx context.Context, sqlClient *sql.DB, es *eventstore.Eventstore, co
}
OrgProjection = newOrgProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["orgs"]))
OrgMetadataProjection = newOrgMetadataProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["org_metadata"]))
ActionProjection = newActionProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["actions"]))
FlowProjection = newFlowProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["flows"]))
ProjectProjection = newProjectProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["projects"]))