TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-12 04:57:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: project quotas and usages (#6441)

Browse Source 
* project quota added

* project quota removed

* add periods table

* make log record generic

* accumulate usage

* query usage

* count action run seconds

* fix filter in ReportQuotaUsage

* fix existing tests

* fix logstore tests

* fix typo

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* move notifications into debouncer and improve limit querying

* cleanup

* comment

* fix: add quota unit tests command side

* fix remaining quota usage query

* implement InmemLogStorage

* cleanup and linting

* improve test

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* action notifications and fixes for notifications query

* revert console prefix

* fix: add quota unit tests command side

* fix: add quota integration tests

* improve accountable requests

* improve accountable requests

* fix: add quota integration tests

* fix: add quota integration tests

* fix: add quota integration tests

* comment

* remove ability to store logs in db and other changes requested from review

* changes requested from review

* changes requested from review

* Update internal/api/http/middleware/access_interceptor.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* tests: fix quotas integration tests

* improve incrementUsageStatement

* linting

* fix: delete e2e tests as intergation tests cover functionality

* Update internal/api/http/middleware/access_interceptor.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* backup

* fix conflict

* create rc

* create prerelease

* remove issue release labeling

* fix tracing

---------

Co-authored-by: Livio Spring <livio.a@gmail.com>
Co-authored-by: Stefan Benz <stefan@caos.ch>
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
This commit is contained in:
Elio Bischof
2023-09-15 16:58:45 +02:00
committed by GitHub
parent b4d0d2c9a7
commit 1a49b7d298
66 changed files with 3423 additions and 1413 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/query/projection/main_test.go
View File

@@ -13,6 +13,7 @@ import (
key_repo "github.com/zitadel/zitadel/internal/repository/keypair"
"github.com/zitadel/zitadel/internal/repository/org"
proj_repo "github.com/zitadel/zitadel/internal/repository/project"
quota_repo "github.com/zitadel/zitadel/internal/repository/quota"
usr_repo "github.com/zitadel/zitadel/internal/repository/user"
"github.com/zitadel/zitadel/internal/repository/usergrant"
)
@@ -29,6 +30,7 @@ func eventstoreExpect(t *testing.T, expects ...expect) *eventstore.Eventstore {
org.RegisterEventMappers(es)
usr_repo.RegisterEventMappers(es)
proj_repo.RegisterEventMappers(es)
quota_repo.RegisterEventMappers(es)
usergrant.RegisterEventMappers(es)
key_repo.RegisterEventMappers(es)
action_repo.RegisterEventMappers(es)

3
internal/query/projection/projection.go
View File

@@ -69,6 +69,7 @@ var (
SessionProjection *sessionProjection
AuthRequestProjection *authRequestProjection
MilestoneProjection *milestoneProjection
QuotaProjection *quotaProjection
)
type projection interface {
@@ -148,6 +149,7 @@ func Create(ctx context.Context, sqlClient *database.DB, es *eventstore.Eventsto
SessionProjection = newSessionProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["sessions"]))
AuthRequestProjection = newAuthRequestProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["auth_requests"]))
MilestoneProjection = newMilestoneProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["milestones"]))
QuotaProjection = newQuotaProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["quotas"]))
newProjectionsList()
return nil
}
@@ -247,5 +249,6 @@ func newProjectionsList() {
SessionProjection,
AuthRequestProjection,
MilestoneProjection,
QuotaProjection,
}
}

285
internal/query/projection/quota.go Normal file
View File

@@ -0,0 +1,285 @@
package projection
import (
"context"
"time"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/errors"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/eventstore/handler"
"github.com/zitadel/zitadel/internal/eventstore/handler/crdb"
"github.com/zitadel/zitadel/internal/repository/instance"
"github.com/zitadel/zitadel/internal/repository/quota"
)
const (
QuotasProjectionTable = "projections.quotas"
QuotaPeriodsProjectionTable = QuotasProjectionTable + "_" + quotaPeriodsTableSuffix
QuotaNotificationsTable = QuotasProjectionTable + "_" + quotaNotificationsTableSuffix
QuotaColumnID = "id"
QuotaColumnInstanceID = "instance_id"
QuotaColumnUnit = "unit"
QuotaColumnAmount = "amount"
QuotaColumnFrom = "from_anchor"
QuotaColumnInterval = "interval"
QuotaColumnLimit = "limit_usage"
quotaPeriodsTableSuffix = "periods"
QuotaPeriodColumnInstanceID = "instance_id"
QuotaPeriodColumnUnit = "unit"
QuotaPeriodColumnStart = "start"
QuotaPeriodColumnUsage = "usage"
quotaNotificationsTableSuffix = "notifications"
QuotaNotificationColumnInstanceID = "instance_id"
QuotaNotificationColumnUnit = "unit"
QuotaNotificationColumnID = "id"
QuotaNotificationColumnCallURL = "call_url"
QuotaNotificationColumnPercent = "percent"
QuotaNotificationColumnRepeat = "repeat"
QuotaNotificationColumnLatestDuePeriodStart = "latest_due_period_start"
QuotaNotificationColumnNextDueThreshold = "next_due_threshold"
)
const (
incrementQuotaStatement = `INSERT INTO projections.quotas_periods` +
` (instance_id, unit, start, usage)` +
` VALUES ($1, $2, $3, $4) ON CONFLICT (instance_id, unit, start)` +
` DO UPDATE SET usage = projections.quotas_periods.usage + excluded.usage RETURNING usage`
)
type quotaProjection struct {
crdb.StatementHandler
client *database.DB
}
func newQuotaProjection(ctx context.Context, config crdb.StatementHandlerConfig) *quotaProjection {
p := new(quotaProjection)
config.ProjectionName = QuotasProjectionTable
config.Reducers = p.reducers()
config.InitCheck = crdb.NewMultiTableCheck(
crdb.NewTable(
[]*crdb.Column{
crdb.NewColumn(QuotaColumnID, crdb.ColumnTypeText),
crdb.NewColumn(QuotaColumnInstanceID, crdb.ColumnTypeText),
crdb.NewColumn(QuotaColumnUnit, crdb.ColumnTypeEnum),
crdb.NewColumn(QuotaColumnAmount, crdb.ColumnTypeInt64),
crdb.NewColumn(QuotaColumnFrom, crdb.ColumnTypeTimestamp),
crdb.NewColumn(QuotaColumnInterval, crdb.ColumnTypeInterval),
crdb.NewColumn(QuotaColumnLimit, crdb.ColumnTypeBool),
},
crdb.NewPrimaryKey(QuotaColumnInstanceID, QuotaColumnUnit),
),
crdb.NewSuffixedTable(
[]*crdb.Column{
crdb.NewColumn(QuotaPeriodColumnInstanceID, crdb.ColumnTypeText),
crdb.NewColumn(QuotaPeriodColumnUnit, crdb.ColumnTypeEnum),
crdb.NewColumn(QuotaPeriodColumnStart, crdb.ColumnTypeTimestamp),
crdb.NewColumn(QuotaPeriodColumnUsage, crdb.ColumnTypeInt64),
},
crdb.NewPrimaryKey(QuotaPeriodColumnInstanceID, QuotaPeriodColumnUnit, QuotaPeriodColumnStart),
quotaPeriodsTableSuffix,
),
crdb.NewSuffixedTable(
[]*crdb.Column{
crdb.NewColumn(QuotaNotificationColumnInstanceID, crdb.ColumnTypeText),
crdb.NewColumn(QuotaNotificationColumnUnit, crdb.ColumnTypeEnum),
crdb.NewColumn(QuotaNotificationColumnID, crdb.ColumnTypeText),
crdb.NewColumn(QuotaNotificationColumnCallURL, crdb.ColumnTypeText),
crdb.NewColumn(QuotaNotificationColumnPercent, crdb.ColumnTypeInt64),
crdb.NewColumn(QuotaNotificationColumnRepeat, crdb.ColumnTypeBool),
crdb.NewColumn(QuotaNotificationColumnLatestDuePeriodStart, crdb.ColumnTypeTimestamp, crdb.Nullable()),
crdb.NewColumn(QuotaNotificationColumnNextDueThreshold, crdb.ColumnTypeInt64, crdb.Nullable()),
},
crdb.NewPrimaryKey(QuotaNotificationColumnInstanceID, QuotaNotificationColumnUnit, QuotaNotificationColumnID),
quotaNotificationsTableSuffix,
),
)
p.StatementHandler = crdb.NewStatementHandler(ctx, config)
p.client = config.Client
return p
}
func (q *quotaProjection) reducers() []handler.AggregateReducer {
return []handler.AggregateReducer{
{
Aggregate: instance.AggregateType,
EventRedusers: []handler.EventReducer{
{
Event: instance.InstanceRemovedEventType,
Reduce: q.reduceInstanceRemoved,
},
},
},
{
Aggregate: quota.AggregateType,
EventRedusers: []handler.EventReducer{
{
Event: quota.AddedEventType,
Reduce: q.reduceQuotaAdded,
},
},
},
{
Aggregate: quota.AggregateType,
EventRedusers: []handler.EventReducer{
{
Event: quota.RemovedEventType,
Reduce: q.reduceQuotaRemoved,
},
},
},
{
Aggregate: quota.AggregateType,
EventRedusers: []handler.EventReducer{
{
Event: quota.NotificationDueEventType,
Reduce: q.reduceQuotaNotificationDue,
},
},
},
{
Aggregate: quota.AggregateType,
EventRedusers: []handler.EventReducer{
{
Event: quota.NotifiedEventType,
Reduce: q.reduceQuotaNotified,
},
},
},
}
}
func (q *quotaProjection) reduceQuotaNotified(event eventstore.Event) (*handler.Statement, error) {
return crdb.NewNoOpStatement(event), nil
}
func (q *quotaProjection) reduceQuotaAdded(event eventstore.Event) (*handler.Statement, error) {
e, err := assertEvent[*quota.AddedEvent](event)
if err != nil {
return nil, err
}
createStatements := make([]func(e eventstore.Event) crdb.Exec, len(e.Notifications)+1)
createStatements[0] = crdb.AddCreateStatement(
[]handler.Column{
handler.NewCol(QuotaColumnID, e.Aggregate().ID),
handler.NewCol(QuotaColumnInstanceID, e.Aggregate().InstanceID),
handler.NewCol(QuotaColumnUnit, e.Unit),
handler.NewCol(QuotaColumnAmount, e.Amount),
handler.NewCol(QuotaColumnFrom, e.From),
handler.NewCol(QuotaColumnInterval, e.ResetInterval),
handler.NewCol(QuotaColumnLimit, e.Limit),
})
for i := range e.Notifications {
notification := e.Notifications[i]
createStatements[i+1] = crdb.AddCreateStatement(
[]handler.Column{
handler.NewCol(QuotaNotificationColumnInstanceID, e.Aggregate().InstanceID),
handler.NewCol(QuotaNotificationColumnUnit, e.Unit),
handler.NewCol(QuotaNotificationColumnID, notification.ID),
handler.NewCol(QuotaNotificationColumnCallURL, notification.CallURL),
handler.NewCol(QuotaNotificationColumnPercent, notification.Percent),
handler.NewCol(QuotaNotificationColumnRepeat, notification.Repeat),
},
crdb.WithTableSuffix(quotaNotificationsTableSuffix),
)
}
return crdb.NewMultiStatement(e, createStatements...), nil
}
func (q *quotaProjection) reduceQuotaNotificationDue(event eventstore.Event) (*handler.Statement, error) {
e, err := assertEvent[*quota.NotificationDueEvent](event)
if err != nil {
return nil, err
}
return crdb.NewUpdateStatement(e,
[]handler.Column{
handler.NewCol(QuotaNotificationColumnLatestDuePeriodStart, e.PeriodStart),
handler.NewCol(QuotaNotificationColumnNextDueThreshold, e.Threshold+100), // next due_threshold is always the reached + 100 => percent (e.g. 90) in the next bucket (e.g. 190)
},
[]handler.Condition{
handler.NewCond(QuotaNotificationColumnInstanceID, e.Aggregate().InstanceID),
handler.NewCond(QuotaNotificationColumnUnit, e.Unit),
handler.NewCond(QuotaNotificationColumnID, e.ID),
},
crdb.WithTableSuffix(quotaNotificationsTableSuffix),
), nil
}
func (q *quotaProjection) reduceQuotaRemoved(event eventstore.Event) (*handler.Statement, error) {
e, err := assertEvent[*quota.RemovedEvent](event)
if err != nil {
return nil, err
}
return crdb.NewMultiStatement(
e,
crdb.AddDeleteStatement(
[]handler.Condition{
handler.NewCond(QuotaPeriodColumnInstanceID, e.Aggregate().InstanceID),
handler.NewCond(QuotaPeriodColumnUnit, e.Unit),
},
crdb.WithTableSuffix(quotaPeriodsTableSuffix),
),
crdb.AddDeleteStatement(
[]handler.Condition{
handler.NewCond(QuotaNotificationColumnInstanceID, e.Aggregate().InstanceID),
handler.NewCond(QuotaNotificationColumnUnit, e.Unit),
},
crdb.WithTableSuffix(quotaNotificationsTableSuffix),
),
crdb.AddDeleteStatement(
[]handler.Condition{
handler.NewCond(QuotaColumnInstanceID, e.Aggregate().InstanceID),
handler.NewCond(QuotaColumnUnit, e.Unit),
},
),
), nil
}
func (q *quotaProjection) reduceInstanceRemoved(event eventstore.Event) (*handler.Statement, error) {
// we only assert the event to make sure it is the correct type
e, err := assertEvent[*instance.InstanceRemovedEvent](event)
if err != nil {
return nil, err
}
return crdb.NewMultiStatement(
e,
crdb.AddDeleteStatement(
[]handler.Condition{
handler.NewCond(QuotaPeriodColumnInstanceID, e.Aggregate().InstanceID),
},
crdb.WithTableSuffix(quotaPeriodsTableSuffix),
),
crdb.AddDeleteStatement(
[]handler.Condition{
handler.NewCond(QuotaNotificationColumnInstanceID, e.Aggregate().InstanceID),
},
crdb.WithTableSuffix(quotaNotificationsTableSuffix),
),
crdb.AddDeleteStatement(
[]handler.Condition{
handler.NewCond(QuotaColumnInstanceID, e.Aggregate().InstanceID),
},
),
), nil
}
func (q *quotaProjection) IncrementUsage(ctx context.Context, unit quota.Unit, instanceID string, periodStart time.Time, count uint64) (sum uint64, err error) {
if count == 0 {
return 0, nil
}
err = q.client.DB.QueryRowContext(
ctx,
incrementQuotaStatement,
instanceID, unit, periodStart, count,
).Scan(&sum)
if err != nil {
return 0, errors.ThrowInternalf(err, "PROJ-SJL3h", "incrementing usage for unit %d failed for at least one quota period", unit)
}
return sum, err
}

321
internal/query/projection/quota_test.go Normal file
View File

@@ -0,0 +1,321 @@
package projection
import (
"context"
"regexp"
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
"github.com/stretchr/testify/assert"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/errors"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/eventstore/handler"
"github.com/zitadel/zitadel/internal/eventstore/repository"
"github.com/zitadel/zitadel/internal/repository/instance"
"github.com/zitadel/zitadel/internal/repository/quota"
)
func TestQuotasProjection_reduces(t *testing.T) {
type args struct {
event func(t *testing.T) eventstore.Event
}
tests := []struct {
name string
args args
reduce func(event eventstore.Event) (*handler.Statement, error)
want wantReduce
}{
{
name: "reduceQuotaAdded",
args: args{
event: getEvent(testEvent(
repository.EventType(quota.AddedEventType),
quota.AggregateType,
[]byte(`{
"unit": 1,
"amount": 10,
"limit": true,
"from": "2023-01-01T00:00:00Z",
"interval": 300000000000
}`),
), quota.AddedEventMapper),
},
reduce: (&quotaProjection{}).reduceQuotaAdded,
want: wantReduce{
aggregateType: eventstore.AggregateType("quota"),
sequence: 15,
previousSequence: 10,
executer: &testExecuter{
executions: []execution{
{
expectedStmt: "INSERT INTO projections.quotas (id, instance_id, unit, amount, from_anchor, interval, limit_usage) VALUES ($1, $2, $3, $4, $5, $6, $7)",
expectedArgs: []interface{}{
"agg-id",
"instance-id",
quota.RequestsAllAuthenticated,
uint64(10),
time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC),
time.Minute * 5,
true,
},
},
},
},
},
},
{
name: "reduceQuotaAdded with notification",
args: args{
event: getEvent(testEvent(
repository.EventType(quota.AddedEventType),
quota.AggregateType,
[]byte(`{
"unit": 1,
"amount": 10,
"limit": true,
"from": "2023-01-01T00:00:00Z",
"interval": 300000000000,
"notifications": [
{
"id": "id",
"percent": 100,
"repeat": true,
"callURL": "url"
}
]
}`),
), quota.AddedEventMapper),
},
reduce: (&quotaProjection{}).reduceQuotaAdded,
want: wantReduce{
aggregateType: eventstore.AggregateType("quota"),
sequence: 15,
previousSequence: 10,
executer: &testExecuter{
executions: []execution{
{
expectedStmt: "INSERT INTO projections.quotas (id, instance_id, unit, amount, from_anchor, interval, limit_usage) VALUES ($1, $2, $3, $4, $5, $6, $7)",
expectedArgs: []interface{}{
"agg-id",
"instance-id",
quota.RequestsAllAuthenticated,
uint64(10),
time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC),
time.Minute * 5,
true,
},
},
{
expectedStmt: "INSERT INTO projections.quotas_notifications (instance_id, unit, id, call_url, percent, repeat) VALUES ($1, $2, $3, $4, $5, $6)",
expectedArgs: []interface{}{
"instance-id",
quota.RequestsAllAuthenticated,
"id",
"url",
uint16(100),
true,
},
},
},
},
},
},
{
name: "reduceQuotaNotificationDue",
args: args{
event: getEvent(testEvent(
repository.EventType(quota.NotificationDueEventType),
quota.AggregateType,
[]byte(`{
"id": "id",
"unit": 1,
"callURL": "url",
"periodStart": "2023-01-01T00:00:00Z",
"threshold": 200,
"usage": 100
}`),
), quota.NotificationDueEventMapper),
},
reduce: (&quotaProjection{}).reduceQuotaNotificationDue,
want: wantReduce{
aggregateType: eventstore.AggregateType("quota"),
sequence: 15,
previousSequence: 10,
executer: &testExecuter{
executions: []execution{
{
expectedStmt: "UPDATE projections.quotas_notifications SET (latest_due_period_start, next_due_threshold) = ($1, $2) WHERE (instance_id = $3) AND (unit = $4) AND (id = $5)",
expectedArgs: []interface{}{
time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC),
uint16(300),
"instance-id",
quota.RequestsAllAuthenticated,
"id",
},
},
},
},
},
},
{
name: "reduceQuotaRemoved",
args: args{
event: getEvent(testEvent(
repository.EventType(quota.RemovedEventType),
quota.AggregateType,
[]byte(`{
"unit": 1
}`),
), quota.RemovedEventMapper),
},
reduce: (&quotaProjection{}).reduceQuotaRemoved,
want: wantReduce{
aggregateType: eventstore.AggregateType("quota"),
sequence: 15,
previousSequence: 10,
executer: &testExecuter{
executions: []execution{
{
expectedStmt: "DELETE FROM projections.quotas_periods WHERE (instance_id = $1) AND (unit = $2)",
expectedArgs: []interface{}{
"instance-id",
quota.RequestsAllAuthenticated,
},
},
{
expectedStmt: "DELETE FROM projections.quotas_notifications WHERE (instance_id = $1) AND (unit = $2)",
expectedArgs: []interface{}{
"instance-id",
quota.RequestsAllAuthenticated,
},
},
{
expectedStmt: "DELETE FROM projections.quotas WHERE (instance_id = $1) AND (unit = $2)",
expectedArgs: []interface{}{
"instance-id",
quota.RequestsAllAuthenticated,
},
},
},
},
},
}, {
name: "reduceInstanceRemoved",
args: args{
event: getEvent(testEvent(
repository.EventType(instance.InstanceRemovedEventType),
instance.AggregateType,
[]byte(`{
"name": "name"
}`),
), instance.InstanceRemovedEventMapper),
},
reduce: (&quotaProjection{}).reduceInstanceRemoved,
want: wantReduce{
aggregateType: eventstore.AggregateType("instance"),
sequence: 15,
previousSequence: 10,
executer: &testExecuter{
executions: []execution{
{
expectedStmt: "DELETE FROM projections.quotas_periods WHERE (instance_id = $1)",
expectedArgs: []interface{}{
"instance-id",
},
},
{
expectedStmt: "DELETE FROM projections.quotas_notifications WHERE (instance_id = $1)",
expectedArgs: []interface{}{
"instance-id",
},
},
{
expectedStmt: "DELETE FROM projections.quotas WHERE (instance_id = $1)",
expectedArgs: []interface{}{
"instance-id",
},
},
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
event := baseEvent(t)
got, err := tt.reduce(event)
if !errors.IsErrorInvalidArgument(err) {
t.Errorf("no wrong event mapping: %v, got: %v", err, got)
}
event = tt.args.event(t)
got, err = tt.reduce(event)
assertReduce(t, got, err, QuotasProjectionTable, tt.want)
})
}
}
func Test_quotaProjection_IncrementUsage(t *testing.T) {
testNow := time.Now()
type fields struct {
client *database.DB
}
type args struct {
ctx context.Context
unit quota.Unit
instanceID string
periodStart time.Time
count uint64
}
type res struct {
sum uint64
err error
}
tests := []struct {
name string
fields fields
args args
res res
}{
{
name: "",
fields: fields{
client: func() *database.DB {
db, mock, _ := sqlmock.New()
mock.ExpectQuery(regexp.QuoteMeta(incrementQuotaStatement)).
WithArgs(
"instance_id",
1,
testNow,
2,
).
WillReturnRows(sqlmock.NewRows([]string{"key"}).
AddRow(3))
return &database.DB{DB: db}
}(),
},
args: args{
ctx: context.Background(),
unit: quota.RequestsAllAuthenticated,
instanceID: "instance_id",
periodStart: testNow,
count: 2,
},
res: res{
sum: 3,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
q := &quotaProjection{
client: tt.fields.client,
}
gotSum, err := q.IncrementUsage(tt.args.ctx, tt.args.unit, tt.args.instanceID, tt.args.periodStart, tt.args.count)
assert.Equal(t, tt.res.sum, gotSum)
assert.ErrorIs(t, err, tt.res.err)
})
}
}