zitadel/internal/logstore/emitters/execution/database.go
Elio Bischof 5823fdbef9
perf: project quotas and usages (#6441)
* project quota added

* project quota removed

* add periods table

* make log record generic

* accumulate usage

* query usage

* count action run seconds

* fix filter in ReportQuotaUsage

* fix existing tests

* fix logstore tests

* fix typo

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* move notifications into debouncer and improve limit querying

* cleanup

* comment

* fix: add quota unit tests command side

* fix remaining quota usage query

* implement InmemLogStorage

* cleanup and linting

* improve test

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* action notifications and fixes for notifications query

* revert console prefix

* fix: add quota unit tests command side

* fix: add quota integration tests

* improve accountable requests

* improve accountable requests

* fix: add quota integration tests

* fix: add quota integration tests

* fix: add quota integration tests

* comment

* remove ability to store logs in db and other changes requested from review

* changes requested from review

* changes requested from review

* Update internal/api/http/middleware/access_interceptor.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* tests: fix quotas integration tests

* improve incrementUsageStatement

* linting

* fix: delete e2e tests as intergation tests cover functionality

* Update internal/api/http/middleware/access_interceptor.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* backup

* fix conflict

* create rc

* create prerelease

* remove issue release labeling

* fix tracing

---------

Co-authored-by: Livio Spring <livio.a@gmail.com>
Co-authored-by: Stefan Benz <stefan@caos.ch>
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
(cherry picked from commit 1a49b7d298)
2023-09-15 17:00:12 +02:00

87 lines
2.7 KiB
Go

package execution
import (
"context"
"database/sql"
"errors"
"math"
"time"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/command"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/logstore"
"github.com/zitadel/zitadel/internal/logstore/record"
"github.com/zitadel/zitadel/internal/query"
"github.com/zitadel/zitadel/internal/query/projection"
"github.com/zitadel/zitadel/internal/repository/quota"
)
var _ logstore.UsageStorer[*record.ExecutionLog] = (*databaseLogStorage)(nil)
type databaseLogStorage struct {
dbClient *database.DB
commands *command.Commands
queries *query.Queries
}
func NewDatabaseLogStorage(dbClient *database.DB, commands *command.Commands, queries *query.Queries) *databaseLogStorage {
return &databaseLogStorage{dbClient: dbClient, commands: commands, queries: queries}
}
func (l *databaseLogStorage) QuotaUnit() quota.Unit {
return quota.ActionsAllRunsSeconds
}
func (l *databaseLogStorage) Emit(ctx context.Context, bulk []*record.ExecutionLog) error {
if len(bulk) == 0 {
return nil
}
return l.incrementUsage(ctx, bulk)
}
func (l *databaseLogStorage) incrementUsage(ctx context.Context, bulk []*record.ExecutionLog) (err error) {
byInstance := make(map[string][]*record.ExecutionLog)
for _, r := range bulk {
if r.InstanceID != "" {
byInstance[r.InstanceID] = append(byInstance[r.InstanceID], r)
}
}
for instanceID, instanceBulk := range byInstance {
q, getQuotaErr := l.queries.GetQuota(ctx, instanceID, quota.ActionsAllRunsSeconds)
if errors.Is(getQuotaErr, sql.ErrNoRows) {
continue
}
err = errors.Join(err, getQuotaErr)
if getQuotaErr != nil {
continue
}
sum, incrementErr := l.incrementUsageFromExecutionLogs(ctx, instanceID, q.CurrentPeriodStart, instanceBulk)
err = errors.Join(err, incrementErr)
if incrementErr != nil {
continue
}
notifications, getNotificationErr := l.queries.GetDueQuotaNotifications(ctx, instanceID, quota.ActionsAllRunsSeconds, q, q.CurrentPeriodStart, sum)
err = errors.Join(err, getNotificationErr)
if getNotificationErr != nil || len(notifications) == 0 {
continue
}
ctx = authz.WithInstanceID(ctx, instanceID)
reportErr := l.commands.ReportQuotaUsage(ctx, notifications)
err = errors.Join(err, reportErr)
if reportErr != nil {
continue
}
}
return err
}
func (l *databaseLogStorage) incrementUsageFromExecutionLogs(ctx context.Context, instanceID string, periodStart time.Time, records []*record.ExecutionLog) (sum uint64, err error) {
var total time.Duration
for _, r := range records {
total += r.Took
}
return projection.QuotaProjection.IncrementUsage(ctx, quota.ActionsAllRunsSeconds, instanceID, periodStart, uint64(math.Floor(total.Seconds())))
}