mirror of
https://github.com/zitadel/zitadel.git
synced 2025-08-11 21:37:32 +00:00
perf: project quotas and usages (#6441)
* project quota added * project quota removed * add periods table * make log record generic * accumulate usage * query usage * count action run seconds * fix filter in ReportQuotaUsage * fix existing tests * fix logstore tests * fix typo * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * move notifications into debouncer and improve limit querying * cleanup * comment * fix: add quota unit tests command side * fix remaining quota usage query * implement InmemLogStorage * cleanup and linting * improve test * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * action notifications and fixes for notifications query * revert console prefix * fix: add quota unit tests command side * fix: add quota integration tests * improve accountable requests * improve accountable requests * fix: add quota integration tests * fix: add quota integration tests * fix: add quota integration tests * comment * remove ability to store logs in db and other changes requested from review * changes requested from review * changes requested from review * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * tests: fix quotas integration tests * improve incrementUsageStatement * linting * fix: delete e2e tests as intergation tests cover functionality * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * backup * fix conflict * create rc * create prerelease * remove issue release labeling * fix tracing --------- Co-authored-by: Livio Spring <livio.a@gmail.com> Co-authored-by: Stefan Benz <stefan@caos.ch> Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
This commit is contained in:
Elio Bischof
@@ -3,167 +3,91 @@ package access
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/Masterminds/squirrel"
|
||||
"github.com/zitadel/logging"
|
||||
"google.golang.org/grpc/codes"
|
||||
|
||||
"github.com/zitadel/zitadel/internal/api/call"
|
||||
zitadel_http "github.com/zitadel/zitadel/internal/api/http"
|
||||
"github.com/zitadel/zitadel/internal/api/authz"
|
||||
"github.com/zitadel/zitadel/internal/command"
|
||||
"github.com/zitadel/zitadel/internal/database"
|
||||
caos_errors "github.com/zitadel/zitadel/internal/errors"
|
||||
"github.com/zitadel/zitadel/internal/logstore"
|
||||
"github.com/zitadel/zitadel/internal/logstore/record"
|
||||
"github.com/zitadel/zitadel/internal/query"
|
||||
"github.com/zitadel/zitadel/internal/query/projection"
|
||||
"github.com/zitadel/zitadel/internal/repository/quota"
|
||||
"github.com/zitadel/zitadel/internal/telemetry/tracing"
|
||||
)
|
||||
|
||||
const (
|
||||
accessLogsTable = "logstore.access"
|
||||
accessTimestampCol = "log_date"
|
||||
accessProtocolCol = "protocol"
|
||||
accessRequestURLCol = "request_url"
|
||||
accessResponseStatusCol = "response_status"
|
||||
accessRequestHeadersCol = "request_headers"
|
||||
accessResponseHeadersCol = "response_headers"
|
||||
accessInstanceIdCol = "instance_id"
|
||||
accessProjectIdCol = "project_id"
|
||||
accessRequestedDomainCol = "requested_domain"
|
||||
accessRequestedHostCol = "requested_host"
|
||||
)
|
||||
|
||||
var _ logstore.UsageQuerier = (*databaseLogStorage)(nil)
|
||||
var _ logstore.LogCleanupper = (*databaseLogStorage)(nil)
|
||||
var _ logstore.UsageStorer[*record.AccessLog] = (*databaseLogStorage)(nil)
|
||||
|
||||
type databaseLogStorage struct {
|
||||
dbClient *database.DB
|
||||
commands *command.Commands
|
||||
queries *query.Queries
|
||||
}
|
||||
|
||||
func NewDatabaseLogStorage(dbClient *database.DB) *databaseLogStorage {
|
||||
return &databaseLogStorage{dbClient: dbClient}
|
||||
func NewDatabaseLogStorage(dbClient *database.DB, commands *command.Commands, queries *query.Queries) *databaseLogStorage {
|
||||
return &databaseLogStorage{dbClient: dbClient, commands: commands, queries: queries}
|
||||
}
|
||||
|
||||
func (l *databaseLogStorage) QuotaUnit() quota.Unit {
|
||||
return quota.RequestsAllAuthenticated
|
||||
}
|
||||
|
||||
func (l *databaseLogStorage) Emit(ctx context.Context, bulk []logstore.LogRecord) error {
|
||||
func (l *databaseLogStorage) Emit(ctx context.Context, bulk []*record.AccessLog) error {
|
||||
if len(bulk) == 0 {
|
||||
return nil
|
||||
}
|
||||
builder := squirrel.Insert(accessLogsTable).
|
||||
Columns(
|
||||
accessTimestampCol,
|
||||
accessProtocolCol,
|
||||
accessRequestURLCol,
|
||||
accessResponseStatusCol,
|
||||
accessRequestHeadersCol,
|
||||
accessResponseHeadersCol,
|
||||
accessInstanceIdCol,
|
||||
accessProjectIdCol,
|
||||
accessRequestedDomainCol,
|
||||
accessRequestedHostCol,
|
||||
).
|
||||
PlaceholderFormat(squirrel.Dollar)
|
||||
|
||||
for idx := range bulk {
|
||||
item := bulk[idx].(*Record)
|
||||
builder = builder.Values(
|
||||
item.LogDate,
|
||||
item.Protocol,
|
||||
item.RequestURL,
|
||||
item.ResponseStatus,
|
||||
item.RequestHeaders,
|
||||
item.ResponseHeaders,
|
||||
item.InstanceID,
|
||||
item.ProjectID,
|
||||
item.RequestedDomain,
|
||||
item.RequestedHost,
|
||||
)
|
||||
}
|
||||
|
||||
stmt, args, err := builder.ToSql()
|
||||
if err != nil {
|
||||
return caos_errors.ThrowInternal(err, "ACCESS-KOS7I", "Errors.Internal")
|
||||
}
|
||||
|
||||
result, err := l.dbClient.ExecContext(ctx, stmt, args...)
|
||||
if err != nil {
|
||||
return caos_errors.ThrowInternal(err, "ACCESS-alnT9", "Errors.Access.StorageFailed")
|
||||
}
|
||||
|
||||
rows, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return caos_errors.ThrowInternal(err, "ACCESS-7KIpL", "Errors.Internal")
|
||||
}
|
||||
|
||||
logging.WithFields("rows", rows).Debug("successfully stored access logs")
|
||||
return nil
|
||||
return l.incrementUsage(ctx, bulk)
|
||||
}
|
||||
|
||||
func (l *databaseLogStorage) QueryUsage(ctx context.Context, instanceId string, start time.Time) (uint64, error) {
|
||||
stmt, args, err := squirrel.Select(
|
||||
fmt.Sprintf("count(%s)", accessInstanceIdCol),
|
||||
).
|
||||
From(accessLogsTable + l.dbClient.Timetravel(call.Took(ctx))).
|
||||
Where(squirrel.And{
|
||||
squirrel.Eq{accessInstanceIdCol: instanceId},
|
||||
squirrel.GtOrEq{accessTimestampCol: start},
|
||||
squirrel.Expr(fmt.Sprintf(`%s #>> '{%s,0}' = '[REDACTED]'`, accessRequestHeadersCol, strings.ToLower(zitadel_http.Authorization))),
|
||||
squirrel.NotLike{accessRequestURLCol: "%/zitadel.system.v1.SystemService/%"},
|
||||
squirrel.NotLike{accessRequestURLCol: "%/system/v1/%"},
|
||||
squirrel.Or{
|
||||
squirrel.And{
|
||||
squirrel.Eq{accessProtocolCol: HTTP},
|
||||
squirrel.NotEq{accessResponseStatusCol: http.StatusForbidden},
|
||||
squirrel.NotEq{accessResponseStatusCol: http.StatusInternalServerError},
|
||||
squirrel.NotEq{accessResponseStatusCol: http.StatusTooManyRequests},
|
||||
},
|
||||
squirrel.And{
|
||||
squirrel.Eq{accessProtocolCol: GRPC},
|
||||
squirrel.NotEq{accessResponseStatusCol: codes.PermissionDenied},
|
||||
squirrel.NotEq{accessResponseStatusCol: codes.Internal},
|
||||
squirrel.NotEq{accessResponseStatusCol: codes.ResourceExhausted},
|
||||
},
|
||||
},
|
||||
}).
|
||||
PlaceholderFormat(squirrel.Dollar).
|
||||
ToSql()
|
||||
func (l *databaseLogStorage) incrementUsage(ctx context.Context, bulk []*record.AccessLog) (err error) {
|
||||
ctx, span := tracing.NewSpan(ctx)
|
||||
defer func() { span.EndWithError(err) }()
|
||||
|
||||
if err != nil {
|
||||
return 0, caos_errors.ThrowInternal(err, "ACCESS-V9Sde", "Errors.Internal")
|
||||
byInstance := make(map[string][]*record.AccessLog)
|
||||
for _, r := range bulk {
|
||||
if r.InstanceID != "" {
|
||||
byInstance[r.InstanceID] = append(byInstance[r.InstanceID], r)
|
||||
}
|
||||
}
|
||||
|
||||
var count uint64
|
||||
err = l.dbClient.
|
||||
QueryRowContext(ctx,
|
||||
func(row *sql.Row) error {
|
||||
return row.Scan(&count)
|
||||
},
|
||||
stmt, args...,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return 0, caos_errors.ThrowInternal(err, "ACCESS-pBPrM", "Errors.Logstore.Access.ScanFailed")
|
||||
for instanceID, instanceBulk := range byInstance {
|
||||
q, getQuotaErr := l.queries.GetQuota(ctx, instanceID, quota.RequestsAllAuthenticated)
|
||||
if errors.Is(getQuotaErr, sql.ErrNoRows) {
|
||||
continue
|
||||
}
|
||||
err = errors.Join(err, getQuotaErr)
|
||||
if getQuotaErr != nil {
|
||||
continue
|
||||
}
|
||||
sum, incrementErr := l.incrementUsageFromAccessLogs(ctx, instanceID, q.CurrentPeriodStart, instanceBulk)
|
||||
err = errors.Join(err, incrementErr)
|
||||
if incrementErr != nil {
|
||||
continue
|
||||
}
|
||||
notifications, getNotificationErr := l.queries.GetDueQuotaNotifications(ctx, instanceID, quota.RequestsAllAuthenticated, q, q.CurrentPeriodStart, sum)
|
||||
err = errors.Join(err, getNotificationErr)
|
||||
if getNotificationErr != nil || len(notifications) == 0 {
|
||||
continue
|
||||
}
|
||||
ctx = authz.WithInstanceID(ctx, instanceID)
|
||||
reportErr := l.commands.ReportQuotaUsage(ctx, notifications)
|
||||
err = errors.Join(err, reportErr)
|
||||
if reportErr != nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func (l *databaseLogStorage) Cleanup(ctx context.Context, keep time.Duration) error {
|
||||
stmt, args, err := squirrel.Delete(accessLogsTable).
|
||||
Where(squirrel.LtOrEq{accessTimestampCol: time.Now().Add(-keep)}).
|
||||
PlaceholderFormat(squirrel.Dollar).
|
||||
ToSql()
|
||||
|
||||
if err != nil {
|
||||
return caos_errors.ThrowInternal(err, "ACCESS-2oTh6", "Errors.Internal")
|
||||
}
|
||||
|
||||
execCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
_, err = l.dbClient.ExecContext(execCtx, stmt, args...)
|
||||
return err
|
||||
}
|
||||
|
||||
func (l *databaseLogStorage) incrementUsageFromAccessLogs(ctx context.Context, instanceID string, periodStart time.Time, records []*record.AccessLog) (sum uint64, err error) {
|
||||
ctx, span := tracing.NewSpan(ctx)
|
||||
defer func() { span.EndWithError(err) }()
|
||||
|
||||
var count uint64
|
||||
for _, r := range records {
|
||||
if r.IsAuthenticated() {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return projection.QuotaProjection.IncrementUsage(ctx, quota.RequestsAllAuthenticated, instanceID, periodStart, count)
|
||||
}
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
package access
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
zitadel_http "github.com/zitadel/zitadel/internal/api/http"
|
||||
"github.com/zitadel/zitadel/internal/logstore"
|
||||
)
|
||||
|
||||
var _ logstore.LogRecord = (*Record)(nil)
|
||||
|
||||
type Record struct {
|
||||
LogDate time.Time `json:"logDate"`
|
||||
Protocol Protocol `json:"protocol"`
|
||||
RequestURL string `json:"requestUrl"`
|
||||
ResponseStatus uint32 `json:"responseStatus"`
|
||||
// RequestHeaders are plain maps so varying implementation
|
||||
// between HTTP and gRPC don't interfere with each other
|
||||
RequestHeaders map[string][]string `json:"requestHeaders"`
|
||||
// ResponseHeaders are plain maps so varying implementation
|
||||
// between HTTP and gRPC don't interfere with each other
|
||||
ResponseHeaders map[string][]string `json:"responseHeaders"`
|
||||
InstanceID string `json:"instanceId"`
|
||||
ProjectID string `json:"projectId"`
|
||||
RequestedDomain string `json:"requestedDomain"`
|
||||
RequestedHost string `json:"requestedHost"`
|
||||
}
|
||||
|
||||
type Protocol uint8
|
||||
|
||||
const (
|
||||
GRPC Protocol = iota
|
||||
HTTP
|
||||
|
||||
redacted = "[REDACTED]"
|
||||
)
|
||||
|
||||
func (a Record) Normalize() logstore.LogRecord {
|
||||
a.RequestedDomain = cutString(a.RequestedDomain, 200)
|
||||
a.RequestURL = cutString(a.RequestURL, 200)
|
||||
a.RequestHeaders = normalizeHeaders(a.RequestHeaders, strings.ToLower(zitadel_http.Authorization), "grpcgateway-authorization", "cookie", "grpcgateway-cookie")
|
||||
a.ResponseHeaders = normalizeHeaders(a.ResponseHeaders, "set-cookie")
|
||||
return &a
|
||||
}
|
||||
|
||||
// normalizeHeaders lowers all header keys and redacts secrets
|
||||
func normalizeHeaders(header map[string][]string, redactKeysLower ...string) map[string][]string {
|
||||
return pruneKeys(redactKeys(lowerKeys(header), redactKeysLower...))
|
||||
}
|
||||
|
||||
func lowerKeys(header map[string][]string) map[string][]string {
|
||||
lower := make(map[string][]string, len(header))
|
||||
for k, v := range header {
|
||||
lower[strings.ToLower(k)] = v
|
||||
}
|
||||
return lower
|
||||
}
|
||||
|
||||
func redactKeys(header map[string][]string, redactKeysLower ...string) map[string][]string {
|
||||
redactedKeys := make(map[string][]string, len(header))
|
||||
for k, v := range header {
|
||||
redactedKeys[k] = v
|
||||
}
|
||||
for _, redactKey := range redactKeysLower {
|
||||
if _, ok := redactedKeys[redactKey]; ok {
|
||||
redactedKeys[redactKey] = []string{redacted}
|
||||
}
|
||||
}
|
||||
return redactedKeys
|
||||
}
|
||||
|
||||
const maxValuesPerKey = 10
|
||||
|
||||
func pruneKeys(header map[string][]string) map[string][]string {
|
||||
prunedKeys := make(map[string][]string, len(header))
|
||||
for key, value := range header {
|
||||
valueItems := make([]string, 0, maxValuesPerKey)
|
||||
for i, valueItem := range value {
|
||||
// Max 10 header values per key
|
||||
if i > maxValuesPerKey {
|
||||
break
|
||||
}
|
||||
// Max 200 value length
|
||||
valueItems = append(valueItems, cutString(valueItem, 200))
|
||||
}
|
||||
prunedKeys[key] = valueItems
|
||||
}
|
||||
return prunedKeys
|
||||
}
|
||||
|
||||
func cutString(str string, pos int) string {
|
||||
if len(str) <= pos {
|
||||
return str
|
||||
}
|
||||
return str[:pos-1]
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
package access_test
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/zitadel/zitadel/internal/logstore/emitters/access"
|
||||
)
|
||||
|
||||
func TestRecord_Normalize(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
record access.Record
|
||||
want *access.Record
|
||||
}{{
|
||||
name: "headers with certain keys should be redacted",
|
||||
record: access.Record{
|
||||
RequestHeaders: map[string][]string{
|
||||
"authorization": {"AValue"},
|
||||
"grpcgateway-authorization": {"AValue"},
|
||||
"cookie": {"AValue"},
|
||||
"grpcgateway-cookie": {"AValue"},
|
||||
}, ResponseHeaders: map[string][]string{
|
||||
"set-cookie": {"AValue"},
|
||||
},
|
||||
},
|
||||
want: &access.Record{
|
||||
RequestHeaders: map[string][]string{
|
||||
"authorization": {"[REDACTED]"},
|
||||
"grpcgateway-authorization": {"[REDACTED]"},
|
||||
"cookie": {"[REDACTED]"},
|
||||
"grpcgateway-cookie": {"[REDACTED]"},
|
||||
}, ResponseHeaders: map[string][]string{
|
||||
"set-cookie": {"[REDACTED]"},
|
||||
},
|
||||
},
|
||||
}, {
|
||||
name: "header keys should be lower cased",
|
||||
record: access.Record{
|
||||
RequestHeaders: map[string][]string{"AKey": {"AValue"}},
|
||||
ResponseHeaders: map[string][]string{"AKey": {"AValue"}}},
|
||||
want: &access.Record{
|
||||
RequestHeaders: map[string][]string{"akey": {"AValue"}},
|
||||
ResponseHeaders: map[string][]string{"akey": {"AValue"}}},
|
||||
}, {
|
||||
name: "an already prune record should stay unchanged",
|
||||
record: access.Record{
|
||||
RequestURL: "https://my.zitadel.cloud/",
|
||||
RequestHeaders: map[string][]string{
|
||||
"authorization": {"[REDACTED]"},
|
||||
},
|
||||
ResponseHeaders: map[string][]string{},
|
||||
},
|
||||
want: &access.Record{
|
||||
RequestURL: "https://my.zitadel.cloud/",
|
||||
RequestHeaders: map[string][]string{
|
||||
"authorization": {"[REDACTED]"},
|
||||
},
|
||||
ResponseHeaders: map[string][]string{},
|
||||
},
|
||||
}, {
|
||||
name: "empty record should stay empty",
|
||||
record: access.Record{
|
||||
RequestHeaders: map[string][]string{},
|
||||
ResponseHeaders: map[string][]string{},
|
||||
},
|
||||
want: &access.Record{
|
||||
RequestHeaders: map[string][]string{},
|
||||
ResponseHeaders: map[string][]string{},
|
||||
},
|
||||
}}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := tt.record.Normalize(); !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("Normalize() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user