feat: add quotas (#4779)

adds possibilities to cap authenticated requests and execution seconds of actions on a defined intervall

internal/logstore/emitters/access/database.go

@@ -0,0 +1,159 @@
+package access
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/Masterminds/squirrel"
+	"github.com/zitadel/logging"
+	"google.golang.org/grpc/codes"
+
+	zitadel_http "github.com/zitadel/zitadel/internal/api/http"
+	caos_errors "github.com/zitadel/zitadel/internal/errors"
+	"github.com/zitadel/zitadel/internal/logstore"
+	"github.com/zitadel/zitadel/internal/repository/quota"
+)
+
+const (
+	accessLogsTable          = "logstore.access"
+	accessTimestampCol       = "log_date"
+	accessProtocolCol        = "protocol"
+	accessRequestURLCol      = "request_url"
+	accessResponseStatusCol  = "response_status"
+	accessRequestHeadersCol  = "request_headers"
+	accessResponseHeadersCol = "response_headers"
+	accessInstanceIdCol      = "instance_id"
+	accessProjectIdCol       = "project_id"
+	accessRequestedDomainCol = "requested_domain"
+	accessRequestedHostCol   = "requested_host"
+)
+
+var _ logstore.UsageQuerier = (*databaseLogStorage)(nil)
+var _ logstore.LogCleanupper = (*databaseLogStorage)(nil)
+
+type databaseLogStorage struct {
+	dbClient *sql.DB
+}
+
+func NewDatabaseLogStorage(dbClient *sql.DB) *databaseLogStorage {
+	return &databaseLogStorage{dbClient: dbClient}
+}
+
+func (l *databaseLogStorage) QuotaUnit() quota.Unit {
+	return quota.RequestsAllAuthenticated
+}
+
+func (l *databaseLogStorage) Emit(ctx context.Context, bulk []logstore.LogRecord) error {
+	builder := squirrel.Insert(accessLogsTable).
+		Columns(
+			accessTimestampCol,
+			accessProtocolCol,
+			accessRequestURLCol,
+			accessResponseStatusCol,
+			accessRequestHeadersCol,
+			accessResponseHeadersCol,
+			accessInstanceIdCol,
+			accessProjectIdCol,
+			accessRequestedDomainCol,
+			accessRequestedHostCol,
+		).
+		PlaceholderFormat(squirrel.Dollar)
+
+	for idx := range bulk {
+		item := bulk[idx].(*Record)
+		builder = builder.Values(
+			item.LogDate,
+			item.Protocol,
+			item.RequestURL,
+			item.ResponseStatus,
+			item.RequestHeaders,
+			item.ResponseHeaders,
+			item.InstanceID,
+			item.ProjectID,
+			item.RequestedDomain,
+			item.RequestedHost,
+		)
+	}
+
+	stmt, args, err := builder.ToSql()
+	if err != nil {
+		return caos_errors.ThrowInternal(err, "ACCESS-KOS7I", "Errors.Internal")
+	}
+
+	result, err := l.dbClient.ExecContext(ctx, stmt, args...)
+	if err != nil {
+		return caos_errors.ThrowInternal(err, "ACCESS-alnT9", "Errors.Access.StorageFailed")
+	}
+
+	rows, err := result.RowsAffected()
+	if err != nil {
+		return caos_errors.ThrowInternal(err, "ACCESS-7KIpL", "Errors.Internal")
+	}
+
+	logging.WithFields("rows", rows).Debug("successfully stored access logs")
+	return nil
+}
+
+// TODO: AS OF SYSTEM TIME
+func (l *databaseLogStorage) QueryUsage(ctx context.Context, instanceId string, start time.Time) (uint64, error) {
+	stmt, args, err := squirrel.Select(
+		fmt.Sprintf("count(%s)", accessInstanceIdCol),
+	).
+		From(accessLogsTable).
+		Where(squirrel.And{
+			squirrel.Eq{accessInstanceIdCol: instanceId},
+			squirrel.GtOrEq{accessTimestampCol: start},
+			squirrel.Expr(fmt.Sprintf(`%s #>> '{%s,0}' = '[REDACTED]'`, accessRequestHeadersCol, strings.ToLower(zitadel_http.Authorization))),
+			squirrel.NotLike{accessRequestURLCol: "%/zitadel.system.v1.SystemService/%"},
+			squirrel.NotLike{accessRequestURLCol: "%/system/v1/%"},
+			squirrel.Or{
+				squirrel.And{
+					squirrel.Eq{accessProtocolCol: HTTP},
+					squirrel.NotEq{accessResponseStatusCol: http.StatusForbidden},
+					squirrel.NotEq{accessResponseStatusCol: http.StatusInternalServerError},
+					squirrel.NotEq{accessResponseStatusCol: http.StatusTooManyRequests},
+				},
+				squirrel.And{
+					squirrel.Eq{accessProtocolCol: GRPC},
+					squirrel.NotEq{accessResponseStatusCol: codes.PermissionDenied},
+					squirrel.NotEq{accessResponseStatusCol: codes.Internal},
+					squirrel.NotEq{accessResponseStatusCol: codes.ResourceExhausted},
+				},
+			},
+		}).
+		PlaceholderFormat(squirrel.Dollar).
+		ToSql()
+
+	if err != nil {
+		return 0, caos_errors.ThrowInternal(err, "ACCESS-V9Sde", "Errors.Internal")
+	}
+
+	var count uint64
+	if err = l.dbClient.
+		QueryRowContext(ctx, stmt, args...).
+		Scan(&count); err != nil {
+		return 0, caos_errors.ThrowInternal(err, "ACCESS-pBPrM", "Errors.Logstore.Access.ScanFailed")
+	}
+
+	return count, nil
+}
+
+func (l *databaseLogStorage) Cleanup(ctx context.Context, keep time.Duration) error {
+	stmt, args, err := squirrel.Delete(accessLogsTable).
+		Where(squirrel.LtOrEq{accessTimestampCol: time.Now().Add(-keep)}).
+		PlaceholderFormat(squirrel.Dollar).
+		ToSql()
+
+	if err != nil {
+		return caos_errors.ThrowInternal(err, "ACCESS-2oTh6", "Errors.Internal")
+	}
+
+	execCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+	_, err = l.dbClient.ExecContext(execCtx, stmt, args...)
+	return err
+}

internal/logstore/emitters/access/record.go

@@ -0,0 +1,91 @@
+package access
+
+import (
+	"strings"
+	"time"
+
+	zitadel_http "github.com/zitadel/zitadel/internal/api/http"
+	"github.com/zitadel/zitadel/internal/logstore"
+)
+
+var _ logstore.LogRecord = (*Record)(nil)
+
+type Record struct {
+	LogDate        time.Time `json:"logDate"`
+	Protocol       Protocol  `json:"protocol"`
+	RequestURL     string    `json:"requestUrl"`
+	ResponseStatus uint32    `json:"responseStatus"`
+	// RequestHeaders are plain maps so varying implementation
+	// between HTTP and gRPC don't interfere with each other
+	RequestHeaders map[string][]string `json:"requestHeaders"`
+	// ResponseHeaders are plain maps so varying implementation
+	// between HTTP and gRPC don't interfere with each other
+	ResponseHeaders map[string][]string `json:"responseHeaders"`
+	InstanceID      string              `json:"instanceId"`
+	ProjectID       string              `json:"projectId"`
+	RequestedDomain string              `json:"requestedDomain"`
+	RequestedHost   string              `json:"requestedHost"`
+}
+
+type Protocol uint8
+
+const (
+	GRPC Protocol = iota
+	HTTP
+
+	redacted = "[REDACTED]"
+)
+
+func (a Record) Normalize() logstore.LogRecord {
+	a.RequestedDomain = cutString(a.RequestedDomain, 200)
+	a.RequestURL = cutString(a.RequestURL, 200)
+	normalizeHeaders(a.RequestHeaders, strings.ToLower(zitadel_http.Authorization), "grpcgateway-authorization", "cookie", "grpcgateway-cookie")
+	normalizeHeaders(a.ResponseHeaders, "set-cookie")
+	return &a
+}
+
+const maxValuesPerKey = 10
+
+// normalizeHeaders lowers all header keys and redacts secrets
+func normalizeHeaders(header map[string][]string, redactKeysLower ...string) {
+	lowerKeys(header)
+	redactKeys(header, redactKeysLower...)
+	pruneKeys(header)
+}
+
+func lowerKeys(header map[string][]string) {
+	for k, v := range header {
+		delete(header, k)
+		header[strings.ToLower(k)] = v
+	}
+}
+
+func redactKeys(header map[string][]string, redactKeysLower ...string) {
+	for _, redactKey := range redactKeysLower {
+		if _, ok := header[redactKey]; ok {
+			header[redactKey] = []string{redacted}
+		}
+	}
+}
+
+func pruneKeys(header map[string][]string) {
+	for key, value := range header {
+		valueItems := make([]string, 0, maxValuesPerKey)
+		for i, valueItem := range value {
+			// Max 10 header values per key
+			if i > maxValuesPerKey {
+				break
+			}
+			// Max 200 value length
+			valueItems = append(valueItems, cutString(valueItem, 200))
+		}
+		header[key] = valueItems
+	}
+}
+
+func cutString(str string, pos int) string {
+	if len(str) <= pos {
+		return str
+	}
+	return str[:pos-1]
+}

internal/logstore/emitters/execution/database.go

@@ -0,0 +1,135 @@
+package execution
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"time"
+
+	"github.com/Masterminds/squirrel"
+	"github.com/zitadel/logging"
+
+	caos_errors "github.com/zitadel/zitadel/internal/errors"
+	"github.com/zitadel/zitadel/internal/logstore"
+	"github.com/zitadel/zitadel/internal/repository/quota"
+)
+
+const (
+	executionLogsTable     = "logstore.execution"
+	executionTimestampCol  = "log_date"
+	executionTookCol       = "took"
+	executionMessageCol    = "message"
+	executionLogLevelCol   = "loglevel"
+	executionInstanceIdCol = "instance_id"
+	executionActionIdCol   = "action_id"
+	executionMetadataCol   = "metadata"
+)
+
+var _ logstore.UsageQuerier = (*databaseLogStorage)(nil)
+var _ logstore.LogCleanupper = (*databaseLogStorage)(nil)
+
+type databaseLogStorage struct {
+	dbClient *sql.DB
+}
+
+func NewDatabaseLogStorage(dbClient *sql.DB) *databaseLogStorage {
+	return &databaseLogStorage{dbClient: dbClient}
+}
+
+func (l *databaseLogStorage) QuotaUnit() quota.Unit {
+	return quota.ActionsAllRunsSeconds
+}
+
+func (l *databaseLogStorage) Emit(ctx context.Context, bulk []logstore.LogRecord) error {
+	builder := squirrel.Insert(executionLogsTable).
+		Columns(
+			executionTimestampCol,
+			executionTookCol,
+			executionMessageCol,
+			executionLogLevelCol,
+			executionInstanceIdCol,
+			executionActionIdCol,
+			executionMetadataCol,
+		).
+		PlaceholderFormat(squirrel.Dollar)
+
+	for idx := range bulk {
+		item := bulk[idx].(*Record)
+
+		var took interface{}
+		if item.Took > 0 {
+			took = item.Took
+		}
+
+		builder = builder.Values(
+			item.LogDate,
+			took,
+			item.Message,
+			item.LogLevel,
+			item.InstanceID,
+			item.ActionID,
+			item.Metadata,
+		)
+	}
+
+	stmt, args, err := builder.ToSql()
+	if err != nil {
+		return caos_errors.ThrowInternal(err, "EXEC-KOS7I", "Errors.Internal")
+	}
+
+	result, err := l.dbClient.ExecContext(ctx, stmt, args...)
+	if err != nil {
+		return caos_errors.ThrowInternal(err, "EXEC-0j6i5", "Errors.Access.StorageFailed")
+	}
+
+	rows, err := result.RowsAffected()
+	if err != nil {
+		return caos_errors.ThrowInternal(err, "EXEC-MGchJ", "Errors.Internal")
+	}
+
+	logging.WithFields("rows", rows).Debug("successfully stored execution logs")
+	return nil
+}
+
+// TODO: AS OF SYSTEM TIME
+func (l *databaseLogStorage) QueryUsage(ctx context.Context, instanceId string, start time.Time) (uint64, error) {
+	stmt, args, err := squirrel.Select(
+		fmt.Sprintf("COALESCE(SUM(%s)::INT,0)", executionTookCol),
+	).
+		From(executionLogsTable).
+		Where(squirrel.And{
+			squirrel.Eq{executionInstanceIdCol: instanceId},
+			squirrel.GtOrEq{executionTimestampCol: start},
+			squirrel.NotEq{executionTookCol: nil},
+		}).
+		PlaceholderFormat(squirrel.Dollar).
+		ToSql()
+
+	if err != nil {
+		return 0, caos_errors.ThrowInternal(err, "EXEC-DXtzg", "Errors.Internal")
+	}
+
+	var durationSeconds uint64
+	if err = l.dbClient.
+		QueryRowContext(ctx, stmt, args...).
+		Scan(&durationSeconds); err != nil {
+		return 0, caos_errors.ThrowInternal(err, "EXEC-Ad8nP", "Errors.Logstore.Execution.ScanFailed")
+	}
+	return durationSeconds, nil
+}
+
+func (l *databaseLogStorage) Cleanup(ctx context.Context, keep time.Duration) error {
+	stmt, args, err := squirrel.Delete(executionLogsTable).
+		Where(squirrel.LtOrEq{executionTimestampCol: time.Now().Add(-keep)}).
+		PlaceholderFormat(squirrel.Dollar).
+		ToSql()
+
+	if err != nil {
+		return caos_errors.ThrowInternal(err, "EXEC-Bja8V", "Errors.Internal")
+	}
+
+	execCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+	_, err = l.dbClient.ExecContext(execCtx, stmt, args...)
+	return err
+}

internal/logstore/emitters/execution/record.go

@@ -0,0 +1,33 @@
+package execution
+
+import (
+	"time"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/zitadel/zitadel/internal/logstore"
+)
+
+var _ logstore.LogRecord = (*Record)(nil)
+
+type Record struct {
+	LogDate    time.Time              `json:"logDate"`
+	Took       time.Duration          `json:"took"`
+	Message    string                 `json:"message"`
+	LogLevel   logrus.Level           `json:"logLevel"`
+	InstanceID string                 `json:"instanceId"`
+	ActionID   string                 `json:"actionId,omitempty"`
+	Metadata   map[string]interface{} `json:"metadata,omitempty"`
+}
+
+func (e Record) Normalize() logstore.LogRecord {
+	e.Message = cutString(e.Message, 2000)
+	return &e
+}
+
+func cutString(str string, pos int) string {
+	if len(str) <= pos {
+		return str
+	}
+	return str[:pos]
+}

internal/logstore/emitters/mock/inmem.go

@@ -0,0 +1,89 @@
+package mock
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/benbjohnson/clock"
+
+	"github.com/zitadel/zitadel/internal/logstore"
+	"github.com/zitadel/zitadel/internal/repository/quota"
+)
+
+var _ logstore.UsageQuerier = (*InmemLogStorage)(nil)
+var _ logstore.LogCleanupper = (*InmemLogStorage)(nil)
+
+type InmemLogStorage struct {
+	mux     sync.Mutex
+	clock   clock.Clock
+	emitted []*record
+	bulks   []int
+}
+
+func NewInMemoryStorage(clock clock.Clock) *InmemLogStorage {
+	return &InmemLogStorage{
+		clock:   clock,
+		emitted: make([]*record, 0),
+		bulks:   make([]int, 0),
+	}
+}
+
+func (l *InmemLogStorage) QuotaUnit() quota.Unit {
+	return quota.Unimplemented
+}
+
+func (l *InmemLogStorage) Emit(_ context.Context, bulk []logstore.LogRecord) error {
+	if len(bulk) == 0 {
+		return nil
+	}
+	l.mux.Lock()
+	defer l.mux.Unlock()
+	for idx := range bulk {
+		l.emitted = append(l.emitted, bulk[idx].(*record))
+	}
+	l.bulks = append(l.bulks, len(bulk))
+	return nil
+}
+
+func (l *InmemLogStorage) QueryUsage(_ context.Context, _ string, start time.Time) (uint64, error) {
+	l.mux.Lock()
+	defer l.mux.Unlock()
+
+	var count uint64
+	for _, r := range l.emitted {
+		if r.ts.After(start) {
+			count++
+		}
+	}
+	return count, nil
+}
+
+func (l *InmemLogStorage) Cleanup(_ context.Context, keep time.Duration) error {
+	l.mux.Lock()
+	defer l.mux.Unlock()
+
+	clean := make([]*record, 0)
+	from := l.clock.Now().Add(-(keep + 1))
+	for _, r := range l.emitted {
+		if r.ts.After(from) {
+			clean = append(clean, r)
+		}
+	}
+	l.emitted = clean
+	return nil
+}
+
+func (l *InmemLogStorage) Bulks() []int {
+	l.mux.Lock()
+	defer l.mux.Unlock()
+
+	return l.bulks
+}
+
+func (l *InmemLogStorage) Len() int {
+	l.mux.Lock()
+	defer l.mux.Unlock()
+
+	return len(l.emitted)
+}

internal/logstore/emitters/mock/record.go

@@ -0,0 +1,25 @@
+package mock
+
+import (
+	"time"
+
+	"github.com/benbjohnson/clock"
+
+	"github.com/zitadel/zitadel/internal/logstore"
+)
+
+var _ logstore.LogRecord = (*record)(nil)
+
+func NewRecord(clock clock.Clock) *record {
+	return &record{ts: clock.Now()}
+}
+
+type record struct {
+	ts       time.Time
+	redacted bool
+}
+
+func (r record) Normalize() logstore.LogRecord {
+	r.redacted = true
+	return &r
+}

internal/logstore/emitters/stdout/stdout.go

@@ -0,0 +1,23 @@
+package stdout
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/zitadel/logging"
+
+	"github.com/zitadel/zitadel/internal/logstore"
+)
+
+func NewStdoutEmitter() logstore.LogEmitter {
+	return logstore.LogEmitterFunc(func(ctx context.Context, bulk []logstore.LogRecord) error {
+		for idx := range bulk {
+			bytes, err := json.Marshal(bulk[idx])
+			if err != nil {
+				return err
+			}
+			logging.WithFields("record", string(bytes)).Info("log record emitted")
+		}
+		return nil
+	})
+}