feat: limit audit trail (#6744)

* feat: enable limiting audit trail

* support AddExclusiveQuery

* fix invalid condition

* register event mappers

* fix NullDuration validity

* test query side for limits

* lint

* acceptance test audit trail limit

* fix acceptance test

* translate limits not found

* update tests

* fix linting

* add audit log retention to default instance

* fix tests

* update docs

* remove todo

* improve test name
This commit is contained in:
Elio Bischof
2023-10-25 13:42:00 +02:00
committed by GitHub
parent 1c839e308b
commit 385a55bd21
52 changed files with 1778 additions and 172 deletions

View File

@@ -4,7 +4,9 @@ import (
"context"
"time"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/api/call"
"github.com/zitadel/zitadel/internal/errors"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/telemetry/tracing"
)
@@ -26,33 +28,45 @@ type EventEditor struct {
AvatarKey string
}
func (q *Queries) SearchEvents(ctx context.Context, query *eventstore.SearchQueryBuilder, auditLogRetention time.Duration) (_ []*Event, err error) {
ctx, span := tracing.NewSpan(ctx)
defer func() { span.EndWithError(err) }()
events, err := q.eventstore.Filter(ctx, query.AllowTimeTravel())
if err != nil {
return nil, err
}
if auditLogRetention != 0 {
events = filterAuditLogRetention(ctx, events, auditLogRetention)
}
return q.convertEvents(ctx, events), nil
type eventsReducer struct {
ctx context.Context
q *Queries
events []*Event
}
func filterAuditLogRetention(ctx context.Context, events []eventstore.Event, auditLogRetention time.Duration) []eventstore.Event {
func (r *eventsReducer) AppendEvents(events ...eventstore.Event) {
r.events = append(r.events, r.q.convertEvents(r.ctx, events)...)
}
func (r *eventsReducer) Reduce() error { return nil }
func (q *Queries) SearchEvents(ctx context.Context, query *eventstore.SearchQueryBuilder) (_ []*Event, err error) {
ctx, span := tracing.NewSpan(ctx)
defer func() { span.EndWithError(err) }()
auditLogRetention := q.defaultAuditLogRetention
instanceLimits, err := q.Limits(ctx, authz.GetInstance(ctx).InstanceID())
if err != nil && !errors.IsNotFound(err) {
return nil, err
}
if instanceLimits != nil && instanceLimits.AuditLogRetention != nil {
auditLogRetention = *instanceLimits.AuditLogRetention
}
if auditLogRetention != 0 {
query = filterAuditLogRetention(ctx, auditLogRetention, query)
}
reducer := &eventsReducer{ctx: ctx, q: q}
if err = q.eventstore.FilterToReducer(ctx, query, reducer); err != nil {
return nil, err
}
return reducer.events, nil
}
func filterAuditLogRetention(ctx context.Context, auditLogRetention time.Duration, builder *eventstore.SearchQueryBuilder) *eventstore.SearchQueryBuilder {
callTime := call.FromContext(ctx)
if callTime.IsZero() {
callTime = time.Now()
}
filteredEvents := make([]eventstore.Event, 0, len(events))
for _, event := range events {
if event.CreatedAt().After(callTime.Add(-auditLogRetention)) {
filteredEvents = append(filteredEvents, event)
}
}
return filteredEvents
return builder.CreationDateAfter(callTime.Add(-auditLogRetention))
}
func (q *Queries) SearchEventTypes(ctx context.Context) []string {

119
internal/query/limits.go Normal file
View File

@@ -0,0 +1,119 @@
package query
import (
"context"
"database/sql"
errs "errors"
"time"
sq "github.com/Masterminds/squirrel"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/api/call"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/errors"
"github.com/zitadel/zitadel/internal/query/projection"
"github.com/zitadel/zitadel/internal/telemetry/tracing"
)
var (
limitSettingsTable = table{
name: projection.LimitsProjectionTable,
instanceIDCol: projection.LimitsColumnInstanceID,
}
LimitsColumnAggregateID = Column{
name: projection.LimitsColumnAggregateID,
table: limitSettingsTable,
}
LimitsColumnCreationDate = Column{
name: projection.LimitsColumnCreationDate,
table: limitSettingsTable,
}
LimitsColumnChangeDate = Column{
name: projection.LimitsColumnChangeDate,
table: limitSettingsTable,
}
LimitsColumnResourceOwner = Column{
name: projection.LimitsColumnResourceOwner,
table: limitSettingsTable,
}
LimitsColumnInstanceID = Column{
name: projection.LimitsColumnInstanceID,
table: limitSettingsTable,
}
LimitsColumnSequence = Column{
name: projection.LimitsColumnSequence,
table: limitSettingsTable,
}
LimitsColumnAuditLogRetention = Column{
name: projection.LimitsColumnAuditLogRetention,
table: limitSettingsTable,
}
)
type Limits struct {
AggregateID string
CreationDate time.Time
ChangeDate time.Time
ResourceOwner string
Sequence uint64
AuditLogRetention *time.Duration
}
func (q *Queries) Limits(ctx context.Context, resourceOwner string) (limits *Limits, err error) {
ctx, span := tracing.NewSpan(ctx)
defer func() { span.EndWithError(err) }()
stmt, scan := prepareLimitsQuery(ctx, q.client)
query, args, err := stmt.Where(sq.Eq{
LimitsColumnInstanceID.identifier(): authz.GetInstance(ctx).InstanceID(),
LimitsColumnResourceOwner.identifier(): resourceOwner,
}).ToSql()
if err != nil {
return nil, errors.ThrowInternal(err, "QUERY-jJe80", "Errors.Query.SQLStatment")
}
err = q.client.QueryRowContext(ctx, func(row *sql.Row) error {
limits, err = scan(row)
return err
}, query, args...)
return limits, err
}
func prepareLimitsQuery(ctx context.Context, db prepareDatabase) (sq.SelectBuilder, func(*sql.Row) (*Limits, error)) {
return sq.Select(
LimitsColumnAggregateID.identifier(),
LimitsColumnCreationDate.identifier(),
LimitsColumnChangeDate.identifier(),
LimitsColumnResourceOwner.identifier(),
LimitsColumnSequence.identifier(),
LimitsColumnAuditLogRetention.identifier(),
).
From(limitSettingsTable.identifier() + db.Timetravel(call.Took(ctx))).
PlaceholderFormat(sq.Dollar),
func(row *sql.Row) (*Limits, error) {
var (
limits = new(Limits)
auditLogRetention database.NullDuration
)
err := row.Scan(
&limits.AggregateID,
&limits.CreationDate,
&limits.ChangeDate,
&limits.ResourceOwner,
&limits.Sequence,
&auditLogRetention,
)
if err != nil {
if errs.Is(err, sql.ErrNoRows) {
return nil, errors.ThrowNotFound(err, "QUERY-GU1em", "Errors.Limits.NotFound")
}
return nil, errors.ThrowInternal(err, "QUERY-00jgy", "Errors.Internal")
}
if auditLogRetention.Valid {
limits.AuditLogRetention = &auditLogRetention.Duration
}
return limits, nil
}
}

View File

@@ -0,0 +1,116 @@
package query
import (
"database/sql"
"database/sql/driver"
"errors"
"fmt"
"regexp"
"testing"
"time"
"github.com/muhlemmer/gu"
errs "github.com/zitadel/zitadel/internal/errors"
)
var (
expectedLimitsQuery = regexp.QuoteMeta("SELECT projections.limits.aggregate_id," +
" projections.limits.creation_date," +
" projections.limits.change_date," +
" projections.limits.resource_owner," +
" projections.limits.sequence," +
" projections.limits.audit_log_retention" +
" FROM projections.limits" +
" AS OF SYSTEM TIME '-1 ms'",
)
limitsCols = []string{
"aggregate_id",
"creation_date",
"change_date",
"resource_owner",
"sequence",
"audit_log_retention",
}
)
func Test_LimitsPrepare(t *testing.T) {
type want struct {
sqlExpectations sqlExpectation
err checkErr
}
tests := []struct {
name string
prepare interface{}
want want
object interface{}
}{
{
name: "prepareLimitsQuery no result",
prepare: prepareLimitsQuery,
want: want{
sqlExpectations: mockQueriesScanErr(
expectedLimitsQuery,
nil,
nil,
),
err: func(err error) (error, bool) {
if !errs.IsNotFound(err) {
return fmt.Errorf("err should be zitadel.NotFoundError got: %w", err), false
}
return nil, true
},
},
object: (*Limits)(nil),
},
{
name: "prepareLimitsQuery",
prepare: prepareLimitsQuery,
want: want{
sqlExpectations: mockQuery(
expectedLimitsQuery,
limitsCols,
[]driver.Value{
"limits1",
testNow,
testNow,
"instance1",
0,
intervalDriverValue(t, time.Hour),
},
),
},
object: &Limits{
AggregateID: "limits1",
CreationDate: testNow,
ChangeDate: testNow,
ResourceOwner: "instance1",
Sequence: 0,
AuditLogRetention: gu.Ptr(time.Hour),
},
},
{
name: "prepareLimitsQuery sql err",
prepare: prepareLimitsQuery,
want: want{
sqlExpectations: mockQueryErr(
expectedLimitsQuery,
sql.ErrConnDone,
),
err: func(err error) (error, bool) {
if !errors.Is(err, sql.ErrConnDone) {
return fmt.Errorf("err should be sql.ErrConnDone got: %w", err), false
}
return nil, true
},
},
object: (*Limits)(nil),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assertPrepare(t, tt.prepare, tt.object, tt.want.sqlExpectations, tt.want.err, defaultPrepareArgs...)
})
}
}

View File

@@ -13,13 +13,16 @@ import (
"github.com/DATA-DOG/go-sqlmock"
sq "github.com/Masterminds/squirrel"
"github.com/jackc/pgtype"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zitadel/zitadel/internal/database"
)
var (
testNow = time.Now()
dayNow = testNow.Truncate(24 * time.Hour)
)
// assertPrepare checks if the prepare func executes the correct sql query and returns the correct object
@@ -385,6 +388,15 @@ func TestValidatePrepare(t *testing.T) {
}
}
func intervalDriverValue(t *testing.T, src time.Duration) pgtype.Interval {
interval := pgtype.Interval{}
err := interval.Set(src)
if err != nil {
t.Fatal(err)
}
return interval
}
type prepareDB struct{}
const asOfSystemTime = " AS OF SYSTEM TIME '-1 ms' "

View File

@@ -0,0 +1,114 @@
package projection
import (
"context"
"github.com/zitadel/zitadel/internal/eventstore"
old_handler "github.com/zitadel/zitadel/internal/eventstore/handler"
"github.com/zitadel/zitadel/internal/eventstore/handler/v2"
"github.com/zitadel/zitadel/internal/repository/instance"
"github.com/zitadel/zitadel/internal/repository/limits"
)
const (
LimitsProjectionTable = "projections.limits"
LimitsColumnAggregateID = "aggregate_id"
LimitsColumnCreationDate = "creation_date"
LimitsColumnChangeDate = "change_date"
LimitsColumnResourceOwner = "resource_owner"
LimitsColumnInstanceID = "instance_id"
LimitsColumnSequence = "sequence"
LimitsColumnAuditLogRetention = "audit_log_retention"
)
type limitsProjection struct{}
func newLimitsProjection(ctx context.Context, config handler.Config) *handler.Handler {
return handler.NewHandler(ctx, &config, &limitsProjection{})
}
func (*limitsProjection) Name() string {
return LimitsProjectionTable
}
func (*limitsProjection) Init() *old_handler.Check {
return handler.NewTableCheck(
handler.NewTable([]*handler.InitColumn{
handler.NewColumn(LimitsColumnAggregateID, handler.ColumnTypeText),
handler.NewColumn(LimitsColumnCreationDate, handler.ColumnTypeTimestamp),
handler.NewColumn(LimitsColumnChangeDate, handler.ColumnTypeTimestamp),
handler.NewColumn(LimitsColumnResourceOwner, handler.ColumnTypeText),
handler.NewColumn(LimitsColumnInstanceID, handler.ColumnTypeText),
handler.NewColumn(LimitsColumnSequence, handler.ColumnTypeInt64),
handler.NewColumn(LimitsColumnAuditLogRetention, handler.ColumnTypeInterval, handler.Nullable()),
},
handler.NewPrimaryKey(LimitsColumnInstanceID, LimitsColumnResourceOwner),
),
)
}
func (p *limitsProjection) Reducers() []handler.AggregateReducer {
return []handler.AggregateReducer{
{
Aggregate: limits.AggregateType,
EventReducers: []handler.EventReducer{
{
Event: limits.SetEventType,
Reduce: p.reduceLimitsSet,
},
{
Event: limits.ResetEventType,
Reduce: p.reduceLimitsReset,
},
},
},
{
Aggregate: instance.AggregateType,
EventReducers: []handler.EventReducer{
{
Event: instance.InstanceRemovedEventType,
Reduce: reduceInstanceRemovedHelper(LimitsColumnInstanceID),
},
},
},
}
}
func (p *limitsProjection) reduceLimitsSet(event eventstore.Event) (*handler.Statement, error) {
e, err := assertEvent[*limits.SetEvent](event)
if err != nil {
return nil, err
}
conflictCols := []handler.Column{
handler.NewCol(LimitsColumnInstanceID, e.Aggregate().InstanceID),
handler.NewCol(LimitsColumnResourceOwner, e.Aggregate().ResourceOwner),
}
updateCols := []handler.Column{
handler.NewCol(LimitsColumnInstanceID, e.Aggregate().InstanceID),
handler.NewCol(LimitsColumnResourceOwner, e.Aggregate().ResourceOwner),
handler.NewCol(LimitsColumnCreationDate, e.CreationDate()),
handler.NewCol(LimitsColumnChangeDate, e.CreationDate()),
handler.NewCol(LimitsColumnSequence, e.Sequence()),
handler.NewCol(LimitsColumnAggregateID, e.Aggregate().ID),
}
if e.AuditLogRetention != nil {
updateCols = append(updateCols, handler.NewCol(LimitsColumnAuditLogRetention, *e.AuditLogRetention))
}
return handler.NewUpsertStatement(e, conflictCols, updateCols), nil
}
func (p *limitsProjection) reduceLimitsReset(event eventstore.Event) (*handler.Statement, error) {
e, err := assertEvent[*limits.ResetEvent](event)
if err != nil {
return nil, err
}
return handler.NewDeleteStatement(
e,
[]handler.Condition{
handler.NewCond(LimitsColumnInstanceID, e.Aggregate().InstanceID),
handler.NewCond(LimitsColumnResourceOwner, e.Aggregate().ResourceOwner),
},
), nil
}

View File

@@ -0,0 +1,96 @@
package projection
import (
"testing"
"time"
"github.com/zitadel/zitadel/internal/errors"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/eventstore/handler/v2"
"github.com/zitadel/zitadel/internal/repository/limits"
)
func TestLimitsProjection_reduces(t *testing.T) {
type args struct {
event func(t *testing.T) eventstore.Event
}
tests := []struct {
name string
args args
reduce func(event eventstore.Event) (*handler.Statement, error)
want wantReduce
}{
{
name: "reduceLimitsSet",
args: args{
event: getEvent(testEvent(
limits.SetEventType,
limits.AggregateType,
[]byte(`{
"auditLogRetention": 300000000000
}`),
), limits.SetEventMapper),
},
reduce: (&limitsProjection{}).reduceLimitsSet,
want: wantReduce{
aggregateType: eventstore.AggregateType("limits"),
sequence: 15,
executer: &testExecuter{
executions: []execution{
{
expectedStmt: "INSERT INTO projections.limits (instance_id, resource_owner, creation_date, change_date, sequence, aggregate_id, audit_log_retention) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (instance_id, resource_owner) DO UPDATE SET (creation_date, change_date, sequence, aggregate_id, audit_log_retention) = (EXCLUDED.creation_date, EXCLUDED.change_date, EXCLUDED.sequence, EXCLUDED.aggregate_id, EXCLUDED.audit_log_retention)",
expectedArgs: []interface{}{
"instance-id",
"ro-id",
anyArg{},
anyArg{},
uint64(15),
"agg-id",
time.Minute * 5,
},
},
},
},
},
},
{
name: "reduceLimitsReset",
args: args{
event: getEvent(testEvent(
limits.ResetEventType,
limits.AggregateType,
[]byte(`{}`),
), limits.ResetEventMapper),
},
reduce: (&limitsProjection{}).reduceLimitsReset,
want: wantReduce{
aggregateType: eventstore.AggregateType("limits"),
sequence: 15,
executer: &testExecuter{
executions: []execution{
{
expectedStmt: "DELETE FROM projections.limits WHERE (instance_id = $1) AND (resource_owner = $2)",
expectedArgs: []interface{}{
"instance-id",
"ro-id",
},
},
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
event := baseEvent(t)
got, err := tt.reduce(event)
if !errors.IsErrorInvalidArgument(err) {
t.Errorf("no wrong event mapping: %v, got: %v", err, got)
}
event = tt.args.event(t)
got, err = tt.reduce(event)
assertReduce(t, got, err, LimitsProjectionTable, tt.want)
})
}
}

View File

@@ -11,6 +11,7 @@ import (
action_repo "github.com/zitadel/zitadel/internal/repository/action"
iam_repo "github.com/zitadel/zitadel/internal/repository/instance"
key_repo "github.com/zitadel/zitadel/internal/repository/keypair"
"github.com/zitadel/zitadel/internal/repository/limits"
"github.com/zitadel/zitadel/internal/repository/org"
proj_repo "github.com/zitadel/zitadel/internal/repository/project"
quota_repo "github.com/zitadel/zitadel/internal/repository/quota"
@@ -36,6 +37,7 @@ func eventstoreExpect(t *testing.T, expects ...expect) *eventstore.Eventstore {
usr_repo.RegisterEventMappers(es)
proj_repo.RegisterEventMappers(es)
quota_repo.RegisterEventMappers(es)
limits.RegisterEventMappers(es)
usergrant.RegisterEventMappers(es)
key_repo.RegisterEventMappers(es)
action_repo.RegisterEventMappers(es)

View File

@@ -69,6 +69,7 @@ var (
AuthRequestProjection *handler.Handler
MilestoneProjection *handler.Handler
QuotaProjection *quotaProjection
LimitsProjection *handler.Handler
)
type projection interface {
@@ -141,6 +142,7 @@ func Create(ctx context.Context, sqlClient *database.DB, es handler.EventStore,
AuthRequestProjection = newAuthRequestProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["auth_requests"]))
MilestoneProjection = newMilestoneProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["milestones"]), systemUsers)
QuotaProjection = newQuotaProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["quotas"]))
LimitsProjection = newLimitsProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["limits"]))
newProjectionsList()
return nil
}
@@ -244,5 +246,6 @@ func newProjectionsList() {
AuthRequestProjection,
MilestoneProjection,
QuotaProjection.handler,
LimitsProjection,
}
}

View File

@@ -24,6 +24,7 @@ import (
"github.com/zitadel/zitadel/internal/repository/idpintent"
iam_repo "github.com/zitadel/zitadel/internal/repository/instance"
"github.com/zitadel/zitadel/internal/repository/keypair"
"github.com/zitadel/zitadel/internal/repository/limits"
"github.com/zitadel/zitadel/internal/repository/oidcsession"
"github.com/zitadel/zitadel/internal/repository/org"
"github.com/zitadel/zitadel/internal/repository/project"
@@ -50,6 +51,7 @@ type Queries struct {
supportedLangs []language.Tag
zitadelRoles []authz.RoleMapping
multifactors domain.MultifactorConfigs
defaultAuditLogRetention time.Duration
}
func StartQueries(
@@ -62,6 +64,7 @@ func StartQueries(
zitadelRoles []authz.RoleMapping,
sessionTokenVerifier func(ctx context.Context, sessionToken string, sessionID string, tokenID string) (err error),
permissionCheck func(q *Queries) domain.PermissionCheck,
defaultAuditLogRetention time.Duration,
systemAPIUsers map[string]*internal_authz.SystemAPIUser,
) (repo *Queries, err error) {
statikLoginFS, err := fs.NewWithNamespace("login")
@@ -84,6 +87,7 @@ func StartQueries(
NotificationTranslationFileContents: make(map[string][]byte),
zitadelRoles: zitadelRoles,
sessionTokenVerifier: sessionTokenVerifier,
defaultAuditLogRetention: defaultAuditLogRetention,
}
iam_repo.RegisterEventMappers(repo.eventstore)
usr_repo.RegisterEventMappers(repo.eventstore)
@@ -97,6 +101,7 @@ func StartQueries(
authrequest.RegisterEventMappers(repo.eventstore)
oidcsession.RegisterEventMappers(repo.eventstore)
quota.RegisterEventMappers(repo.eventstore)
limits.RegisterEventMappers(repo.eventstore)
repo.idpConfigEncryption = idpConfigEncryption
repo.multifactors = domain.MultifactorConfigs{

View File

@@ -9,8 +9,6 @@ import (
"testing"
"time"
"github.com/jackc/pgtype"
errs "github.com/zitadel/zitadel/internal/errors"
)
@@ -33,19 +31,6 @@ var (
}
)
func dayNow() time.Time {
return time.Now().Truncate(24 * time.Hour)
}
func interval(t *testing.T, src time.Duration) pgtype.Interval {
interval := pgtype.Interval{}
err := interval.Set(src)
if err != nil {
t.Fatal(err)
}
return interval
}
func Test_QuotaPrepare(t *testing.T) {
type want struct {
sqlExpectations sqlExpectation
@@ -84,8 +69,8 @@ func Test_QuotaPrepare(t *testing.T) {
quotaCols,
[]driver.Value{
"quota-id",
dayNow(),
interval(t, time.Hour*24),
dayNow,
intervalDriverValue(t, time.Hour*24),
uint64(1000),
true,
testNow,
@@ -94,9 +79,9 @@ func Test_QuotaPrepare(t *testing.T) {
},
object: &Quota{
ID: "quota-id",
From: dayNow(),
From: dayNow,
ResetInterval: time.Hour * 24,
CurrentPeriodStart: dayNow(),
CurrentPeriodStart: dayNow,
Amount: 1000,
Limit: true,
},