feat: push telemetry (#6027)

* document analytics config

* rework configuration and docs

* describe HandleActiveInstances better

* describe active instances on quotas better

* only projected events are considered

* cleanup

* describe changes at runtime

* push milestones

* stop tracking events

* calculate and push 4 in 6 milestones

* reduce milestone pushed

* remove docs

* fix scheduled pseudo event projection

* push 5 in 6 milestones

* push 6 in 6 milestones

* ignore client ids

* fix text array contains

* push human readable milestone type

* statement unit tests

* improve dev and db performance

* organize imports

* cleanup

* organize imports

* test projection

* check rows.Err()

* test search query

* pass linting

* review

* test 4 milestones

* simplify milestone by instance ids query

* use type NamespacedCondition

* cleanup

* lint

* lint

* dont overwrite original error

* no opt-in in examples

* cleanup

* prerelease

* enable request headers

* make limit configurable

* review fixes

* only requeue special handlers secondly

* include integration tests

* Revert "include integration tests"

This reverts commit 96db9504ec.

* pass reducers

* test handlers

* fix unit test

* feat: increment version

* lint

* remove prerelease

* fix integration tests

internal/repository/milestone/aggregate.go

@@ -0,0 +1,30 @@
+package milestone
+
+import (
+	"context"
+
+	"github.com/zitadel/zitadel/internal/api/authz"
+	"github.com/zitadel/zitadel/internal/eventstore"
+)
+
+const (
+	AggregateType    = "milestone"
+	AggregateVersion = "v1"
+)
+
+type Aggregate struct {
+	eventstore.Aggregate
+}
+
+func NewAggregate(ctx context.Context, id string) *Aggregate {
+	instanceID := authz.GetInstance(ctx).InstanceID()
+	return &Aggregate{
+		Aggregate: eventstore.Aggregate{
+			Type:          AggregateType,
+			Version:       AggregateVersion,
+			ID:            id,
+			ResourceOwner: instanceID,
+			InstanceID:    instanceID,
+		},
+	}
+}

internal/repository/milestone/events.go

@@ -0,0 +1,54 @@
+package milestone
+
+import (
+	"context"
+
+	"github.com/zitadel/zitadel/internal/eventstore"
+)
+
+const (
+	eventTypePrefix = eventstore.EventType("milestone.")
+	PushedEventType = eventTypePrefix + "pushed"
+)
+
+type PushedEvent struct {
+	*eventstore.BaseEvent `json:"-"`
+	MilestoneType         Type     `json:"type"`
+	ExternalDomain        string   `json:"externalDomain"`
+	PrimaryDomain         string   `json:"primaryDomain"`
+	Endpoints             []string `json:"endpoints"`
+}
+
+func (p *PushedEvent) Data() interface{} {
+	return p
+}
+
+func (p *PushedEvent) UniqueConstraints() []*eventstore.EventUniqueConstraint {
+	return nil
+}
+
+func (p *PushedEvent) SetBaseEvent(b *eventstore.BaseEvent) {
+	p.BaseEvent = b
+}
+
+var PushedEventMapper = eventstore.GenericEventMapper[PushedEvent]
+
+func NewPushedEvent(
+	ctx context.Context,
+	aggregate *Aggregate,
+	msType Type,
+	endpoints []string,
+	externalDomain, primaryDomain string,
+) *PushedEvent {
+	return &PushedEvent{
+		BaseEvent: eventstore.NewBaseEventForPush(
+			ctx,
+			&aggregate.Aggregate,
+			PushedEventType,
+		),
+		MilestoneType:  msType,
+		Endpoints:      endpoints,
+		ExternalDomain: externalDomain,
+		PrimaryDomain:  primaryDomain,
+	}
+}

internal/repository/milestone/eventstore.go

@@ -0,0 +1,9 @@
+package milestone
+
+import (
+	"github.com/zitadel/zitadel/internal/eventstore"
+)
+
+func RegisterEventMappers(es *eventstore.Eventstore) {
+	es.RegisterFilterEventMapper(AggregateType, PushedEventType, PushedEventMapper)
+}

internal/repository/milestone/type.go

@@ -0,0 +1,59 @@
+//go:generate stringer -type Type
+
+package milestone
+
+import (
+	"fmt"
+	"strings"
+)
+
+type Type int
+
+const (
+	unknown Type = iota
+
+	InstanceCreated
+	AuthenticationSucceededOnInstance
+	ProjectCreated
+	ApplicationCreated
+	AuthenticationSucceededOnApplication
+	InstanceDeleted
+
+	typesCount
+)
+
+func AllTypes() []Type {
+	types := make([]Type, typesCount-1)
+	for i := Type(1); i < typesCount; i++ {
+		types[i-1] = i
+	}
+	return types
+}
+
+func (t *Type) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`"%s"`, t.String())), nil
+}
+
+func (t *Type) UnmarshalJSON(data []byte) error {
+	*t = typeFromString(strings.Trim(string(data), `"`))
+	return nil
+}
+
+func typeFromString(t string) Type {
+	switch t {
+	case InstanceCreated.String():
+		return InstanceCreated
+	case AuthenticationSucceededOnInstance.String():
+		return AuthenticationSucceededOnInstance
+	case ProjectCreated.String():
+		return ProjectCreated
+	case ApplicationCreated.String():
+		return ApplicationCreated
+	case AuthenticationSucceededOnApplication.String():
+		return AuthenticationSucceededOnApplication
+	case InstanceDeleted.String():
+		return InstanceDeleted
+	default:
+		return unknown
+	}
+}

internal/repository/milestone/type_string.go

@@ -0,0 +1,30 @@
+// Code generated by "stringer -type Type"; DO NOT EDIT.
+
+package milestone
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[unknown-0]
+	_ = x[InstanceCreated-1]
+	_ = x[AuthenticationSucceededOnInstance-2]
+	_ = x[ProjectCreated-3]
+	_ = x[ApplicationCreated-4]
+	_ = x[AuthenticationSucceededOnApplication-5]
+	_ = x[InstanceDeleted-6]
+	_ = x[typesCount-7]
+}
+
+const _Type_name = "unknownInstanceCreatedAuthenticationSucceededOnInstanceProjectCreatedApplicationCreatedAuthenticationSucceededOnApplicationInstanceDeletedtypesCount"
+
+var _Type_index = [...]uint8{0, 7, 22, 55, 69, 87, 123, 138, 148}
+
+func (i Type) String() string {
+	if i < 0 || i >= Type(len(_Type_index)-1) {
+		return "Type(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _Type_name[_Type_index[i]:_Type_index[i+1]]
+}

internal/repository/pseudo/aggregate.go

@@ -0,0 +1,21 @@
+package pseudo
+
+import "github.com/zitadel/zitadel/internal/eventstore"
+
+const (
+	AggregateType    = "pseudo"
+	AggregateVersion = "v1"
+)
+
+type Aggregate struct {
+	eventstore.Aggregate
+}
+
+func NewAggregate() *Aggregate {
+	return &Aggregate{
+		Aggregate: eventstore.Aggregate{
+			Type:    AggregateType,
+			Version: AggregateVersion,
+		},
+	}
+}

internal/repository/pseudo/events.go

@@ -0,0 +1,40 @@
+// Package pseudo contains virtual events, that are not stored in the eventstore.
+package pseudo
+
+import (
+	"context"
+	"time"
+
+	"github.com/zitadel/zitadel/internal/eventstore"
+)
+
+const (
+	eventTypePrefix    = eventstore.EventType("pseudo.")
+	ScheduledEventType = eventTypePrefix + "timestamp"
+)
+
+var _ eventstore.Event = (*ScheduledEvent)(nil)
+
+type ScheduledEvent struct {
+	*eventstore.BaseEvent `json:"-"`
+	Timestamp             time.Time `json:"-"`
+	InstanceIDs           []string  `json:"-"`
+}
+
+// NewScheduledEvent returns an event that can be processed by event handlers like any other event.
+// It receives the current timestamp and an ID list of instances that are active and should be processed.
+func NewScheduledEvent(
+	ctx context.Context,
+	timestamp time.Time,
+	instanceIDs ...string,
+) *ScheduledEvent {
+	return &ScheduledEvent{
+		BaseEvent: eventstore.NewBaseEventForPush(
+			ctx,
+			&NewAggregate().Aggregate,
+			ScheduledEventType,
+		),
+		Timestamp:   timestamp,
+		InstanceIDs: instanceIDs,
+	}
+}