feat: projections auto create their tables (#3324)

* begin init checks for projections

* first projection checks

* debug notification providers with query fixes

* more projections and first index

* more projections

* more projections

* finish projections

* fix tests (remove db name)

* create tables in setup

* fix logging / error handling

* add tenant to views

* rename tenant to instance_id

* add instance_id to all projections

* add instance_id to all queries

* correct instance_id on projections

* add instance_id to failed_events

* use separate context for instance

* implement features projection

* implement features projection

* remove unique constraint from setup when migration failed

* add error to failed setup event

* add instance_id to primary keys

* fix IAM projection

* remove old migrations folder

* fix keysFromYAML test

internal/migration/command.go

@@ -0,0 +1,77 @@
+package migration
+
+import "github.com/caos/zitadel/internal/eventstore"
+
+//SetupStep is the command pushed on the eventstore
+type SetupStep struct {
+	typ       eventstore.EventType
+	migration Migration
+	Name      string `json:"name"`
+	Error     error  `json:"error,omitempty"`
+	done      bool
+}
+
+func setupStartedCmd(migration Migration) eventstore.Command {
+	return &SetupStep{
+		migration: migration,
+		typ:       startedType,
+		Name:      migration.String(),
+	}
+}
+
+func setupDoneCmd(migration Migration, err error) eventstore.Command {
+	s := &SetupStep{
+		typ:       doneType,
+		migration: migration,
+		Name:      migration.String(),
+	}
+
+	if err != nil {
+		s.typ = failedType
+		s.Error = err
+	}
+
+	return s
+}
+
+func (s *SetupStep) Aggregate() eventstore.Aggregate {
+	return eventstore.Aggregate{
+		ID:            aggregateID,
+		Type:          aggregateType,
+		ResourceOwner: "SYSTEM",
+		Version:       "v1",
+	}
+}
+
+func (s *SetupStep) EditorService() string {
+	return "system"
+}
+
+func (s *SetupStep) EditorUser() string {
+	return "system"
+}
+
+func (s *SetupStep) Type() eventstore.EventType {
+	return s.typ
+}
+
+func (s *SetupStep) Data() interface{} {
+	return s
+}
+
+func (s *SetupStep) UniqueConstraints() []*eventstore.EventUniqueConstraint {
+	switch s.typ {
+	case startedType:
+		return []*eventstore.EventUniqueConstraint{
+			eventstore.NewAddEventUniqueConstraint("migration_started", s.migration.String(), "Errors.Step.Started.AlreadyExists"),
+		}
+	case failedType:
+		return []*eventstore.EventUniqueConstraint{
+			eventstore.NewRemoveEventUniqueConstraint("migration_started", s.migration.String()),
+		}
+	default:
+		return []*eventstore.EventUniqueConstraint{
+			eventstore.NewAddEventUniqueConstraint("migration_done", s.migration.String(), "Errors.Step.Done.AlreadyExists"),
+		}
+	}
+}

internal/migration/migration.go

@@ -0,0 +1,84 @@
+package migration
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/caos/logging"
+
+	"github.com/caos/zitadel/internal/errors"
+	"github.com/caos/zitadel/internal/eventstore"
+)
+
+const (
+	startedType   = eventstore.EventType("system.migration.started")
+	doneType      = eventstore.EventType("system.migration.done")
+	failedType    = eventstore.EventType("system.migration.failed")
+	aggregateType = eventstore.AggregateType("system")
+	aggregateID   = "SYSTEM"
+)
+
+type Migration interface {
+	String() string
+	Execute(context.Context) error
+}
+
+func Migrate(ctx context.Context, es *eventstore.Eventstore, migration Migration) (err error) {
+	if should, err := shouldExec(ctx, es, migration); !should || err != nil {
+		return err
+	}
+
+	if _, err = es.Push(ctx, setupStartedCmd(migration)); err != nil {
+		return err
+	}
+
+	err = migration.Execute(ctx)
+	logging.OnError(err).Error("migration failed")
+
+	_, err = es.Push(ctx, setupDoneCmd(migration, err))
+	return err
+}
+
+func shouldExec(ctx context.Context, es *eventstore.Eventstore, migration Migration) (should bool, err error) {
+	events, err := es.Filter(ctx, eventstore.NewSearchQueryBuilder(eventstore.ColumnsEvent).
+		OrderDesc().
+		AddQuery().
+		AggregateTypes(aggregateType).
+		AggregateIDs(aggregateID).
+		EventTypes(startedType, doneType, failedType).
+		Builder())
+	if err != nil {
+		return false, err
+	}
+
+	if len(events) == 0 {
+		return true, nil
+	}
+	if events[len(events)-1].Type() == startedType {
+		return false, nil
+	}
+
+	for _, e := range events {
+		step := new(SetupStep)
+
+		err = json.Unmarshal(e.DataAsBytes(), step)
+		if err != nil {
+			return false, err
+		}
+
+		if step.Name != migration.String() {
+			continue
+		}
+
+		switch e.Type() {
+		case startedType, doneType:
+			//TODO: if started should we wait until done/failed?
+			return false, nil
+		case failedType:
+			//TODO: how to allow retries?
+			logging.WithFields("migration", migration.String()).Error("failed before")
+			return false, errors.ThrowInternal(nil, "MIGRA-mjI2E", "migration failed before")
+		}
+	}
+	return true, nil
+}