feat: V2 alpha import and export of organizations (#3798)

* feat(import): add functionality to import data into an instance * feat(import): move import to admin api and additional checks for nil pointer * fix(export): export implementation with filtered members and grants * fix: export and import implementation * fix: add possibility to export hashed passwords with the user * fix(import): import with structure of v1 and v2 * docs: add v1 proto * fix(import): check im imported user is already existing * fix(import): add otp import function * fix(import): add external idps, domains, custom text and messages * fix(import): correct usage of default values from login policy * fix(export): fix renaming of add project function * fix(import): move checks for unit tests * expect filter * fix(import): move checks for unit tests * fix(import): move checks for unit tests * fix(import): produce prerelease from branch * fix(import): correctly use provided user id for machine user imports * fix(import): corrected otp import and added guide for export and import * fix: import verified and primary domains * fix(import): add reading from gcs, s3 and localfile with tracing * fix(import): gcs and s3, file size correction and error logging * Delete docker-compose.yml * fix(import): progress logging and count of resources * fix(import): progress logging and count of resources * log subscription * fix(import): incorporate review * fix(import): incorporate review * docs: add suggestion for import Co-authored-by: Fabi <38692350+hifabienne@users.noreply.github.com> * fix(import): add verification otp event and handling of deleted but existing users Co-authored-by: Livio Amstutz <livio.a@gmail.com> Co-authored-by: Fabienne <fabienne.gerschwiler@gmail.com> Co-authored-by: Silvan <silvan.reusser@gmail.com> Co-authored-by: Fabi <38692350+hifabienne@users.noreply.github.com>
2025-08-12 03:37:34 +00:00 · 2022-07-28 15:42:35 +02:00
parent d620126aab
commit bc9a85daf3
51 changed files with 4430 additions and 648 deletions
--- a/internal/query/idp.go
+++ b/internal/query/idp.go
@@ -502,3 +502,15 @@ func prepareIDPsQuery() (sq.SelectBuilder, func(*sql.Rows) (*IDPs, error)) {
 			}, nil
 		}
 }
+
+func (q *Queries) GetOIDCIDPClientSecret(ctx context.Context, shouldRealTime bool, resourceowner, idpID string) (string, error) {
+	idp, err := q.IDPByIDAndResourceOwner(ctx, shouldRealTime, idpID, resourceowner)
+	if err != nil {
+		return "", err
+	}
+
+	if idp.ClientSecret != nil && idp.ClientSecret.Crypted != nil {
+		return crypto.DecryptString(idp.ClientSecret, q.idpConfigEncryption)
+	}
+	return "", errors.ThrowNotFound(nil, "QUERY-bsm2o", "Errors.Query.NotFound")
+}
--- a/internal/query/query.go
+++ b/internal/query/query.go
@@ -4,6 +4,8 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	sd "github.com/zitadel/zitadel/internal/config/systemdefaults"
+	"github.com/zitadel/zitadel/internal/domain"
 	"net/http"
 	"sync"

@@ -27,6 +29,8 @@ type Queries struct {
 	eventstore *eventstore.Eventstore
 	client     *sql.DB

+	idpConfigEncryption crypto.EncryptionAlgorithm
+
 	DefaultLanguage                     language.Tag
 	LoginDir                            http.FileSystem
 	NotificationDir                     http.FileSystem
@@ -35,9 +39,10 @@ type Queries struct {
 	NotificationTranslationFileContents map[string][]byte
 	supportedLangs                      []language.Tag
 	zitadelRoles                        []authz.RoleMapping
+	multifactors                        domain.MultifactorConfigs
 }

-func StartQueries(ctx context.Context, es *eventstore.Eventstore, sqlClient *sql.DB, projections projection.Config, keyEncryptionAlgorithm crypto.EncryptionAlgorithm, zitadelRoles []authz.RoleMapping) (repo *Queries, err error) {
+func StartQueries(ctx context.Context, es *eventstore.Eventstore, sqlClient *sql.DB, projections projection.Config, defaults sd.SystemDefaults, idpConfigEncryption, otpEncryption, keyEncryptionAlgorithm crypto.EncryptionAlgorithm, zitadelRoles []authz.RoleMapping) (repo *Queries, err error) {
 	statikLoginFS, err := fs.NewWithNamespace("login")
 	if err != nil {
 		return nil, fmt.Errorf("unable to start login statik dir")
@@ -66,6 +71,14 @@ func StartQueries(ctx context.Context, es *eventstore.Eventstore, sqlClient *sql
 	keypair.RegisterEventMappers(repo.eventstore)
 	usergrant.RegisterEventMappers(repo.eventstore)

+	repo.idpConfigEncryption = idpConfigEncryption
+	repo.multifactors = domain.MultifactorConfigs{
+		OTP: domain.OTPConfig{
+			CryptoMFA: otpEncryption,
+			Issuer:    defaults.Multifactors.OTP.Issuer,
+		},
+	}
+
 	err = projection.Start(ctx, sqlClient, es, projections, keyEncryptionAlgorithm)
 	if err != nil {
 		return nil, err
--- a/internal/query/user_otp.go
+++ b/internal/query/user_otp.go
@@ -0,0 +1,91 @@
+package query
+
+import (
+	"context"
+	"github.com/zitadel/zitadel/internal/crypto"
+	"github.com/zitadel/zitadel/internal/domain"
+	caos_errs "github.com/zitadel/zitadel/internal/errors"
+	"github.com/zitadel/zitadel/internal/eventstore"
+	"github.com/zitadel/zitadel/internal/repository/user"
+	"github.com/zitadel/zitadel/internal/telemetry/tracing"
+)
+
+func (q *Queries) GetHumanOTPSecret(ctx context.Context, userID, resourceowner string) (string, error) {
+	if userID == "" {
+		return "", caos_errs.ThrowPreconditionFailed(nil, "QUERY-8N9ds", "Errors.User.UserIDMissing")
+	}
+	existingOTP, err := q.otpWriteModelByID(ctx, userID, resourceowner)
+	if err != nil {
+		return "", err
+	}
+	if existingOTP.State != domain.MFAStateReady {
+		return "", caos_errs.ThrowNotFound(nil, "QUERY-01982h", "Errors.User.NotFound")
+	}
+
+	return crypto.DecryptString(existingOTP.Secret, q.multifactors.OTP.CryptoMFA)
+}
+
+func (q *Queries) otpWriteModelByID(ctx context.Context, userID, resourceOwner string) (writeModel *HumanOTPWriteModel, err error) {
+	ctx, span := tracing.NewSpan(ctx)
+	defer func() { span.EndWithError(err) }()
+
+	writeModel = NewHumanOTPWriteModel(userID, resourceOwner)
+	err = q.eventstore.FilterToQueryReducer(ctx, writeModel)
+	if err != nil {
+		return nil, err
+	}
+	return writeModel, nil
+}
+
+type HumanOTPWriteModel struct {
+	eventstore.WriteModel
+
+	State  domain.MFAState
+	Secret *crypto.CryptoValue
+}
+
+func NewHumanOTPWriteModel(userID, resourceOwner string) *HumanOTPWriteModel {
+	return &HumanOTPWriteModel{
+		WriteModel: eventstore.WriteModel{
+			AggregateID:   userID,
+			ResourceOwner: resourceOwner,
+		},
+	}
+}
+
+func (wm *HumanOTPWriteModel) Reduce() error {
+	for _, event := range wm.Events {
+		switch e := event.(type) {
+		case *user.HumanOTPAddedEvent:
+			wm.Secret = e.Secret
+			wm.State = domain.MFAStateNotReady
+		case *user.HumanOTPVerifiedEvent:
+			wm.State = domain.MFAStateReady
+		case *user.HumanOTPRemovedEvent:
+			wm.State = domain.MFAStateRemoved
+		case *user.UserRemovedEvent:
+			wm.State = domain.MFAStateRemoved
+		}
+	}
+	return wm.WriteModel.Reduce()
+}
+
+func (wm *HumanOTPWriteModel) Query() *eventstore.SearchQueryBuilder {
+	query := eventstore.NewSearchQueryBuilder(eventstore.ColumnsEvent).
+		AddQuery().
+		AggregateTypes(user.AggregateType).
+		AggregateIDs(wm.AggregateID).
+		EventTypes(user.HumanMFAOTPAddedType,
+			user.HumanMFAOTPVerifiedType,
+			user.HumanMFAOTPRemovedType,
+			user.UserRemovedType,
+			user.UserV1MFAOTPAddedType,
+			user.UserV1MFAOTPVerifiedType,
+			user.UserV1MFAOTPRemovedType).
+		Builder()
+
+	if wm.ResourceOwner != "" {
+		query.ResourceOwner(wm.ResourceOwner)
+	}
+	return query
+}
--- a/internal/query/user_password.go
+++ b/internal/query/user_password.go
@@ -0,0 +1,140 @@
+package query
+
+import (
+	"context"
+	"github.com/zitadel/zitadel/internal/crypto"
+	"github.com/zitadel/zitadel/internal/domain"
+	caos_errs "github.com/zitadel/zitadel/internal/errors"
+	"github.com/zitadel/zitadel/internal/eventstore"
+	"github.com/zitadel/zitadel/internal/repository/user"
+	"github.com/zitadel/zitadel/internal/telemetry/tracing"
+	"time"
+)
+
+type HumanPasswordWriteModel struct {
+	eventstore.WriteModel
+
+	Secret               *crypto.CryptoValue
+	SecretChangeRequired bool
+
+	Code                     *crypto.CryptoValue
+	CodeCreationDate         time.Time
+	CodeExpiry               time.Duration
+	PasswordCheckFailedCount uint64
+
+	UserState domain.UserState
+}
+
+func (q *Queries) GetHumanPassword(ctx context.Context, orgID, userID string) (passwordHash []byte, algorithm string, err error) {
+	if userID == "" {
+		return nil, "", caos_errs.ThrowInvalidArgument(nil, "QUERY-4Mfsf", "Errors.User.UserIDMissing")
+	}
+	existingPassword, err := q.passwordWriteModel(ctx, userID, orgID)
+	if err != nil {
+		return nil, "", caos_errs.ThrowInternal(nil, "QUERY-p1k1n2i", "Errors.User.NotFound")
+	}
+	if existingPassword.UserState == domain.UserStateUnspecified || existingPassword.UserState == domain.UserStateDeleted {
+		return nil, "", caos_errs.ThrowPreconditionFailed(nil, "QUERY-3n77z", "Errors.User.NotFound")
+	}
+
+	if existingPassword.Secret != nil && existingPassword.Secret.Crypted != nil {
+		return existingPassword.Secret.Crypted, existingPassword.Secret.Algorithm, nil
+	}
+
+	return nil, "", nil
+}
+
+func (q *Queries) passwordWriteModel(ctx context.Context, userID, resourceOwner string) (writeModel *HumanPasswordWriteModel, err error) {
+	ctx, span := tracing.NewSpan(ctx)
+	defer func() { span.EndWithError(err) }()
+
+	writeModel = NewHumanPasswordWriteModel(userID, resourceOwner)
+	err = q.eventstore.FilterToQueryReducer(ctx, writeModel)
+	if err != nil {
+		return nil, err
+	}
+	return writeModel, nil
+}
+
+func NewHumanPasswordWriteModel(userID, resourceOwner string) *HumanPasswordWriteModel {
+	return &HumanPasswordWriteModel{
+		WriteModel: eventstore.WriteModel{
+			AggregateID:   userID,
+			ResourceOwner: resourceOwner,
+		},
+	}
+}
+
+func (wm *HumanPasswordWriteModel) Reduce() error {
+	for _, event := range wm.Events {
+		switch e := event.(type) {
+		case *user.HumanAddedEvent:
+			wm.Secret = e.Secret
+			wm.SecretChangeRequired = e.ChangeRequired
+			wm.UserState = domain.UserStateActive
+		case *user.HumanRegisteredEvent:
+			wm.Secret = e.Secret
+			wm.SecretChangeRequired = e.ChangeRequired
+			wm.UserState = domain.UserStateActive
+		case *user.HumanInitialCodeAddedEvent:
+			wm.UserState = domain.UserStateInitial
+		case *user.HumanInitializedCheckSucceededEvent:
+			wm.UserState = domain.UserStateActive
+		case *user.HumanPasswordChangedEvent:
+			wm.Secret = e.Secret
+			wm.SecretChangeRequired = e.ChangeRequired
+			wm.Code = nil
+			wm.PasswordCheckFailedCount = 0
+		case *user.HumanPasswordCodeAddedEvent:
+			wm.Code = e.Code
+			wm.CodeCreationDate = e.CreationDate()
+			wm.CodeExpiry = e.Expiry
+		case *user.HumanEmailVerifiedEvent:
+			if wm.UserState == domain.UserStateInitial {
+				wm.UserState = domain.UserStateActive
+			}
+		case *user.HumanPasswordCheckFailedEvent:
+			wm.PasswordCheckFailedCount += 1
+		case *user.HumanPasswordCheckSucceededEvent:
+			wm.PasswordCheckFailedCount = 0
+		case *user.UserUnlockedEvent:
+			wm.PasswordCheckFailedCount = 0
+		case *user.UserRemovedEvent:
+			wm.UserState = domain.UserStateDeleted
+		}
+	}
+	return wm.WriteModel.Reduce()
+}
+
+func (wm *HumanPasswordWriteModel) Query() *eventstore.SearchQueryBuilder {
+	query := eventstore.NewSearchQueryBuilder(eventstore.ColumnsEvent).
+		AddQuery().
+		AggregateTypes(user.AggregateType).
+		AggregateIDs(wm.AggregateID).
+		EventTypes(user.HumanAddedType,
+			user.HumanRegisteredType,
+			user.HumanInitialCodeAddedType,
+			user.HumanInitializedCheckSucceededType,
+			user.HumanPasswordChangedType,
+			user.HumanPasswordCodeAddedType,
+			user.HumanEmailVerifiedType,
+			user.HumanPasswordCheckFailedType,
+			user.HumanPasswordCheckSucceededType,
+			user.UserRemovedType,
+			user.UserUnlockedType,
+			user.UserV1AddedType,
+			user.UserV1RegisteredType,
+			user.UserV1InitialCodeAddedType,
+			user.UserV1InitializedCheckSucceededType,
+			user.UserV1PasswordChangedType,
+			user.UserV1PasswordCodeAddedType,
+			user.UserV1EmailVerifiedType,
+			user.UserV1PasswordCheckFailedType,
+			user.UserV1PasswordCheckSucceededType).
+		Builder()
+
+	if wm.ResourceOwner != "" {
+		query.ResourceOwner(wm.ResourceOwner)
+	}
+	return query
+}