chore: integration test base (#5739)

Add integrations tests for the gRPC API, primarily targeted at `v2` but
can also be used for legacy. Provides a crude framework that runs a
server, prepares a client and exposes the underlying resources such as
`command` and `query` handlers to the tester. The code in
`internal/integration` is written just as a proof of concept and
probably needs to be split up in more reusable chunks when we need more
functionality, like multiple users, organisations, instances etc.

Integrations tests for `user/v2alpha` are also included.

See the added documentation for more details.

Related to #5598

.github/workflows/integration.yml

@@ -0,0 +1,51 @@
+name: Integration tests
+
+on:
+  push:
+    tags-ignore:
+      - '**'
+  pull_request:
+    branches:
+      - '**'
+
+jobs:
+  run:
+    strategy:
+      matrix:
+        db: [cockroach, postgres]
+    runs-on: ubuntu-20.04
+    env:
+      DOCKER_BUILDKIT: 1
+      INTEGRATION_DB_FLAVOR: ${{ matrix.db }}
+      ZITADEL_MASTERKEY: MasterkeyNeedsToHave32Characters
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+      - name: Source checkout
+        uses: actions/checkout@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          driver: docker
+          install: true
+      - name: Generate gRPC definitions
+        run: docker build -f build/grpc/Dockerfile -t zitadel-base:local .
+      - name: Copy gRPC definitions
+        run: docker build -f build/zitadel/Dockerfile . -t zitadel-go-base --target go-copy -o .
+      - name: Download Go modules
+        run: go mod download
+      - name: Start ${{ matrix.db }} database
+        run: docker compose -f internal/integration/config/docker-compose.yaml up --wait ${INTEGRATION_DB_FLAVOR}
+      - name: Run zitadel init and setup
+        run: |
+          go run main.go init --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml
+          go run main.go setup --masterkeyFromEnv --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml
+      - name: Run integration tests
+        run: go test -tags=integration -race -parallel 1 -v -coverprofile=profile.cov -coverpkg=./... ./internal/integration ./internal/api/grpc/...
+      - name: Publish go coverage
+        uses: codecov/codecov-action@v3.1.0
+        with:
+          file: profile.cov
+          name: integration-tests

.github/workflows/test-code.yml

@@ -44,7 +44,7 @@ jobs:
         uses: codecov/codecov-action@v3.1.0
         with:
           file: .artifacts/codecov/profile.cov
-          name: go-codecov
+          name: unit-tests
       # As goreleaser doesn't build a dockerfile in snapshot mode, we have to build it here
       - name: Build Docker Image
         run: docker build -t zitadel:pr --file build/Dockerfile .artifacts/zitadel

CONTRIBUTING.md

@@ -199,6 +199,21 @@ When you are happy with your changes, you can cleanup your environment.
 docker compose --file ./e2e/config/host.docker.internal/docker-compose.yaml down
 ```
 
+#### Integration tests
+
+In order to run the integrations tests for the gRPC API, PostgreSQL and CockroachDB must be started and initialized:
+
+```bash
+export INTEGRATION_DB_FLAVOR="cockroach" ZITADEL_MASTERKEY="MasterkeyNeedsToHave32Characters"
+docker compose -f internal/integration/config/docker-compose.yaml up --wait ${INTEGRATION_DB_FLAVOR}
+go run main.go init --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml
+go run main.go setup --masterkeyFromEnv --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml
+go test -tags=integration -race -parallel 1 ./internal/integration ./internal/api/grpc/...
+docker compose -f internal/integration/config/docker-compose.yaml down
+```
+
+The above can be repeated with `INTEGRATION_DB_FLAVOR="postgres"`.
+
 ### Console
 
 By executing the commands from this section, you run everything you need to develop the console locally.

cmd/admin/admin.go

@@ -26,8 +26,8 @@ func New() *cobra.Command {
 	adminCMD.AddCommand(
 		initialise.New(),
 		setup.New(),
-		start.New(),
+		start.New(nil),
-		start.NewStartFromInit(),
+		start.NewStartFromInit(nil),
 		key.New(),
 	)
 

cmd/start/start.go

@@ -21,6 +21,7 @@ import (
 	"github.com/zitadel/saml/pkg/provider"
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/h2c"
+	"golang.org/x/sys/unix"
 
 	"github.com/zitadel/zitadel/cmd/key"
 	cmd_tls "github.com/zitadel/zitadel/cmd/tls"
@@ -45,6 +46,7 @@ import (
 	"github.com/zitadel/zitadel/internal/authz"
 	authz_repo "github.com/zitadel/zitadel/internal/authz/repository"
 	"github.com/zitadel/zitadel/internal/command"
+	"github.com/zitadel/zitadel/internal/crypto"
 	cryptoDB "github.com/zitadel/zitadel/internal/crypto/database"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/eventstore"
@@ -60,7 +62,7 @@ import (
 	"github.com/zitadel/zitadel/openapi"
 )
 
-func New() *cobra.Command {
+func New(server chan<- *Server) *cobra.Command {
 	start := &cobra.Command{
 		Use:   "start",
 		Short: "starts ZITADEL instance",
@@ -78,7 +80,7 @@ Requirements:
 				return err
 			}
 
-			return startZitadel(config, masterKey)
+			return startZitadel(config, masterKey, server)
 		},
 	}
 
@@ -87,7 +89,23 @@ Requirements:
 	return start
 }
 
-func startZitadel(config *Config, masterKey string) error {
+type Server struct {
+	Config     *Config
+	DB         *database.DB
+	KeyStorage crypto.KeyStorage
+	Keys       *encryptionKeys
+	Eventstore *eventstore.Eventstore
+	Queries    *query.Queries
+	AuthzRepo  authz_repo.Repository
+	Storage    static.Storage
+	Commands   *command.Commands
+	LogStore   *logstore.Service
+	Router     *mux.Router
+	TLSConfig  *tls.Config
+	Shutdown   chan<- os.Signal
+}
+
+func startZitadel(config *Config, masterKey string, server chan<- *Server) error {
 	ctx := context.Background()
 
 	dbClient, err := database.Connect(config.Database, false)
@@ -180,7 +198,30 @@ func startZitadel(config *Config, masterKey string) error {
 	if err != nil {
 		return err
 	}
-	return listen(ctx, router, config.Port, tlsConfig)
+
+	shutdown := make(chan os.Signal, 1)
+	signal.Notify(shutdown, os.Interrupt, syscall.SIGTERM)
+
+	if server != nil {
+		server <- &Server{
+			Config:     config,
+			DB:         dbClient,
+			KeyStorage: keyStorage,
+			Keys:       keys,
+			Eventstore: eventstoreClient,
+			Queries:    queries,
+			AuthzRepo:  authZRepo,
+			Storage:    storage,
+			Commands:   commands,
+			LogStore:   actionsLogstoreSvc,
+			Router:     router,
+			TLSConfig:  tlsConfig,
+			Shutdown:   shutdown,
+		}
+		close(server)
+	}
+
+	return listen(ctx, router, config.Port, tlsConfig, shutdown)
 }
 
 func startAPIs(
@@ -301,10 +342,21 @@ func startAPIs(
 	return nil
 }
 
-func listen(ctx context.Context, router *mux.Router, port uint16, tlsConfig *tls.Config) error {
+func reusePort(network, address string, conn syscall.RawConn) error {
+	return conn.Control(func(descriptor uintptr) {
+		err := syscall.SetsockoptInt(int(descriptor), syscall.SOL_SOCKET, unix.SO_REUSEPORT, 1)
+		if err != nil {
+			panic(err)
+		}
+	})
+}
+
+func listen(ctx context.Context, router *mux.Router, port uint16, tlsConfig *tls.Config, shutdown <-chan os.Signal) error {
 	http2Server := &http2.Server{}
 	http1Server := &http.Server{Handler: h2c.NewHandler(router, http2Server), TLSConfig: tlsConfig}
-	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
+
+	lc := &net.ListenConfig{Control: reusePort}
+	lis, err := lc.Listen(ctx, "tcp", fmt.Sprintf(":%d", port))
 	if err != nil {
 		return fmt.Errorf("tcp listener on %d failed: %w", port, err)
 	}
@@ -321,9 +373,6 @@ func listen(ctx context.Context, router *mux.Router, port uint16, tlsConfig *tls
 		}
 	}()
 
-	shutdown := make(chan os.Signal, 1)
-	signal.Notify(shutdown, os.Interrupt, syscall.SIGTERM)
-
 	select {
 	case err := <-errCh:
 		return fmt.Errorf("error starting server: %w", err)

cmd/start/start_from_init.go

@@ -11,7 +11,7 @@ import (
 	"github.com/zitadel/zitadel/cmd/tls"
 )
 
-func NewStartFromInit() *cobra.Command {
+func NewStartFromInit(server chan<- *Server) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "start-from-init",
 		Short: "cold starts zitadel",
@@ -37,7 +37,7 @@ Requirements:
 
 			startConfig := MustNewConfig(viper.GetViper())
 
-			err = startZitadel(startConfig, masterKey)
+			err = startZitadel(startConfig, masterKey, server)
 			logging.OnError(err).Fatal("unable to start zitadel")
 		},
 	}

cmd/start/start_from_setup.go

@@ -10,7 +10,7 @@ import (
 	"github.com/zitadel/zitadel/cmd/tls"
 )
 
-func NewStartFromSetup() *cobra.Command {
+func NewStartFromSetup(server chan<- *Server) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "start-from-setup",
 		Short: "cold starts zitadel",
@@ -35,7 +35,7 @@ Requirements:
 
 			startConfig := MustNewConfig(viper.GetViper())
 
-			err = startZitadel(startConfig, masterKey)
+			err = startZitadel(startConfig, masterKey, server)
 			logging.OnError(err).Fatal("unable to start zitadel")
 		},
 	}

cmd/zitadel.go

@@ -26,7 +26,7 @@ var (
 	defaultConfig []byte
 )
 
-func New(out io.Writer, in io.Reader, args []string) *cobra.Command {
+func New(out io.Writer, in io.Reader, args []string, server chan<- *start.Server) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "zitadel",
 		Short: "The ZITADEL CLI lets you interact with ZITADEL",
@@ -51,9 +51,9 @@ func New(out io.Writer, in io.Reader, args []string) *cobra.Command {
 		admin.New(), //is now deprecated, remove later on
 		initialise.New(),
 		setup.New(),
-		start.New(),
+		start.New(server),
-		start.NewStartFromInit(),
+		start.NewStartFromInit(server),
-		start.NewStartFromSetup(),
+		start.NewStartFromSetup(server),
 		key.New(),
 	)
 

internal/api/grpc/admin/information_integration_test.go

@@ -0,0 +1,36 @@
+//go:build integration
+
+package admin_test
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zitadel/zitadel/internal/integration"
+	"github.com/zitadel/zitadel/pkg/grpc/admin"
+)
+
+var (
+	Tester *integration.Tester
+)
+
+func TestMain(m *testing.M) {
+	os.Exit(func() int {
+		ctx, _, cancel := integration.Contexts(time.Minute)
+		defer cancel()
+		Tester = integration.NewTester(ctx)
+		defer Tester.Done()
+
+		return m.Run()
+	}())
+}
+
+func TestServer_Healthz(t *testing.T) {
+	client := admin.NewAdminServiceClient(Tester.GRPCClientConn)
+	_, err := client.Healthz(context.TODO(), &admin.HealthzRequest{})
+	require.NoError(t, err)
+}

internal/api/grpc/user/v2/email.go

@@ -21,11 +21,7 @@ func (s *Server) SetEmail(ctx context.Context, req *user.SetEmailRequest) (resp
 	case *user.SetEmailRequest_ReturnCode:
 		email, err = s.command.ChangeUserEmailReturnCode(ctx, req.GetUserId(), resourceOwner, req.GetEmail(), s.userCodeAlg)
 	case *user.SetEmailRequest_IsVerified:
-		if v.IsVerified {
+		email, err = s.command.ChangeUserEmailVerified(ctx, req.GetUserId(), resourceOwner, req.GetEmail())
-			email, err = s.command.ChangeUserEmailVerified(ctx, req.GetUserId(), resourceOwner, req.GetEmail())
-		} else {
-			email, err = s.command.ChangeUserEmail(ctx, req.GetUserId(), resourceOwner, req.GetEmail(), s.userCodeAlg)
-		}
 	case nil:
 		email, err = s.command.ChangeUserEmail(ctx, req.GetUserId(), resourceOwner, req.GetEmail(), s.userCodeAlg)
 	default:

internal/api/grpc/user/v2/email_integration_test.go

@@ -0,0 +1,210 @@
+//go:build integration
+
+package user_test
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/muhlemmer/gu"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/zitadel/zitadel/internal/integration"
+	object "github.com/zitadel/zitadel/pkg/grpc/object/v2alpha"
+	user "github.com/zitadel/zitadel/pkg/grpc/user/v2alpha"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+func createHumanUser(t *testing.T) *user.AddHumanUserResponse {
+	resp, err := Client.AddHumanUser(CTX, &user.AddHumanUserRequest{
+		Organisation: &object.Organisation{
+			Org: &object.Organisation_OrgId{
+				OrgId: Tester.Organisation.ID,
+			},
+		},
+		Profile: &user.SetHumanProfile{
+			FirstName: "Mickey",
+			LastName:  "Mouse",
+		},
+		Email: &user.SetHumanEmail{
+			Email: fmt.Sprintf("%d@mouse.com", time.Now().UnixNano()),
+			Verification: &user.SetHumanEmail_ReturnCode{
+				ReturnCode: &user.ReturnEmailVerificationCode{},
+			},
+		},
+	})
+	require.NoError(t, err)
+	require.NotEmpty(t, resp.GetUserId())
+	return resp
+}
+
+func TestServer_SetEmail(t *testing.T) {
+	userID := createHumanUser(t).GetUserId()
+
+	tests := []struct {
+		name    string
+		req     *user.SetEmailRequest
+		want    *user.SetEmailResponse
+		wantErr bool
+	}{
+		{
+			name: "default verfication",
+			req: &user.SetEmailRequest{
+				UserId: userID,
+				Email:  "default-verifier@mouse.com",
+			},
+			want: &user.SetEmailResponse{
+				Details: &object.Details{
+					Sequence:      1,
+					ChangeDate:    timestamppb.Now(),
+					ResourceOwner: Tester.Organisation.ID,
+				},
+			},
+		},
+		{
+			name: "custom url template",
+			req: &user.SetEmailRequest{
+				UserId: userID,
+				Email:  "custom-url@mouse.com",
+				Verification: &user.SetEmailRequest_SendCode{
+					SendCode: &user.SendEmailVerificationCode{
+						UrlTemplate: gu.Ptr("https://example.com/email/verify?userID={{.UserID}}&code={{.Code}}&orgID={{.OrgID}}"),
+					},
+				},
+			},
+			want: &user.SetEmailResponse{
+				Details: &object.Details{
+					Sequence:      1,
+					ChangeDate:    timestamppb.Now(),
+					ResourceOwner: Tester.Organisation.ID,
+				},
+			},
+		},
+		{
+			name: "template error",
+			req: &user.SetEmailRequest{
+				UserId: userID,
+				Email:  "custom-url@mouse.com",
+				Verification: &user.SetEmailRequest_SendCode{
+					SendCode: &user.SendEmailVerificationCode{
+						UrlTemplate: gu.Ptr("{{"),
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "return code",
+			req: &user.SetEmailRequest{
+				UserId: userID,
+				Email:  "return-code@mouse.com",
+				Verification: &user.SetEmailRequest_ReturnCode{
+					ReturnCode: &user.ReturnEmailVerificationCode{},
+				},
+			},
+			want: &user.SetEmailResponse{
+				Details: &object.Details{
+					Sequence:      1,
+					ChangeDate:    timestamppb.Now(),
+					ResourceOwner: Tester.Organisation.ID,
+				},
+				VerificationCode: gu.Ptr("xxx"),
+			},
+		},
+		{
+			name: "is verified true",
+			req: &user.SetEmailRequest{
+				UserId: userID,
+				Email:  "verified-true@mouse.com",
+				Verification: &user.SetEmailRequest_IsVerified{
+					IsVerified: true,
+				},
+			},
+			want: &user.SetEmailResponse{
+				Details: &object.Details{
+					Sequence:      1,
+					ChangeDate:    timestamppb.Now(),
+					ResourceOwner: Tester.Organisation.ID,
+				},
+			},
+		},
+		{
+			name: "is verified false",
+			req: &user.SetEmailRequest{
+				UserId: userID,
+				Email:  "verified-false@mouse.com",
+				Verification: &user.SetEmailRequest_IsVerified{
+					IsVerified: false,
+				},
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := Client.SetEmail(CTX, tt.req)
+			if tt.wantErr {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+			integration.AssertDetails(t, tt.want, got)
+			if tt.want.GetVerificationCode() != "" {
+				assert.NotEmpty(t, got.GetVerificationCode())
+			}
+		})
+	}
+}
+
+func TestServer_VerifyEmail(t *testing.T) {
+	userResp := createHumanUser(t)
+	tests := []struct {
+		name    string
+		req     *user.VerifyEmailRequest
+		want    *user.VerifyEmailResponse
+		wantErr bool
+	}{
+		{
+			name: "wrong code",
+			req: &user.VerifyEmailRequest{
+				UserId:           userResp.GetUserId(),
+				VerificationCode: "xxx",
+			},
+			wantErr: true,
+		},
+		{
+			name: "wrong user",
+			req: &user.VerifyEmailRequest{
+				UserId:           "xxx",
+				VerificationCode: userResp.GetEmailCode(),
+			},
+			wantErr: true,
+		},
+		{
+			name: "verify user",
+			req: &user.VerifyEmailRequest{
+				UserId:           userResp.GetUserId(),
+				VerificationCode: userResp.GetEmailCode(),
+			},
+			want: &user.VerifyEmailResponse{
+				Details: &object.Details{
+					Sequence:      1,
+					ChangeDate:    timestamppb.Now(),
+					ResourceOwner: Tester.Organisation.ID,
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := Client.VerifyEmail(CTX, tt.req)
+			if tt.wantErr {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+			integration.AssertDetails(t, tt.want, got)
+		})
+	}
+}

internal/api/grpc/user/v2/user_integration_test.go

@@ -0,0 +1,317 @@
+//go:build integration
+
+package user_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/muhlemmer/gu"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/zitadel/zitadel/internal/integration"
+	object "github.com/zitadel/zitadel/pkg/grpc/object/v2alpha"
+	user "github.com/zitadel/zitadel/pkg/grpc/user/v2alpha"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+var (
+	CTX    context.Context
+	ErrCTX context.Context
+	Tester *integration.Tester
+	Client user.UserServiceClient
+)
+
+func TestMain(m *testing.M) {
+	os.Exit(func() int {
+		ctx, errCtx, cancel := integration.Contexts(time.Hour)
+		defer cancel()
+
+		Tester = integration.NewTester(ctx)
+		defer Tester.Done()
+
+		CTX, ErrCTX = Tester.WithSystemAuthorization(ctx, integration.OrgOwner), errCtx
+		Client = user.NewUserServiceClient(Tester.GRPCClientConn)
+		return m.Run()
+	}())
+}
+
+func TestServer_AddHumanUser(t *testing.T) {
+	type args struct {
+		ctx context.Context
+		req *user.AddHumanUserRequest
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    *user.AddHumanUserResponse
+		wantErr bool
+	}{
+		{
+			name: "default verification",
+			args: args{
+				CTX,
+				&user.AddHumanUserRequest{
+					Organisation: &object.Organisation{
+						Org: &object.Organisation_OrgId{
+							OrgId: Tester.Organisation.ID,
+						},
+					},
+					Profile: &user.SetHumanProfile{
+						FirstName:         "Donald",
+						LastName:          "Duck",
+						NickName:          gu.Ptr("Dukkie"),
+						DisplayName:       gu.Ptr("Donald Duck"),
+						PreferredLanguage: gu.Ptr("en"),
+						Gender:            user.Gender_GENDER_DIVERSE.Enum(),
+					},
+					Email: &user.SetHumanEmail{},
+					Metadata: []*user.SetMetadataEntry{
+						{
+							Key:   "somekey",
+							Value: []byte("somevalue"),
+						},
+					},
+					PasswordType: &user.AddHumanUserRequest_Password{
+						Password: &user.Password{
+							Password:       "DifficultPW666!",
+							ChangeRequired: true,
+						},
+					},
+				},
+			},
+			want: &user.AddHumanUserResponse{
+				Details: &object.Details{
+					ChangeDate:    timestamppb.Now(),
+					ResourceOwner: Tester.Organisation.ID,
+				},
+			},
+		},
+		{
+			name: "return verification code",
+			args: args{
+				CTX,
+				&user.AddHumanUserRequest{
+					Organisation: &object.Organisation{
+						Org: &object.Organisation_OrgId{
+							OrgId: Tester.Organisation.ID,
+						},
+					},
+					Profile: &user.SetHumanProfile{
+						FirstName:         "Donald",
+						LastName:          "Duck",
+						NickName:          gu.Ptr("Dukkie"),
+						DisplayName:       gu.Ptr("Donald Duck"),
+						PreferredLanguage: gu.Ptr("en"),
+						Gender:            user.Gender_GENDER_DIVERSE.Enum(),
+					},
+					Email: &user.SetHumanEmail{
+						Verification: &user.SetHumanEmail_ReturnCode{
+							ReturnCode: &user.ReturnEmailVerificationCode{},
+						},
+					},
+					Metadata: []*user.SetMetadataEntry{
+						{
+							Key:   "somekey",
+							Value: []byte("somevalue"),
+						},
+					},
+					PasswordType: &user.AddHumanUserRequest_Password{
+						Password: &user.Password{
+							Password:       "DifficultPW666!",
+							ChangeRequired: true,
+						},
+					},
+				},
+			},
+			want: &user.AddHumanUserResponse{
+				Details: &object.Details{
+					ChangeDate:    timestamppb.Now(),
+					ResourceOwner: Tester.Organisation.ID,
+				},
+				EmailCode: gu.Ptr("something"),
+			},
+		},
+		{
+			name: "custom template",
+			args: args{
+				CTX,
+				&user.AddHumanUserRequest{
+					Organisation: &object.Organisation{
+						Org: &object.Organisation_OrgId{
+							OrgId: Tester.Organisation.ID,
+						},
+					},
+					Profile: &user.SetHumanProfile{
+						FirstName:         "Donald",
+						LastName:          "Duck",
+						NickName:          gu.Ptr("Dukkie"),
+						DisplayName:       gu.Ptr("Donald Duck"),
+						PreferredLanguage: gu.Ptr("en"),
+						Gender:            user.Gender_GENDER_DIVERSE.Enum(),
+					},
+					Email: &user.SetHumanEmail{
+						Verification: &user.SetHumanEmail_SendCode{
+							SendCode: &user.SendEmailVerificationCode{
+								UrlTemplate: gu.Ptr("https://example.com/email/verify?userID={{.UserID}}&code={{.Code}}&orgID={{.OrgID}}"),
+							},
+						},
+					},
+					Metadata: []*user.SetMetadataEntry{
+						{
+							Key:   "somekey",
+							Value: []byte("somevalue"),
+						},
+					},
+					PasswordType: &user.AddHumanUserRequest_Password{
+						Password: &user.Password{
+							Password:       "DifficultPW666!",
+							ChangeRequired: true,
+						},
+					},
+				},
+			},
+			want: &user.AddHumanUserResponse{
+				Details: &object.Details{
+					ChangeDate:    timestamppb.Now(),
+					ResourceOwner: Tester.Organisation.ID,
+				},
+			},
+		},
+		{
+			name: "custom template error",
+			args: args{
+				CTX,
+				&user.AddHumanUserRequest{
+					Organisation: &object.Organisation{
+						Org: &object.Organisation_OrgId{
+							OrgId: Tester.Organisation.ID,
+						},
+					},
+					Profile: &user.SetHumanProfile{
+						FirstName:         "Donald",
+						LastName:          "Duck",
+						NickName:          gu.Ptr("Dukkie"),
+						DisplayName:       gu.Ptr("Donald Duck"),
+						PreferredLanguage: gu.Ptr("en"),
+						Gender:            user.Gender_GENDER_DIVERSE.Enum(),
+					},
+					Email: &user.SetHumanEmail{
+						Verification: &user.SetHumanEmail_SendCode{
+							SendCode: &user.SendEmailVerificationCode{
+								UrlTemplate: gu.Ptr("{{"),
+							},
+						},
+					},
+					Metadata: []*user.SetMetadataEntry{
+						{
+							Key:   "somekey",
+							Value: []byte("somevalue"),
+						},
+					},
+					PasswordType: &user.AddHumanUserRequest_Password{
+						Password: &user.Password{
+							Password:       "DifficultPW666!",
+							ChangeRequired: true,
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "missing REQUIRED profile",
+			args: args{
+				CTX,
+				&user.AddHumanUserRequest{
+					Organisation: &object.Organisation{
+						Org: &object.Organisation_OrgId{
+							OrgId: Tester.Organisation.ID,
+						},
+					},
+					Email: &user.SetHumanEmail{
+						Verification: &user.SetHumanEmail_ReturnCode{
+							ReturnCode: &user.ReturnEmailVerificationCode{},
+						},
+					},
+					Metadata: []*user.SetMetadataEntry{
+						{
+							Key:   "somekey",
+							Value: []byte("somevalue"),
+						},
+					},
+					PasswordType: &user.AddHumanUserRequest_Password{
+						Password: &user.Password{
+							Password:       "DifficultPW666!",
+							ChangeRequired: true,
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "missing REQUIRED email",
+			args: args{
+				CTX,
+				&user.AddHumanUserRequest{
+					Organisation: &object.Organisation{
+						Org: &object.Organisation_OrgId{
+							OrgId: Tester.Organisation.ID,
+						},
+					},
+					Profile: &user.SetHumanProfile{
+						FirstName:         "Donald",
+						LastName:          "Duck",
+						NickName:          gu.Ptr("Dukkie"),
+						DisplayName:       gu.Ptr("Donald Duck"),
+						PreferredLanguage: gu.Ptr("en"),
+						Gender:            user.Gender_GENDER_DIVERSE.Enum(),
+					},
+					Metadata: []*user.SetMetadataEntry{
+						{
+							Key:   "somekey",
+							Value: []byte("somevalue"),
+						},
+					},
+					PasswordType: &user.AddHumanUserRequest_Password{
+						Password: &user.Password{
+							Password:       "DifficultPW666!",
+							ChangeRequired: true,
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+	}
+	for i, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			userID := fmt.Sprint(time.Now().UnixNano() + int64(i))
+			tt.args.req.UserId = &userID
+			if email := tt.args.req.GetEmail(); email != nil {
+				email.Email = fmt.Sprintf("%s@me.now", userID)
+			}
+
+			if tt.want != nil {
+				tt.want.UserId = userID
+			}
+
+			got, err := Client.AddHumanUser(tt.args.ctx, tt.args.req)
+			if tt.wantErr {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+
+			assert.Equal(t, tt.want.GetUserId(), got.GetUserId())
+			if tt.want.GetEmailCode() != "" {
+				assert.NotEmpty(t, got.GetEmailCode())
+			}
+			integration.AssertDetails(t, tt.want, got)
+		})
+	}
+}

internal/integration/assert.go

@@ -0,0 +1,41 @@
+package integration
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	object "github.com/zitadel/zitadel/pkg/grpc/object/v2alpha"
+)
+
+type DetailsMsg interface {
+	GetDetails() *object.Details
+}
+
+// AssertDetails asserts values in a message's object Details,
+// if the object Details in expected is a non-nil value.
+// It targets API v2 messages that have the `GetDetails()` method.
+//
+// Dynamically generated values are not compared with expected.
+// Instead a sanity check is performed.
+// For the sequence a non-zero value is expected.
+// The change date has to be now, with a tollerance of 1 second.
+//
+// The resource owner is compared with expected and is
+// therefore the only value that has to be set.
+func AssertDetails[D DetailsMsg](t testing.TB, exptected, actual D) {
+	wantDetails, gotDetails := exptected.GetDetails(), actual.GetDetails()
+	if wantDetails == nil {
+		assert.Nil(t, gotDetails)
+		return
+	}
+
+	assert.NotZero(t, gotDetails.GetSequence())
+
+	gotCD := gotDetails.GetChangeDate().AsTime()
+	now := time.Now()
+	assert.WithinRange(t, gotCD, now.Add(-time.Second), now.Add(time.Second))
+
+	assert.Equal(t, wantDetails.GetResourceOwner(), gotDetails.GetResourceOwner())
+}

internal/integration/assert_test.go

@@ -0,0 +1,51 @@
+package integration
+
+import (
+	"testing"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	object "github.com/zitadel/zitadel/pkg/grpc/object/v2alpha"
+)
+
+type myMsg struct {
+	details *object.Details
+}
+
+func (m myMsg) GetDetails() *object.Details {
+	return m.details
+}
+
+func TestAssertDetails(t *testing.T) {
+	tests := []struct {
+		name      string
+		exptected myMsg
+		actual    myMsg
+	}{
+		{
+			name:      "nil",
+			exptected: myMsg{},
+			actual:    myMsg{},
+		},
+		{
+			name: "values",
+			exptected: myMsg{
+				details: &object.Details{
+					ResourceOwner: "me",
+				},
+			},
+			actual: myMsg{
+				details: &object.Details{
+					Sequence:      123,
+					ChangeDate:    timestamppb.Now(),
+					ResourceOwner: "me",
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			AssertDetails(t, tt.exptected, tt.actual)
+		})
+	}
+}

internal/integration/config/cockroach.yaml

@@ -0,0 +1,10 @@
+Database:
+  cockroach:
+    Host: localhost
+    Port: 26257
+    Database: zitadel
+    Options: ""
+    User:
+      Username: zitadel
+    Admin:
+      Username: root

internal/integration/config/docker-compose.yaml

@@ -0,0 +1,24 @@
+version: '3.8'
+
+services:
+  cockroach:
+    extends:
+      file: '../../../e2e/config/localhost/docker-compose.yaml'
+      service: 'db'
+  
+  postgres:
+    restart: 'always'
+    image: 'postgres:15'
+    environment:
+      - POSTGRES_USER=zitadel
+      - PGUSER=zitadel
+      - POSTGRES_DB=zitadel
+      - POSTGRES_HOST_AUTH_METHOD=trust
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready"]
+      interval: '10s'
+      timeout: '30s'
+      retries: 5
+      start_period: '20s'
+    ports:
+      - 5432:5432

internal/integration/config/postgres.yaml

@@ -0,0 +1,15 @@
+Database:
+  postgres:
+    Host: localhost
+    Port: 5432
+    Database: zitadel
+    MaxOpenConns: 20
+    MaxIdleConns: 10
+    User:
+      Username: zitadel
+      SSL:
+        Mode: disable
+    Admin:
+      Username: zitadel
+      SSL:
+        Mode: disable

internal/integration/config/zitadel.yaml

@@ -0,0 +1,37 @@
+Log:
+  Level: debug
+
+TLS:
+  Enabled: false
+
+FirstInstance:
+  Org:
+    Human:
+      PasswordChangeRequired: false
+
+LogStore:
+  Access:
+    Database:
+      Enabled: true
+      Debounce:
+        MinFrequency: 0s
+        MaxBulkSize: 0
+  Execution:
+    Database:
+      Enabled: true
+    Stdout:
+      Enabled: true
+
+Quotas:
+  Access:
+    ExhaustedCookieKey: "zitadel.quota.limiting"
+    ExhaustedCookieMaxAge: "60s"
+
+Projections:
+  Customizations:
+    NotificationsQuotas:
+      RequeueEvery: 1s
+
+DefaultInstance:
+  LoginPolicy:
+    MfaInitSkipLifetime: "0"

internal/integration/integration.go

@@ -0,0 +1,250 @@
+// Package integration provides helpers for integration testing.
+package integration
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	_ "embed"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/spf13/viper"
+	"github.com/zitadel/logging"
+	"github.com/zitadel/oidc/v2/pkg/oidc"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/metadata"
+
+	"github.com/zitadel/zitadel/cmd"
+	"github.com/zitadel/zitadel/cmd/start"
+	"github.com/zitadel/zitadel/internal/api/authz"
+	z_oidc "github.com/zitadel/zitadel/internal/api/oidc"
+	"github.com/zitadel/zitadel/internal/command"
+	"github.com/zitadel/zitadel/internal/domain"
+	caos_errs "github.com/zitadel/zitadel/internal/errors"
+	"github.com/zitadel/zitadel/internal/eventstore/v1/models"
+	"github.com/zitadel/zitadel/internal/query"
+	"github.com/zitadel/zitadel/pkg/grpc/admin"
+)
+
+var (
+	//go:embed config/zitadel.yaml
+	zitadelYAML []byte
+	//go:embed config/cockroach.yaml
+	cockroachYAML []byte
+	//go:embed config/postgres.yaml
+	postgresYAML []byte
+)
+
+// UserType provides constants that give
+// a short explinanation with the purpose
+// a serverice user.
+// This allows to pre-create users with
+// different permissions and reuse them.
+type UserType int
+
+//go:generate stringer -type=UserType
+const (
+	Unspecified UserType = iota
+	OrgOwner
+)
+
+// User information with a Personal Access Token.
+type User struct {
+	*query.User
+	Token string
+}
+
+// Tester is a Zitadel server and client with all resources available for testing.
+type Tester struct {
+	*start.Server
+
+	Instance     authz.Instance
+	Organisation *query.Org
+	Users        map[UserType]User
+
+	GRPCClientConn *grpc.ClientConn
+	wg             sync.WaitGroup // used for shutdown
+}
+
+const commandLine = `start --masterkeyFromEnv`
+
+func (s *Tester) Host() string {
+	return fmt.Sprintf("%s:%d", s.Config.ExternalDomain, s.Config.Port)
+}
+
+func (s *Tester) createClientConn(ctx context.Context) {
+	target := s.Host()
+	cc, err := grpc.DialContext(ctx, target,
+		grpc.WithBlock(), grpc.WithTransportCredentials(insecure.NewCredentials()),
+	)
+	if err != nil {
+		s.Shutdown <- os.Interrupt
+		s.wg.Wait()
+	}
+	logging.OnError(err).Fatal("integration tester client dial")
+	logging.New().WithField("target", target).Info("finished dialing grpc client conn")
+
+	s.GRPCClientConn = cc
+	err = s.pollHealth(ctx)
+	logging.OnError(err).Fatal("integration tester health")
+}
+
+// pollHealth waits until a healthy status is reported.
+// TODO: remove when we make the setup blocking on all
+// projections completed.
+func (s *Tester) pollHealth(ctx context.Context) (err error) {
+	client := admin.NewAdminServiceClient(s.GRPCClientConn)
+
+	for {
+		err = func(ctx context.Context) error {
+			ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+			defer cancel()
+
+			_, err := client.Healthz(ctx, &admin.HealthzRequest{})
+			return err
+		}(ctx)
+		if err == nil {
+			return nil
+		}
+		logging.WithError(err).Info("poll healthz")
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(time.Second):
+			continue
+		}
+	}
+}
+
+const (
+	SystemUser = "integration"
+)
+
+func (s *Tester) createSystemUser(ctx context.Context) {
+	var err error
+
+	s.Instance, err = s.Queries.InstanceByHost(ctx, s.Host())
+	logging.OnError(err).Fatal("query instance")
+	ctx = authz.WithInstance(ctx, s.Instance)
+
+	s.Organisation, err = s.Queries.OrgByID(ctx, true, s.Instance.DefaultOrganisationID())
+	logging.OnError(err).Fatal("query organisation")
+
+	query, err := query.NewUserUsernameSearchQuery(SystemUser, query.TextEquals)
+	logging.OnError(err).Fatal("user query")
+	user, err := s.Queries.GetUser(ctx, true, true, query)
+
+	if errors.Is(err, sql.ErrNoRows) {
+		_, err = s.Commands.AddMachine(ctx, &command.Machine{
+			ObjectRoot: models.ObjectRoot{
+				ResourceOwner: s.Organisation.ID,
+			},
+			Username:        SystemUser,
+			Name:            SystemUser,
+			Description:     "who cares?",
+			AccessTokenType: domain.OIDCTokenTypeJWT,
+		})
+		logging.OnError(err).Fatal("add machine user")
+		user, err = s.Queries.GetUser(ctx, true, true, query)
+
+	}
+	logging.OnError(err).Fatal("get user")
+
+	_, err = s.Commands.AddOrgMember(ctx, s.Organisation.ID, user.ID, "ORG_OWNER")
+	target := new(caos_errs.AlreadyExistsError)
+	if !errors.As(err, &target) {
+		logging.OnError(err).Fatal("add org member")
+	}
+
+	scopes := []string{oidc.ScopeOpenID, z_oidc.ScopeUserMetaData, z_oidc.ScopeResourceOwner}
+	pat := command.NewPersonalAccessToken(user.ResourceOwner, user.ID, time.Now().Add(time.Hour), scopes, domain.UserTypeMachine)
+	_, err = s.Commands.AddPersonalAccessToken(ctx, pat)
+	logging.OnError(err).Fatal("add pat")
+
+	s.Users = map[UserType]User{
+		OrgOwner: {
+			User:  user,
+			Token: pat.Token,
+		},
+	}
+}
+
+func (s *Tester) WithSystemAuthorization(ctx context.Context, u UserType) context.Context {
+	return metadata.AppendToOutgoingContext(ctx, "Authorization", fmt.Sprintf("Bearer %s", s.Users[u].Token))
+}
+
+// Done send an interrupt signal to cleanly shutdown the server.
+func (s *Tester) Done() {
+	err := s.GRPCClientConn.Close()
+	logging.OnError(err).Error("integration tester client close")
+
+	s.Shutdown <- os.Interrupt
+	s.wg.Wait()
+}
+
+// NewTester start a new Zitadel server by passing the default commandline.
+// The server will listen on the configured port.
+// The database configuration that will be used can be set by the
+// INTEGRATION_DB_FLAVOR environment variable and can have the values "cockroach"
+// or "postgres". Defaults to "cockroach".
+//
+// The deault Instance and Organisation are read from the DB and system
+// users are created as needed.
+//
+// After the server is started, a [grpc.ClientConn] will be created and
+// the server is polled for it's health status.
+//
+// Note: the database must already be setup and intialized before
+// using NewTester. See the CONTRIBUTING.md document for details.
+func NewTester(ctx context.Context) *Tester {
+	args := strings.Split(commandLine, " ")
+
+	sc := make(chan *start.Server)
+	//nolint:contextcheck
+	cmd := cmd.New(os.Stdout, os.Stdin, args, sc)
+	cmd.SetArgs(args)
+	err := viper.MergeConfig(bytes.NewBuffer(zitadelYAML))
+	logging.OnError(err).Fatal()
+
+	flavor := os.Getenv("INTEGRATION_DB_FLAVOR")
+	switch flavor {
+	case "cockroach", "":
+		err = viper.MergeConfig(bytes.NewBuffer(cockroachYAML))
+	case "postgres":
+		err = viper.MergeConfig(bytes.NewBuffer(postgresYAML))
+	default:
+		logging.New().WithField("flavor", flavor).Fatal("unknown db flavor set in INTEGRATION_DB_FLAVOR")
+	}
+	logging.OnError(err).Fatal()
+
+	tester := new(Tester)
+	tester.wg.Add(1)
+	go func(wg *sync.WaitGroup) {
+		logging.OnError(cmd.Execute()).Fatal()
+		wg.Done()
+	}(&tester.wg)
+
+	select {
+	case tester.Server = <-sc:
+	case <-ctx.Done():
+		logging.OnError(ctx.Err()).Fatal("waiting for integration tester server")
+	}
+	tester.createClientConn(ctx)
+	tester.createSystemUser(ctx)
+
+	return tester
+}
+
+func Contexts(timeout time.Duration) (ctx, errCtx context.Context, cancel context.CancelFunc) {
+	errCtx, cancel = context.WithCancel(context.Background())
+	cancel()
+	ctx, cancel = context.WithTimeout(context.Background(), timeout)
+	return ctx, errCtx, cancel
+}

internal/integration/integration_test.go

@@ -0,0 +1,16 @@
+//go:build integration
+
+package integration
+
+import (
+	"testing"
+	"time"
+)
+
+func TestNewTester(t *testing.T) {
+	ctx, _, cancel := Contexts(time.Hour)
+	defer cancel()
+
+	s := NewTester(ctx)
+	defer s.Done()
+}

internal/integration/usertype_string.go

@@ -0,0 +1,24 @@
+// Code generated by "stringer -type=UserType"; DO NOT EDIT.
+
+package integration
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[Unspecified-0]
+	_ = x[OrgOwner-1]
+}
+
+const _UserType_name = "UnspecifiedOrgOwner"
+
+var _UserType_index = [...]uint8{0, 11, 19}
+
+func (i UserType) String() string {
+	if i < 0 || i >= UserType(len(_UserType_index)-1) {
+		return "UserType(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _UserType_name[_UserType_index[i]:_UserType_index[i+1]]
+}

main.go

@@ -10,6 +10,6 @@ import (
 
 func main() {
 	args := os.Args[1:]
-	rootCmd := cmd.New(os.Stdout, os.Stdin /*, int(os.Stdin.Fd())*/, args)
+	rootCmd := cmd.New(os.Stdout, os.Stdin, args, nil)
 	cobra.CheckErr(rootCmd.Execute())
 }