chore(tests): use a coverage server binary (#8407)

# Which Problems Are Solved

Use a single server instance for API integration tests. This optimizes
the time taken for the integration test pipeline,
because it allows running tests on multiple packages in parallel. Also,
it saves time by not start and stopping a zitadel server for every
package.

# How the Problems Are Solved

- Build a binary with `go build -race -cover ....`
- Integration tests only construct clients. The server remains running
in the background.
- The integration package and tested packages now fully utilize the API.
No more direct database access trough `query` and `command` packages.
- Use Makefile recipes to setup, start and stop the server in the
background.
- The binary has the race detector enabled
- Init and setup jobs are configured to halt immediately on race
condition
- Because the server runs in the background, races are only logged. When
the server is stopped and race logs exist, the Makefile recipe will
throw an error and print the logs.
- Makefile recipes include logic to print logs and convert coverage
reports after the server is stopped.
- Some tests need a downstream HTTP server to make requests, like quota
and milestones. A new `integration/sink` package creates an HTTP server
and uses websockets to forward HTTP request back to the test packages.
The package API uses Go channels for abstraction and easy usage.

# Additional Changes

- Integration test files already used the `//go:build integration`
directive. In order to properly split integration from unit tests,
integration test files need to be in a `integration_test` subdirectory
of their package.
- `UseIsolatedInstance` used to overwrite the `Tester.Client` for each
instance. Now a `Instance` object is returned with a gRPC client that is
connected to the isolated instance's hostname.
- The `Tester` type is now `Instance`. The object is created for the
first instance, used by default in any test. Isolated instances are also
`Instance` objects and therefore benefit from the same methods and
values. The first instance and any other us capable of creating an
isolated instance over the system API.
- All test packages run in an Isolated instance by calling
`NewInstance()`
- Individual tests that use an isolated instance use `t.Parallel()`

# Additional Context

- Closes #6684
- https://go.dev/doc/articles/race_detector
- https://go.dev/doc/build-cover

---------

Co-authored-by: Stefan Benz <46600784+stebenz@users.noreply.github.com>

internal/notification/handlers/handlers_integration_test.go

@@ -1,36 +0,0 @@
-//go:build integration
-
-package handlers_test
-
-import (
-	"context"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/zitadel/zitadel/internal/integration"
-)
-
-var (
-	CTX       context.Context
-	SystemCTX context.Context
-	Tester    *integration.Tester
-)
-
-func TestMain(m *testing.M) {
-	os.Exit(func() int {
-		ctx, _, cancel := integration.Contexts(5 * time.Minute)
-		defer cancel()
-		CTX = ctx
-
-		Tester = integration.NewTester(ctx, `
-Quotas:
-  Access:
-    Enabled: true
-`)
-		defer Tester.Done()
-
-		SystemCTX = Tester.WithAuthorization(ctx, integration.SystemUser)
-		return m.Run()
-	}())
-}

internal/notification/handlers/integration_test/handlers_test.go

@@ -0,0 +1,23 @@
+//go:build integration
+
+package handlers_test
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+)
+
+var (
+	CTX context.Context
+)
+
+func TestMain(m *testing.M) {
+	os.Exit(func() int {
+		ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
+		defer cancel()
+		CTX = ctx
+		return m.Run()
+	}())
+}

internal/notification/handlers/integration_test/telemetry_pusher_test.go

@@ -4,7 +4,6 @@ package handlers_test
 
 import (
 	"bytes"
-	"context"
 	"encoding/json"
 	"net/url"
 	"testing"
@@ -16,6 +15,7 @@ import (
 	"github.com/zitadel/oidc/v3/pkg/oidc"
 
 	"github.com/zitadel/zitadel/internal/integration"
+	"github.com/zitadel/zitadel/internal/integration/sink"
 	"github.com/zitadel/zitadel/pkg/grpc/app"
 	"github.com/zitadel/zitadel/pkg/grpc/management"
 	"github.com/zitadel/zitadel/pkg/grpc/object"
@@ -25,16 +25,20 @@ import (
 )
 
 func TestServer_TelemetryPushMilestones(t *testing.T) {
-	primaryDomain, instanceID, adminID, iamOwnerCtx := Tester.UseIsolatedInstance(t, CTX, SystemCTX)
-	t.Log("testing against instance with primary domain", primaryDomain)
-	awaitMilestone(t, Tester.MilestoneChan, primaryDomain, "InstanceCreated")
+	sub := sink.Subscribe(CTX, sink.ChannelMilestone)
+	defer sub.Close()
 
-	projectAdded, err := Tester.Client.Mgmt.AddProject(iamOwnerCtx, &management.AddProjectRequest{Name: "integration"})
+	instance := integration.NewInstance(CTX)
+	iamOwnerCtx := instance.WithAuthorization(CTX, integration.UserTypeIAMOwner)
+	t.Log("testing against instance with primary domain", instance.Domain)
+	awaitMilestone(t, sub, instance.Domain, "InstanceCreated")
+
+	projectAdded, err := instance.Client.Mgmt.AddProject(iamOwnerCtx, &management.AddProjectRequest{Name: "integration"})
 	require.NoError(t, err)
-	awaitMilestone(t, Tester.MilestoneChan, primaryDomain, "ProjectCreated")
+	awaitMilestone(t, sub, instance.Domain, "ProjectCreated")
 
 	redirectURI := "http://localhost:8888"
-	application, err := Tester.Client.Mgmt.AddOIDCApp(iamOwnerCtx, &management.AddOIDCAppRequest{
+	application, err := instance.Client.Mgmt.AddOIDCApp(iamOwnerCtx, &management.AddOIDCAppRequest{
 		ProjectId:       projectAdded.GetId(),
 		Name:            "integration",
 		RedirectUris:    []string{redirectURI},
@@ -46,35 +50,37 @@ func TestServer_TelemetryPushMilestones(t *testing.T) {
 		AccessTokenType: app.OIDCTokenType_OIDC_TOKEN_TYPE_JWT,
 	})
 	require.NoError(t, err)
-	awaitMilestone(t, Tester.MilestoneChan, primaryDomain, "ApplicationCreated")
+	awaitMilestone(t, sub, instance.Domain, "ApplicationCreated")
 
 	// create the session to be used for the authN of the clients
-	sessionID, sessionToken, _, _ := Tester.CreatePasswordSession(t, iamOwnerCtx, adminID, "Password1!")
+	sessionID, sessionToken, _, _ := instance.CreatePasswordSession(t, iamOwnerCtx, instance.AdminUserID, "Password1!")
 
-	console := consoleOIDCConfig(iamOwnerCtx, t)
-	loginToClient(iamOwnerCtx, t, primaryDomain, console.GetClientId(), instanceID, console.GetRedirectUris()[0], sessionID, sessionToken)
-	awaitMilestone(t, Tester.MilestoneChan, primaryDomain, "AuthenticationSucceededOnInstance")
+	console := consoleOIDCConfig(t, instance)
+	loginToClient(t, instance, console.GetClientId(), console.GetRedirectUris()[0], sessionID, sessionToken)
+	awaitMilestone(t, sub, instance.Domain, "AuthenticationSucceededOnInstance")
 
 	// make sure the client has been projected
 	require.EventuallyWithT(t, func(collectT *assert.CollectT) {
-		_, err := Tester.Client.Mgmt.GetAppByID(iamOwnerCtx, &management.GetAppByIDRequest{
+		_, err := instance.Client.Mgmt.GetAppByID(iamOwnerCtx, &management.GetAppByIDRequest{
 			ProjectId: projectAdded.GetId(),
 			AppId:     application.GetAppId(),
 		})
 		assert.NoError(collectT, err)
-	}, 1*time.Minute, 100*time.Millisecond, "app not found")
-	loginToClient(iamOwnerCtx, t, primaryDomain, application.GetClientId(), instanceID, redirectURI, sessionID, sessionToken)
-	awaitMilestone(t, Tester.MilestoneChan, primaryDomain, "AuthenticationSucceededOnApplication")
+	}, time.Minute, time.Second, "app not found")
+	loginToClient(t, instance, application.GetClientId(), redirectURI, sessionID, sessionToken)
+	awaitMilestone(t, sub, instance.Domain, "AuthenticationSucceededOnApplication")
 
-	_, err = Tester.Client.System.RemoveInstance(SystemCTX, &system.RemoveInstanceRequest{InstanceId: instanceID})
+	_, err = integration.SystemClient().RemoveInstance(CTX, &system.RemoveInstanceRequest{InstanceId: instance.ID()})
 	require.NoError(t, err)
-	awaitMilestone(t, Tester.MilestoneChan, primaryDomain, "InstanceDeleted")
+	awaitMilestone(t, sub, instance.Domain, "InstanceDeleted")
 }
 
-func loginToClient(iamOwnerCtx context.Context, t *testing.T, primaryDomain, clientID, instanceID, redirectURI, sessionID, sessionToken string) {
-	authRequestID, err := Tester.CreateOIDCAuthRequestWithDomain(iamOwnerCtx, primaryDomain, clientID, Tester.Users.Get(instanceID, integration.IAMOwner).ID, redirectURI, "openid")
+func loginToClient(t *testing.T, instance *integration.Instance, clientID, redirectURI, sessionID, sessionToken string) {
+	iamOwnerCtx := instance.WithAuthorization(CTX, integration.UserTypeIAMOwner)
+
+	authRequestID, err := instance.CreateOIDCAuthRequestWithDomain(iamOwnerCtx, instance.Domain, clientID, instance.Users.Get(integration.UserTypeIAMOwner).ID, redirectURI, "openid")
 	require.NoError(t, err)
-	callback, err := Tester.Client.OIDCv2.CreateCallback(iamOwnerCtx, &oidc_v2.CreateCallbackRequest{
+	callback, err := instance.Client.OIDCv2.CreateCallback(iamOwnerCtx, &oidc_v2.CreateCallbackRequest{
 		AuthRequestId: authRequestID,
 		CallbackKind: &oidc_v2.CreateCallbackRequest_Session{Session: &oidc_v2.Session{
 			SessionId:    sessionID,
@@ -82,7 +88,7 @@ func loginToClient(iamOwnerCtx context.Context, t *testing.T, primaryDomain, cli
 		}},
 	})
 	require.NoError(t, err)
-	provider, err := Tester.CreateRelyingPartyForDomain(iamOwnerCtx, primaryDomain, clientID, redirectURI)
+	provider, err := instance.CreateRelyingPartyForDomain(iamOwnerCtx, instance.Domain, clientID, redirectURI)
 	require.NoError(t, err)
 	callbackURL, err := url.Parse(callback.GetCallbackUrl())
 	require.NoError(t, err)
@@ -91,8 +97,10 @@ func loginToClient(iamOwnerCtx context.Context, t *testing.T, primaryDomain, cli
 	require.NoError(t, err)
 }
 
-func consoleOIDCConfig(iamOwnerCtx context.Context, t *testing.T) *app.OIDCConfig {
-	projects, err := Tester.Client.Mgmt.ListProjects(iamOwnerCtx, &management.ListProjectsRequest{
+func consoleOIDCConfig(t *testing.T, instance *integration.Instance) *app.OIDCConfig {
+	iamOwnerCtx := instance.WithAuthorization(CTX, integration.UserTypeIAMOwner)
+
+	projects, err := instance.Client.Mgmt.ListProjects(iamOwnerCtx, &management.ListProjectsRequest{
 		Queries: []*project.ProjectQuery{
 			{
 				Query: &project.ProjectQuery_NameQuery{
@@ -106,7 +114,7 @@ func consoleOIDCConfig(iamOwnerCtx context.Context, t *testing.T) *app.OIDCConfi
 	})
 	require.NoError(t, err)
 	require.Len(t, projects.GetResult(), 1)
-	apps, err := Tester.Client.Mgmt.ListApps(iamOwnerCtx, &management.ListAppsRequest{
+	apps, err := instance.Client.Mgmt.ListApps(iamOwnerCtx, &management.ListAppsRequest{
 		ProjectId: projects.GetResult()[0].GetId(),
 		Queries: []*app.AppQuery{
 			{
@@ -124,12 +132,12 @@ func consoleOIDCConfig(iamOwnerCtx context.Context, t *testing.T) *app.OIDCConfi
 	return apps.GetResult()[0].GetOidcConfig()
 }
 
-func awaitMilestone(t *testing.T, bodies chan []byte, primaryDomain, expectMilestoneType string) {
+func awaitMilestone(t *testing.T, sub *sink.Subscription, primaryDomain, expectMilestoneType string) {
 	for {
 		select {
-		case body := <-bodies:
+		case req := <-sub.Recv():
 			plain := new(bytes.Buffer)
-			if err := json.Indent(plain, body, "", "  "); err != nil {
+			if err := json.Indent(plain, req.Body, "", "  "); err != nil {
 				t.Fatal(err)
 			}
 			t.Log("received milestone", plain.String())
@@ -137,13 +145,13 @@ func awaitMilestone(t *testing.T, bodies chan []byte, primaryDomain, expectMiles
 				Type          string `json:"type"`
 				PrimaryDomain string `json:"primaryDomain"`
 			}{}
-			if err := json.Unmarshal(body, &milestone); err != nil {
+			if err := json.Unmarshal(req.Body, &milestone); err != nil {
 				t.Error(err)
 			}
 			if milestone.Type == expectMilestoneType && milestone.PrimaryDomain == primaryDomain {
 				return
 			}
-		case <-time.After(60 * time.Second):
+		case <-time.After(2 * time.Minute): // why does it take so long to get a milestone !?
 			t.Fatalf("timed out waiting for milestone %s in domain %s", expectMilestoneType, primaryDomain)
 		}
 	}

internal/notification/handlers/quota_notifier_test.go

@@ -1,155 +0,0 @@
-//go:build integration
-
-package handlers_test
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"strconv"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
-	"google.golang.org/protobuf/types/known/durationpb"
-	"google.golang.org/protobuf/types/known/timestamppb"
-
-	"github.com/zitadel/zitadel/internal/repository/quota"
-	"github.com/zitadel/zitadel/pkg/grpc/admin"
-	quota_pb "github.com/zitadel/zitadel/pkg/grpc/quota"
-	"github.com/zitadel/zitadel/pkg/grpc/system"
-)
-
-func TestServer_QuotaNotification_Limit(t *testing.T) {
-	_, instanceID, _, iamOwnerCtx := Tester.UseIsolatedInstance(t, CTX, SystemCTX)
-	amount := 10
-	percent := 50
-	percentAmount := amount * percent / 100
-
-	_, err := Tester.Client.System.AddQuota(SystemCTX, &system.AddQuotaRequest{
-		InstanceId:    instanceID,
-		Unit:          quota_pb.Unit_UNIT_REQUESTS_ALL_AUTHENTICATED,
-		From:          timestamppb.Now(),
-		ResetInterval: durationpb.New(time.Minute * 5),
-		Amount:        uint64(amount),
-		Limit:         true,
-		Notifications: []*quota_pb.Notification{
-			{
-				Percent: uint32(percent),
-				Repeat:  true,
-				CallUrl: "http://localhost:8082",
-			},
-			{
-				Percent: 100,
-				Repeat:  true,
-				CallUrl: "http://localhost:8082",
-			},
-		},
-	})
-	require.NoError(t, err)
-
-	for i := 0; i < percentAmount; i++ {
-		_, err := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{})
-		if err != nil {
-			require.NoError(t, fmt.Errorf("error in %d call of %d: %f", i, percentAmount, err))
-		}
-	}
-	awaitNotification(t, time.Now(), Tester.QuotaNotificationChan, quota.RequestsAllAuthenticated, percent)
-
-	for i := 0; i < (amount - percentAmount); i++ {
-		_, err := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{})
-		require.NoError(t, err)
-	}
-	awaitNotification(t, time.Now(), Tester.QuotaNotificationChan, quota.RequestsAllAuthenticated, 100)
-
-	_, limitErr := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{})
-	require.Error(t, limitErr)
-}
-
-func TestServer_QuotaNotification_NoLimit(t *testing.T) {
-	_, instanceID, _, iamOwnerCtx := Tester.UseIsolatedInstance(t, CTX, SystemCTX)
-	amount := 10
-	percent := 50
-	percentAmount := amount * percent / 100
-
-	_, err := Tester.Client.System.AddQuota(SystemCTX, &system.AddQuotaRequest{
-		InstanceId:    instanceID,
-		Unit:          quota_pb.Unit_UNIT_REQUESTS_ALL_AUTHENTICATED,
-		From:          timestamppb.Now(),
-		ResetInterval: durationpb.New(time.Minute * 5),
-		Amount:        uint64(amount),
-		Limit:         false,
-		Notifications: []*quota_pb.Notification{
-			{
-				Percent: uint32(percent),
-				Repeat:  false,
-				CallUrl: "http://localhost:8082",
-			},
-			{
-				Percent: 100,
-				Repeat:  true,
-				CallUrl: "http://localhost:8082",
-			},
-		},
-	})
-	require.NoError(t, err)
-
-	for i := 0; i < percentAmount; i++ {
-		_, err := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{})
-		if err != nil {
-			require.NoError(t, fmt.Errorf("error in %d call of %d: %f", i, percentAmount, err))
-		}
-	}
-	awaitNotification(t, time.Now(), Tester.QuotaNotificationChan, quota.RequestsAllAuthenticated, percent)
-
-	for i := 0; i < (amount - percentAmount); i++ {
-		_, err := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{})
-		if err != nil {
-			require.NoError(t, fmt.Errorf("error in %d call of %d: %f", percentAmount+i, amount, err))
-		}
-	}
-	awaitNotification(t, time.Now(), Tester.QuotaNotificationChan, quota.RequestsAllAuthenticated, 100)
-
-	for i := 0; i < amount; i++ {
-		_, err := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{})
-		if err != nil {
-			require.NoError(t, fmt.Errorf("error in %d call of %d over limit: %f", i, amount, err))
-		}
-	}
-	awaitNotification(t, time.Now(), Tester.QuotaNotificationChan, quota.RequestsAllAuthenticated, 200)
-
-	_, limitErr := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{})
-	require.NoError(t, limitErr)
-}
-
-func awaitNotification(t *testing.T, start time.Time, bodies chan []byte, unit quota.Unit, percent int) {
-	for {
-		select {
-		case body := <-bodies:
-			plain := new(bytes.Buffer)
-			if err := json.Indent(plain, body, "", "  "); err != nil {
-				t.Fatal(err)
-			}
-			t.Log("received notificationDueEvent", plain.String())
-			event := struct {
-				Unit        quota.Unit `json:"unit"`
-				ID          string     `json:"id"`
-				CallURL     string     `json:"callURL"`
-				PeriodStart time.Time  `json:"periodStart"`
-				Threshold   uint16     `json:"threshold"`
-				Usage       uint64     `json:"usage"`
-			}{}
-			if err := json.Unmarshal(body, &event); err != nil {
-				t.Error(err)
-			}
-			if event.ID == "" {
-				continue
-			}
-			if event.Unit == unit && event.Threshold == uint16(percent) {
-				return
-			}
-		case <-time.After(20 * time.Second):
-			t.Fatalf("start %s stop %s timed out waiting for unit %s and percent %d", start.Format(time.RFC3339), time.Now().Format(time.RFC3339), strconv.Itoa(int(unit)), percent)
-		}
-	}
-}