chore: improve development for non-Linux contributors (#5288)

* test: fix e2e against console dev server

* chore: get rid of network_mode host

* explain e2e commands

* chore: fix pipelines

* fix e2e paths

* fix dockerized e2e

* chore: map cypress run service ports

* simplify localhost

* access db via compose service

* access db via compose service

* fix npm run open:angular and e2e:angular

* docs: add empty line

* chore: remove unused file

* docs:  update contrib

---------

Co-authored-by: adlerhurst <silvan.reusser@gmail.com>

.github/workflows/e2e.yml

@@ -40,18 +40,18 @@ jobs:
           driver: docker
           install: true
       - name: Test ${{ matrix.browser }}
-        run: docker compose run e2e --browser ${{ matrix.browser }}
+        run: docker compose run --service-ports e2e --browser ${{ matrix.browser }}
-        working-directory: e2e
+        working-directory: e2e/config/host.docker.internal
       - name: Ensure Artifacts Directory Exists
         run: mkdir -p ./.artifacts
       - name: Save ZITADEL Logs
         if: always()
-        run: docker compose logs zitadel > ../.artifacts/e2e-compose-zitadel.log
+        run: docker compose logs zitadel > ../../../.artifacts/e2e-compose-zitadel.log
-        working-directory: e2e
+        working-directory: e2e/config/host.docker.internal
       - name: Save Prepare Logs
         if: always()
-        run: docker compose logs prepare > ../.artifacts/e2e-compose-prepare.log
+        run: docker compose logs prepare > ../../../.artifacts/e2e-compose-prepare.log
-        working-directory: e2e
+        working-directory: e2e/config/host.docker.internal
       - name: Archive production tests ${{ matrix.browser }}
         if: always()
         uses: actions/upload-artifact@v3

.github/workflows/test-code.yml

@@ -49,18 +49,18 @@ jobs:
       - name: Build Docker Image
         run: docker build -t zitadel:pr --file build/Dockerfile .artifacts/zitadel
       - name: Run E2E Tests
-        run: docker compose run e2e --browser chrome
+        run: docker compose run --service-ports e2e --browser chrome
-        working-directory: e2e
+        working-directory: e2e/config/host.docker.internal
         env:
           ZITADEL_IMAGE: zitadel:pr
       - name: Save ZITADEL Logs
         if: always()
-        run: docker compose logs zitadel > ../.artifacts/e2e-compose-zitadel.log
+        run: docker compose logs zitadel > ../../../.artifacts/e2e-compose-zitadel.log
-        working-directory: e2e
+        working-directory: e2e/config/host.docker.internal
       - name: Save Prepare Logs
         if: always()
-        run: docker compose logs prepare > ../.artifacts/e2e-compose-prepare.log
+        run: docker compose logs prepare > ../../../.artifacts/e2e-compose-prepare.log
-        working-directory: e2e
+        working-directory: e2e/config/host.docker.internal
       - name: Archive Test Results
         if: always()
         uses: actions/upload-artifact@v3

CONTRIBUTING.md

@@ -151,6 +151,10 @@ Also, you can verify the data by running `cockroach sql --database zitadel --ins
 
 As soon as you are ready to battle test your changes, run the end-to-end tests.
 
+#### Running the tests with docker
+
+Running the tests with docker doesn't require you to take care of other dependencies than docker and goreleaser.
+
 ```bash
 # Build the production binary (unit tests are executed, too)
 goreleaser build --id prod --snapshot --single-target --rm-dist --output .artifacts/zitadel/zitadel
@@ -162,14 +166,37 @@ DOCKER_BUILDKIT=1 docker build --file build/Dockerfile .artifacts/zitadel -t zit
 (cd ./e2e && npm run lint:fix)
 
 # Run the tests
-ZITADEL_IMAGE=zitadel:local docker compose --file ./e2e/docker-compose.yaml run e2e
+ZITADEL_IMAGE=zitadel:local docker compose --file ./e2e/config/host.docker.internal/docker-compose.yaml run --service-ports e2e
 ```
 
 When you are happy with your changes, you can cleanup your environment.
 
 ```bash
 # Stop and remove the docker containers for zitadel and the database
-docker compose --file ./e2e/docker-compose.yaml down
+docker compose --file ./e2e/config/host.docker.internal/docker-compose.yaml down
+```
+
+#### Running the tests without docker
+
+If you also make [changes to the console](#console), you can run the test suite against your locally built backend code and frontend server.
+But you will have to install the relevant node dependencies.
+
+```bash
+# Install dependencies
+(cd ./e2e && npm install)
+
+# Run the tests interactively
+(cd ./e2e && npm run open:golangangular)
+
+# Run the tests non-interactively
+(cd ./e2e && npm run e2e:golangangular)
+```
+
+When you are happy with your changes, you can cleanup your environment.
+
+```bash
+# Stop and remove the docker containers for zitadel and the database
+docker compose --file ./e2e/config/host.docker.internal/docker-compose.yaml down
 ```
 
 ### Console
@@ -208,7 +235,7 @@ Run the database and the latest backend locally.
 cd ./console
 
 # You just need the db and the zitadel services to develop the console against.
-docker compose --file ../e2e/docker-compose.yaml up --detach db zitadel
+docker compose --file ../e2e/docker-compose.yaml up --detach zitadel
 ```
 
 When the backend is ready, you have the latest zitadel exposed at http://localhost:8080.
@@ -260,14 +287,21 @@ npm run lint:fix
 npm install
 
 # Run all e2e tests
-npm run e2e:dev -- --headed
+npm run e2e:angular -- --headed
 ```
 
-You can also open the test suite interactively for fast success feedback on specific tests.
+You can also open the test suite interactively for fast feedback on specific tests.
 
 ```bash
 # Run tests interactively
-npm run open:dev
+npm run open:angular
+```
+
+If you also make [changes to the backend code](#backend--login), you can run the test against your locally built backend code and frontend server
+
+```bash
+npm run open:golangangular
+npm run e2e:golangangular
 ```
 
 When you are happy with your changes, you can format your code and cleanup your environment

e2e/config/host.docker.internal/docker-compose.yaml

@@ -0,0 +1,44 @@
+version: '3.8'
+
+services:
+  zitadel:
+    extends:
+      file: '../localhost/docker-compose.yaml'
+      service: 'zitadel'
+    volumes:
+      - ./zitadel.yaml:/zitadel.yaml
+
+  db:
+    extends:
+      file: '../localhost/docker-compose.yaml'
+      service: 'db'
+
+  prepare:
+    image: node:18-alpine3.15
+    working_dir: /e2e
+    user: '$UID'
+    volumes:
+      - ../../:/e2e
+    command: 'sh -c "npm ci --omit=dev && npm run lint && npx wait-on http://zitadel:8080/debug/ready"'
+
+  e2e:
+    image: cypress/included:12.2.0
+    depends_on:
+      zitadel:
+        condition: 'service_started'
+      db:
+        condition: 'service_healthy'
+      prepare:
+        condition: 'service_completed_successfully'
+    working_dir: /e2e
+    user: '$UID'
+    volumes:
+      - ../../:/e2e
+    environment:
+      CYPRESS_BASE_URL: http://host.docker.internal:8080/ui/console
+      CYPRESS_WEBHOOK_HANDLER_HOST: host.docker.internal
+      CYPRESS_DATABASE_CONNECTION_URL: 'postgresql://root@db:26257/zitadel'
+    ports:
+      - "8900:8900"
+    extra_hosts:
+    - "host.docker.internal:host-gateway"

e2e/config/host.docker.internal/zitadel.yaml

@@ -0,0 +1,41 @@
+ExternalDomain: host.docker.internal
+ExternalSecure: false
+
+Database:
+  cockroach:
+    Host:  db
+
+TLS:
+  Enabled: false
+
+FirstInstance:
+  Org:
+    Human:
+      PasswordChangeRequired: false
+
+LogStore:
+  Access:
+    Database:
+      Enabled: true
+      Debounce:
+        MinFrequency: 0s
+        MaxBulkSize: 0
+  Execution:
+    Database:
+      Enabled: true
+    Stdout:
+      Enabled: false
+
+Quotas:
+  Access:
+    ExhaustedCookieKey: "zitadel.quota.limiting"
+    ExhaustedCookieMaxAge: "60s"
+
+DefaultInstance:
+  LoginPolicy:
+    MfaInitSkipLifetime: "0"
+
+SystemAPIUsers:
+- cypress:
+    KeyData: "LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF6aStGRlNKTDdmNXl3NEtUd3pnTQpQMzRlUEd5Y20vTStrVDBNN1Y0Q2d4NVYzRWFESXZUUUtUTGZCYUVCNDV6YjlMdGpJWHpEdzByWFJvUzJoTzZ0CmgrQ1lRQ3ozS0N2aDA5QzBJenhaaUIySVMzSC9hVCs1Qng5RUZZK3ZuQWtaamNjYnlHNVlOUnZtdE9sbnZJZUkKSDdxWjB0RXdrUGZGNUdFWk5QSlB0bXkzVUdWN2lvZmRWUVMxeFJqNzMrYU13NXJ2SDREOElkeWlBQzNWZWtJYgpwdDBWajBTVVgzRHdLdG9nMzM3QnpUaVBrM2FYUkYwc2JGaFFvcWRKUkk4TnFnWmpDd2pxOXlmSTV0eXhZc3duCitKR3pIR2RIdlczaWRPRGxtd0V0NUsycGFzaVJJV0syT0dmcSt3MEVjbHRRSGFidXFFUGdabG1oQ2tSZE5maXgKQndJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg=="
+

e2e/config/localhost/docker-compose.yaml

@@ -0,0 +1,30 @@
+version: '3.8'
+
+services:
+  zitadel:
+    user: '$UID'
+    restart: 'always'
+    image: '${ZITADEL_IMAGE:-ghcr.io/zitadel/zitadel:latest}'
+    command: 'start-from-init --masterkey "MasterkeyNeedsToHave32Characters" --tlsMode disabled --config /zitadel.yaml --steps /zitadel.yaml'
+    depends_on:
+      db:
+        condition: 'service_healthy'
+    volumes:
+      - ./zitadel.yaml:/zitadel.yaml
+    ports:
+    - "8080:8080"
+    extra_hosts:
+    - "host.docker.internal:host-gateway"
+
+  db:
+    restart: 'always'
+    image: 'cockroachdb/cockroach:v22.2.2'
+    command: 'start-single-node --insecure --http-addr :9090'
+    healthcheck:
+      test: ['CMD', 'curl', '-f', 'http://localhost:9090/health?ready=1']
+      interval: '10s'
+      timeout: '30s'
+      retries: 5
+      start_period: '20s'
+    ports:
+      - "26257:26257"

e2e/config/localhost/zitadel.yaml

@@ -1,3 +1,13 @@
+ExternalDomain: localhost
+ExternalSecure: false
+
+Database:
+  cockroach:
+    Host:  db
+
+TLS:
+  Enabled: false
+
 FirstInstance:
   Org:
     Human:
@@ -23,7 +33,7 @@ Quotas:
 
 DefaultInstance:
   LoginPolicy:
-    MfaInitSkipLifetime: 0
+    MfaInitSkipLifetime: "0"
 
 SystemAPIUsers:
 - cypress:

e2e/cypress/support/login/users.ts

@@ -58,7 +58,7 @@ export function login(
           times: 1,
         }).as('password');
 
-        cy.visit(loginUrl, { retryOnNetworkFailure: true });
+        cy.visit(Cypress.config('baseUrl'), { retryOnNetworkFailure: true });
 
         onUsernameScreen ? onUsernameScreen() : null;
         cy.get('#loginName').type(creds.username);
@@ -69,7 +69,7 @@ export function login(
         cy.get('#submit-button').click();
 
         cy.wait('@password').then((interception) => {
-          if (interception.response.body.indexOf('/ui/login/mfa/prompt') === -1) {
+          if (interception.response.body.indexOf(`${loginUrl}/mfa/prompt`) === -1) {
             return;
           }
 

e2e/docker-compose.yaml

@@ -2,52 +2,11 @@ version: '3.8'
 
 services:
   zitadel:
-    user: '$UID'
+    extends:
-    restart: 'always'
+      file: './config/localhost/docker-compose.yaml'
-    image: '${ZITADEL_IMAGE:-ghcr.io/zitadel/zitadel:latest}'
+      service: 'zitadel'
-    command: 'start-from-init --masterkey "MasterkeyNeedsToHave32Characters" --tlsMode disabled --config /zitadel.yaml --steps /zitadel.yaml'
-    depends_on:
-      db:
-        condition: 'service_healthy'
-    volumes:
-      - ./docker-compose-zitadel.yaml:/zitadel.yaml
-    network_mode: host
 
   db:
-    restart: 'always'
+    extends:
-    image: 'cockroachdb/cockroach:v22.2.2'
+      file: './config/localhost/docker-compose.yaml'
-    command: 'start-single-node --insecure --http-addr :9090'
+      service: 'db'
-    healthcheck:
-      test: ['CMD', 'curl', '-f', 'http://localhost:9090/health?ready=1']
-      interval: '10s'
-      timeout: '30s'
-      retries: 5
-      start_period: '20s'
-    network_mode: host
-
-  prepare:
-    image: node:18-alpine3.15
-    working_dir: /e2e
-    user: '$UID'
-    volumes:
-      - .:/e2e
-    command: 'sh -c "npm ci --omit=dev && npm run lint && npx wait-on http://localhost:8080/debug/ready"'
-    network_mode: host
-
-  e2e:
-    image: cypress/included:12.2.0
-    depends_on:
-      zitadel:
-        condition: 'service_started'
-      db:
-        condition: 'service_healthy'
-      prepare:
-        condition: 'service_completed_successfully'
-    working_dir: /e2e
-    user: '$UID'
-    volumes:
-      - .:/e2e
-    network_mode: host
-
-networks:
-  zitadel:

e2e/package.json

@@ -4,8 +4,12 @@
   "scripts": {
     "open": "npx cypress open",
     "e2e": "npx cypress run",
-    "open:dev": "CYPRESS_BASE_URL=http://localhost:4200 CYPRESS_BACKEND_URL=http://localhost:8080 npm run open --",
+    "open:golang": "npm run open --",
-    "e2e:dev": "CYPRESS_BASE_URL=http://localhost:4200 CYPRESS_BACKEND_URL=http://localhost:8080 npm run e2e --",
+    "e2e:golang": "npm run e2e --",
+    "open:golangangular": "CYPRESS_BASE_URL=http://localhost:4200 CYPRESS_BACKEND_URL=http://localhost:8080 npm run open --",
+    "e2e:golangangular": "CYPRESS_BASE_URL=http://localhost:4200 CYPRESS_BACKEND_URL=http://localhost:8080 npm run e2e --",
+    "open:angular": "CYPRESS_BASE_URL=http://localhost:4200 CYPRESS_BACKEND_URL=http://localhost:8080 CYPRESS_WEBHOOK_HANDLER_HOST=host.docker.internal npm run open --",
+    "e2e:angular": "CYPRESS_BASE_URL=http://localhost:4200 CYPRESS_BACKEND_URL=http://localhost:8080 CYPRESS_WEBHOOK_HANDLER_HOST=host.docker.internal npm run e2e --",
     "lint": "prettier --check cypress",
     "lint:fix": "prettier --write cypress"
   },