chore: build image for digest and release on workflow_dispatch (#6287)

.github/workflows/build.yml

@@ -2,6 +2,10 @@ name: ZITADEL CI/CD
 
 on:
   pull_request:
+    paths-ignore:
+      - 'docs/**'
+      - 'guides/**'
+      - '**.md'
   workflow_dispatch:
 
 permissions:
@@ -72,24 +76,30 @@ jobs:
   container:
     needs: [compile]
     uses: ./.github/workflows/container.yml
+    secrets: inherit
+    permissions:
+      packages: write
+    if: ${{ github.event_name == 'workflow_dispatch' }}
     with:
-      image_name: 'ghcr.io/zitadel/zitadel'
+      build_image_name: 'ghcr.io/zitadel/zitadel-build'
 
   e2e:
     uses: ./.github/workflows/e2e.yml
-    needs: [container]
+    needs: [compile]
-    with:
-      image: ${{ needs.container.outputs.image }}-debug
 
   release:
     uses: ./.github/workflows/release.yml
+    permissions:
+      packages: write
+      contents: write
+      issues: write
+      pull-requests: write
     needs: [version, core-unit-test, core-integration-test, lint, container, e2e]
-    # TODO: trigger release on workflow_dispatch if: ${{ github.event_name == 'workflow_dispatch' }}
+    if: ${{ needs.version.outputs.published == 'true' && github.event_name == 'workflow_dispatch' }}
-    if: ${{ needs.version.outputs.published == 'true' }}
     secrets:
       GCR_JSON_KEY_BASE64: ${{ secrets.GCR_JSON_KEY_BASE64 }}
     with:
-      image: ${{ needs.container.outputs.image }}
+      build_image_name: ${{ needs.container.outputs.build_image }}
       semantic_version: '19.0.2'
       image_name: 'ghcr.io/zitadel/zitadel'
       google_image_name: "europe-docker.pkg.dev/zitadel-common/zitadel-repo/zitadel"

.github/workflows/container.yml

@@ -3,12 +3,15 @@ name: Container
 on:
   workflow_call:
     inputs:
-      image_name:
+      build_image_name:
         required: true
         type: string
     outputs:
-      image:
+      build_image:
-        value: '${{ inputs.image_name }}:${{ github.sha }}'
+        value: '${{ inputs.build_image_name }}:${{ github.sha }}'
+
+permissions:
+  packages: write
 
 env:
   default_labels: |
@@ -31,7 +34,7 @@ jobs:
         id: scratch-meta
         uses: docker/metadata-action@v4
         with:
-          images: ${{ inputs.image_name }}
+          images: ${{ inputs.build_image_name }}
           labels: ${{ env.default_labels}}
           tags: |
             type=sha,prefix=,suffix=,format=long
@@ -40,7 +43,7 @@ jobs:
         id: debug-meta
         uses: docker/metadata-action@v4
         with:
-          images: ${{ inputs.image_name }}
+          images: ${{ inputs.build_image_name }}
           labels: ${{ env.default_labels}}
           tags: |
             type=sha,prefix=,suffix=-debug,format=long
@@ -76,12 +79,14 @@ jobs:
         timeout-minutes: 3
         with:
           context: .
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
           file: build/Dockerfile
           target: artifact
           platforms: linux/${{ matrix.arch }}
           push: true
           labels: ${{ steps.debug-meta.outputs.labels }}
-          outputs: type=image,name=${{ inputs.image_name }},push-by-digest=true,name-canonical=true,push=true
+          outputs: type=image,name=${{ inputs.build_image_name }},push-by-digest=true,name-canonical=true,push=true
       - 
         name: Scratch
         id: build-scratch
@@ -89,12 +94,14 @@ jobs:
         timeout-minutes: 3
         with:
           context: .
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
           file: build/Dockerfile
           target: final
           platforms: linux/${{ matrix.arch }}
           push: true
           labels: ${{ steps.scratch-meta.outputs.labels }}
-          outputs: type=image,name=${{ inputs.image_name }},push-by-digest=true,name-canonical=true,push=true
+          outputs: type=image,name=${{ inputs.build_image_name }},push-by-digest=true,name-canonical=true,push=true
       -
         name: Export debug digest
         run: |
@@ -141,14 +148,6 @@ jobs:
         uses: docker/setup-buildx-action@v2
         with:
           driver-opts: 'image=moby/buildkit:v0.11.6'
-      -
-        name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: '${{ inputs.image_name }}'
-          tags: |
-            type=sha,prefix=,suffix=${{ matrix.suffix }},format=long
       -
         name: Login to Docker registry
         uses: docker/login-action@v2
@@ -157,12 +156,21 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       -
-        name: Create manifest list and push
+        name: Docker meta
+        id: build-meta
+        uses: docker/metadata-action@v4
+        with:
+          images: '${{ inputs.build_image_name }}'
+          tags: |
+            type=sha,prefix=,suffix=${{ matrix.suffix }},format=long
+      -
+        name: Create build manifest list and push
         working-directory: /tmp/digests/${{ matrix.image }}
         run: |
-          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '${{ steps.build-meta.outputs.json }}') \
-            $(printf '${{ inputs.image_name }}@sha256:%s ' *)
+            $(printf '${{ inputs.build_image_name }}@sha256:%s ' *)
       -
-        name: Inspect image
+        name: Inspect build image
         run: |
-          docker buildx imagetools inspect ${{ inputs.image_name }}:${{ github.sha }}${{ matrix.suffix }}
+          docker buildx imagetools inspect ${{ inputs.build_image_name }}:${{ github.sha }}${{ matrix.suffix }}
+

.github/workflows/e2e.yml

@@ -2,10 +2,6 @@ name: "ZITADEL e2e Tests"
 
 on:
   workflow_call:
-    inputs:
-      image:
-        required: true
-        type: string
 
 jobs:
   test:
@@ -13,79 +9,41 @@ jobs:
       fail-fast: false
       matrix:
         browser: [firefox, chrome]
-    services:
-      # we currently use postgres because cockroach doesn't work
-      postgres:
-        image: postgres
-        ports:
-          - 5432:5432
-        env:
-          POSTGRES_USER: zitadel
-          PGUSER: zitadel
-          POSTGRES_DB: zitadel
-          POSTGRES_HOST_AUTH_METHOD: trust
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-          --health-start-period 10s
-      zitadel:
-        image: ${{ inputs.image }}
-        credentials:
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-        options: >-
-          --health-cmd "zitadel ready"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-          --health-start-period 10s
-          --add-host host.docker.internal:host-gateway
-        ports:
-          - 8080:8080
-        env:
-          ZITADEL_ARGS: "start-from-init --masterkeyFromEnv"
-          ZITADEL_MASTERKEY: MasterkeyNeedsToHave32Characters
-          ZITADEL_LOG_LEVEL: debug
-          ZITADEL_EXTERNALDOMAIN: localhost
-          ZITADEL_EXTERNALSECURE: "false"
-          ZITADEL_TLS_ENABLED: "false"
-          ZITADEL_DATABASE_POSTGRES_HOST: postgres
-          ZITADEL_DATABASE_POSTGRES_PORT: "5432"
-          ZITADEL_DATABASE_POSTGRES_DATABASE: zitadel
-          ZITADEL_DATABASE_POSTGRES_MAXOPENCONNS: "20"
-          ZITADEL_DATABASE_POSTGRES_MAXIDLECONNS: "10"
-          ZITADEL_DATABASE_POSTGRES_USER_USERNAME: zitadel
-          ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE: disable
-          ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME: zitadel
-          ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_MODE: disable
-          ZITADEL_FIRSTINSTANCE_ORG_HUMAN_PASSWORDCHANGEREQUIRED: "false"
-          ZITADEL_LOGSTORE_ACCESS_DATABASE_ENABLED: "true"
-          ZITADEL_LOGSTORE_ACCESS_DATABASE_DEBOUNCE_MINFREQUENCY: 0s
-          ZITADEL_LOGSTORE_ACCESS_DATABASE_DEBOUNCE_MAXBULKSIZE: "0"
-          ZITADEL_LOGSTORE_EXECUTION_DATABASE_ENABLED: "true"
-          ZITADEL_LOGSTORE_EXECUTION_STDOUT_ENABLED: "false"
-          ZITADEL_QUOTAS_ACCESS_EXHAUSTEDCOOKIEKEY: "zitadel.quota.limiting"
-          ZITADEL_QUOTAS_ACCESS_EXHAUSTEDCOOKIEMAXAGE: 60s
-          ZITADEL_PROJECTIONS_CUSTOMIZATIONS_NOTIFICATIONSQUOTAS_REQUEUEEVERY: 1s
-          ZITADEL_DEFAULTINSTANCE_LOGINPOLICY_MFAINITSKIPLIFETIME: "0"
-          ZITADEL_SYSTEMAPIUSERS: "{\"cypress\": {\"keyData\": \"LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF6aStGRlNKTDdmNXl3NEtUd3pnTQpQMzRlUEd5Y20vTStrVDBNN1Y0Q2d4NVYzRWFESXZUUUtUTGZCYUVCNDV6YjlMdGpJWHpEdzByWFJvUzJoTzZ0CmgrQ1lRQ3ozS0N2aDA5QzBJenhaaUIySVMzSC9hVCs1Qng5RUZZK3ZuQWtaamNjYnlHNVlOUnZtdE9sbnZJZUkKSDdxWjB0RXdrUGZGNUdFWk5QSlB0bXkzVUdWN2lvZmRWUVMxeFJqNzMrYU13NXJ2SDREOElkeWlBQzNWZWtJYgpwdDBWajBTVVgzRHdLdG9nMzM3QnpUaVBrM2FYUkYwc2JGaFFvcWRKUkk4TnFnWmpDd2pxOXlmSTV0eXhZc3duCitKR3pIR2RIdlczaWRPRGxtd0V0NUsycGFzaVJJV0syT0dmcSt3MEVjbHRRSGFidXFFUGdabG1oQ2tSZE5maXgKQndJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg==\"}}"
-
     runs-on: ubuntu-latest
-    env:
-      ZITADEL_IMAGE: ${{ inputs.image }}
     steps:
       - 
         name: Checkout Repository
         uses: actions/checkout@v3
+      -
+        uses: actions/download-artifact@v3
+        with:
+          path: .artifacts
+          name: zitadel-linux-amd64
+      -
+        name: Unpack executable
+        run: |
+          tar -xvf .artifacts/zitadel-linux-amd64.tar
+          mv zitadel-linux-amd64/zitadel ./zitadel
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          driver-opts: 'image=moby/buildkit:v0.11.6'
+      -
+        name: Start DB and ZITADEL
+        run: |
+          cd ./e2e
+          ZITADEL_IMAGE=zitadel:local docker compose up --detach --wait
       - 
         name: Cypress run
         uses: cypress-io/github-action@v5
         env:
           CYPRESS_BASE_URL: http://localhost:8080/ui/console
           CYPRESS_WEBHOOK_HANDLER_HOST: host.docker.internal
-          CYPRESS_DATABASE_CONNECTION_URL: 'postgresql://zitadel@localhost:5432/zitadel'
+          CYPRESS_DATABASE_CONNECTION_URL: 'postgresql://root@localhost:26257/zitadel'
           CYPRESS_BACKEND_URL: http://localhost:8080
         with:
           working-directory: e2e

.github/workflows/release-channels.yml

@@ -35,11 +35,6 @@ jobs:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - uses: tibdex/github-app-token@v1
-        id: generate-token
-        with:
-          app_id: ${{ secrets.APP_ID }}
-          private_key: ${{ secrets.APP_PRIVATE_KEY }}
       - name: Google Artifact Registry Login
         uses: docker/login-action@v2
         with:

.github/workflows/release.yml

@@ -6,7 +6,7 @@ on:
       semantic_version:
         required: true
         type: string
-      image:
+      build_image_name:
         required: true
         type: string
       image_name:
@@ -56,20 +56,20 @@ jobs:
       run: |
         docker buildx imagetools create \
           --tag ${{ inputs.image_name }}:${{ needs.version.outputs.version }} \
-          ${{ inputs.image }}
+          ${{ inputs.build_image_name }}
         docker buildx imagetools create \
           --tag ${{ inputs.image_name }}:${{ needs.version.outputs.version }}-debug \
-          ${{ inputs.image }}-debug        
+          ${{ inputs.build_image_name }}-debug        
         docker buildx imagetools create \
           --tag ${{ inputs.google_image_name }}:${{ needs.version.outputs.version }} \
-          ${{ inputs.image }}
+          ${{ inputs.build_image_name }}
     -
       name: Publish latest
       if: ${{ github.ref_name == 'next' }}
       run: |
         docker buildx imagetools create \
           --tag ${{ inputs.image_name }}:latest \
-          ${{ inputs.image }}
+          ${{ inputs.build_image_name }}
         docker buildx imagetools create \
           --tag ${{ inputs.image_name }}:latest-debug \
-          ${{ inputs.image }}-debug
+          ${{ inputs.build_image_name }}-debug

.releaserc.js

@@ -7,5 +7,5 @@ module.exports = {
     "@semantic-release/commit-analyzer",
     "@semantic-release/release-notes-generator",
     "@semantic-release/github"
-  ],
+  ]
 };

e2e/config/host.docker.internal/docker-compose.yaml

@@ -16,24 +16,13 @@ services:
       service: 'zitadel'
     volumes:
       - ./zitadel.yaml:/zitadel.yaml
-
-  prepare:
-    image: node:18-alpine3.15
-    working_dir: /e2e
-    user: '$UID'
-    volumes:
-      - ../../:/e2e
-    command: 'sh -c "npm ci --omit=dev && npm run lint && npx wait-on http://zitadel:8080/debug/ready"'
-
   e2e:
     image: cypress/included:12.2.0
     depends_on:
       zitadel:
-        condition: 'service_started'
+        condition: 'service_healthy'
       db:
         condition: 'service_healthy'
-      prepare:
-        condition: 'service_completed_successfully'
     working_dir: /e2e
     user: '$UID'
     volumes:

e2e/config/localhost/docker-compose.yaml

@@ -5,6 +5,14 @@ services:
     user: '$UID'
     restart: 'always'
     image: '${ZITADEL_IMAGE:-ghcr.io/zitadel/zitadel:latest}'
+    build:
+      context: ../../..
+      dockerfile: ./build/Dockerfile
+      target: artifact
+      cache_from:
+      - type=gha
+      cache_to:
+      - type=gha,mode=max
     command: 'start-from-init --masterkey "MasterkeyNeedsToHave32Characters" --tlsMode disabled --config /zitadel.yaml --steps /zitadel.yaml'
     depends_on:
       db:
@@ -13,6 +21,12 @@ services:
       - ./zitadel.yaml:/zitadel.yaml
     ports:
     - "8080:8080"
+    healthcheck:
+      test: ["CMD", "zitadel", "ready"]
+      interval: '10s'
+      timeout: '5s'
+      retries: 5
+      start_period: '10s'
     extra_hosts:
     - "host.docker.internal:host-gateway"