chore(ci): release pipeline for v2 (#3197)

* not working

* test ci

* correct path

* test

* test

* test

* remove k8s & test

* test

* test image

* typo

* test

* fix

* test

* fix

* test

* fix

* fix windows

* fix

* fix

* fix codecov

* fix with arch

* test

* test

* test

* test

* trial

* trial

* trial

* trial

* trial

* trial

* trial

* try

* try

* first improvement

* trial

* improve

* improve

* improve

* use bash

* trial

* random test

* wip

* test with goreleaser and semrel

* fix typo

* trial

* add buildx

* define default

* force buildkit

* use buildx

* debug GH docker

* use images

* debug docker

* work around image name issues

* trial

* test

* test with prebuilt image

* fix

* use load

* test

* use only docker

* trial

* test

* use load

* test

* test with docker driver

* trial

* trial

* trial

* use arm

* upload artifacts

* try to cache grpc base

* try to use a cache

* fall back

* test

* fix

* improve

* upload artifacts

* fix

* name things

* add tag if poc

* args for protos

* remove v2 build

Co-authored-by: Livio Amstutz <livio.a@gmail.com>

.dockerignore

@@ -17,3 +17,5 @@ README.md
 SECURITY.md
 pkg/grpc/*/*.pb.*
 pkg/grpc/*/*.swagger.json
+.goreleaser.yaml
+.artifacts/

.github/workflows/zitadel.yml

@@ -1,27 +1,15 @@
 name: ZITADEL Release
-on:
-  push:
-    branches:
-      - main
-    tags-ignore:
-      - '**'
-  pull_request:
-    branches:
-      - '**'
-  workflow_dispatch:
 
-env:
+on:
-  REGISTRY: ghcr.io
+  #pull_request:
-  ARTIFACTS_FOLDER: './artifacts'
+  push:
+
+permissions:
+  contents: write
 
 jobs:
-  refs:
+  Build:
-    name: Prepare CI Vars
+    runs-on: ubuntu-20.04
-    runs-on: ubuntu-18.04
-    outputs:
-      sha_short: ${{ steps.refs.outputs.sha_short }}
-      short_ref: ${{ steps.refs.outputs.short_ref }}
-      version: ${{ steps.refs.outputs.version }}
     steps:
       - name: Source checkout
         uses: actions/checkout@v2
@@ -30,224 +18,55 @@ jobs:
         uses: cycjimmy/semantic-release-action@v2
         with:
           dry_run: true
-          semantic_version: 17.0.4
+          semantic_version: 19.0.2
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Set outputs
+      - name: Set up Go
-        id: refs
+        uses: actions/setup-go@v2
-        run: |
+        with:
-          export BRANCH=${GITHUB_REF#refs/*/}
+          go-version: 1.17
-          echo "branch: ${BRANCH}"
+      - name: Set up QEMU
-
+        uses: docker/setup-qemu-action@v1
-          export BRANCHTRANSLATED=$(echo ${BRANCH} | tr '/' '_')
-          echo "short_ref: ${BRANCHTRANSLATED}"
-
-          export VERSION=""
-          if ${{ steps.semantic.outputs.new_release_published == 'true' }}; then
-            export VERSION=${{ steps.semantic.outputs.new_release_version }}
-          else
-            export VERSION=${BRANCHTRANSLATED}
-          fi
-          echo "New semantic release: ${{ steps.semantic.outputs.new_release_published }}"
-          echo "version: ${VERSION}"
-
-          echo "::set-output name=short_ref::${BRANCHTRANSLATED}"
-          echo "::set-output name=sha_short::SHA-$(git rev-parse --short=12 HEAD)"
-          echo "::set-output name=version::${VERSION}"
-
-  zitadel:
-    name: Build ZITADEL
-    needs: refs
-    runs-on: ubuntu-18.04
-    steps:
-      - name: Source checkout
-        uses: actions/checkout@v2
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
-      - uses: docker/build-push-action@v2
         with:
-          context: .
+          driver: docker
-          file: ./build/zitadel/Dockerfile
+          install: true
-          platforms: linux/amd64
+      - name: Run GoReleaser Snapshot
-          cache-from: type=gha,scope=${{ github.workflow }}
+        if: ${{ !startsWith(github.ref, 'refs/tags/') }}
-          cache-to: type=gha,scope=${{ github.workflow }},mode=max
+        uses: goreleaser/goreleaser-action@v2
-          outputs: type=local,dest=/tmp/zitadel
+        with:
-          build-args: |
+          distribution: goreleaser
-            VERSION=${{ needs.refs.outputs.version }}
+          version: v1.5.0
+          args: release --snapshot
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Run GoReleaser Release on tag
+        if: ${{ startsWith(github.ref, 'refs/tags/') }}
+        uses: goreleaser/goreleaser-action@v2
+        with:
+          distribution: goreleaser
+          version: v1.5.0
+          args: release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions/upload-artifact@v2
+        name: Upload goreleaser dist
         with:
-          name: zitadel
+          name: goreleaser-dist
-          path: /tmp/zitadel/
+          path: .artifacts/goreleaser
-
+      - uses: actions/upload-artifact@v2
-  zitadel-codecov:
+        name: Upload gRPC clients
-    name: Upload ZITADEL Codecov
-    needs: [refs, zitadel]
-    runs-on: ubuntu-18.04
-    steps:
-      - name: Source checkout
-        uses: actions/checkout@v2
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - uses: docker/build-push-action@v2
         with:
-          context: .
+          name: grpc-clients
-          file: ./build/zitadel/Dockerfile
+          path: .artifacts/grpc
-          platforms: linux/amd64
+      - uses: actions/upload-artifact@v2
-          target: go-codecov
+        name: Upload compiled console
-          cache-from: type=gha,scope=${{ github.workflow }}
+        with:
-          outputs: type=local,dest=/tmp/zitadel
+          name: console
+          path: .artifacts/console
       - uses: codecov/codecov-action@v1
+        name: Upload go coverage
         with:
-          name: zitadel-codecov
+          files: .artifacts/codecov/profile.cov
-          files: /tmp/zitadel/profile.cov
+          name: go-codecov
-
-  zitadel-console:
-    name: Build ZITADEL Console
-    needs: refs
-    runs-on: ubuntu-18.04
-    steps:
-      - name: Source checkout
-        uses: actions/checkout@v2
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: ./build/console/Dockerfile
-          platforms: linux/amd64
-          cache-from: type=gha,scope=${{ github.workflow }}
-          cache-to: type=gha,scope=${{ github.workflow }},mode=max
-          outputs: type=local,dest=/tmp/zitadel-console
-      - uses: actions/upload-artifact@v2
-        with:
-          name: zitadel-console
-          path: /tmp/zitadel-console/
-
-  zitadel-image:
-    name: Package ZITADEL Image
-    needs: [refs, zitadel, zitadel-console]
-    runs-on: ubuntu-18.04
-    if: ${{ github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main' }}
-    steps:
-      - name: Source checkout
-        uses: actions/checkout@v2
-      - uses: actions/download-artifact@v2
-        with:
-          name: zitadel
-          path: .download/zitadel
-      - uses: actions/download-artifact@v2
-        with:
-          name: zitadel-console
-          path: .download/zitadel/console
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-          registry: ${{ env.REGISTRY }}
-      - uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: ./build/Dockerfile
-          platforms: linux/amd64
-          tags: ${{ env.REGISTRY }}/${{ github.repository }}:${{ needs.refs.outputs.sha_short }},${{ env.REGISTRY }}/${{ github.repository }}:${{ needs.refs.outputs.short_ref }}
-          push: true
-          cache-from: type=gha,scope=${{ github.workflow }}
-          cache-to: type=gha,scope=${{ github.workflow }},mode=max
-  release:
-    name: Semantic Release Images and Artifacts
-    runs-on: ubuntu-18.04
-    needs: [ refs, zitadel-image ]
-    if: ${{ github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main' }}
-    env:
-      DOCKER_USERNAME: ${{ github.repository_owner }}
-      DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
-    steps:
-      - name: Source checkout
-        uses: actions/checkout@v2
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-          registry: ${{ env.REGISTRY }}
-      - name: Docker Pull ZITADEL Image
-        run: docker pull $REGISTRY/$GITHUB_REPOSITORY:${{ needs.refs.outputs.sha_short }}
-      - name: Download zitadelctl Artifacts
-        uses: actions/download-artifact@v2
-        with:
-          path: ${{ env.ARTIFACTS_FOLDER }}/
-      - name: Workaround for exe
-        run: |
-          mv ${ARTIFACTS_FOLDER}/zitadelctl-windows-amd64/zitadelctl-windows-amd64 ${ARTIFACTS_FOLDER}/zitadelctl-windows-amd64/zitadelctl-windows-amd64.exe
-          find ${ARTIFACTS_FOLDER}
-
-      - name: Run Semantic Release
-        id: semantic
-        uses: cycjimmy/semantic-release-action@v2
-        with:
-          dry_run: false
-          semantic_version: 17.0.4
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Echo Semantic Release Versions
-        if: steps.semantic.outputs.new_release_published == 'true'
-        run: |
-          echo ${{ steps.semantic.outputs.new_release_version }}
-          echo ${{ steps.semantic.outputs.new_release_major_version }}
-          echo ${{ steps.semantic.outputs.new_release_minor_version }}
-          echo ${{ steps.semantic.outputs.new_release_patch_version }}
-      - name: Docker Tag Version
-        run: |
-          docker tag $REGISTRY/$GITHUB_REPOSITORY:${{ needs.refs.outputs.sha_short }} $REGISTRY/$GITHUB_REPOSITORY:${{ steps.semantic.outputs.new_release_version }}
-        if: steps.semantic.outputs.new_release_published == 'true'
-      - name: Docker Tag Latest
-        run: |
-          docker tag $REGISTRY/$GITHUB_REPOSITORY:${{ needs.refs.outputs.sha_short }} $REGISTRY/$GITHUB_REPOSITORY:latest
-        if: steps.semantic.outputs.new_release_published == 'true'
-      - name: Docker Push Version
-        run: |
-          docker push $REGISTRY/$GITHUB_REPOSITORY:${{ steps.semantic.outputs.new_release_version }}
-        if: steps.semantic.outputs.new_release_published == 'true'
-      - name: Docker Push Latest
-        run: |
-          docker push $REGISTRY/$GITHUB_REPOSITORY:latest
-        if: steps.semantic.outputs.new_release_published == 'true'
-
-      - name: Docker Tag Version
-        run: |
-          docker tag $REGISTRY/$GITHUB_REPOSITORY:${{ needs.refs.outputs.sha_short }} $REGISTRY/$GITHUB_REPOSITORY:${{ needs.refs.outputs.short_ref }}
-        if: steps.semantic.outputs.new_release_published != 'true' && needs.refs.outputs.short_ref != 'main' && needs.refs.outputs.short_ref != ''
-      - name: Docker Push Version
-        run: |
-          docker push $REGISTRY/$GITHUB_REPOSITORY:${{ needs.refs.outputs.short_ref }}
-        if: steps.semantic.outputs.new_release_published != 'true' && needs.refs.outputs.short_ref != 'main' && needs.refs.outputs.short_ref != ''
-      - name: Development Release
-        id: create_release
-        uses: ncipollo/release-action@v1.8.4
-        if: steps.semantic.outputs.new_release_published != 'true' && needs.refs.outputs.short_ref != 'main' && needs.refs.outputs.short_ref != ''
-        with:
-          artifacts: "${{ env.ARTIFACTS_FOLDER }}/zitadelctl-darwin-amd64/zitadelctl-darwin-amd64,${{ env.ARTIFACTS_FOLDER }}/zitadelctl-linux-amd64/zitadelctl-linux-amd64,${{ env.ARTIFACTS_FOLDER }}/zitadelctl-windows-amd64/zitadelctl-windows-amd64.exe"
-          body: |
-            This is a release from a development branch.
-            Do not use these artifacts in production.
-          tag: ${{ needs.refs.outputs.short_ref }}-dev
-          commit: ${{ needs.refs.outputs.short_ref }}
-          name: Branch ${{ needs.refs.outputs.short_ref }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-          replacesArtifacts: true
-          prerelease: true
-          draft: false
-          allowUpdates: true
-      - name: Create Sentry release
-        if: ${{ github.ref == 'refs/heads/main' }}
-        uses: getsentry/action-release@v1
-        env:
-          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
-          SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
-          SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
-        with:
-          version: zitadel-${{ needs.refs.outputs.version }}
-          projects: "console zitadel zitadelctl"

.gitignore

@@ -59,3 +59,5 @@ openapi/**/*.json
 build/local/cloud.env
 migrations/cockroach/migrate_cloud.go
 .notifications
+
+.artifacts

.goreleaser.yaml

@@ -0,0 +1,55 @@
+project_name: zitadel
+release:
+  github:
+    owner: caos
+    name: zitadel
+  draft: true
+  prerelease: auto
+before:
+  hooks:
+    - docker build -f build/grpc/Dockerfile -t zitadel-base:local .
+    - docker build -f build/zitadel/Dockerfile . -t zitadel-go-test --target go-codecov -o .artifacts/codecov
+    - docker build -f build/zitadel/Dockerfile . -t zitadel-go-base --target go-copy -o .artifacts/grpc/go-client
+    - docker build -f build/console/Dockerfile . -t zitadel-npm-base --target npm-copy -o .artifacts/grpc/js-client
+    - docker build -f build/console/Dockerfile . -t zitadel-npm-base --target angular-export -o .artifacts/console
+builds:
+  - env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - windows
+      - darwin
+    goarch:
+      - amd64
+      - arm64
+dist: .artifacts/goreleaser
+dockers:
+- image_templates:
+  - "caos/zitadel:{{ .Version }}-amd64"
+  use: buildx
+  dockerfile: build/Dockerfile
+  build_flag_templates:
+  - "--platform=linux/amd64"
+- image_templates:
+  - "caos/zitadel:{{ .Version }}-arm64"
+  use: buildx
+  dockerfile: build/Dockerfile
+  build_flag_templates:
+  - "--platform=linux/arm64"
+archives:
+- format: binary
+gomod:
+  proxy: true
+  env:
+    - GOPROXY=https://proxy.golang.org,direct
+    - GOSUMDB=sum.golang.org
+checksum:
+  name_template: 'checksums.txt'
+snapshot:
+  name_template: "{{ incpatch .Version }}-next"
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - '^docs:'
+      - '^test:'

.releaserc.js

@@ -1,23 +1,9 @@
 module.exports = {
-    branches: ["main"],
+    branches: [
+        {name: 'main'},
+        {name: '1.x.x', range: '1.x.x', channel: '1.x.x'}, 
+      ],
     plugins: [
-        "@semantic-release/commit-analyzer",
+        "@semantic-release/commit-analyzer"
-        "@semantic-release/release-notes-generator",
-        ["@semantic-release/github", {
-            "assets": [
-                {
-                    "path": "./artifacts/zitadelctl-darwin-amd64/zitadelctl-darwin-amd64",
-                    "label": "Zitadelctl Darwin x86_64"
-                },
-                {
-                    "path": "./artifacts/zitadelctl-linux-amd64/zitadelctl-linux-amd64",
-                    "label": "Zitadelctl Linux x86_64"
-                },
-                {
-                    "path": "./artifacts/zitadelctl-windows-amd64/zitadelctl-windows-amd64.exe",
-                    "label": "Zitadelctl Windows x86_64"
-                }
-            ]
-        }],
     ]
 };

build/Dockerfile

@@ -2,12 +2,8 @@
 ## Final Production Image
 #######################
 FROM alpine:3 as artifact
-ARG BUILDARCH
+COPY zitadel /app/zitadel
 RUN adduser -D zitadel
-COPY cmd/zitadel/*.yaml /app/
-COPY .download/zitadel/zitadel-linux-${BUILDARCH} /app/zitadel
-COPY .download/zitadel/console /app/console/
-RUN chmod a+x /app/zitadel
 
 #######################
 ## Scratch Image

build/console/Dockerfile

@@ -1,49 +1,5 @@
 ARG NODE_VERSION=14
 
-#######################
-## These steps set platform / arch type specific variables
-#######################
-FROM alpine:3 AS arm64-base
-ENV PROTOC_ARCH aarch_64
-
-FROM alpine:3 AS amd64-base
-ENV PROTOC_ARCH x86_64
-
-#######################
-## This step sets up the folder structure,
-## initalices go mods,
-## downloads the protofiles,
-## protoc and protoc-gen-grpc-web for later use
-#######################
-FROM ${BUILDARCH}-base AS base
-ARG PROTOC_VERSION=3.13.0
-ARG PROTOC_ZIP=protoc-${PROTOC_VERSION}-linux-${PROTOC_ARCH}.zip
-ARG GRPC_WEB_VERSION=1.3.0
-# no arm specific version available and x86 works fine at the moment:
-ARG GRPC_WEB=protoc-gen-grpc-web-${GRPC_WEB_VERSION}-linux-x86_64
-
-RUN apk add tar curl
-WORKDIR /proto
-
-#protoc
-RUN apk add tar curl \
-    && curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/$PROTOC_ZIP \
-    && unzip -o $PROTOC_ZIP -d /usr/local bin/protoc \
-    && unzip -o $PROTOC_ZIP -d /proto include/* \
-    && rm -f $PROTOC_ZIP \
-    && curl -OL https://github.com/grpc/grpc-web/releases/download/${GRPC_WEB_VERSION}/${GRPC_WEB} \
-    && mv ${GRPC_WEB} /usr/local/bin/protoc-gen-grpc-web \
-    && chmod +x /usr/local/bin/protoc-gen-grpc-web \
-    && curl https://raw.githubusercontent.com/envoyproxy/protoc-gen-validate/v0.6.2/validate/validate.proto --create-dirs -o include/validate/validate.proto  \
-    && curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v2.2.0/protoc-gen-openapiv2/options/annotations.proto --create-dirs -o include/protoc-gen-openapiv2/options/annotations.proto \
-    && curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v2.2.0/protoc-gen-openapiv2/options/openapiv2.proto --create-dirs -o include/protoc-gen-openapiv2/options/openapiv2.proto \
-    && curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/annotations.proto --create-dirs -o include/google/api/annotations.proto \
-    && curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto --create-dirs -o include/google/api/http.proto \
-    && curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/field_behavior.proto --create-dirs -o include/google/api/field_behavior.proto
-
-#zitadel protos
-COPY proto/ include/.
-
 #######################
 ## With this step we prepare all node_modules, this helps caching the build
 ## Speed up this step by mounting your local node_modules directory
@@ -52,11 +8,11 @@ FROM node:${NODE_VERSION} as npm-base
 WORKDIR /console
 
 COPY console/package.json console/package-lock.json ./
-RUN npm install
+RUN npm ci
 
 COPY console .
-COPY --from=base /proto /proto
+COPY --from=zitadel-base:local /proto /proto
-COPY --from=base /usr/local/bin /usr/local/bin/.
+COPY --from=zitadel-base:local /usr/local/bin /usr/local/bin/.
 COPY build/console build/console/
 RUN build/console/generate-grpc.sh
 
@@ -64,23 +20,18 @@ RUN build/console/generate-grpc.sh
 ## copy for local dev
 #######################
 FROM scratch as npm-copy
-COPY --from=npm-base /console/src/app/proto/generated ./console/src/app/proto/generated
+COPY --from=npm-base /console/src/app/proto/generated .
-
-#######################
-## angular dev build
-#######################
-FROM npm-base as dev-angular-build
-RUN npm install -g @angular/cli
 
 #######################
 ## angular lint workspace and prod build
 #######################
-FROM npm-base as prod-angular-build
+FROM npm-base as angular-build
 RUN npm run lint
 RUN npm run prodbuild
+RUN ls -la /console/dist/console
 
 #######################
 ## Only Copy Assets
 #######################
-FROM scratch as prod-angular-export
+FROM scratch as angular-export
-COPY --from=prod-angular-build /console/dist/console .
+COPY --from=angular-build /console/dist/console .

build/grpc/Dockerfile

@@ -0,0 +1,47 @@
+#ARG BUILDARCH=x86_64
+
+#######################
+## These steps set platform / arch type specific variables
+#######################
+FROM alpine:3 AS arm64-base
+ENV PROTOC_ARCH aarch_64
+
+FROM alpine:3 AS amd64-base
+ENV PROTOC_ARCH x86_64
+
+#######################
+## This step sets up the folder structure,
+## initalices go mods,
+## downloads the protofiles,
+## protoc and protoc-gen-grpc-web for later use
+#######################
+FROM ${BUILDARCH}-base
+ARG PROTOC_VERSION=3.18.0
+ARG PROTOC_ZIP=protoc-${PROTOC_VERSION}-linux-${PROTOC_ARCH}.zip
+ARG GRPC_WEB_VERSION=1.3.0
+ARG GATEWAY_VERSION=2.6.0
+ARG VALIDATOR_VERSION=0.6.2
+# no arm specific version available and x86 works fine at the moment:
+ARG GRPC_WEB=protoc-gen-grpc-web-${GRPC_WEB_VERSION}-linux-x86_64
+
+RUN apk add tar curl
+WORKDIR /proto
+
+#protoc
+RUN apk add tar curl \
+    && curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/$PROTOC_ZIP \
+    && unzip -o $PROTOC_ZIP -d /usr/local bin/protoc \
+    && unzip -o $PROTOC_ZIP -d /proto include/* \
+    && rm -f $PROTOC_ZIP \
+    && curl -OL https://github.com/grpc/grpc-web/releases/download/${GRPC_WEB_VERSION}/${GRPC_WEB} \
+    && mv ${GRPC_WEB} /usr/local/bin/protoc-gen-grpc-web \
+    && chmod +x /usr/local/bin/protoc-gen-grpc-web \
+    && curl https://raw.githubusercontent.com/envoyproxy/protoc-gen-validate/v${VALIDATOR_VERSION}/validate/validate.proto --create-dirs -o include/validate/validate.proto  \
+    && curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v${GATEWAY_VERSION}/protoc-gen-openapiv2/options/annotations.proto --create-dirs -o include/protoc-gen-openapiv2/options/annotations.proto \
+    && curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v${GATEWAY_VERSION}/protoc-gen-openapiv2/options/openapiv2.proto --create-dirs -o include/protoc-gen-openapiv2/options/openapiv2.proto \
+    && curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/annotations.proto --create-dirs -o include/google/api/annotations.proto \
+    && curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto --create-dirs -o include/google/api/http.proto \
+    && curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/field_behavior.proto --create-dirs -o include/google/api/field_behavior.proto
+
+#zitadel protos
+COPY proto/ include/.

build/local/Dockerfile.clientid

@@ -1,11 +0,0 @@
-FROM ubuntu:latest AS client-id
-#install dependencies
-RUN apt-get update \
-    && apt-get install curl -y \
-    && apt-get install jq -y
-
-#prepare script
-COPY build/local/clientid.sh clientid.sh
-RUN chmod +x /clientid.sh
-
-ENTRYPOINT [ "/clientid.sh" ]

build/local/Dockerfile.gateway

@@ -1,33 +0,0 @@
-# copy from https://raw.githubusercontent.com/grpc/grpc-web/master/net/grpc/gateway/docker/grpcwebproxy/Dockerfile
-
-FROM golang:1.17-alpine3.13
-
-RUN apk add --no-cache curl git ca-certificates && \
-  rm -rf /var/lib/apt/lists/*
-
-ARG VERSION=0.14.0
-
-WORKDIR /tmp
-
-RUN curl -sS https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
-RUN wget https://github.com/improbable-eng/grpc-web/archive/v$VERSION.tar.gz
-
-WORKDIR /go/src/github.com/improbable-eng/
-
-RUN tar -zxf /tmp/v$VERSION.tar.gz -C .
-RUN mv grpc-web-$VERSION grpc-web
-
-WORKDIR /go/src/github.com/improbable-eng/grpc-web
-
-RUN dep ensure && \
-  go env -w GO111MODULE=auto && \
-  go install ./go/grpcwebproxy
-
-ENV BKD_HOST=backend-run
-ENV BKD_PORT=50001
-
-ENTRYPOINT [ "/bin/sh", "-c", "exec /go/bin/grpcwebproxy \
-  --backend_addr=${BKD_HOST}:${BKD_PORT} \
-  --run_tls_server=false \
-  --use_websockets \
-  --allow_all_origins " ]

build/local/Dockerfile.keys

@@ -1,7 +0,0 @@
-FROM alpine:latest AS gen-keys
-COPY build/local/keys.sh keys.sh
-RUN chmod +x /keys.sh
-ENTRYPOINT [ "/keys.sh" ]
-
-FROM scratch AS copy-keys
-COPY --from=gen-keys /.keys /.keys

build/local/clientid.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-# ------------------------------
-# sets the client id in environment.json
-# ------------------------------
-
-clientid=""
-while [ -z $clientid ]; do
-    echo "no from zitadel ==> retrying in 5 seconds"
-    sleep 5
-    clientid=$(curl -s http://${HOST}:${PORT}/clientID)
-    if [[ "$clientid" != *@zitadel* ]]; then
-        echo "invalid response from zitadel ==> retrying in 5 seconds"
-        clientid=""
-    fi
-done
-
-echo "$(jq ".clientid = $clientid" /environment.json)" > environment.json

build/local/docker-compose-local.yml

@@ -2,7 +2,6 @@ version: '3.8'
 
 services:
   db:
-    profiles: ['backend', 'storage']
     restart: always
     networks:
       - zitadel
@@ -18,106 +17,11 @@ services:
       - 8080:8080
       - 26257:26257
 
-# schema changes on the database
-  db-migrations:
-    profiles: ['backend', 'storage']
-    restart: on-failure
-    networks:
-      - zitadel
-    depends_on:
-      db: 
-        condition: service_healthy
-    image: flyway/flyway:latest
-    volumes:
-      - ../../migrations/cockroach:/flyway/sql
-    environment:
-      - FLYWAY_PLACEHOLDERS_eventstorepassword=NULL
-      - FLYWAY_PLACEHOLDERS_managementpassword=NULL
-      - FLYWAY_PLACEHOLDERS_adminapipassword=NULL
-      - FLYWAY_PLACEHOLDERS_authpassword=NULL
-      - FLYWAY_PLACEHOLDERS_notificationpassword=NULL
-      - FLYWAY_PLACEHOLDERS_authzpassword=NULL
-      - FLYWAY_PLACEHOLDERS_queriespassword=NULL
-    command: -url=jdbc:postgresql://db:26257/defaultdb -user=root -password= -connectRetries=5 migrate
-
-# minio is used to store assets
-  minio:
-    profiles: ['backend', 'storage']
-    image: minio/minio:RELEASE.2021-06-14T01-29-23Z
-    restart: on-failure
-    networks:
-      - zitadel
-    environment:
-      - MINIO_ACCESS_KEY=access_key
-      - MINIO_SECRET_KEY=secret_key
-    healthcheck:
-      test: ['CMD', 'curl', '-f', 'http://localhost:9000/minio/health/live']
-      interval: 30s
-      timeout: 5s
-      retries: 3
-      start_period: 5s
-    command:
-      - gateway
-      - nas
-      - /export
-
-# ZITADEL needs several keys to encrypt data
-# this container generates the required keys
-# and stores them into zitadel/.keys
-  keys:
-    profiles: ['backend', 'backend-stub']
-    restart: on-failure
-    networks:
-      - zitadel
-    build:
-      context: ../..
-      dockerfile: build/local/Dockerfile.keys
-      target: gen-keys
-    volumes:
-      - ../../.:/zitadel
-    env_file:
-      - ./local.env
-
-# To interact with ZITADEL requires some data setted up.
-# Due to the evolution of ZITADEL it's required to add additional 
-# setup steps, because of this fact it's recommended to rerun the setup
-# on each restart, at least after a new version got released
-  backend-setup:
-    profiles: ['backend']
-    restart: on-failure
-    networks:
-      - zitadel
-    depends_on:
-      db-migrations:
-        condition: service_completed_successfully
-      keys:
-        condition: service_completed_successfully
-    build:
-      context: ../..
-      dockerfile: build/zitadel/Dockerfile
-      target: dev-go-build
-      args:
-        ENV: dev
-    volumes:
-      - ../../.keys:/go/src/github.com/caos/zitadel/.keys
-    env_file:
-      - ./local.env
-    environment:
-      - ZITADEL_EVENTSTORE_HOST=db
-    command:
-      [
-        '-setup-files=cmd/zitadel/setup.yaml',
-        '-setup-files=cmd/zitadel/system-defaults.yaml',
-        '-setup-files=cmd/zitadel/authz.yaml',
-        'setup',
-      ]
-
 # starts the backend (API's) of ZITADEL
 # Port 50001 serves the GRPC API
 # Port 50002 serves the REST API
 # Port 50003 serves the login GUI
   backend-run:
-    profiles: ['backend']
     restart: on-failure
     networks:
       - zitadel
@@ -130,99 +34,15 @@ services:
     depends_on:
       db:
         condition: service_healthy
-      minio: 
-        condition: service_healthy
-      backend-setup:
-        condition: service_completed_successfully
     build:
       context: ../..
       dockerfile: build/zitadel/Dockerfile
       target: dev-go-build
-      args:
-        ENV: dev
-    volumes:
-      - ../../.keys:/go/src/github.com/caos/zitadel/.keys
-      - ../../.notifications:/go/src/github.com/caos/zitadel/.notifications
-    env_file:
-      - ./local.env
     environment:
       - ZITADEL_EVENTSTORE_HOST=db
     ports:
       - 50001:50001
-      - 50002:50002
-      - 50003:50003
     command:
       [
-        '-console=false',
-        '-localDevMode=true',
-        '-config-files=cmd/zitadel/startup.yaml',
-        '-config-files=cmd/zitadel/system-defaults.yaml',
-        '-config-files=cmd/zitadel/authz.yaml',
         'start',
       ]
-
-# the GRPC web gateway proxies the 
-# GRPC web calls to GRPC
-# it's used in console (frontend) for example
-  grpc-web-gateway:
-    profiles: ['backend']
-    restart: on-failure
-    logging:
-      driver: none
-    depends_on:
-      backend-run:
-        condition: service_healthy
-    networks:
-      - zitadel
-    build:
-      context: ../..
-      dockerfile: build/local/Dockerfile.gateway
-    image: grpcweb/grpcwebproxy
-    ports:
-      - '50000:8080'
-
-# this service generates the environemnt.json
-# needed in console.
-# It curl's the client id of console and writes it to 
-# the environment.json
-  client-id:
-    profiles: ['frontend', 'console-stub']
-    depends_on:
-      backend-run:
-        condition: service_healthy
-    networks:
-      - zitadel
-    build:
-      context: ../..
-      dockerfile: build/local/Dockerfile.clientid
-      target: client-id
-    volumes:
-      - ./environment.json:/environment.json
-    environment:
-      - HOST=backend-run
-      - PORT=50002
-
-# starts console in development mode
-  frontend-run:
-    profiles: ['frontend']
-    networks:
-      - zitadel
-    depends_on:
-      grpc-web-gateway:
-        condition: service_started
-      client-id:
-        condition: service_completed_successfully
-    build:
-      context: ../..
-      dockerfile: build/console/Dockerfile
-      target: dev-angular-build
-      args:
-        ENV: dev
-    volumes:
-      - ./environment.json:/console/src/assets/environment.json
-    command: sh -c "ng serve --host 0.0.0.0 --disable-host-check"
-    ports:
-      - 4200:4200
-
-networks:
-  zitadel: {}

build/local/keys.sh

@@ -1,23 +0,0 @@
-#!/bin/sh
-
-# ----------------------------------------------------------------
-# generates necessary ZITADEL keys
-# ----------------------------------------------------------------
-
-set -e
-
-
-KEY_PATH=$(echo "/zitadel/$(dirname ${ZITADEL_KEY_PATH})")
-KEY_FILE=${KEY_PATH}/local_keys.yaml
-
-mkdir -p ${KEY_PATH}
-if [ ! -f ${KEY_FILE} ]; then
-    touch ${KEY_FILE}
-fi
-
-for key in $(env | grep "ZITADEL_.*_KEY" | cut -d'=' -f2); do
-    if [ $(grep -L ${key} ${KEY_FILE}) ]; then
-        echo "create key for ${key} in ${KEY_FILE}"
-        echo -e "${key}: $(head -c22 /dev/urandom | base64)" >> ${KEY_FILE}
-    fi
-done

build/zitadel/Dockerfile

@@ -1,44 +1,5 @@
 ARG GO_VERSION=1.17
 
-#######################
-## These steps set platform / arch type specific variables
-#######################
-FROM alpine:3 AS arm64-base
-ENV PROTOC_ARCH aarch_64
-
-FROM alpine:3 AS amd64-base
-ENV PROTOC_ARCH x86_64
-
-#######################
-## This step sets up the folder structure,
-## initalices go mods,
-## downloads the protofiles and protoc for later use
-#######################
-FROM ${BUILDARCH}-base AS base
-ARG PROTOC_VERSION=3.18.0
-ARG PROTOC_ZIP=protoc-${PROTOC_VERSION}-linux-${PROTOC_ARCH}.zip
-ARG GATEWAY_VERSION=2.6.0
-ARG VALIDATOR_VERSION=0.6.2
-
-RUN apk add tar curl
-WORKDIR /proto
-
-#protoc
-RUN apk add tar curl \
-    && curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/$PROTOC_ZIP \
-    && unzip -o $PROTOC_ZIP -d /usr/local bin/protoc \
-    && unzip -o $PROTOC_ZIP -d /proto include/* \
-    && rm -f $PROTOC_ZIP \
-    && curl https://raw.githubusercontent.com/envoyproxy/protoc-gen-validate/v${VALIDATOR_VERSION}/validate/validate.proto --create-dirs -o include/validate/validate.proto  \
-    && curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v${GATEWAY_VERSION}/protoc-gen-openapiv2/options/annotations.proto --create-dirs -o include/protoc-gen-openapiv2/options/annotations.proto \
-    && curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v${GATEWAY_VERSION}/protoc-gen-openapiv2/options/openapiv2.proto --create-dirs -o include/protoc-gen-openapiv2/options/openapiv2.proto \
-    && curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/annotations.proto --create-dirs -o include/google/api/annotations.proto \
-    && curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto --create-dirs -o include/google/api/http.proto \
-    && curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/field_behavior.proto --create-dirs -o include/google/api/field_behavior.proto
-
-#zitadel protos
-COPY proto/ include/.
-
 #######################
 ## Go dependencies
 ## Speed up this step by mounting your local go mod pkg directory
@@ -48,7 +9,7 @@ RUN mkdir -p src/github.com/caos/zitadel
 WORKDIR /go/src/github.com/caos/zitadel
 
 #download modules
-COPY ./go.* .
+COPY . .
 RUN go mod download
 
 # install tools
@@ -84,8 +45,8 @@ RUN go generate internal/ui/login/statik/generate.go \
 ## generates grpc stub
 #######################
 FROM go-static AS go-stub
-COPY --from=base /proto /proto
+COPY --from=zitadel-base:local /proto /proto
-COPY --from=base /usr/local/bin /usr/local/bin/.
+COPY --from=zitadel-base:local /usr/local/bin /usr/local/bin/.
 
 COPY build/zitadel/generate-grpc.sh build/zitadel/generate-grpc.sh
 COPY internal/protoc internal/protoc
@@ -140,25 +101,3 @@ RUN go install github.com/rakyll/statik \
 #######################
 FROM scratch as go-codecov
 COPY --from=go-test /go/src/github.com/caos/zitadel/profile.cov profile.cov
-
-#######################
-## Go prod build
-#######################
-FROM go-test as prod-go-build
-ARG BUILDARCH
-ARG VERSION=""
-
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=${BUILDARCH} go build -a -installsuffix cgo -ldflags "-X main.version=${VERSION:-'dev'} -extldflags \"-static\"" -o zitadel-linux-${BUILDARCH} main.go
-
-#######################
-## Go dev build
-#######################
-FROM go-base as dev-go-build
-ENTRYPOINT [ "go", "run", "main.go" ]
-
-#######################
-## Only Copy Assets
-#######################
-FROM scratch as prod-go-export
-ARG BUILDARCH
-COPY --from=prod-go-build /go/src/github.com/caos/zitadel/zitadel-linux-${BUILDARCH} .

console/package.json

@@ -5,7 +5,7 @@
     "ng": "ng",
     "start": "ng serve",
     "build": "ng build",
-    "prodbuild": "ng build --configuration production",
+    "prodbuild": "ng build --aot=true --buildOptimizer=true",
     "lint": "ng lint && stylelint './src/**/*.scss' --syntax scss"
   },
   "private": true,

guides/development.md

@@ -24,7 +24,7 @@ This part is relevant if you start the backend or console without docker compose
 This command generates the grpc stub for console into the folder console/src/app/proto/generated for local development.
 
 ```bash
-DOCKER_BUILDKIT=1 docker build -f build/console/Dockerfile . -t zitadel:gen-fe --target npm-copy -o .
+DOCKER_BUILDKIT=1 docker build -f build/zitadel/Dockerfile . -t zitadel:gen-fe --target js-client -o .
 ```
 
 ### Start the Backend
@@ -33,7 +33,7 @@ With these commands you can generate the stub for the backend.
 
 ```bash
 # generates grpc stub
-DOCKER_BUILDKIT=1 docker build -f build/zitadel/Dockerfile . -t zitadel:gen-be --target go-copy -o .
+DOCKER_BUILDKIT=1 docker build -f build/zitadel/Dockerfile . -t zitadel:gen-be --target go-client -o .
 # generates keys for cryptography
 COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 \
 && docker compose -f ./build/local/docker-compose-local.yml --profile backend-stub up --exit-code-from keys

k8s/base/deployment.yaml

@@ -1,85 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: zitadel
-  labels:
-    app: zitadel
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: zitadel
-      version: v1
-  template:
-    metadata:
-      labels:
-        app: zitadel
-        version: v1
-    spec:
-      containers:
-      - name: zitadel
-        image: docker.pkg.github.com/caos/zitadel/zitadel:latest
-        imagePullPolicy: IfNotPresent
-        args: ["start"]
-        ports:
-        - name: grpc
-          containerPort: 50001
-        - name: http
-          containerPort: 50002
-        - name: ui
-          containerPort: 50003
-        readinessProbe:
-          httpGet:
-            path: /ready
-            port: http
-          periodSeconds: 5
-          failureThreshold: 2
-        livenessProbe:
-          httpGet:
-            path: /healthz
-            port: http
-          periodSeconds: 5
-          failureThreshold: 2
-        env:
-        - name: POD_IP
-          valueFrom:
-            fieldRef:
-              fieldPath: status.podIP
-        - name: CHAT_URL
-          valueFrom:
-            secretKeyRef:
-              name: zitadel-secrets-vars
-              key: ZITADEL_GOOGLE_CHAT_URL
-        - name: TWILIO_TOKEN
-          valueFrom:
-            secretKeyRef:
-              name: zitadel-secrets-vars
-              key: ZITADEL_TWILIO_AUTH_TOKEN
-        - name: TWILIO_SERVICE_SID
-          valueFrom:
-            secretKeyRef:
-              name: zitadel-secrets-vars
-              key: ZITADEL_TWILIO_SID
-        - name: SMTP_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              name: zitadel-secrets-vars
-              key: ZITADEL_EMAILAPPKEY
-        envFrom:
-          - configMapRef:
-              name: zitadel-vars
-        volumeMounts:
-          - name: zitadel-secret
-            mountPath: /secret
-          - name: console-config
-            mountPath: /console/environment.json
-            subPath: environment.json
-      imagePullSecrets:
-      - name: githubsecret
-      volumes:
-        - name: zitadel-secret
-          secret:
-            secretName: zitadel-secret
-        - name: console-config
-          configMap:
-            name: console-config

k8s/base/flow.yaml

@@ -1,18 +0,0 @@
-apiVersion: logging.banzaicloud.io/v1beta1
-kind: Flow
-metadata:
-    name: flow-zitadel
-spec:
-    filters:
-      - parser:
-            remove_key_name_field: true
-            reserve_data: true
-            parse:
-                type: logfmt
-      - tag_normaliser:
-            metadata: ${namespace}.${container}.${pod}
-            metadata_name: ${namespace_name}.${container_name}.${pod_name}
-    selectors:
-        app: zitadel
-    outputRefs:
-      - output-loki

k8s/base/kustomization.yaml

@@ -1,11 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-namespace: zitadel
-
-resources:
--  deployment.yaml
--  service-grpc.yaml
--  service-http.yaml
--  service-ui.yaml
--  flow.yaml

k8s/base/output.yaml

@@ -1,13 +0,0 @@
-
-apiVersion: logging.banzaicloud.io/v1beta1
-kind: Output
-metadata:
-  name: output-loki
-spec:
-  loki:
-    buffer:
-      timekey: 1m
-      timekey_use_utc: true
-      timekey_wait: 30s
-    configure_kubernetes_labels: true
-    url: http://loki.caos-system:3100

k8s/base/service-grpc.yaml

@@ -1,13 +0,0 @@
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: grpc-v1
-spec:
-  ports:
-  - name: grpc
-    port: 80
-    targetPort: grpc
-  selector:
-    app: zitadel
-    version: v1

k8s/base/service-http.yaml

@@ -1,13 +0,0 @@
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: http-v1
-spec:
-  ports:
-  - name: http
-    port: 80
-    targetPort: http
-  selector:
-    app: zitadel
-    version: v1

k8s/base/service-ui.yaml

@@ -1,13 +0,0 @@
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: ui-v1
-spec:
-  ports:
-  - name: ui
-    port: 80
-    targetPort: ui
-  selector:
-    app: zitadel
-    version: v1