chore: change build process to use local and server the same (#955)

* initital ide

* init

* it is alive

* go test is running

* wip with buildkit

* it definitly alive

* all done, now the github actions

* pretty

* before i delete the generated proto stub

* delete generated code

* mount of grpc does not yet work

* save before

* level of insanity 1000%

* huhur

* gh action new dockerfile

* fix

* fix

* fix

* fix

* amd64 only

* try caching

* try with reg

* test with artifact upload

* trial

* add comments

* publish branchname

* correct vars

* correct id

* mode max for cache

* remove unused code

* cleanup

* test action which uploads coverage

* use

* proper path

* debug file location

* path

* test if ci still works

* correct docker file name

.github/dependabot.yml

@@ -17,7 +17,7 @@ updates:
     prefix: chore
     include: scope
 - package-ecosystem: "docker"
-  directory: "/build/docker"
+  directory: "/build/"
   schedule:
     interval: "weekly"
   open-pull-requests-limit: 10

.github/scripts/githelper.sh

@@ -1,54 +0,0 @@
-#!/bin/bash
-#debugger
-set -x
-
-source ./.github/scripts/variables.env
-
-############################
-function setup_git {
-############################
-  echo "###############"
-  echo "set git config"
-  echo "###############"
-
-  git config --global user.email "$GIT_USER_MAIL"
-  git config --global user.name "$GIT_USER_NAME"
-}
-
-############################
-function checkout_project {
-############################
-  echo "###############"
-  echo "clone repository $GIT_URL"
-  echo "###############"
-
-  # clone opsrepo
-  git clone $GIT_URL $LOCAL_TMP_DIR/$GIT_OPSREPO
-}
-
-############################
-function change_image_version {
-############################
-  echo "###############"
-  echo "checkout master"
-  echo "###############"
-
-  cd $LOCAL_TMP_DIR/$GIT_OPSREPO/$GIT_OPSREPO_APPFOLDER/$GIT_OPSREPO_APPLICATION_NAME/overlay/$TARGET_ENVIRONMENT
-  git checkout master
-  git pull
-  echo "###############"
-  echo "change image version and commit"
-  echo "###############"
-  sed -i "s#image: $REGISTRY_IMAGE:.*#image: $REGISTRY_IMAGE:$CAOS_NEXT_VERSION#g" $GIT_OPSREPO_IMAGEFILE
-  git add $GIT_OPSREPO_IMAGEFILE
-  git commit --message "Github Workflow: $GITHUB_WORKFLOW"
-}
-
-############################
-function upload_files {
-############################
-  echo "###############"
-  echo "git push"
-  echo "###############"
-  git push --quiet --set-upstream origin
-}

.github/scripts/variables.env

@@ -1,21 +0,0 @@
-### local vars
-export LOCAL_TMP_DIR="/tmp"
-
-### git settings for cloning operations repository
-export GIT_USER_MAIL="hi@caos.ch"
-export GIT_USER_NAME="zitadel-pipeline"
-
-#path of opsrepository
-export GIT_URL="https://$GIT_OPSREPO_DEPLOYTOKEN@github.com/caos/zitadel-ops.git"
-export GIT_OPSREPO="citadel-ops"
-
-### application settings
-export GIT_OPSREPO_APPFOLDER="k8s/workload"
-export GIT_OPSREPO_APPLICATION_NAME="zitadel"
-export GIT_OPSREPO_IMAGEFILE="imageversion.yaml"
-export REGISTRY_IMAGE="$REGISTRY/$GITHUB_REPOSITORY/$IMAGE"
-
-### environment settings
-#export TARGET_ENVIRONMENT="dev"
-
-

.github/workflows/codecov.yml

@@ -0,0 +1,40 @@
+name: Code Coverage
+on: push
+
+env:
+  REGISTRY: ghcr.io
+  NODE_VERSION: '12'
+  GO_VERSION: '1.15'
+
+jobs:
+  container:
+    runs-on: ubuntu-18.04
+    steps:
+    - name: Source checkout
+      uses: actions/checkout@v2
+    - name: Cache Docker layers
+      uses: actions/cache@v2
+      with:
+        path: /tmp/.buildx-cache
+        key: ${{ runner.os }}-buildx-${{ github.sha }}
+        restore-keys: |
+          ${{ runner.os }}-buildx-
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+    - uses: docker/build-push-action@v2
+      with:
+        context: .
+        file: ./build/dockerfile
+        platforms: linux/amd64
+        tags: ${{ env.REGISTRY }}/${{ github.repository }}:coverage
+        push: false
+        cache-from: type=local,src=/tmp/.buildx-cache
+        cache-to: type=local,mode=max,dest=/tmp/.buildx-cache
+        target: go-codecov
+        outputs: type=local,dest=.
+    - uses: codecov/codecov-action@v1
+      with:
+        file: ./profile.cov
+        name: codecov-go

.github/workflows/release.yml

@@ -8,128 +8,49 @@ env:
   GO_VERSION: '1.15'
 
 jobs:
-
-  ## Angular test, will be added later
-
-  angular-lint: 
+  container:
     runs-on: ubuntu-18.04
-    defaults:
-      run:
-        working-directory: ./console
-    steps:
-    - uses: actions/checkout@v2
-    - name: Install Protoc
-      uses: arduino/setup-protoc@master
-      with:
-        version: '3.x'
-    - run: wget -O protoc-gen-grpc-web https://github.com/grpc/grpc-web/releases/download/1.2.0/protoc-gen-grpc-web-1.2.0-linux-x86_64
-    - run: sudo mv protoc-gen-grpc-web /usr/local/bin/protoc-gen-grpc-web
-    - run: sudo chmod +x /usr/local/bin/protoc-gen-grpc-web
-    - uses: actions/checkout@v2
-    - uses: actions/setup-node@v1
-      with:
-        node-version: ${{ env.NODE_VERSION }}
-    - run: npm ci
-    - run: npm run lint
-
-  angular-build:
-    runs-on: ubuntu-18.04
-    defaults:
-      run:
-        working-directory: ./console
-    steps:
-    - uses: actions/checkout@v2
-    - name: Install Protoc
-      uses: arduino/setup-protoc@master
-      with:
-        version: '3.x'
-    - run: wget -O protoc-gen-grpc-web https://github.com/grpc/grpc-web/releases/download/1.2.0/protoc-gen-grpc-web-1.2.0-linux-x86_64
-    - run: sudo mv protoc-gen-grpc-web /usr/local/bin/protoc-gen-grpc-web
-    - run: sudo chmod +x /usr/local/bin/protoc-gen-grpc-web
-    - uses: actions/setup-node@v1
-      with:
-        node-version: ${{ env.NODE_VERSION }}
-    - run: npm ci
-    - run: npm run prodbuild
-    - uses: actions/upload-artifact@v1
-      with:
-        name: angular
-        path: console/dist/console
-
-  go-test:
-    runs-on: ubuntu-18.04
-    steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-go@v2-beta
-      with:
-        go-version: ${{ env.GO_VERSION }}
-    - run: go test -race -v -coverprofile=profile.cov ./...
-    - uses: actions/upload-artifact@v1
-      with:
-        name: go-coverage
-        path: profile.cov
-    - uses: codecov/codecov-action@v1
-      with:
-        file: ./profile.cov
-        name: codecov-go
-      
-  ## go lint, will be added later
-  
-  go-build:
-    runs-on: ubuntu-18.04
-    needs: [angular-build, angular-lint, go-test] ### We need the artifact from the angular build and that's why we wait here
-    name: Build ${{ matrix.goos }}-${{ matrix.goarch }}
-    strategy:
-      matrix:
-        goos: [ 'linux', 'darwin', 'windows' ]
-        goarch: ['amd64']
-    steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-go@v2-beta
-      with:
-        go-version: ${{ env.GO_VERSION }}
-    - uses: actions/download-artifact@v2
-      with:
-        name: angular
-        path: console/dist/console
-    - run: go get github.com/rakyll/statik
-    - run: ./build/console/generate-static.sh
-    - run: cat internal/ui/console/statik/statik.go
-    - run: ./build/login/generate-static.sh
-    - run: cat internal/ui/login/statik/statik.go
-    - run: ./build/notification/generate-static.sh
-    - run: cat internal/notification/statik/statik.go
-    - run: ./build/zitadel/generate-static.sh
-    - run: cat internal/statik/statik.go
-    - run: CGO_ENABLED=0 GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o zitadel-${{ matrix.goos }}-${{ matrix.goarch }} cmd/zitadel/main.go
-    - uses: actions/upload-artifact@v1
-      with:
-        name: zitadel-${{ matrix.goos }}-${{ matrix.goarch }}
-        path: zitadel-${{ matrix.goos }}-${{ matrix.goarch }}
-
-  container-prod:
-    runs-on: ubuntu-18.04
-    needs: go-build
     steps:
     - name: Source checkout
       uses: actions/checkout@v2
-    - uses: actions/download-artifact@v2
+    - name: Set output
+      id: branch
+      run: echo ::set-output name=short_ref::${GITHUB_REF#refs/*/}
+    - name: Check output
+      run: echo ${{ steps.branch.outputs.short_ref }}
+    - name: Generate Short SHA Container Tag
+      id: vars
+      run: echo "::set-output name=sha_short::SHA-$(git rev-parse --short HEAD)"
+    - name: Cache Docker layers
+      uses: actions/cache@v2
       with:
-        name: zitadel-linux-amd64
-        path: .artifacts
-    - uses: docker/build-push-action@v1
+        path: /tmp/.buildx-cache
+        key: ${{ runner.os }}-buildx-${{ github.sha }}
+        restore-keys: |
+          ${{ runner.os }}-buildx-
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+    - name: Login to DockerHub
+      uses: docker/login-action@v1 
       with:
-        dockerfile: build/docker/Dockerfile
         username: ${{ github.actor }}
         password: ${{ secrets.CR_PAT }}
         registry: ${{ env.REGISTRY }}
-        repository: ${{ github.repository }}
-        tag_with_ref: true
-        tag_with_sha: true
+    - uses: docker/build-push-action@v2
+      with:
+        context: .
+        file: ./build/dockerfile
+        platforms: linux/amd64
+        tags: ${{ env.REGISTRY }}/${{ github.repository }}:${{ steps.vars.outputs.sha_short }},${{ env.REGISTRY }}/${{ github.repository }}:${{ steps.branch.outputs.short_ref }}
+        push: true
+        cache-from: type=local,src=/tmp/.buildx-cache
+        cache-to: type=local,mode=max,dest=/tmp/.buildx-cache
 
   release:
     runs-on: ubuntu-18.04
-    needs: [container-prod]
+    needs: [container]
     env:
       DOCKER_USERNAME: ${{ github.actor }}
       DOCKER_PASSWORD: ${{ secrets.CR_PAT }}
@@ -139,12 +60,6 @@ jobs:
     - name: Generate Short SHA Container Tag
       id: vars
       run: echo "::set-output name=sha_short::SHA-$(git rev-parse --short HEAD)"
-    - uses: actions/download-artifact@v2
-      with:
-        path: .artifacts
-    - name: Display structure of downloaded files
-      run: ls -R
-      working-directory: .artifacts
     - name: Docker Login
       run: docker login $REGISTRY -u $GITHUB_ACTOR -p $GITHUB_TOKEN
     - name: Docker Pull short-sha