feat(operator): zitadel and database operator (#1208)

* feat(operator): add base for zitadel operator

* fix(operator): changed pipeline to release operator

* fix(operator): fmt with only one parameter

* fix(operator): corrected workflow job name

* fix(zitadelctl): added restore and backuplist command

* fix(zitadelctl): scale for restore

* chore(container): use scratch for deploy container

* fix(zitadelctl): limit image to scratch

* fix(migration): added migration scripts for newer version

* fix(operator): changed handling of kubeconfig in operator logic

* fix(operator): changed handling of secrets in operator logic

* fix(operator): use new version of zitadel

* fix(operator): added path for migrations

* fix(operator): delete doublets of migration scripts

* fix(operator): delete subpaths and integrate logic into init container

* fix(operator): corrected path in dockerfile for local migrations

* fix(operator): added migrations for cockroachdb-secure

* fix(operator): delete logic for ambassador module

* fix(operator): added read and write secret commands

* fix(operator): correct and align operator pipeline with zitadel pipeline

* fix(operator): correct yaml error in operator pipeline

* fix(operator): correct action name in operator pipeline

* fix(operator): correct case-sensitive filename in operator pipeline

* fix(operator): upload artifacts from buildx output

* fix(operator): corrected attribute spelling error

* fix(operator): combined jobs for operator binary and image

* fix(operator): added missing comma in operator pipeline

* fix(operator): added codecov for operator image

* fix(operator): added codecov for operator image

* fix(testing): code changes for testing and several unit-tests (#1009)

* fix(operator): usage of interface of kubernetes client for testing and several unit-tests

* fix(operator): several unit-tests

* fix(operator): several unit-tests

* fix(operator): changed order for the operator logic

* fix(operator): added version of zitadelctl from semantic release

* fix(operator): corrected function call with version of zitadelctl

* fix(operator): corrected function call with version of zitadelctl

* fix(operator): add check output to operator release pipeline

* fix(operator): set --short length everywhere to 12

* fix(operator): zitadel setup in job instead of exec with several unit tests

* fix(operator): fixes to combine newest zitadel and testing branch

* fix(operator): corrected path in Dockerfile

* fix(operator): fixed unit-test that was ignored during changes

* fix(operator): fixed unit-test that was ignored during changes

* fix(operator): corrected Dockerfile to correctly use env variable

* fix(operator): quickfix takeoff deployment

* fix(operator): corrected the clusterrolename in the applied artifacts

* fix: update secure migrations

* fix(operator): migrations (#1057)

* fix(operator): copied migrations from orbos repository

* fix(operator): newest migrations

* chore: use cockroach-secure

* fix: rename migration

* fix: remove insecure cockroach migrations

Co-authored-by: Stefan Benz <stefan@caos.ch>

* fix: finalize labels

* fix(operator): cli logging concurrent and fixe deployment of operator during restore

* fix: finalize labels and cli commands

* fix: restore

* chore: cockroachdb is always secure

* chore: use orbos consistent-labels latest commit

* test: make tests compatible with new labels

* fix: default to sa token for start command

* fix: use cockroachdb v12.02

* fix: don't delete flyway user

* test: fix migration test

* fix: use correct table qualifiers

* fix: don't alter sequence ownership

* fix: upgrade flyway

* fix: change ownership of all dbs and tables to admin user

* fix: change defaultdb user

* fix: treat clientid status codes >= 400 as errors

* fix: reconcile specified ZITADEL version, not binary version

* fix: add ca-certs

* fix: use latest orbos code

* fix: use orbos with fixed race condition

* fix: use latest ORBOS code

* fix: use latest ORBOS code

* fix: make migration and scaling around restoring work

* fix(operator): move zitadel operator

* chore(migrations): include owner change migration

* feat(db): add code base for database operator

* fix(db): change used image registry for database operator

* fix(db): generated mock

* fix(db): add accidentally ignored file

* fix(db): add cockroachdb backup image to pipeline

* fix(db): correct pipeline and image versions

* fix(db): correct version of used orbos

* fix(db): correct database import

* fix(db): go mod tidy

* fix(db): use new version for orbos

* fix(migrations): include migrations into zitadelctl binary (#1211)

* fix(db): use statik to integrate migrations into binary

* fix(migrations): corrections unit tests and pipeline for integrated migrations into zitadelctl binary

* fix(migrations): correction in dockerfile for pipeline build

* fix(migrations): correction in dockerfile for pipeline build

* fix(migrations):  dockerfile changes for cache optimization

* fix(database): correct used part-of label in database operator

* fix(database): correct used selectable label in zitadel operator

* fix(operator): correct lables for user secrets in zitadel operator

* fix(operator): correct lables for service test in zitadel operator

* fix: don't enable database features for user operations (#1227)

* fix: don't enable database features for user operations

* fix: omit database feature for connection info adapter

* fix: use latest orbos version

* fix: update ORBOS (#1240)

Co-authored-by: Florian Forster <florian@caos.ch>
Co-authored-by: Elio Bischof <eliobischof@gmail.com>

.github/dependabot.yml

@@ -17,7 +17,15 @@ updates:
     prefix: chore
     include: scope
 - package-ecosystem: "docker"
-  directory: "/build/"
+  directory: "/build"
+  schedule:
+    interval: "weekly"
+  open-pull-requests-limit: 10
+  commit-message:
+    prefix: chore
+    include: scope
+- package-ecosystem: "docker"
+  directory: "/build/operator"
   schedule:
     interval: "weekly"
   open-pull-requests-limit: 10

.github/workflows/codecov.yml

@@ -19,6 +19,13 @@ jobs:
         key: ${{ runner.os }}-buildx-${{ github.sha }}
         restore-keys: |
           ${{ runner.os }}-buildx-
+    - name: Cache Docker layers
+      uses: actions/cache@v2
+      with:
+        path: /tmp/.buildx-cache-op
+        key: ${{ runner.os }}-buildx-op-${{ github.sha }}
+        restore-keys: |
+          ${{ runner.os }}-buildx-op-
     - name: Set up QEMU
       uses: docker/setup-qemu-action@v1
     - name: Set up Docker Buildx
@@ -32,8 +39,18 @@ jobs:
         push: false
         cache-from: type=local,src=/tmp/.buildx-cache
         target: go-codecov
-        outputs: type=local,dest=.
+        outputs: type=local,dest=/tmp/zitadel
+    - uses: docker/build-push-action@v2
+      with:
+        context: .
+        file: ./build/operator/Dockerfile
+        platforms: linux/amd64
+        tags: ${{ env.REGISTRY }}/${{ github.repository }}:coverage
+        push: false
+        cache-from: type=local,src=/tmp/.buildx-cache-op
+        target: go-codecov
+        outputs: type=local,dest=/tmp/operator
     - uses: codecov/codecov-action@v1
       with:
-        file: ./profile.cov
+        files: /tmp/zitadel/profile.cov,/tmp/operator/profile.cov
         name: codecov-go

.github/workflows/operator.yml

@@ -0,0 +1,165 @@
+name: Operator Release
+on: push
+
+env:
+  GITHUB_TOKEN: ${{ secrets.CR_PAT }}
+  REGISTRY: ghcr.io
+  GO_VERSION: '1.15'
+  DOCKER_IMAGE_NAME: ${{ github.repository }}-operator
+  BACKUP_IMAGE_NAME: ${{ github.repository }}-crbackup
+
+jobs:
+  container:
+    runs-on: ubuntu-18.04
+    name: Build ${{ matrix.goos }}-${{ matrix.goarch }}
+    strategy:
+      matrix:
+        goos: [ 'linux', 'darwin', 'windows' ]
+        goarch: [ 'amd64' ]
+    steps:
+      - name: Source checkout
+        uses: actions/checkout@v2
+      - name: Set output
+        id: branch
+        run: echo ::set-output name=short_ref::${GITHUB_REF#refs/*/}
+      - name: Semantic Release
+        id: semantic
+        uses: cycjimmy/semantic-release-action@v2
+        with:
+          dry_run: true
+          semantic_version: 17.0.4
+      - name: Set version
+        id: version
+        run: |
+          if ${{ steps.semantic.outputs.new_release_published == 'true' }}; then
+            echo ::set-output name=version::${{ steps.semantic.outputs.new_release_version }}
+          else
+            echo ::set-output name=version::${{ steps.branch.outputs.short_ref }}
+          fi
+      - name: Check outputs
+        run: |
+          echo ${{ steps.branch.outputs.short_ref }}
+          echo ${{ steps.version.outputs.version }}
+      - name: Generate Short SHA Container Tag
+        id: vars
+        run: echo "::set-output name=sha_short::SHA-$(git rev-parse --short=12 HEAD)"
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-op-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-op-
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ github.actor }}
+          password: ${{ secrets.CR_PAT }}
+          registry: ${{ env.REGISTRY }}
+      - uses: docker/build-push-action@v2
+        name: onlybuild
+        with:
+          context: .
+          file: ./build/operator/Dockerfile
+          platforms: linux/amd64
+          tags: ${{ env.REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}:${{ steps.vars.outputs.sha_short }},${{ env.REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}:${{ steps.branch.outputs.short_ref }}
+          push: false
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache
+          outputs: type=local,dest=/tmp/operator
+          build-args: |
+            OS=${{ matrix.goos }}
+            ARCH=${{ matrix.goarch }}
+            VERSION=${{ steps.version.outputs.version }}
+      - uses: actions/upload-artifact@v2
+        with:
+          name: zitadelctl-${{ matrix.goos }}-${{ matrix.goarch }}
+          path: /tmp/operator/zitadelctl
+      - uses: docker/build-push-action@v2
+        if: ${{ matrix.goos == 'linux' && matrix.goarch == 'amd64' }}
+        name: buildandpush
+        with:
+          context: .
+          file: ./build/operator/Dockerfile
+          platforms: linux/amd64
+          tags: ${{ env.REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}:${{ steps.vars.outputs.sha_short }},${{ env.REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}:${{ steps.branch.outputs.short_ref }}
+          push: true
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache
+          build-args: |
+            OS=${{ matrix.goos }}
+            ARCH=${{ matrix.goarch }}
+            VERSION=${{ steps.version.outputs.version }}
+      - uses: docker/build-push-action@v2
+        if: ${{ matrix.goos == 'linux' && matrix.goarch == 'amd64' }}
+        name: buildandpushcrbackup
+        with:
+          context: .
+          file: ./build/cr-backup/Dockerfile
+          platforms: linux/amd64
+          tags: ${{ env.REGISTRY }}/${{ env.BACKUP_IMAGE_NAME }}:${{ steps.vars.outputs.sha_short }},${{ env.REGISTRY }}/${{ env.BACKUP_IMAGE_NAME }}:${{ steps.branch.outputs.short_ref }}
+          push: true
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache
+
+  release:
+    runs-on: ubuntu-18.04
+    needs: [ container ]
+    env:
+      DOCKER_USERNAME: ${{ github.actor }}
+      DOCKER_PASSWORD: ${{ secrets.CR_PAT }}
+    steps:
+      - name: Source checkout
+        uses: actions/checkout@v2
+      - name: Generate Short SHA Container Tag
+        id: vars
+        run: echo "::set-output name=sha_short::SHA-$(git rev-parse --short=12 HEAD)"
+      - name: Check output
+        run: echo ${{ steps.vars.outputs.sha_short }}
+      - name: Docker Login
+        run: docker login $REGISTRY -u $GITHUB_ACTOR -p $GITHUB_TOKEN
+      - name: Docker Pull short-sha
+        run: docker pull $REGISTRY/$DOCKER_IMAGE_NAME:${{ steps.vars.outputs.sha_short }}
+      - name: Docker Pull short-sha
+        run: docker pull $REGISTRY/$BACKUP_IMAGE_NAME:${{ steps.vars.outputs.sha_short }}
+      - name: Download binaries
+        uses: actions/download-artifact@v2
+        with:
+          path: .artifacts
+      - name: Semantic Release
+        id: semantic
+        uses: cycjimmy/semantic-release-action@v2
+        with:
+          dry_run: false
+          semantic_version: 17.0.4
+      - name: Do something when a new release published
+        if: steps.semantic.outputs.new_release_published == 'true'
+        run: |
+          echo ${{ steps.semantic.outputs.new_release_version }}
+          echo ${{ steps.semantic.outputs.new_release_major_version }}
+          echo ${{ steps.semantic.outputs.new_release_minor_version }}
+          echo ${{ steps.semantic.outputs.new_release_patch_version }}
+      - name: Docker Tag Version
+        run: |
+          docker tag $REGISTRY/$DOCKER_IMAGE_NAME:${{ steps.vars.outputs.sha_short }} $REGISTRY/$DOCKER_IMAGE_NAME:${{ steps.semantic.outputs.new_release_version }}
+          docker tag $REGISTRY/$BACKUP_IMAGE_NAME:${{ steps.vars.outputs.sha_short }} $REGISTRY/$BACKUP_IMAGE_NAME:${{ steps.semantic.outputs.new_release_version }}
+        if: steps.semantic.outputs.new_release_published == 'true'
+      - name: Docker Tag Latest
+        run: |
+          docker tag $REGISTRY/$DOCKER_IMAGE_NAME:${{ steps.vars.outputs.sha_short }} $REGISTRY/$DOCKER_IMAGE_NAME:latest
+          docker tag $REGISTRY/$BACKUP_IMAGE_NAME:${{ steps.vars.outputs.sha_short }} $REGISTRY/$BACKUP_IMAGE_NAME:latest
+        if: steps.semantic.outputs.new_release_published == 'true'
+      - name: Docker Push Version
+        run: |
+          docker push $REGISTRY/$DOCKER_IMAGE_NAME:${{ steps.semantic.outputs.new_release_version }}
+          docker push $REGISTRY/$BACKUP_IMAGE_NAME:${{ steps.semantic.outputs.new_release_version }}
+        if: steps.semantic.outputs.new_release_published == 'true'
+      - name: Docker Push Latest
+        run: |
+          docker push $REGISTRY/$DOCKER_IMAGE_NAME:latest
+          docker push $REGISTRY/$BACKUP_IMAGE_NAME:latest
+        if: steps.semantic.outputs.new_release_published == 'true'

.github/workflows/zitadel.yml

@@ -1,4 +1,4 @@
-name: Release
+name: Zitadel Release
 on: push
 
 env:
@@ -20,7 +20,7 @@ jobs:
       run: echo ${{ steps.branch.outputs.short_ref }}
     - name: Generate Short SHA Container Tag
       id: vars
-      run: echo "::set-output name=sha_short::SHA-$(git rev-parse --short HEAD)"
+      run: echo "::set-output name=sha_short::SHA-$(git rev-parse --short=12 HEAD)"
     - name: Cache Docker layers
       uses: actions/cache@v2
       with:
@@ -33,7 +33,7 @@ jobs:
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v1
     - name: Login to DockerHub
-      uses: docker/login-action@v1 
+      uses: docker/login-action@v1
       with:
         username: ${{ github.actor }}
         password: ${{ secrets.CR_PAT }}
@@ -59,7 +59,7 @@ jobs:
       uses: actions/checkout@v2
     - name: Generate Short SHA Container Tag
       id: vars
-      run: echo "::set-output name=sha_short::SHA-$(git rev-parse --short HEAD)"
+      run: echo "::set-output name=sha_short::SHA-$(git rev-parse --short=12 HEAD)"
     - name: Docker Login
       run: docker login $REGISTRY -u $GITHUB_ACTOR -p $GITHUB_TOKEN
     - name: Docker Pull short-sha