chore(ci): release pipeline for v2 (#3197)

* not working * test ci * correct path * test * test * test * remove k8s & test * test * test image * typo * test * fix * test * fix * test * fix * fix windows * fix * fix * fix codecov * fix with arch * test * test * test * test * trial * trial * trial * trial * trial * trial * trial * try * try * first improvement * trial * improve * improve * improve * use bash * trial * random test * wip * test with goreleaser and semrel * fix typo * trial * add buildx * define default * force buildkit * use buildx * debug GH docker * use images * debug docker * work around image name issues * trial * test * test with prebuilt image * fix * use load * test * use only docker * trial * test * use load * test * test with docker driver * trial * trial * trial * use arm * upload artifacts * try to cache grpc base * try to use a cache * fall back * test * fix * improve * upload artifacts * fix * name things * add tag if poc * args for protos * remove v2 build Co-authored-by: Livio Amstutz <livio.a@gmail.com>
2025-08-11 14:37:34 +00:00 · 2022-02-14 15:35:16 +01:00
parent 2d208dd8ba
commit 2f3a482ade
27 changed files with 217 additions and 17490 deletions
--- a/build/Dockerfile
+++ b/build/Dockerfile
@@ -2,12 +2,8 @@
 ## Final Production Image
 #######################
 FROM alpine:3 as artifact
-ARG BUILDARCH
+COPY zitadel /app/zitadel
 RUN adduser -D zitadel
-COPY cmd/zitadel/*.yaml /app/
-COPY .download/zitadel/zitadel-linux-${BUILDARCH} /app/zitadel
-COPY .download/zitadel/console /app/console/
-RUN chmod a+x /app/zitadel

 #######################
 ## Scratch Image
--- a/build/console/Dockerfile
+++ b/build/console/Dockerfile
@@ -1,49 +1,5 @@
 ARG NODE_VERSION=14

-#######################
-## These steps set platform / arch type specific variables
-#######################
-FROM alpine:3 AS arm64-base
-ENV PROTOC_ARCH aarch_64
-
-FROM alpine:3 AS amd64-base
-ENV PROTOC_ARCH x86_64
-
-#######################
-## This step sets up the folder structure,
-## initalices go mods,
-## downloads the protofiles,
-## protoc and protoc-gen-grpc-web for later use
-#######################
-FROM ${BUILDARCH}-base AS base
-ARG PROTOC_VERSION=3.13.0
-ARG PROTOC_ZIP=protoc-${PROTOC_VERSION}-linux-${PROTOC_ARCH}.zip
-ARG GRPC_WEB_VERSION=1.3.0
-# no arm specific version available and x86 works fine at the moment:
-ARG GRPC_WEB=protoc-gen-grpc-web-${GRPC_WEB_VERSION}-linux-x86_64
-
-RUN apk add tar curl
-WORKDIR /proto
-
-#protoc
-RUN apk add tar curl \
-    && curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/$PROTOC_ZIP \
-    && unzip -o $PROTOC_ZIP -d /usr/local bin/protoc \
-    && unzip -o $PROTOC_ZIP -d /proto include/* \
-    && rm -f $PROTOC_ZIP \
-    && curl -OL https://github.com/grpc/grpc-web/releases/download/${GRPC_WEB_VERSION}/${GRPC_WEB} \
-    && mv ${GRPC_WEB} /usr/local/bin/protoc-gen-grpc-web \
-    && chmod +x /usr/local/bin/protoc-gen-grpc-web \
-    && curl https://raw.githubusercontent.com/envoyproxy/protoc-gen-validate/v0.6.2/validate/validate.proto --create-dirs -o include/validate/validate.proto  \
-    && curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v2.2.0/protoc-gen-openapiv2/options/annotations.proto --create-dirs -o include/protoc-gen-openapiv2/options/annotations.proto \
-    && curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v2.2.0/protoc-gen-openapiv2/options/openapiv2.proto --create-dirs -o include/protoc-gen-openapiv2/options/openapiv2.proto \
-    && curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/annotations.proto --create-dirs -o include/google/api/annotations.proto \
-    && curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto --create-dirs -o include/google/api/http.proto \
-    && curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/field_behavior.proto --create-dirs -o include/google/api/field_behavior.proto
-
-#zitadel protos
-COPY proto/ include/.
-
 #######################
 ## With this step we prepare all node_modules, this helps caching the build
 ## Speed up this step by mounting your local node_modules directory
@@ -52,11 +8,11 @@ FROM node:${NODE_VERSION} as npm-base
 WORKDIR /console

 COPY console/package.json console/package-lock.json ./
-RUN npm install
+RUN npm ci

 COPY console .
-COPY --from=base /proto /proto
-COPY --from=base /usr/local/bin /usr/local/bin/.
+COPY --from=zitadel-base:local /proto /proto
+COPY --from=zitadel-base:local /usr/local/bin /usr/local/bin/.
 COPY build/console build/console/
 RUN build/console/generate-grpc.sh

@@ -64,23 +20,18 @@ RUN build/console/generate-grpc.sh
 ## copy for local dev
 #######################
 FROM scratch as npm-copy
-COPY --from=npm-base /console/src/app/proto/generated ./console/src/app/proto/generated
-
-#######################
-## angular dev build
-#######################
-FROM npm-base as dev-angular-build
-RUN npm install -g @angular/cli
+COPY --from=npm-base /console/src/app/proto/generated .

 #######################
 ## angular lint workspace and prod build
 #######################
-FROM npm-base as prod-angular-build
+FROM npm-base as angular-build
 RUN npm run lint
 RUN npm run prodbuild
+RUN ls -la /console/dist/console

 #######################
 ## Only Copy Assets
 #######################
-FROM scratch as prod-angular-export
-COPY --from=prod-angular-build /console/dist/console .
+FROM scratch as angular-export
+COPY --from=angular-build /console/dist/console .
--- a/build/console/generate-grpc.sh
+++ b/build/console/generate-grpc.sh
@@ -1,4 +1,4 @@
-#! /bin/sh
+#!/bin/sh

 set -eux

--- a/build/grpc/Dockerfile
+++ b/build/grpc/Dockerfile
@@ -0,0 +1,47 @@
+#ARG BUILDARCH=x86_64
+
+#######################
+## These steps set platform / arch type specific variables
+#######################
+FROM alpine:3 AS arm64-base
+ENV PROTOC_ARCH aarch_64
+
+FROM alpine:3 AS amd64-base
+ENV PROTOC_ARCH x86_64
+
+#######################
+## This step sets up the folder structure,
+## initalices go mods,
+## downloads the protofiles,
+## protoc and protoc-gen-grpc-web for later use
+#######################
+FROM ${BUILDARCH}-base
+ARG PROTOC_VERSION=3.18.0
+ARG PROTOC_ZIP=protoc-${PROTOC_VERSION}-linux-${PROTOC_ARCH}.zip
+ARG GRPC_WEB_VERSION=1.3.0
+ARG GATEWAY_VERSION=2.6.0
+ARG VALIDATOR_VERSION=0.6.2
+# no arm specific version available and x86 works fine at the moment:
+ARG GRPC_WEB=protoc-gen-grpc-web-${GRPC_WEB_VERSION}-linux-x86_64
+
+RUN apk add tar curl
+WORKDIR /proto
+
+#protoc
+RUN apk add tar curl \
+    && curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/$PROTOC_ZIP \
+    && unzip -o $PROTOC_ZIP -d /usr/local bin/protoc \
+    && unzip -o $PROTOC_ZIP -d /proto include/* \
+    && rm -f $PROTOC_ZIP \
+    && curl -OL https://github.com/grpc/grpc-web/releases/download/${GRPC_WEB_VERSION}/${GRPC_WEB} \
+    && mv ${GRPC_WEB} /usr/local/bin/protoc-gen-grpc-web \
+    && chmod +x /usr/local/bin/protoc-gen-grpc-web \
+    && curl https://raw.githubusercontent.com/envoyproxy/protoc-gen-validate/v${VALIDATOR_VERSION}/validate/validate.proto --create-dirs -o include/validate/validate.proto  \
+    && curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v${GATEWAY_VERSION}/protoc-gen-openapiv2/options/annotations.proto --create-dirs -o include/protoc-gen-openapiv2/options/annotations.proto \
+    && curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v${GATEWAY_VERSION}/protoc-gen-openapiv2/options/openapiv2.proto --create-dirs -o include/protoc-gen-openapiv2/options/openapiv2.proto \
+    && curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/annotations.proto --create-dirs -o include/google/api/annotations.proto \
+    && curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto --create-dirs -o include/google/api/http.proto \
+    && curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/field_behavior.proto --create-dirs -o include/google/api/field_behavior.proto
+
+#zitadel protos
+COPY proto/ include/.
--- a/build/local/Dockerfile.clientid
+++ b/build/local/Dockerfile.clientid
@@ -1,11 +0,0 @@
-FROM ubuntu:latest AS client-id
-#install dependencies
-RUN apt-get update \
-    && apt-get install curl -y \
-    && apt-get install jq -y
-
-#prepare script
-COPY build/local/clientid.sh clientid.sh
-RUN chmod +x /clientid.sh
-
-ENTRYPOINT [ "/clientid.sh" ]
--- a/build/local/Dockerfile.gateway
+++ b/build/local/Dockerfile.gateway
@@ -1,33 +0,0 @@
-# copy from https://raw.githubusercontent.com/grpc/grpc-web/master/net/grpc/gateway/docker/grpcwebproxy/Dockerfile
-
-FROM golang:1.17-alpine3.13
-
-RUN apk add --no-cache curl git ca-certificates && \
-  rm -rf /var/lib/apt/lists/*
-
-ARG VERSION=0.14.0
-
-WORKDIR /tmp
-
-RUN curl -sS https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
-RUN wget https://github.com/improbable-eng/grpc-web/archive/v$VERSION.tar.gz
-
-WORKDIR /go/src/github.com/improbable-eng/
-
-RUN tar -zxf /tmp/v$VERSION.tar.gz -C .
-RUN mv grpc-web-$VERSION grpc-web
-
-WORKDIR /go/src/github.com/improbable-eng/grpc-web
-
-RUN dep ensure && \
-  go env -w GO111MODULE=auto && \
-  go install ./go/grpcwebproxy
-
-ENV BKD_HOST=backend-run
-ENV BKD_PORT=50001
-
-ENTRYPOINT [ "/bin/sh", "-c", "exec /go/bin/grpcwebproxy \
-  --backend_addr=${BKD_HOST}:${BKD_PORT} \
-  --run_tls_server=false \
-  --use_websockets \
-  --allow_all_origins " ]
--- a/build/local/Dockerfile.keys
+++ b/build/local/Dockerfile.keys
@@ -1,7 +0,0 @@
-FROM alpine:latest AS gen-keys
-COPY build/local/keys.sh keys.sh
-RUN chmod +x /keys.sh
-ENTRYPOINT [ "/keys.sh" ]
-
-FROM scratch AS copy-keys
-COPY --from=gen-keys /.keys /.keys
--- a/build/local/clientid.sh
+++ b/build/local/clientid.sh
@@ -1,18 +0,0 @@
-#!/bin/bash
-
-# ------------------------------
-# sets the client id in environment.json
-# ------------------------------
-
-clientid=""
-while [ -z $clientid ]; do
-    echo "no from zitadel ==> retrying in 5 seconds"
-    sleep 5
-    clientid=$(curl -s http://${HOST}:${PORT}/clientID)
-    if [[ "$clientid" != *@zitadel* ]]; then
-        echo "invalid response from zitadel ==> retrying in 5 seconds"
-        clientid=""
-    fi
-done
-
-echo "$(jq ".clientid = $clientid" /environment.json)" > environment.json
--- a/build/local/docker-compose-local.yml
+++ b/build/local/docker-compose-local.yml
@@ -2,7 +2,6 @@ version: '3.8'

 services:
  db:
-    profiles: ['backend', 'storage']
    restart: always
    networks:
      - zitadel
@@ -18,106 +17,11 @@ services:
      - 8080:8080
      - 26257:26257

-# schema changes on the database
-  db-migrations:
-    profiles: ['backend', 'storage']
-    restart: on-failure
-    networks:
-      - zitadel
-    depends_on:
-      db: 
-        condition: service_healthy
-    image: flyway/flyway:latest
-    volumes:
-      - ../../migrations/cockroach:/flyway/sql
-    environment:
-      - FLYWAY_PLACEHOLDERS_eventstorepassword=NULL
-      - FLYWAY_PLACEHOLDERS_managementpassword=NULL
-      - FLYWAY_PLACEHOLDERS_adminapipassword=NULL
-      - FLYWAY_PLACEHOLDERS_authpassword=NULL
-      - FLYWAY_PLACEHOLDERS_notificationpassword=NULL
-      - FLYWAY_PLACEHOLDERS_authzpassword=NULL
-      - FLYWAY_PLACEHOLDERS_queriespassword=NULL
-    command: -url=jdbc:postgresql://db:26257/defaultdb -user=root -password= -connectRetries=5 migrate
-
-# minio is used to store assets
-  minio:
-    profiles: ['backend', 'storage']
-    image: minio/minio:RELEASE.2021-06-14T01-29-23Z
-    restart: on-failure
-    networks:
-      - zitadel
-    environment:
-      - MINIO_ACCESS_KEY=access_key
-      - MINIO_SECRET_KEY=secret_key
-    healthcheck:
-      test: ['CMD', 'curl', '-f', 'http://localhost:9000/minio/health/live']
-      interval: 30s
-      timeout: 5s
-      retries: 3
-      start_period: 5s
-    command:
-      - gateway
-      - nas
-      - /export
-
-# ZITADEL needs several keys to encrypt data
-# this container generates the required keys
-# and stores them into zitadel/.keys
-  keys:
-    profiles: ['backend', 'backend-stub']
-    restart: on-failure
-    networks:
-      - zitadel
-    build:
-      context: ../..
-      dockerfile: build/local/Dockerfile.keys
-      target: gen-keys
-    volumes:
-      - ../../.:/zitadel
-    env_file:
-      - ./local.env
-
-# To interact with ZITADEL requires some data setted up.
-# Due to the evolution of ZITADEL it's required to add additional 
-# setup steps, because of this fact it's recommended to rerun the setup
-# on each restart, at least after a new version got released
-  backend-setup:
-    profiles: ['backend']
-    restart: on-failure
-    networks:
-      - zitadel
-    depends_on:
-      db-migrations:
-        condition: service_completed_successfully
-      keys:
-        condition: service_completed_successfully
-    build:
-      context: ../..
-      dockerfile: build/zitadel/Dockerfile
-      target: dev-go-build
-      args:
-        ENV: dev
-    volumes:
-      - ../../.keys:/go/src/github.com/caos/zitadel/.keys
-    env_file:
-      - ./local.env
-    environment:
-      - ZITADEL_EVENTSTORE_HOST=db
-    command:
-      [
-        '-setup-files=cmd/zitadel/setup.yaml',
-        '-setup-files=cmd/zitadel/system-defaults.yaml',
-        '-setup-files=cmd/zitadel/authz.yaml',
-        'setup',
-      ]
-
 # starts the backend (API's) of ZITADEL
 # Port 50001 serves the GRPC API
 # Port 50002 serves the REST API
 # Port 50003 serves the login GUI
  backend-run:
-    profiles: ['backend']
    restart: on-failure
    networks:
      - zitadel
@@ -130,99 +34,15 @@ services:
    depends_on:
      db:
        condition: service_healthy
-      minio: 
-        condition: service_healthy
-      backend-setup:
-        condition: service_completed_successfully
    build:
      context: ../..
      dockerfile: build/zitadel/Dockerfile
      target: dev-go-build
-      args:
-        ENV: dev
-    volumes:
-      - ../../.keys:/go/src/github.com/caos/zitadel/.keys
-      - ../../.notifications:/go/src/github.com/caos/zitadel/.notifications
-    env_file:
-      - ./local.env
    environment:
      - ZITADEL_EVENTSTORE_HOST=db
    ports:
      - 50001:50001
-      - 50002:50002
-      - 50003:50003
    command:
      [
-        '-console=false',
-        '-localDevMode=true',
-        '-config-files=cmd/zitadel/startup.yaml',
-        '-config-files=cmd/zitadel/system-defaults.yaml',
-        '-config-files=cmd/zitadel/authz.yaml',
        'start',
      ]
-
-# the GRPC web gateway proxies the 
-# GRPC web calls to GRPC
-# it's used in console (frontend) for example
-  grpc-web-gateway:
-    profiles: ['backend']
-    restart: on-failure
-    logging:
-      driver: none
-    depends_on:
-      backend-run:
-        condition: service_healthy
-    networks:
-      - zitadel
-    build:
-      context: ../..
-      dockerfile: build/local/Dockerfile.gateway
-    image: grpcweb/grpcwebproxy
-    ports:
-      - '50000:8080'
-
-# this service generates the environemnt.json
-# needed in console.
-# It curl's the client id of console and writes it to 
-# the environment.json
-  client-id:
-    profiles: ['frontend', 'console-stub']
-    depends_on:
-      backend-run:
-        condition: service_healthy
-    networks:
-      - zitadel
-    build:
-      context: ../..
-      dockerfile: build/local/Dockerfile.clientid
-      target: client-id
-    volumes:
-      - ./environment.json:/environment.json
-    environment:
-      - HOST=backend-run
-      - PORT=50002
-
-# starts console in development mode
-  frontend-run:
-    profiles: ['frontend']
-    networks:
-      - zitadel
-    depends_on:
-      grpc-web-gateway:
-        condition: service_started
-      client-id:
-        condition: service_completed_successfully
-    build:
-      context: ../..
-      dockerfile: build/console/Dockerfile
-      target: dev-angular-build
-      args:
-        ENV: dev
-    volumes:
-      - ./environment.json:/console/src/assets/environment.json
-    command: sh -c "ng serve --host 0.0.0.0 --disable-host-check"
-    ports:
-      - 4200:4200
-
-networks:
-  zitadel: {}
--- a/build/local/keys.sh
+++ b/build/local/keys.sh
@@ -1,23 +0,0 @@
-#!/bin/sh
-
-# ----------------------------------------------------------------
-# generates necessary ZITADEL keys
-# ----------------------------------------------------------------
-
-set -e
-
-
-KEY_PATH=$(echo "/zitadel/$(dirname ${ZITADEL_KEY_PATH})")
-KEY_FILE=${KEY_PATH}/local_keys.yaml
-
-mkdir -p ${KEY_PATH}
-if [ ! -f ${KEY_FILE} ]; then
-    touch ${KEY_FILE}
-fi
-
-for key in $(env | grep "ZITADEL_.*_KEY" | cut -d'=' -f2); do
-    if [ $(grep -L ${key} ${KEY_FILE}) ]; then
-        echo "create key for ${key} in ${KEY_FILE}"
-        echo -e "${key}: $(head -c22 /dev/urandom | base64)" >> ${KEY_FILE}
-    fi
-done
--- a/build/zitadel/Dockerfile
+++ b/build/zitadel/Dockerfile
@@ -1,44 +1,5 @@
 ARG GO_VERSION=1.17

-#######################
-## These steps set platform / arch type specific variables
-#######################
-FROM alpine:3 AS arm64-base
-ENV PROTOC_ARCH aarch_64
-
-FROM alpine:3 AS amd64-base
-ENV PROTOC_ARCH x86_64
-
-#######################
-## This step sets up the folder structure,
-## initalices go mods,
-## downloads the protofiles and protoc for later use
-#######################
-FROM ${BUILDARCH}-base AS base
-ARG PROTOC_VERSION=3.18.0
-ARG PROTOC_ZIP=protoc-${PROTOC_VERSION}-linux-${PROTOC_ARCH}.zip
-ARG GATEWAY_VERSION=2.6.0
-ARG VALIDATOR_VERSION=0.6.2
-
-RUN apk add tar curl
-WORKDIR /proto
-
-#protoc
-RUN apk add tar curl \
-    && curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/$PROTOC_ZIP \
-    && unzip -o $PROTOC_ZIP -d /usr/local bin/protoc \
-    && unzip -o $PROTOC_ZIP -d /proto include/* \
-    && rm -f $PROTOC_ZIP \
-    && curl https://raw.githubusercontent.com/envoyproxy/protoc-gen-validate/v${VALIDATOR_VERSION}/validate/validate.proto --create-dirs -o include/validate/validate.proto  \
-    && curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v${GATEWAY_VERSION}/protoc-gen-openapiv2/options/annotations.proto --create-dirs -o include/protoc-gen-openapiv2/options/annotations.proto \
-    && curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v${GATEWAY_VERSION}/protoc-gen-openapiv2/options/openapiv2.proto --create-dirs -o include/protoc-gen-openapiv2/options/openapiv2.proto \
-    && curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/annotations.proto --create-dirs -o include/google/api/annotations.proto \
-    && curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto --create-dirs -o include/google/api/http.proto \
-    && curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/field_behavior.proto --create-dirs -o include/google/api/field_behavior.proto
-
-#zitadel protos
-COPY proto/ include/.
-
 #######################
 ## Go dependencies
 ## Speed up this step by mounting your local go mod pkg directory
@@ -48,7 +9,7 @@ RUN mkdir -p src/github.com/caos/zitadel
 WORKDIR /go/src/github.com/caos/zitadel

 #download modules
-COPY ./go.* .
+COPY . .
 RUN go mod download

 # install tools
@@ -84,8 +45,8 @@ RUN go generate internal/ui/login/statik/generate.go \
 ## generates grpc stub
 #######################
 FROM go-static AS go-stub
-COPY --from=base /proto /proto
-COPY --from=base /usr/local/bin /usr/local/bin/.
+COPY --from=zitadel-base:local /proto /proto
+COPY --from=zitadel-base:local /usr/local/bin /usr/local/bin/.

 COPY build/zitadel/generate-grpc.sh build/zitadel/generate-grpc.sh
 COPY internal/protoc internal/protoc
@@ -140,25 +101,3 @@ RUN go install github.com/rakyll/statik \
 #######################
 FROM scratch as go-codecov
 COPY --from=go-test /go/src/github.com/caos/zitadel/profile.cov profile.cov
-
-#######################
-## Go prod build
-#######################
-FROM go-test as prod-go-build
-ARG BUILDARCH
-ARG VERSION=""
-
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=${BUILDARCH} go build -a -installsuffix cgo -ldflags "-X main.version=${VERSION:-'dev'} -extldflags \"-static\"" -o zitadel-linux-${BUILDARCH} main.go
-
-#######################
-## Go dev build
-#######################
-FROM go-base as dev-go-build
-ENTRYPOINT [ "go", "run", "main.go" ]
-
-#######################
-## Only Copy Assets
-#######################
-FROM scratch as prod-go-export
-ARG BUILDARCH
-COPY --from=prod-go-build /go/src/github.com/caos/zitadel/zitadel-linux-${BUILDARCH} .
--- a/build/zitadel/generate-grpc.sh
+++ b/build/zitadel/generate-grpc.sh
@@ -1,4 +1,4 @@
-#! /bin/sh
+#!/bin/sh

 set -eux