chore: change build process to use local and server the same (#955)

* initital ide * init * it is alive * go test is running * wip with buildkit * it definitly alive * all done, now the github actions * pretty * before i delete the generated proto stub * delete generated code * mount of grpc does not yet work * save before * level of insanity 1000% * huhur * gh action new dockerfile * fix * fix * fix * fix * amd64 only * try caching * try with reg * test with artifact upload * trial * add comments * publish branchname * correct vars * correct id * mode max for cache * remove unused code * cleanup * test action which uploads coverage * use * proper path * debug file location * path * test if ci still works * correct docker file name
2025-08-11 18:07:31 +00:00 · 2020-11-17 09:33:07 +01:00
parent e8db038839
commit 84f0e24c63
35 changed files with 420 additions and 99996 deletions
--- a/build/README.md
+++ b/build/README.md
@@ -0,0 +1,60 @@
+
+# Development
+
+## Prerequisite
+
+- Buildkit compatible docker installation
+
+## Generate Proto Clients
+
+### Angular
+
+This command generates the grpc stub for angular into the folder console/src/app/proto/generated for local development
+
+```Bash
+DOCKER_BUILDKIT=1 docker build -f build/dockerfile . -t zitadel:local --target npm-copy -o console/src/app/proto/generated
+```
+
+### Go
+
+With this command you can generate the stub for golang into the correct dir pkg/
+
+```Bash
+DOCKER_BUILDKIT=1 docker build -f build/dockerfile . -t zitadel:local --target go-copy -o pkg
+```
+
+## Run
+
+### Run Angular
+
+```Bash
+COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose -f build/docker-compose-dev.yml up --build angular
+```
+
+### Run Go
+
+```Bash
+COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose -f build/docker-compose-dev.yml up --build  go
+```
+
+### Fullstack including database
+
+```Bash
+COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose -f build/docker-compose.yml up --build
+```
+
+## Debug
+
+### Debug Go
+
+```Bash
+COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose -f build/docker-compose-debug.yml up --build  go
+```
+
+## Production Build
+
+This can also be run locally!
+
+```Bash
+DOCKER_BUILDKIT=1 docker build -f build/dockerfile . -t zitadel:local --build-arg ENV=prod
+```
--- a/build/console/generate-grpc.sh
+++ b/build/console/generate-grpc.sh
@@ -4,38 +4,24 @@ set -eux

 GEN_PATH=src/app/proto/generated

-echo "Remove old files"
-rm -rf $GEN_PATH
-
 echo "Create folders"
 mkdir -p $GEN_PATH

-targetcurl () {
-   mkdir -p $1 && cd $1 && { curl -O $2; cd -; }
-}
-
-echo "Download additional protofiles"
-targetcurl tmp/validate https://raw.githubusercontent.com/envoyproxy/protoc-gen-validate/v0.4.0/validate/validate.proto
-targetcurl tmp/protoc-gen-swagger/options https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v1.14.6/protoc-gen-swagger/options/annotations.proto
-targetcurl tmp/protoc-gen-swagger/options https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v1.14.6/protoc-gen-swagger/options/openapiv2.proto
-
 echo "Generate grpc"

 protoc \
-  -I=/usr/local/include \
-  -I=../pkg/grpc/message \
-  -I=../pkg/grpc/management/proto \
-  -I=../pkg/grpc/auth/proto \
-  -I=../pkg/grpc/admin/proto \
-  -I=../internal/protoc/protoc-gen-authoption \
+  -I=.tmp/protos/message \
+  -I=.tmp/protos/admin/proto \
+  -I=.tmp/protos/management/proto \
+  -I=.tmp/protos/auth/proto \
  -I=node_modules/google-proto-files \
-  -I=tmp \
+  -I=.tmp/protos \
  --js_out=import_style=commonjs,binary:$GEN_PATH \
  --grpc-web_out=import_style=commonjs+dts,mode=grpcweb:$GEN_PATH \
-  ../pkg/grpc/message/proto/*.proto \
-  ../pkg/grpc/management/proto/*.proto \
-  ../pkg/grpc/admin/proto/*.proto \
-  ../pkg/grpc/auth/proto/*.proto
+  .tmp/protos/message/proto/*.proto \
+  .tmp/protos/admin/proto/*.proto \
+  .tmp/protos/auth/proto/*.proto \
+  .tmp/protos/management/proto/*.proto

 echo "Generate annotations js file (compatibility)"

--- a/build/docker-compose-debug.yml
+++ b/build/docker-compose-debug.yml
@@ -0,0 +1,30 @@
+version: "3.8"
+
+services:
+  angular:
+    build:
+      context: ..
+      dockerfile: build/Dockerfile
+      target: dev-angular-build
+      args:
+        ENV: dev
+    command: sh -c "ng serve --host 0.0.0.0"
+    ports:
+      - 4200:4200
+  go:
+    build:
+      context: ..
+      dockerfile: build/Dockerfile
+      target: dev-go-build
+      args:
+        ENV: dev
+    command: dlv --listen=:2345 --headless=true --log=true --log-output=debugger,debuglineerr,gdbwire,lldbout,rpc --accept-multiclient --api-version=2 debug cmd/zitadel/main.go
+    ports:
+      - 2345:2345
+      - 50000:50000
+  db:
+    image: cockroachdb/cockroach:v20.2.0
+    command: start-single-node --insecure
+    ports:
+      - 8080:8080
+      - 26257:26257
--- a/build/docker-compose-dev.yml
+++ b/build/docker-compose-dev.yml
@@ -0,0 +1,31 @@
+version: "3.8"
+
+services:
+  angular:
+    build:
+      context: ..
+      dockerfile: build/Dockerfile
+      target: dev-angular-build
+      args:
+        ENV: dev
+    command: sh -c "ng serve --host 0.0.0.0"
+    ports:
+      - 4200:4200
+  go:
+    build:
+      context: ..
+      dockerfile: build/Dockerfile
+      target: dev-go-build
+      args:
+        ENV: dev
+    command: go run cmd/zitadel/main.go
+    ports:
+      - 50000:50000
+  db:
+    image: cockroachdb/cockroach:v20.2.0
+    command: start-single-node --insecure
+    ports:
+      - 8080:8080
+      - 26257:26257
+    volumes:
+      - "../cockroach-data/zitadel1:/cockroach/cockroach-data"
--- a/build/docker/.dockerignore
+++ b/build/docker/.dockerignore
@@ -1,5 +0,0 @@
-# Exclude system dirs
-
-.dependabot
-.github
-.git
--- a/build/docker/Dockerfile
+++ b/build/docker/Dockerfile
@@ -1,14 +0,0 @@
-# This Stage prepares the user in the container and copies the files
-FROM alpine:latest as prepare
-RUN adduser -D zitadel
-COPY .artifacts/zitadel-linux-amd64 /zitadel
-COPY cmd/zitadel/*.yaml /
-RUN chmod a+x /zitadel
-
-# This Stage is intended as production image
-FROM scratch as final
-COPY --from=prepare /etc/passwd /etc/passwd
-COPY --from=prepare / /
-USER zitadel
-HEALTHCHECK NONE
-ENTRYPOINT ["/zitadel"]
--- a/build/dockerfile
+++ b/build/dockerfile
@@ -0,0 +1,127 @@
+#######################
+## By default we build the prod enviroment
+ARG ENV=prod
+
+#######################
+## This step downloads the protofiles, protoc and protoc-gen-grpc-web for later use
+#######################
+FROM alpine as base
+RUN apk add tar curl
+WORKDIR /.tmp
+RUN wget -O protoc https://github.com/protocolbuffers/protobuf/releases/download/v3.13.0/protoc-3.13.0-linux-x86_64.zip \
+    && unzip protoc \
+    && wget -O bin/protoc-gen-grpc-web https://github.com/grpc/grpc-web/releases/download/1.2.0/protoc-gen-grpc-web-1.2.0-linux-x86_64 \
+    && chmod +x bin/protoc-gen-grpc-web
+RUN curl https://raw.githubusercontent.com/envoyproxy/protoc-gen-validate/v0.4.0/validate/validate.proto --create-dirs -o validate/validate.proto  \
+    && curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v1.14.6/protoc-gen-swagger/options/annotations.proto --create-dirs -o protoc-gen-swagger/options/annotations.proto \
+    && curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v1.14.6/protoc-gen-swagger/options/openapiv2.proto --create-dirs -o protoc-gen-swagger/options/openapiv2.proto \
+    && curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/annotations.proto --create-dirs -o google/api/annotations.proto \
+    && curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto --create-dirs -o google/api/http.proto \
+    && curl https://raw.githubusercontent.com/protocolbuffers/protobuf/master/src/google/protobuf/empty.proto --create-dirs -o google/protobuf/empty.proto \
+    && curl https://raw.githubusercontent.com/protocolbuffers/protobuf/master/src/google/protobuf/timestamp.proto --create-dirs -o google/protobuf/timestamp.proto \
+    && curl https://raw.githubusercontent.com/protocolbuffers/protobuf/master/src/google/protobuf/descriptor.proto --create-dirs -o google/protobuf/descriptor.proto \
+    && curl https://raw.githubusercontent.com/protocolbuffers/protobuf/master/src/google/protobuf/duration.proto --create-dirs -o google/protobuf/duration.proto \
+    && curl https://raw.githubusercontent.com/protocolbuffers/protobuf/master/src/google/protobuf/any.proto --create-dirs -o google/protobuf/any.proto \
+    && curl https://raw.githubusercontent.com/protocolbuffers/protobuf/master/src/google/protobuf/struct.proto --create-dirs -o google/protobuf/struct.proto
+
+COPY pkg/grpc/admin/proto/admin.proto admin/proto/admin.proto
+COPY pkg/grpc/auth/proto/auth.proto auth/proto/auth.proto
+COPY pkg/grpc/management/proto/management.proto management/proto/management.proto
+COPY pkg/grpc/message/proto/message.proto message/proto/message.proto
+COPY internal/protoc/protoc-gen-authoption/authoption/options.proto authoption/options.proto
+
+#######################
+## With this step we prepare all node_modules, this helps caching the build
+## Speed up this step by mounting your local node_modules directory
+#######################
+FROM node:12 as npm-base
+WORKDIR console
+COPY console/package.json console/package-lock.json ./
+RUN npm install \
+    && mkdir .tmp
+COPY console .
+COPY --from=base /.tmp/bin /usr/local/bin/
+COPY --from=base /.tmp .tmp/protos/
+COPY build/console build/console/
+RUN build/console/generate-grpc.sh
+
+FROM scratch as npm-copy
+COPY --from=npm-base /console/src/app/proto/generated .
+
+## anular dev build
+FROM npm-base as dev-angular-build
+RUN npm install -g @angular/cli
+
+## anular prod build
+FROM npm-base as prod-angular-build
+RUN npm run prodbuild
+
+#######################
+## Go base build
+## Speed up this step by mounting your local go mod pkg directory
+#######################
+FROM golang:1.15 as go-base
+WORKDIR src/github.com/caos/zitadel/
+COPY go.mod go.sum ./
+RUN go mod download
+COPY --from=base /.tmp .tmp/protos/
+COPY --from=base /.tmp/bin /usr/local/bin/
+COPY internal/protoc/protoc-base internal/protoc/protoc-base/
+COPY internal/protoc/protoc-gen-authoption internal/protoc/protoc-gen-authoption/
+
+RUN go install \
+    github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway \
+    github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger \
+    github.com/golang/protobuf/protoc-gen-go \
+    github.com/envoyproxy/protoc-gen-validate
+
+RUN go get -u github.com/go-bindata/go-bindata/...
+
+RUN go-bindata ./internal/protoc/protoc-gen-authoption/templates \
+    && go install ./internal/protoc/protoc-gen-authoption
+    
+COPY build/zitadel build/zitadel/
+RUN build/zitadel/generate-grpc.sh
+
+FROM scratch as go-copy
+COPY --from=go-base /go/src/github.com/caos/zitadel/pkg/ .
+
+## Go test
+FROM go-base as go-test
+COPY . .
+RUN go test -race -v -coverprofile=profile.cov ./...
+
+## Go test
+FROM scratch as go-codecov
+COPY --from=go-test /go/src/github.com/caos/zitadel/profile.cov profile.cov
+
+## Go prod build
+FROM go-test as prod-go-build
+COPY --from=prod-angular-build console/dist/console console/dist/console/
+RUN go get github.com/rakyll/statik \
+    && ./build/console/generate-static.sh \
+    && ./build/login/generate-static.sh \
+    && ./build/notification/generate-static.sh \
+    && ./build/zitadel/generate-static.sh
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o zitadel-linux-amd64 cmd/zitadel/main.go
+
+## Go dev build
+FROM go-base as dev-go-build
+RUN go get github.com/go-delve/delve/cmd/dlv
+
+#######################
+## Final Production Image
+#######################
+FROM ${ENV}-go-build as artifact
+RUN useradd zitadel
+COPY cmd/zitadel/*.yaml app/
+RUN chmod a+x zitadel-linux-amd64
+RUN mv zitadel-linux-amd64 app/zitadel
+
+## Scratch Image
+FROM scratch as final
+COPY --from=artifact /etc/passwd /etc/passwd
+COPY --from=artifact /go/src/github.com/caos/zitadel/app app/
+USER zitadel
+HEALTHCHECK NONE
+ENTRYPOINT ["/app/zitadel"]
--- a/build/zitadel/generate-grpc.sh
+++ b/build/zitadel/generate-grpc.sh
@@ -0,0 +1,65 @@
+#! /bin/sh
+
+set -eux
+
+echo "Generate grpc"
+
+protoc \
+  -I=.tmp/protos/message \
+  -I=.tmp/protos/admin/proto \
+  -I=.tmp/protos/management/proto \
+  -I=.tmp/protos/auth/proto \
+  -I=.tmp/protos \
+  -I=${GOPATH}/src \
+  --go_out=plugins=grpc:$GOPATH/src \
+  .tmp/protos/message/proto/message.proto
+
+protoc \
+  -I=.tmp/protos/message \
+  -I=.tmp/protos/admin/proto \
+  -I=.tmp/protos/management/proto \
+  -I=.tmp/protos/auth/proto \
+  -I=.tmp/protos \
+  -I=${GOPATH}/src \
+  --go_out=plugins=grpc:$GOPATH/src \
+  --grpc-gateway_out=logtostderr=true:$GOPATH/src \
+  --swagger_out=logtostderr=true:. \
+  --authoption_out=. \
+  --validate_out=lang=go:${GOPATH}/src \
+  .tmp/protos/admin/proto/admin.proto
+
+mv admin* $GOPATH/src/github.com/caos/zitadel/pkg/grpc/admin/
+
+protoc \
+  -I=.tmp/protos/message \
+  -I=.tmp/protos/admin/proto \
+  -I=.tmp/protos/management/proto \
+  -I=.tmp/protos/auth/proto \
+  -I=.tmp/protos \
+  -I=${GOPATH}/src \
+  --go_out=plugins=grpc:$GOPATH/src \
+  --grpc-gateway_out=logtostderr=true,allow_delete_body=true:${GOPATH}/src \
+  --swagger_out=logtostderr=true,allow_delete_body=true:. \
+  --authoption_out=. \
+  --validate_out=lang=go:${GOPATH}/src \
+  .tmp/protos/management/proto/management.proto
+
+mv management* $GOPATH/src/github.com/caos/zitadel/pkg/grpc/management/
+
+protoc \
+  -I=.tmp/protos/message \
+  -I=.tmp/protos/admin/proto \
+  -I=.tmp/protos/management/proto \
+  -I=.tmp/protos/auth/proto \
+  -I=.tmp/protos \
+  -I=${GOPATH}/src \
+  --go_out=plugins=grpc:$GOPATH/src \
+  --grpc-gateway_out=logtostderr=true:$GOPATH/src \
+  --swagger_out=logtostderr=true:. \
+  --authoption_out=. \
+  --validate_out=lang=go:${GOPATH}/src \
+  .tmp/protos/auth/proto/auth.proto
+
+mv auth* $GOPATH/src/github.com/caos/zitadel/pkg/grpc/auth/
+
+echo "done generating"