ci: improve performance (#5953)

* pipeline runs on ubuntu instead of docker
* added Makefile to build zitadel core (backend) and console (frontend)
* pipeline runs in parallel where possible
* pipeline is split into multiple jobs
* removed goreleaser
* added command to check if zitadel instance is running
This commit is contained in:
Silvan
2023-07-17 10:08:20 +02:00
committed by GitHub
parent bcf4bfc585
commit 1c354ca977
64 changed files with 12290 additions and 21117 deletions

View File

@@ -1,21 +1,32 @@
#######################
## Final Production Image
#######################
FROM alpine:3 as artifact
COPY zitadel /app/zitadel
RUN adduser -D zitadel && \
chown zitadel /app/zitadel && \
chmod +x /app/zitadel
FROM --platform=$TARGETPLATFORM debian:latest as artifact
ENV ZITADEL_ARGS=
ARG TARGETPLATFORM
RUN apt-get update && apt-get install ca-certificates -y
COPY build/entrypoint.sh /app/entrypoint.sh
COPY zitadel /app/zitadel
RUN useradd -s "" --home / zitadel && \
chown zitadel /app/zitadel && \
chmod +x /app/zitadel && \
chown zitadel /app/entrypoint.sh && \
chmod +x /app/entrypoint.sh
WORKDIR /app
ENV PATH="/app:${PATH}"
USER zitadel
ENTRYPOINT ["/app/entrypoint.sh"]
FROM --platform=$TARGETPLATFORM scratch as final
ARG TARGETPLATFORM
#######################
## Scratch Image
#######################
FROM scratch as final
LABEL org.opencontainers.image.source="https://github.com/zitadel/zitadel"
COPY --from=artifact /etc/passwd /etc/passwd
COPY --from=artifact /etc/ssl/certs /etc/ssl/certs
COPY --from=artifact /app /
USER zitadel
HEALTHCHECK NONE
ENTRYPOINT ["/zitadel"]
COPY --from=artifact /app/zitadel /app/zitadel
HEALTHCHECK NONE
USER zitadel
ENTRYPOINT ["/app/zitadel"]

View File

@@ -1,30 +0,0 @@
ARG NODE_VERSION=18
#######################
## With this step we prepare all node_modules, this helps caching the build
## Speed up this step by mounting your local node_modules directory
## We also copy and generate the source code
#######################
FROM node:${NODE_VERSION} as npm-base
WORKDIR /console
# Dependencies
COPY console/package.json console/package-lock.json ./
RUN npm ci
# Sources
COPY console .
COPY proto/ /proto/
#######################
## angular lint workspace and prod build
#######################
FROM npm-base as angular-build
RUN npm run lint
RUN npm run build
#######################
## Only Copy Assets
#######################
FROM scratch as angular-export
COPY --from=angular-build /console/dist/console .

17
build/entrypoint.sh Executable file
View File

@@ -0,0 +1,17 @@
#!/bin/bash
case $@ in
sh*)
${@:3}
;;
bash*)
${@:5}
;;
*)
if [[ ! -z "$@" ]]
then
ZITADEL_ARGS="$@"
fi
/app/zitadel ${ZITADEL_ARGS}
;;
esac

View File

@@ -1,47 +0,0 @@
#ARG BUILDARCH=x86_64
#######################
## These steps set platform / arch type specific variables
#######################
FROM alpine:3 AS arm64-base
ENV PROTOC_ARCH aarch_64
FROM alpine:3 AS amd64-base
ENV PROTOC_ARCH x86_64
#######################
## This step sets up the folder structure,
## initalices go mods,
## downloads the protofiles,
## protoc and protoc-gen-grpc-web for later use
#######################
FROM ${BUILDARCH}-base
ARG PROTOC_VERSION=22.3
ARG PROTOC_ZIP=protoc-${PROTOC_VERSION}-linux-${PROTOC_ARCH}.zip
ARG GRPC_WEB_VERSION=1.3.0
ARG GATEWAY_VERSION=2.15.2
ARG VALIDATOR_VERSION=0.10.1
# no arm specific version available and x86 works fine at the moment:
ARG GRPC_WEB=protoc-gen-grpc-web-${GRPC_WEB_VERSION}-linux-x86_64
RUN apk add tar curl
WORKDIR /proto
#protoc
RUN apk add tar curl \
&& curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/$PROTOC_ZIP \
&& unzip -o $PROTOC_ZIP -d /usr/local bin/protoc \
&& unzip -o $PROTOC_ZIP -d /proto include/* \
&& rm -f $PROTOC_ZIP \
&& curl -OL https://github.com/grpc/grpc-web/releases/download/${GRPC_WEB_VERSION}/${GRPC_WEB} \
&& mv ${GRPC_WEB} /usr/local/bin/protoc-gen-grpc-web \
&& chmod +x /usr/local/bin/protoc-gen-grpc-web \
&& curl https://raw.githubusercontent.com/envoyproxy/protoc-gen-validate/v${VALIDATOR_VERSION}/validate/validate.proto --create-dirs -o include/validate/validate.proto \
&& curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v${GATEWAY_VERSION}/protoc-gen-openapiv2/options/annotations.proto --create-dirs -o include/protoc-gen-openapiv2/options/annotations.proto \
&& curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v${GATEWAY_VERSION}/protoc-gen-openapiv2/options/openapiv2.proto --create-dirs -o include/protoc-gen-openapiv2/options/openapiv2.proto \
&& curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/annotations.proto --create-dirs -o include/google/api/annotations.proto \
&& curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto --create-dirs -o include/google/api/http.proto \
&& curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/field_behavior.proto --create-dirs -o include/google/api/field_behavior.proto
#zitadel protos
COPY proto/ include/.

302
build/workflow.Dockerfile Normal file
View File

@@ -0,0 +1,302 @@
# ##############################################################################
# core
# ##############################################################################
# #######################################
# download dependencies
# #######################################
FROM golang:buster AS core-deps
WORKDIR /go/src/github.com/zitadel/zitadel
COPY go.mod .
COPY go.sum .
RUN go mod download
# #######################################
# compile custom protoc plugins
# #######################################
FROM golang:buster AS core-api-generator
WORKDIR /go/src/github.com/zitadel/zitadel
COPY go.mod .
COPY go.sum .
COPY internal/protoc internal/protoc
COPY pkg/grpc/protoc/v2 pkg/grpc/protoc/v2
RUN go install internal/protoc/protoc-gen-authoption/main.go \
&& mv $(go env GOPATH)/bin/main $(go env GOPATH)/bin/protoc-gen-authoption \
&& go install internal/protoc/protoc-gen-zitadel/main.go \
&& mv $(go env GOPATH)/bin/main $(go env GOPATH)/bin/protoc-gen-zitadel
# #######################################
# build backend stub
# #######################################
FROM golang:buster AS core-api
WORKDIR /go/src/github.com/zitadel/zitadel
COPY go.mod .
COPY go.sum .
COPY proto proto
COPY buf.*.yaml .
COPY Makefile Makefile
COPY --from=core-api-generator /go/bin /usr/local/bin
RUN make grpc
# #######################################
# generate code for login ui
# #######################################
FROM golang:buster AS core-login
WORKDIR /go/src/github.com/zitadel/zitadel
COPY Makefile Makefile
COPY internal/api/ui/login/static internal/api/ui/login/static
COPY internal/api/ui/login/statik internal/api/ui/login/statik
COPY internal/notification/static internal/notification/static
COPY internal/notification/statik internal/notification/statik
COPY internal/static internal/static
COPY internal/statik internal/statik
RUN make static
# #######################################
# generate code for assets
# #######################################
FROM golang:buster AS core-assets
WORKDIR /go/src/github.com/zitadel/zitadel
COPY go.mod .
COPY go.sum .
COPY Makefile Makefile
COPY internal/api/assets/generator internal/api/assets/generator
COPY internal/config internal/config
COPY internal/errors internal/errors
COPY --from=core-api /go/src/github.com/zitadel/zitadel/openapi/v2 openapi/v2
RUN make assets
# #######################################
# Gather all core files
# #######################################
FROM core-deps AS core-gathered
COPY cmd cmd
COPY internal internal
COPY pkg pkg
COPY proto proto
COPY openapi openapi
COPY statik statik
COPY main.go main.go
COPY --from=core-api /go/src/github.com/zitadel/zitadel .
COPY --from=core-login /go/src/github.com/zitadel/zitadel .
COPY --from=core-assets /go/src/github.com/zitadel/zitadel/internal ./internal
# ##############################################################################
# build console
# ##############################################################################
# #######################################
# download console dependencies
# #######################################
FROM node:18-buster AS console-deps
WORKDIR /zitadel/console
COPY console/package.json .
COPY console/yarn.lock .
RUN yarn install --frozen-lockfile
# #######################################
# generate console client
# #######################################
FROM node:18-buster AS console-client
WORKDIR /zitadel/console
# install buf
COPY --from=bufbuild/buf:latest /usr/local/bin/* /usr/local/bin/
ENV PATH="/usr/local/bin:${PATH}"
COPY console/package.json .
COPY console/buf.*.yaml .
COPY proto ../proto
RUN yarn generate
# #######################################
# Gather all console files
# #######################################
FROM console-deps as console-gathered
COPY --from=console-client /zitadel/console/src/app/proto/generated src/app/proto/generated
COPY console/src src
COPY console/angular.json .
COPY console/ngsw-config.json .
COPY console/tsconfig* .
# #######################################
# Build console
# #######################################
FROM console-gathered AS console
RUN yarn build
# ##############################################################################
# build the executable
# ##############################################################################
# #######################################
# build executable
# #######################################
FROM core-gathered AS compile
ARG GOOS
ARG GOARCH
COPY --from=console /zitadel/console/dist/console internal/api/ui/console/static/
RUN go build -o zitadel -ldflags="-s -w -race" \
&& chmod +x zitadel
ENTRYPOINT [ "./zitadel" ]
# #######################################
# copy executable
# #######################################
FROM scratch AS copy-executable
ARG GOOS
ARG GOARCH
COPY --from=compile /go/src/github.com/zitadel/zitadel/zitadel /.artifacts/zitadel
# ##############################################################################
# tests
# ##############################################################################
FROM ubuntu/postgres:latest AS test-core-base
ARG DEBIAN_FRONTEND=noninteractive
RUN apt-get update && \
apt-get install -y --no-install-recommends \
gcc \
make \
ca-certificates \
gcc \
&& \
update-ca-certificates; \
rm -rf /var/lib/apt/lists/*
# install go
COPY --from=golang:latest /usr/local/go/ /usr/local/go/
ENV PATH="/go/bin:/usr/local/go/bin:${PATH}"
WORKDIR /go/src/github.com/zitadel/zitadel
# default vars
ENV DB_FLAVOR=postgres
ENV POSTGRES_USER=zitadel
ENV POSTGRES_DB=zitadel
ENV POSTGRES_PASSWORD=postgres
ENV POSTGRES_HOST_AUTH_METHOD=trust
ENV PGUSER=zitadel
ENV PGDATABASE=zitadel
ENV PGPASSWORD=postgres
ENV CGO_ENABLED=1
# copy zitadel files
COPY --from=core-deps /go/pkg/mod /root/go/pkg/mod
COPY --from=core-gathered /go/src/github.com/zitadel/zitadel .
# #######################################
# unit test core
# #######################################
FROM test-core-base AS test-core-unit
RUN go test -race -v -coverprofile=profile.cov ./...
# #######################################
# coverage output
# #######################################
FROM scratch AS coverage-core-unit
COPY --from=test-core-unit /go/src/github.com/zitadel/zitadel/profile.cov /coverage/
# #######################################
# integration test core
# #######################################
FROM test-core-base AS test-core-integration
ENV DB_FLAVOR=cockroach
# install cockroach
COPY --from=cockroachdb/cockroach:latest /cockroach/cockroach /usr/local/bin/
ENV COCKROACH_BINARY=/cockroach/cockroach
ENV ZITADEL_MASTERKEY=MasterkeyNeedsToHave32Characters
COPY build/core-integration-test.sh /usr/local/bin/run-tests.sh
RUN chmod +x /usr/local/bin/run-tests.sh
RUN run-tests.sh
# #######################################
# coverage output
# #######################################
FROM scratch AS coverage-core-integration
COPY --from=test-core-integration /go/src/github.com/zitadel/zitadel/profile.cov /coverage/
# ##############################################################################
# linting
# ##############################################################################
# #######################################
# api
# #######################################
FROM bufbuild/buf:latest AS lint-api
COPY proto proto
COPY buf.*.yaml .
RUN buf lint
# #######################################
# console
# #######################################
FROM console-gathered AS lint-console
COPY console/.eslintrc.js .
COPY console/.prettier* .
RUN yarn lint
# #######################################
# core
# #######################################
FROM golangci/golangci-lint:latest AS lint-core
ARG LINT_EXIT_CODE=1
WORKDIR /go/src/github.com/zitadel/zitadel
COPY .golangci.yaml .
COPY .git/ .git/
COPY --from=core-deps /go/pkg/mod /go/pkg/mod
COPY --from=core-gathered /go/src/github.com/zitadel/zitadel .
RUN git fetch https://github.com/zitadel/zitadel main:main
RUN golangci-lint run \
--timeout 10m \
--config ./.golangci.yaml \
--out-format=github-actions:report,colored-line-number \
--issues-exit-code=${LINT_EXIT_CODE} \
--concurrency=$(getconf _NPROCESSORS_ONLN)
# #######################################
# report output
# #######################################
FROM scratch AS lint-core-report
COPY --from=lint-core /go/src/github.com/zitadel/zitadel/report .

View File

@@ -1,107 +0,0 @@
ARG GO_VERSION=1.20
#######################
## Go dependencies
## Speed up this step by mounting your local go mod pkg directory
#######################
FROM golang:${GO_VERSION} as go-dep
WORKDIR /go/src/github.com/zitadel/zitadel
#download modules
COPY go.mod ./
COPY go.sum ./
RUN go mod download
# install tools
COPY tools ./tools
RUN ./tools/install.sh
#######################
## generates static files
#######################
FROM go-dep AS go-static
COPY internal/api/ui/login/static internal/api/ui/login/static
COPY internal/api/ui/login/statik internal/api/ui/login/statik
COPY internal/notification/static internal/notification/static
COPY internal/notification/statik internal/notification/statik
COPY internal/static internal/static
COPY internal/statik internal/statik
RUN go generate internal/api/ui/login/statik/generate.go \
&& go generate internal/api/ui/login/static/generate.go \
&& go generate internal/notification/statik/generate.go \
&& go generate internal/statik/generate.go
#######################
## generates grpc stub
#######################
FROM go-static AS go-stub
COPY --from=zitadel-base:local /proto /proto
COPY --from=zitadel-base:local /usr/local/bin /usr/local/bin/.
COPY build/zitadel/generate-grpc.sh build/zitadel/generate-grpc.sh
COPY internal/protoc internal/protoc
COPY openapi/statik openapi/statik
COPY internal/api/assets/generator internal/api/assets/generator
COPY internal/config internal/config
COPY internal/errors internal/errors
RUN build/zitadel/generate-grpc.sh && \
go generate openapi/statik/generate.go && \
mkdir -p docs/apis/assets/ && \
go run internal/api/assets/generator/asset_generator.go -directory=internal/api/assets/generator/ -assets=docs/apis/assets/assets.md
#######################
## Go base build
#######################
FROM go-stub as go-base
# copy remaining zitadel files
COPY cmd cmd
COPY internal internal
COPY pkg pkg
COPY openapi openapi
#######################
## copy for local dev
#######################
FROM scratch as go-copy
COPY --from=go-stub /go/src/github.com/zitadel/zitadel/internal/api/ui/login/statik/statik.go internal/api/ui/login/statik/statik.go
COPY --from=go-stub /go/src/github.com/zitadel/zitadel/internal/notification/statik/statik.go internal/notification/statik/statik.go
COPY --from=go-stub /go/src/github.com/zitadel/zitadel/internal/statik/statik.go internal/statik/statik.go
COPY --from=go-stub /go/src/github.com/zitadel/zitadel/openapi/statik/statik.go openapi/statik/statik.go
COPY --from=go-stub /go/src/github.com/zitadel/zitadel/pkg/grpc pkg/grpc
COPY --from=go-stub /go/src/github.com/zitadel/zitadel/openapi/v2/zitadel openapi/v2/zitadel
COPY --from=go-stub /go/src/github.com/zitadel/zitadel/openapi/statik/statik.go openapi/statik/statik.go
COPY --from=go-stub /go/src/github.com/zitadel/zitadel/internal/protoc/protoc-gen-authoption/authoption/options.pb.go internal/protoc/protoc-gen-authoption/authoption/options.pb.go
COPY --from=go-stub /go/src/github.com/zitadel/zitadel/docs/apis/proto docs/docs/apis/proto
COPY --from=go-stub /go/src/github.com/zitadel/zitadel/docs/apis/assets docs/docs/apis/assets
COPY --from=go-stub /go/src/github.com/zitadel/zitadel/internal/api/assets/authz.go ./internal/api/assets/authz.go
COPY --from=go-stub /go/src/github.com/zitadel/zitadel/internal/api/assets/router.go ./internal/api/assets/router.go
#######################
## Go test
#######################
FROM go-base as go-test
ARG COCKROACH_BINARY=/usr/local/bin/cockroach
ARG COCKROACH_VERSION=v22.2.2
RUN apt install openssl tzdata tar
# cockroach binary used to backup database
RUN mkdir /usr/local/lib/cockroach
RUN wget -qO- https://binaries.cockroachdb.com/cockroach-${COCKROACH_VERSION}.linux-amd64.tgz \
| tar xvz && cp -i cockroach-${COCKROACH_VERSION}.linux-amd64/cockroach /usr/local/bin/
RUN rm -r cockroach-${COCKROACH_VERSION}.linux-amd64
# Migrations for cockroach-secure
RUN go install github.com/rakyll/statik \
&& go test -race -coverprofile=profile.cov $(go list ./... | grep -v /operator/)
#######################
## Go test results
#######################
FROM scratch as go-codecov
COPY --from=go-test /go/src/github.com/zitadel/zitadel/profile.cov profile.cov

View File

@@ -1,123 +0,0 @@
#!/bin/sh
set -eux
echo "Generate grpc"
OPENAPI_PATH=${GOPATH}/src/github.com/zitadel/zitadel/openapi/v2
ZITADEL_PATH=${GOPATH}/src/github.com/zitadel/zitadel
GRPC_PATH=${ZITADEL_PATH}/pkg/grpc
PROTO_PATH=/proto/include/zitadel
DOCS_PATH=${ZITADEL_PATH}/docs/apis/proto
# generate go stub and grpc code for all files
protoc \
-I=/proto/include/ \
--go_out $GOPATH/src \
--go-grpc_out $GOPATH/src \
--validate_out=lang=go:${GOPATH}/src \
$(find ${PROTO_PATH} -iname *.proto)
# install authoption and zitadel proto compiler
go install ${ZITADEL_PATH}/internal/protoc/protoc-gen-auth
go install ${ZITADEL_PATH}/internal/protoc/protoc-gen-zitadel
# output folder for openapi v2
mkdir -p ${OPENAPI_PATH}
mkdir -p ${DOCS_PATH}
# generate additional output
protoc \
-I=/proto/include \
--grpc-gateway_out ${GOPATH}/src \
--grpc-gateway_opt logtostderr=true \
--openapiv2_out ${OPENAPI_PATH} \
--openapiv2_opt logtostderr=true \
--auth_out ${GOPATH}/src \
--validate_out=lang=go:${GOPATH}/src \
${PROTO_PATH}/system.proto
protoc \
-I=/proto/include \
--grpc-gateway_out ${GOPATH}/src \
--grpc-gateway_opt logtostderr=true \
--openapiv2_out ${OPENAPI_PATH} \
--openapiv2_opt logtostderr=true \
--auth_out ${GOPATH}/src \
--validate_out=lang=go:${GOPATH}/src \
${PROTO_PATH}/admin.proto
protoc \
-I=/proto/include \
--grpc-gateway_out ${GOPATH}/src \
--grpc-gateway_opt logtostderr=true \
--grpc-gateway_opt allow_delete_body=true \
--openapiv2_out ${OPENAPI_PATH} \
--openapiv2_opt logtostderr=true \
--openapiv2_opt allow_delete_body=true \
--auth_out ${GOPATH}/src \
--validate_out=lang=go:${GOPATH}/src \
${PROTO_PATH}/management.proto
protoc \
-I=/proto/include \
--grpc-gateway_out ${GOPATH}/src \
--grpc-gateway_opt logtostderr=true \
--grpc-gateway_opt allow_delete_body=true \
--openapiv2_out ${OPENAPI_PATH} \
--openapiv2_opt logtostderr=true \
--openapiv2_opt allow_delete_body=true \
--auth_out=${GOPATH}/src \
--validate_out=lang=go:${GOPATH}/src \
${PROTO_PATH}/auth.proto
protoc \
-I=/proto/include \
--grpc-gateway_out ${GOPATH}/src \
--grpc-gateway_opt logtostderr=true \
--grpc-gateway_opt allow_delete_body=true \
--openapiv2_out ${OPENAPI_PATH} \
--openapiv2_opt logtostderr=true \
--openapiv2_opt allow_delete_body=true \
--zitadel_out=${GOPATH}/src \
--validate_out=lang=go:${GOPATH}/src \
${PROTO_PATH}/user/v2alpha/user_service.proto
protoc \
-I=/proto/include \
--grpc-gateway_out ${GOPATH}/src \
--grpc-gateway_opt logtostderr=true \
--grpc-gateway_opt allow_delete_body=true \
--openapiv2_out ${OPENAPI_PATH} \
--openapiv2_opt logtostderr=true \
--openapiv2_opt allow_delete_body=true \
--zitadel_out=${GOPATH}/src \
--validate_out=lang=go:${GOPATH}/src \
${PROTO_PATH}/session/v2alpha/session_service.proto
protoc \
-I=/proto/include \
--grpc-gateway_out ${GOPATH}/src \
--grpc-gateway_opt logtostderr=true \
--grpc-gateway_opt allow_delete_body=true \
--openapiv2_out ${OPENAPI_PATH} \
--openapiv2_opt logtostderr=true \
--openapiv2_opt allow_delete_body=true \
--zitadel_out=${GOPATH}/src \
--validate_out=lang=go:${GOPATH}/src \
${PROTO_PATH}/settings/v2alpha/settings_service.proto
protoc \
-I=/proto/include \
--grpc-gateway_out ${GOPATH}/src \
--grpc-gateway_opt logtostderr=true \
--grpc-gateway_opt allow_delete_body=true \
--openapiv2_out ${OPENAPI_PATH} \
--openapiv2_opt logtostderr=true \
--openapiv2_opt allow_delete_body=true \
--zitadel_out=${GOPATH}/src \
--validate_out=lang=go:${GOPATH}/src \
${PROTO_PATH}/oidc/v2alpha/oidc_service.proto
echo "done generating grpc"