fix(zitadel-image): refactor dockerfiles and gh action (#2027)

* early prototyp

* fix some errors

* remove docker image cache

* add comment

* add false

* typo

* test cache speed

* upload artifact

* remove tag

* seperate after build step

* debug

* debug

* debug

* debug

* debug

* test gh

* test

* test

* test

* test

* test golang

* test go

* test with release image

* fix

* use scratch to export

* test

* fix path

* ref

* typo

* debug

* test

* debug

* speed up docker

* test

* debug

* debug

* try scope

* test cache

* restore cache

* reenable operator build

* fix duplicate

* fix buildpush version

* test cache

* improve caching

* test release IF

* only run console and zitadel without dispatch

* test with reworked operator

* testing without release step

* lint yaml

* fix if

* improve wording

* enable release step & only notify sentry on main

* console from file system

* update docker file paths

* remove migration line

* add statik for operator

* console files using go 1.15

* fix console default path

* improve code QL

Co-authored-by: Livio Amstutz <livio.a@gmail.com>
This commit is contained in:
Florian Forster
2021-07-27 14:34:56 +02:00
committed by GitHub
parent e546988d2f
commit 8d96f1a895
16 changed files with 459 additions and 604 deletions

86
build/console/Dockerfile Normal file
View File

@@ -0,0 +1,86 @@
ARG NODE_VERSION=14
#######################
## These steps set platform / arch type specific variables
#######################
FROM alpine:3 AS arm64-base
ENV PROTOC_ARCH aarch_64
FROM alpine:3 AS amd64-base
ENV PROTOC_ARCH x86_64
#######################
## This step sets up the folder structure,
## initalices go mods,
## downloads the protofiles,
## protoc and protoc-gen-grpc-web for later use
#######################
FROM ${BUILDARCH}-base AS base
ARG PROTOC_VERSION=3.13.0
ARG PROTOC_ZIP=protoc-${PROTOC_VERSION}-linux-${PROTOC_ARCH}.zip
ARG GRPC_WEB_VERSION=1.2.1
# no arm specific version available and x86 works fine at the moment:
ARG GRPC_WEB=protoc-gen-grpc-web-${GRPC_WEB_VERSION}-linux-x86_64
RUN apk add tar curl
WORKDIR /proto
#protoc
RUN apk add tar curl \
&& curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/$PROTOC_ZIP \
&& unzip -o $PROTOC_ZIP -d /usr/local bin/protoc \
&& unzip -o $PROTOC_ZIP -d /proto include/* \
&& rm -f $PROTOC_ZIP \
&& curl -OL https://github.com/grpc/grpc-web/releases/download/${GRPC_WEB_VERSION}/${GRPC_WEB} \
&& mv ${GRPC_WEB} /usr/local/bin/protoc-gen-grpc-web \
&& chmod +x /usr/local/bin/protoc-gen-grpc-web \
&& curl https://raw.githubusercontent.com/envoyproxy/protoc-gen-validate/v0.4.1/validate/validate.proto --create-dirs -o include/validate/validate.proto \
&& curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v2.2.0/protoc-gen-openapiv2/options/annotations.proto --create-dirs -o include/protoc-gen-openapiv2/options/annotations.proto \
&& curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v2.2.0/protoc-gen-openapiv2/options/openapiv2.proto --create-dirs -o include/protoc-gen-openapiv2/options/openapiv2.proto \
&& curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/annotations.proto --create-dirs -o include/google/api/annotations.proto \
&& curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto --create-dirs -o include/google/api/http.proto \
&& curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/field_behavior.proto --create-dirs -o include/google/api/field_behavior.proto
#zitadel protos
COPY proto/ include/.
#######################
## With this step we prepare all node_modules, this helps caching the build
## Speed up this step by mounting your local node_modules directory
#######################
FROM node:${NODE_VERSION} as npm-base
WORKDIR /console
COPY console/package.json console/package-lock.json ./
RUN npm install
COPY console .
COPY --from=base /proto /proto
COPY --from=base /usr/local/bin /usr/local/bin/.
COPY build/console build/console/
RUN build/console/generate-grpc.sh
#######################
## copy for local dev
#######################
FROM scratch as npm-copy
COPY --from=npm-base /console/src/app/proto/generated ./console/src/app/proto/generated
#######################
## angular dev build
#######################
FROM npm-base as dev-angular-build
RUN npm install -g @angular/cli
#######################
## angular lint workspace and prod build
#######################
FROM npm-base as prod-angular-build
RUN npm run lint
RUN npm run prodbuild
#######################
## Only Copy Assets
#######################
FROM scratch as prod-angular-export
COPY --from=prod-angular-build /console/dist/console .

View File

@@ -1,3 +0,0 @@
{
"lockfileVersion": 1
}

View File

@@ -1,226 +1,13 @@
ARG GO_VERSION=1.15.8
ARG NODE_VERSION=14.17.0
ARG ENV=prod
#######################
## These steps set platform / arch type specific variables
#######################
FROM alpine AS arm64-base
ENV PROTOC_ARCH aarch_64
FROM alpine AS amd64-base
ENV PROTOC_ARCH x86_64
#######################
## This step sets up the folder structure,
## initalices go mods,
## downloads the protofiles,
## protoc and protoc-gen-grpc-web for later use
#######################
FROM ${BUILDARCH}-base AS base
ARG PROTOC_VERSION=3.13.0
ARG PROTOC_ZIP=protoc-${PROTOC_VERSION}-linux-${PROTOC_ARCH}.zip
ARG GRPC_WEB_VERSION=1.2.1
# no arm specific version available and x86 works fine at the moment:
ARG GRPC_WEB=protoc-gen-grpc-web-${GRPC_WEB_VERSION}-linux-x86_64
RUN apk add tar curl
WORKDIR /proto
#protoc
RUN curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/$PROTOC_ZIP \
&& unzip -o $PROTOC_ZIP -d /usr/local bin/protoc \
&& unzip -o $PROTOC_ZIP -d /proto include/* \
&& rm -f $PROTOC_ZIP
#grpc web
RUN curl -OL https://github.com/grpc/grpc-web/releases/download/${GRPC_WEB_VERSION}/${GRPC_WEB} \
&& mv ${GRPC_WEB} /usr/local/bin/protoc-gen-grpc-web \
&& chmod +x /usr/local/bin/protoc-gen-grpc-web
#proto dependencies
RUN curl https://raw.githubusercontent.com/envoyproxy/protoc-gen-validate/v0.4.1/validate/validate.proto --create-dirs -o include/validate/validate.proto \
&& curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v2.2.0/protoc-gen-openapiv2/options/annotations.proto --create-dirs -o include/protoc-gen-openapiv2/options/annotations.proto \
&& curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v2.2.0/protoc-gen-openapiv2/options/openapiv2.proto --create-dirs -o include/protoc-gen-openapiv2/options/openapiv2.proto \
&& curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/annotations.proto --create-dirs -o include/google/api/annotations.proto \
&& curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto --create-dirs -o include/google/api/http.proto \
&& curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/field_behavior.proto --create-dirs -o include/google/api/field_behavior.proto
#zitadel protos
COPY proto/ include/.
#######################
## With this step we prepare all node_modules, this helps caching the build
## Speed up this step by mounting your local node_modules directory
#######################
FROM node:${NODE_VERSION} as npm-base
WORKDIR /console
COPY console/package.json console/package-lock.json ./
RUN npm install
COPY console .
COPY --from=base /proto /proto
COPY --from=base /usr/local/bin /usr/local/bin/.
COPY build/console build/console/
RUN build/console/generate-grpc.sh
#######################
## copy for local dev
#######################
FROM scratch as npm-copy
COPY --from=npm-base /console/src/app/proto/generated ./console/src/app/proto/generated
#######################
## angular dev build
#######################
FROM npm-base as dev-angular-build
RUN npm install -g @angular/cli
#######################
## angular lint workspace and prod build
#######################
FROM npm-base as prod-angular-build
RUN npm run lint
RUN npm run prodbuild
#######################
## Go dependencies
## Speed up this step by mounting your local go mod pkg directory
#######################
FROM golang:${GO_VERSION} as go-dep
RUN mkdir -p src/github.com/caos/zitadel
WORKDIR /go/src/github.com/caos/zitadel
#download modules
COPY ./go.* .
RUN go mod download
# install tools
COPY tools ./tools
RUN ./tools/install.sh
#######################
## generates static files
#######################
FROM go-dep AS go-static
COPY internal/ui/login/static internal/ui/login/static
COPY internal/ui/login/statik internal/ui/login/statik
COPY internal/notification/static internal/notification/static
COPY internal/notification/statik internal/notification/statik
COPY internal/static internal/static
COPY internal/statik internal/statik
RUN go generate internal/ui/login/statik/generate.go \
&& go generate internal/ui/login/static/generate.go \
&& go generate internal/notification/statik/generate.go \
&& go generate internal/statik/generate.go
#######################
## generates grpc stub
#######################
FROM go-static AS go-stub
COPY --from=base /proto /proto
COPY --from=base /usr/local/bin /usr/local/bin/.
COPY build/zitadel/generate-grpc.sh build/zitadel/generate-grpc.sh
COPY internal/protoc internal/protoc
COPY openapi/statik openapi/statik
COPY internal/api/assets/generator internal/api/assets/generator
COPY internal/config internal/config
COPY internal/errors internal/errors
RUN build/zitadel/generate-grpc.sh \
&& go generate openapi/statik/generate.go \
&& mkdir -p docs/apis/assets/ \
&& go run internal/api/assets/generator/asset_generator.go -directory=internal/api/assets/generator/ -assets=docs/apis/assets/assets.md
#######################
## Go base build
#######################
FROM go-stub as go-base
# copy remaining zitadel files
COPY . .
#######################
## copy for local dev
#######################
FROM scratch as go-copy
COPY --from=go-stub /go/src/github.com/caos/zitadel/internal/ui/login/statik/statik.go internal/ui/login/statik/statik.go
COPY --from=go-stub /go/src/github.com/caos/zitadel/internal/notification/statik/statik.go internal/notification/statik/statik.go
COPY --from=go-stub /go/src/github.com/caos/zitadel/internal/statik/statik.go internal/statik/statik.go
COPY --from=go-stub /go/src/github.com/caos/zitadel/openapi/statik/statik.go openapi/statik/statik.go
COPY --from=go-stub /go/src/github.com/caos/zitadel/pkg/grpc pkg/grpc
COPY --from=go-stub /go/src/github.com/caos/zitadel/openapi/v2/zitadel openapi/v2/zitadel
COPY --from=go-stub /go/src/github.com/caos/zitadel/openapi/statik/statik.go openapi/statik/statik.go
COPY --from=go-stub /go/src/github.com/caos/zitadel/internal/protoc/protoc-gen-authoption/templates.gen.go internal/protoc/protoc-gen-authoption/templates.gen.go
COPY --from=go-stub /go/src/github.com/caos/zitadel/internal/protoc/protoc-gen-authoption/authoption/options.pb.go internal/protoc/protoc-gen-authoption/authoption/options.pb.go
COPY --from=go-stub /go/src/github.com/caos/zitadel/docs/apis/proto docs/docs/apis/proto
COPY --from=go-stub /go/src/github.com/caos/zitadel/docs/apis/assets docs/docs/apis/assets
COPY --from=go-stub /go/src/github.com/caos/zitadel/internal/api/assets/authz.go ./internal/api/assets/authz.go
COPY --from=go-stub /go/src/github.com/caos/zitadel/internal/api/assets/router.go ./internal/api/assets/router.go
#######################
## Go test
#######################
FROM go-base as go-test
# Migrations for cockroach-secure
RUN go install github.com/rakyll/statik
RUN ./build/operator/prebuild.sh ./migrations
RUN go test -race -v -coverprofile=profile.cov $(go list ./... | grep -v /operator/)
#######################
## Go test results
#######################
FROM scratch as go-codecov
COPY --from=go-test /go/src/github.com/caos/zitadel/profile.cov profile.cov
#######################
## Go prod build
#######################
FROM go-test as prod-go-build
ARG BUILDARCH
#generate statik code for console
COPY --from=prod-angular-build console/dist/console console/dist/console/
RUN go generate internal/ui/console/statik/generate.go
RUN CGO_ENABLED=0 GOOS=linux GOARCH=${BUILDARCH} go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o zitadel-linux-${BUILDARCH} cmd/zitadel/main.go
#######################
## Go dev build
#######################
FROM go-base as dev-go-build
ENTRYPOINT [ "go", "run", "cmd/zitadel/main.go" ]
#######################
## Final Production Image
#######################
FROM alpine:latest as artifact
FROM alpine:3 as artifact
ARG BUILDARCH
RUN adduser -D zitadel
COPY cmd/zitadel/*.yaml /app/
COPY --from=prod-go-build /go/src/github.com/caos/zitadel/zitadel-linux-${BUILDARCH} /app/zitadel
COPY .download/zitadel/zitadel-linux-${BUILDARCH} /app/zitadel
COPY .download/zitadel/console /app/console/
RUN chmod a+x /app/zitadel
RUN ls -la /
#######################
## Scratch Image

View File

@@ -103,7 +103,7 @@ services:
- minio
build:
context: ../..
dockerfile: build/dockerfile
dockerfile: build/zitadel/Dockerfile
target: dev-go-build
args:
ENV: dev
@@ -164,7 +164,7 @@ services:
- grpc-web-gateway
build:
context: ../..
dockerfile: build/dockerfile
dockerfile: build/console/Dockerfile
target: dev-angular-build
args:
ENV: dev

View File

@@ -58,7 +58,7 @@ ZITADEL_CONSOLE_RESPONSE_TYPE=CODE
ZITADEL_CONSOLE_GRANT_TYPE=AUTHORIZATION_CODE
ZITADEL_CONSOLE_DEV_MODE=true
ZITADEL_CONSOLE_ENV_DIR=console/src/assets/
ZITADEL_CONSOLE_DIR=console/src/assets/
#minio configuration
ZITADEL_ASSET_STORAGE_ACCESS_KEY_ID=access_key

153
build/zitadel/Dockerfile Normal file
View File

@@ -0,0 +1,153 @@
ARG GO_VERSION=1.15
#######################
## These steps set platform / arch type specific variables
#######################
FROM alpine:3 AS arm64-base
ENV PROTOC_ARCH aarch_64
FROM alpine:3 AS amd64-base
ENV PROTOC_ARCH x86_64
#######################
## This step sets up the folder structure,
## initalices go mods,
## downloads the protofiles,
## protoc and protoc-gen-grpc-web for later use
#######################
FROM ${BUILDARCH}-base AS base
ARG PROTOC_VERSION=3.13.0
ARG PROTOC_ZIP=protoc-${PROTOC_VERSION}-linux-${PROTOC_ARCH}.zip
ARG GRPC_WEB_VERSION=1.2.1
# no arm specific version available and x86 works fine at the moment:
ARG GRPC_WEB=protoc-gen-grpc-web-${GRPC_WEB_VERSION}-linux-x86_64
RUN apk add tar curl
WORKDIR /proto
#protoc
RUN apk add tar curl \
&& curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/$PROTOC_ZIP \
&& unzip -o $PROTOC_ZIP -d /usr/local bin/protoc \
&& unzip -o $PROTOC_ZIP -d /proto include/* \
&& rm -f $PROTOC_ZIP \
&& curl -OL https://github.com/grpc/grpc-web/releases/download/${GRPC_WEB_VERSION}/${GRPC_WEB} \
&& mv ${GRPC_WEB} /usr/local/bin/protoc-gen-grpc-web \
&& chmod +x /usr/local/bin/protoc-gen-grpc-web \
&& curl https://raw.githubusercontent.com/envoyproxy/protoc-gen-validate/v0.4.1/validate/validate.proto --create-dirs -o include/validate/validate.proto \
&& curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v2.2.0/protoc-gen-openapiv2/options/annotations.proto --create-dirs -o include/protoc-gen-openapiv2/options/annotations.proto \
&& curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v2.2.0/protoc-gen-openapiv2/options/openapiv2.proto --create-dirs -o include/protoc-gen-openapiv2/options/openapiv2.proto \
&& curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/annotations.proto --create-dirs -o include/google/api/annotations.proto \
&& curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto --create-dirs -o include/google/api/http.proto \
&& curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/field_behavior.proto --create-dirs -o include/google/api/field_behavior.proto
#zitadel protos
COPY proto/ include/.
#######################
## Go dependencies
## Speed up this step by mounting your local go mod pkg directory
#######################
FROM golang:${GO_VERSION} as go-dep
RUN mkdir -p src/github.com/caos/zitadel
WORKDIR /go/src/github.com/caos/zitadel
#download modules
COPY ./go.* .
RUN go mod download
# install tools
COPY tools ./tools
RUN ./tools/install.sh
#######################
## generates static files
#######################
FROM go-dep AS go-static
COPY internal/ui/login/static internal/ui/login/static
COPY internal/ui/login/statik internal/ui/login/statik
COPY internal/notification/static internal/notification/static
COPY internal/notification/statik internal/notification/statik
COPY internal/static internal/static
COPY internal/statik internal/statik
RUN go generate internal/ui/login/statik/generate.go \
&& go generate internal/ui/login/static/generate.go \
&& go generate internal/notification/statik/generate.go \
&& go generate internal/statik/generate.go
#######################
## generates grpc stub
#######################
FROM go-static AS go-stub
COPY --from=base /proto /proto
COPY --from=base /usr/local/bin /usr/local/bin/.
COPY build/zitadel/generate-grpc.sh build/zitadel/generate-grpc.sh
COPY internal/protoc internal/protoc
COPY openapi/statik openapi/statik
COPY internal/api/assets/generator internal/api/assets/generator
COPY internal/config internal/config
COPY internal/errors internal/errors
RUN build/zitadel/generate-grpc.sh \
&& go generate openapi/statik/generate.go \
&& mkdir -p docs/apis/assets/ \
&& go run internal/api/assets/generator/asset_generator.go -directory=internal/api/assets/generator/ -assets=docs/apis/assets/assets.md
#######################
## Go base build
#######################
FROM go-stub as go-base
# copy remaining zitadel files
COPY . .
#######################
## copy for local dev
#######################
FROM scratch as go-copy
COPY --from=go-stub /go/src/github.com/caos/zitadel/internal/ui/login/statik/statik.go internal/ui/login/statik/statik.go
COPY --from=go-stub /go/src/github.com/caos/zitadel/internal/notification/statik/statik.go internal/notification/statik/statik.go
COPY --from=go-stub /go/src/github.com/caos/zitadel/internal/statik/statik.go internal/statik/statik.go
COPY --from=go-stub /go/src/github.com/caos/zitadel/openapi/statik/statik.go openapi/statik/statik.go
COPY --from=go-stub /go/src/github.com/caos/zitadel/pkg/grpc pkg/grpc
COPY --from=go-stub /go/src/github.com/caos/zitadel/openapi/v2/zitadel openapi/v2/zitadel
COPY --from=go-stub /go/src/github.com/caos/zitadel/openapi/statik/statik.go openapi/statik/statik.go
COPY --from=go-stub /go/src/github.com/caos/zitadel/internal/protoc/protoc-gen-authoption/templates.gen.go internal/protoc/protoc-gen-authoption/templates.gen.go
COPY --from=go-stub /go/src/github.com/caos/zitadel/internal/protoc/protoc-gen-authoption/authoption/options.pb.go internal/protoc/protoc-gen-authoption/authoption/options.pb.go
COPY --from=go-stub /go/src/github.com/caos/zitadel/docs/apis/proto docs/docs/apis/proto
COPY --from=go-stub /go/src/github.com/caos/zitadel/docs/apis/assets docs/docs/apis/assets
COPY --from=go-stub /go/src/github.com/caos/zitadel/internal/api/assets/authz.go ./internal/api/assets/authz.go
COPY --from=go-stub /go/src/github.com/caos/zitadel/internal/api/assets/router.go ./internal/api/assets/router.go
#######################
## Go test
#######################
FROM go-base as go-test
# Migrations for cockroach-secure
RUN go install github.com/rakyll/statik \
&& go test -race -v -coverprofile=profile.cov $(go list ./... | grep -v /operator/)
#######################
## Go test results
#######################
FROM scratch as go-codecov
COPY --from=go-test /go/src/github.com/caos/zitadel/profile.cov profile.cov
#######################
## Go prod build
#######################
FROM go-test as prod-go-build
ARG BUILDARCH
RUN CGO_ENABLED=0 GOOS=linux GOARCH=${BUILDARCH} go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o zitadel-linux-${BUILDARCH} cmd/zitadel/main.go
#######################
## Only Copy Assets
#######################
FROM scratch as prod-go-export
ARG BUILDARCH
COPY --from=prod-go-build /go/src/github.com/caos/zitadel/zitadel-linux-${BUILDARCH} .