chore(contribute): add step by step guide (#1754)

* chore(contributing): add startup

* init

* cleanup docker file

* local

* compose works

* markdowns

* add gateway start on readme

* readme done

* finish mds

* rename/delete compose files

* correct docker compose file name

* fix links,
update contribute,
split build/readme into separate files in /guides,
add zitadel startup

* fix(docker compose): allow .keys folder to not exist

* update md's

* use docker-compose instead of docker compose as --profile gets ignored

* write a message if create key

* copy openapi statik.go

* explain how to connect in quickstart

* Apply suggestions from code review

Co-authored-by: Livio Amstutz <livio.a@gmail.com>

* sremove subscription service from env.json

* Delete caos_local.sh

moved to build/local/local.env

Co-authored-by: Livio Amstutz <livio.a@gmail.com>
This commit is contained in:
Silvan
2021-06-01 09:55:08 +02:00
committed by GitHub
parent d61baadfd9
commit a6e4b537fe
28 changed files with 670 additions and 299 deletions

View File

@@ -1,60 +0,0 @@
# Development
## Prerequisite
- Buildkit compatible docker installation
## Generate Proto Clients
### Angular
This command generates the grpc stub for angular into the folder console/src/app/proto/generated for local development
```Bash
DOCKER_BUILDKIT=1 docker build -f build/dockerfile . -t zitadel:local --target npm-copy -o .
```
### Go
With this command you can generate the stub for golang into the zitadel dir
```Bash
DOCKER_BUILDKIT=1 docker build -f build/dockerfile . -t zitadel:local --target go-copy -o .
```
## Run
### Run Angular
```Bash
COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose -f build/docker-compose-dev.yml up --build angular
```
### Run Go
```Bash
COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose -f build/docker-compose-dev.yml up --build go
```
### Fullstack including database
```Bash
COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose -f build/docker-compose-dev.yml up --build
```
## Debug
### Debug Go
```Bash
COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose -f build/docker-compose-debug.yml up --build go
```
## Production Build
This can also be run locally!
```Bash
DOCKER_BUILDKIT=1 docker build -f build/dockerfile . -t zitadel:local --build-arg ENV=prod
```

View File

@@ -1,5 +0,0 @@
#! /bin/sh
set -eux
go generate internal/ui/console/statik/generate.go

View File

@@ -1,30 +0,0 @@
version: "3.8"
services:
angular:
build:
context: ..
dockerfile: dockerfile
target: dev-angular-build
args:
ENV: dev
command: sh -c "ng serve --host 0.0.0.0"
ports:
- 4200:4200
go:
build:
context: ..
dockerfile: dockerfile
target: dev-go-build
args:
ENV: dev
command: dlv --listen=:2345 --headless=true --log=true --log-output=debugger,debuglineerr,gdbwire,lldbout,rpc --accept-multiclient --api-version=2 debug cmd/zitadel/main.go
ports:
- 2345:2345
- 50000:50000
db:
image: cockroachdb/cockroach:v20.2.0
command: start-single-node --insecure
ports:
- 8080:8080
- 26257:26257

View File

@@ -1,31 +0,0 @@
version: "3.8"
services:
angular:
build:
context: ..
dockerfile: dockerfile
target: dev-angular-build
args:
ENV: dev
command: sh -c "ng serve --host 0.0.0.0"
ports:
- 4200:4200
go:
build:
context: ..
dockerfile: dockerfile
target: dev-go-build
args:
ENV: dev
command: go run cmd/zitadel/main.go
ports:
- 50000:50000
db:
image: cockroachdb/cockroach:v20.2.0
command: start-single-node --insecure
ports:
- 8080:8080
- 26257:26257
volumes:
- "../cockroach-data/zitadel1:/cockroach/cockroach-data"

View File

@@ -11,6 +11,7 @@ ENV PROTOC_ARCH aarch_64
FROM alpine AS amd64-base
ENV PROTOC_ARCH x86_64
#######################
## This step sets up the folder structure,
## initalices go mods,
@@ -24,7 +25,6 @@ ARG GRPC_WEB_VERSION=1.2.1
# no arm specific version available and x86 works fine at the moment:
ARG GRPC_WEB=protoc-gen-grpc-web-${GRPC_WEB_VERSION}-linux-x86_64
RUN apk add tar curl
WORKDIR /proto
@@ -51,7 +51,6 @@ RUN curl https://raw.githubusercontent.com/envoyproxy/protoc-gen-validate/v0.4.1
COPY proto/ include/.
#######################
## With this step we prepare all node_modules, this helps caching the build
## Speed up this step by mounting your local node_modules directory
@@ -75,12 +74,14 @@ RUN build/console/generate-grpc.sh
FROM scratch as npm-copy
COPY --from=npm-base /console/src/app/proto/generated ./console/src/app/proto/generated
#######################
## angular dev build
#######################
FROM npm-base as dev-angular-build
RUN npm install -g @angular/cli
#######################
## angular lint workspace and prod build
#######################
@@ -88,44 +89,79 @@ FROM npm-base as prod-angular-build
RUN npm run lint
RUN npm run prodbuild
#######################
## Go dependencies
## Speed up this step by mounting your local go mod pkg directory
#######################
FROM golang:${GO_VERSION} as go-dep
RUN mkdir -p src/github.com/caos/zitadel
# copy mod definitions
COPY tools src/github.com/caos/zitadel/tools
COPY ./go.* src/github.com/caos/zitadel
# install all dependencies
WORKDIR /go/src/github.com/caos/zitadel
#download modules
COPY ./go.* .
RUN go mod download
# install tools
COPY tools ./tools
RUN ./tools/install.sh
FROM go-dep AS go-gen
#######################
## generates static files
#######################
FROM go-dep AS go-static
COPY internal/ui/login/static internal/ui/login/static
COPY internal/ui/login/statik internal/ui/login/statik
COPY internal/notification/static internal/notification/static
COPY internal/notification/statik internal/notification/statik
COPY internal/static internal/static
COPY internal/statik internal/statik
RUN go generate internal/ui/login/statik/generate.go \
&& go generate internal/ui/login/static/generate.go \
&& go generate internal/notification/statik/generate.go \
&& go generate internal/statik/generate.go
#######################
## generates grpc stub
#######################
FROM go-static AS go-stub
COPY --from=base /proto /proto
COPY --from=base /usr/local/bin /usr/local/bin/.
COPY build/zitadel/generate-grpc.sh build/zitadel/generate-grpc.sh
COPY internal/protoc internal/protoc
RUN build/zitadel/generate-grpc.sh
COPY openapi/statik openapi/statik
RUN build/zitadel/generate-grpc.sh \
&& go generate openapi/statik/generate.go
#######################
## Go base build
#######################
FROM go-gen as go-base
# copy all zitadel files
FROM go-stub as go-base
# copy remaining zitadel files
COPY . .
#######################
## copy for local dev
#######################
FROM scratch as go-copy
COPY --from=go-gen /go/src/github.com/caos/zitadel/pkg/grpc ./pkg/grpc
COPY --from=go-gen /go/src/github.com/caos/zitadel/openapi/v2/zitadel ./openapi/v2/zitadel
COPY --from=go-gen /go/src/github.com/caos/zitadel/internal/protoc/protoc-gen-authoption/templates.gen.go ./internal/protoc/protoc-gen-authoption/templates.gen.go
COPY --from=go-gen /go/src/github.com/caos/zitadel/internal/protoc/protoc-gen-authoption/authoption/options.pb.go ./internal/protoc/protoc-gen-authoption/authoption/options.pb.go
COPY --from=go-gen /go/src/github.com/caos/zitadel/docs/apis/proto ./docs/docs/apis/proto
COPY --from=go-static /go/src/github.com/caos/zitadel/internal/ui/login/statik/statik.go internal/ui/login/statik/statik.go
COPY --from=go-static /go/src/github.com/caos/zitadel/internal/notification/statik/statik.go internal/notification/statik/statik.go
COPY --from=go-static /go/src/github.com/caos/zitadel/internal/statik/statik.go internal/statik/statik.go
COPY --from=go-static /go/src/github.com/caos/zitadel/openapi/statik/statik.go openapi/statik/statik.go
COPY --from=go-stub /go/src/github.com/caos/zitadel/pkg/grpc pkg/grpc
COPY --from=go-stub /go/src/github.com/caos/zitadel/openapi/v2/zitadel openapi/v2/zitadel
COPY --from=go-stub /go/src/github.com/caos/zitadel/openapi/statik/statik.go openapi/statik/statik.go
COPY --from=go-stub /go/src/github.com/caos/zitadel/internal/protoc/protoc-gen-authoption/templates.gen.go internal/protoc/protoc-gen-authoption/templates.gen.go
COPY --from=go-stub /go/src/github.com/caos/zitadel/internal/protoc/protoc-gen-authoption/authoption/options.pb.go internal/protoc/protoc-gen-authoption/authoption/options.pb.go
COPY --from=go-stub /go/src/github.com/caos/zitadel/docs/apis/proto docs/docs/apis/proto
#######################
@@ -151,13 +187,11 @@ COPY --from=go-test /go/src/github.com/caos/zitadel/profile.cov profile.cov
#######################
FROM go-test as prod-go-build
ARG BUILDARCH
#generate statik code for console
COPY --from=prod-angular-build console/dist/console console/dist/console/
RUN go get github.com/rakyll/statik \
&& ./build/console/generate-static.sh \
&& ./build/login/generate-static.sh \
&& ./build/notification/generate-static.sh \
&& ./build/zitadel/generate-static.sh \
&& ./build/zitadel/generate-openapi-static.sh
RUN go generate internal/statik/generate.go
RUN CGO_ENABLED=0 GOOS=linux GOARCH=${BUILDARCH} go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o zitadel-linux-${BUILDARCH} cmd/zitadel/main.go
@@ -165,8 +199,7 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=${BUILDARCH} go build -a -installsuffix cgo
## Go dev build
#######################
FROM go-base as dev-go-build
RUN go get github.com/go-delve/delve/cmd/dlv
ENTRYPOINT [ "go", "run", "cmd/zitadel/main.go" ]
#######################
## Final Production Image

View File

@@ -0,0 +1,11 @@
FROM ubuntu:latest AS client-id
#install dependencies
RUN apt-get update \
&& apt-get install curl -y \
&& apt-get install jq -y
#prepare script
COPY build/local/clientid.sh clientid.sh
RUN chmod +x /clientid.sh
ENTRYPOINT [ "/clientid.sh" ]

View File

@@ -0,0 +1,36 @@
# copy from https://raw.githubusercontent.com/grpc/grpc-web/master/net/grpc/gateway/docker/grpcwebproxy/Dockerfile
FROM golang:1.16-alpine3.13
RUN apk add --no-cache curl git ca-certificates && \
rm -rf /var/lib/apt/lists/*
ARG VERSION=0.14.0
WORKDIR /tmp
RUN curl -sS https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
RUN wget https://github.com/improbable-eng/grpc-web/archive/v$VERSION.tar.gz
WORKDIR /go/src/github.com/improbable-eng/
RUN tar -zxf /tmp/v$VERSION.tar.gz -C .
RUN mv grpc-web-$VERSION grpc-web
WORKDIR /go/src/github.com/improbable-eng/grpc-web
RUN dep ensure && \
go env -w GO111MODULE=auto && \
go install ./go/grpcwebproxy
# ADD ./etc/localhost.crt /etc
# ADD ./etc/localhost.key /etc
ENV BKD_HOST=backend-run
ENV BKD_PORT=50001
ENTRYPOINT [ "/bin/sh", "-c", "exec /go/bin/grpcwebproxy \
--backend_addr=${BKD_HOST}:${BKD_PORT} \
--run_tls_server=false \
--use_websockets \
--allow_all_origins " ]

View File

@@ -0,0 +1,7 @@
FROM alpine:latest AS gen-keys
COPY build/local/keys.sh keys.sh
RUN chmod +x /keys.sh
ENTRYPOINT [ "/keys.sh" ]
FROM scratch AS copy-keys
COPY --from=gen-keys /.keys /.keys

View File

@@ -0,0 +1,10 @@
FROM ubuntu:latest AS started
#install dependencies
RUN apt-get update \
&& apt-get install curl -y
#prepare script
COPY build/local/zitadel-started.sh zitadel-started.sh
RUN chmod +x /zitadel-started.sh
ENTRYPOINT [ "/zitadel-started.sh" ]

18
build/local/clientid.sh Normal file
View File

@@ -0,0 +1,18 @@
#!/bin/bash
# ------------------------------
# sets the client id in environment.json
# ------------------------------
clientid=""
while [ -z $clientid ]; do
echo "no from zitadel ==> retry"
sleep 2
clientid=$(curl -s http://${HOST}:${PORT}/clientID)
if [[ "$clientid" != *@zitadel* ]]; then
echo "invalid response from zitadel ==> retry"
clientid=""
fi
done
echo "$(jq ".clientid = $clientid" /environment.json)" > environment.json

View File

@@ -0,0 +1,157 @@
version: "3.8"
services:
db:
profiles: ["database"]
restart: always
networks:
- zitadel
image: cockroachdb/cockroach:v21.1.0
command: start-single-node --insecure --listen-addr=0.0.0.0
ports:
- 8080:8080
- 26257:26257
db-migrations:
profiles: ["database"]
restart: on-failure
networks:
- zitadel
depends_on:
- db
image: flyway/flyway:latest
volumes:
- ../../migrations/cockroach:/flyway/sql
environment:
- FLYWAY_PLACEHOLDERS_eventstorepassword=NULL
- FLYWAY_PLACEHOLDERS_managementpassword=NULL
- FLYWAY_PLACEHOLDERS_adminapipassword=NULL
- FLYWAY_PLACEHOLDERS_authpassword=NULL
- FLYWAY_PLACEHOLDERS_notificationpassword=NULL
- FLYWAY_PLACEHOLDERS_authzpassword=NULL
- FLYWAY_PLACEHOLDERS_queriespassword=NULL
command: -url=jdbc:postgresql://db:26257/defaultdb -user=root -password= -connectRetries=5 migrate
keys:
profiles: ["init-backend"]
restart: on-failure
networks:
- zitadel
build:
context: ../..
dockerfile: build/local/Dockerfile.keys
target: gen-keys
volumes:
- ../../.:/zitadel
env_file:
- ./local.env
backend-setup:
profiles: ["init-backend"]
restart: on-failure
networks:
- zitadel
depends_on:
- keys
build:
context: ../..
dockerfile: build/dockerfile
target: dev-go-build
args:
ENV: dev
volumes:
- ../../.keys:/go/src/github.com/caos/zitadel/.keys
env_file:
- ./local.env
environment:
- ZITADEL_EVENTSTORE_HOST=db
command: [ "-setup-files=cmd/zitadel/setup.yaml", "-setup-files=cmd/zitadel/system-defaults.yaml", "-setup-files=cmd/zitadel/authz.yaml", "setup" ]
backend-run:
profiles: ["backend"]
restart: on-failure
networks:
- zitadel
depends_on:
- db
build:
context: ../..
dockerfile: build/dockerfile
target: dev-go-build
args:
ENV: dev
volumes:
- ../../.keys:/go/src/github.com/caos/zitadel/.keys
env_file:
- ./local.env
environment:
- ZITADEL_EVENTSTORE_HOST=db
ports:
- 50002:50002
- 50003:50003
command: [ "-console=false", "-localDevMode=true", "-config-files=cmd/zitadel/startup.yaml", "-config-files=cmd/zitadel/system-defaults.yaml", "-config-files=cmd/zitadel/authz.yaml", "start" ]
zitadel-setted-up:
profiles: ["setup"]
networks:
- zitadel
build:
context: ../..
dockerfile: build/local/Dockerfile.started
volumes:
- ./environment.json:/environment.json
environment:
- BE_PORT=50002
- FE_PORT=4200
grpc-web-gateway:
profiles: ["frontend"]
restart: on-failure
logging:
driver: none
networks:
- zitadel
build:
context: ../..
dockerfile: build/local/Dockerfile.gateway
image: grpcweb/grpcwebproxy
ports:
- "50000:8080"
environment:
- BKD_HOST=backend-run
- BKD_PORT=50001
frontend-local-run:
profiles: ["frontend"]
networks:
- zitadel
depends_on:
- grpc-web-gateway
build:
context: ../..
dockerfile: build/dockerfile
target: dev-angular-build
args:
ENV: dev
volumes:
- ./environment.json:/console/src/assets/environment.json
command: sh -c "ng serve --host 0.0.0.0"
ports:
- 4200:4200
client-id:
profiles: ["init-frontend"]
networks:
- zitadel
build:
context: ../..
dockerfile: build/local/Dockerfile.clientid
target: client-id
volumes:
- ./environment.json:/environment.json
environment:
- HOST=backend-run
- PORT=50002
networks:
zitadel: {}

View File

@@ -0,0 +1,7 @@
{
"authServiceUrl": "http://localhost:50000",
"mgmtServiceUrl": "http://localhost:50000",
"adminServiceUrl": "http://localhost:50000",
"issuer": "http://localhost:50002/oauth/v2",
"clientid": "@zitadel"
}

23
build/local/keys.sh Executable file
View File

@@ -0,0 +1,23 @@
#!/bin/sh
# ----------------------------------------------------------------
# generates necessary ZITADEL keys
# ----------------------------------------------------------------
set -e
KEY_PATH=$(echo "/zitadel/$(dirname ${ZITADEL_KEY_PATH})")
KEY_FILE=${KEY_PATH}/local_keys.yaml
mkdir -p ${KEY_PATH}
if [ ! -f ${KEY_FILE} ]; then
touch ${KEY_FILE}
fi
for key in $(env | grep "ZITADEL_.*_KEY" | cut -d'=' -f2); do
if [ $(grep -L ${key} ${KEY_FILE}) ]; then
echo "create key for ${key} in ${KEY_FILE}"
echo -e "${key}: $(head -c22 /dev/urandom | base64)" >> ${KEY_FILE}
fi
done

60
build/local/local.env Normal file
View File

@@ -0,0 +1,60 @@
#tracing is disabled locally
ZITADEL_TRACING_TYPE=none
#metrics is disabled locally
ZITADEL_METRICS_TYPE=none
#recommended log level for local is debug
ZITADEL_LOG_LEVEL=debug
#database connection (cockroach insecure)
ZITADEL_EVENTSTORE_HOST=localhost
ZITADEL_EVENTSTORE_PORT=26257
CR_SSL_MODE=disable
#keys for cryptography
ZITADEL_KEY_PATH=.keys/local_keys.yaml
ZITADEL_USER_VERIFICATION_KEY=userverificationkey_1
ZITADEL_OTP_VERIFICATION_KEY=OTPVerificationKey_1
ZITADEL_OIDC_KEYS_ID=oidckey_1
ZITADEL_COOKIE_KEY=cookiekey_1
ZITADEL_CSRF_KEY=cookiekey_1
ZITADEL_IDP_CONFIG_VERIFICATION_KEY=idpconfigverificationkey_1
ZITADEL_DOMAIN_VERIFICATION_KEY=domainverificationkey_1
#debug mode is used for notifications
DEBUG_MODE=true
#used in the oidc library
#true enables usage of (insecure) http for localhost as issuer
CAOS_OIDC_DEV=true
#sets the cookies insecure in login (never use this in production!)
ZITADEL_CSRF_DEV=true
#currently needed
TWILIO_SENDER_NAME=ZITADEL developer
SMTP_HOST=smtp.gmail.com:465
SMTP_USER=zitadel-dev@caos.ch
EMAIL_SENDER_ADDRESS=noreply@caos.ch
EMAIL_SENDER_NAME=CAOS AG
SMTP_TLS=true
#configuration for api/browser calls
ZITADEL_DEFAULT_DOMAIN=zitadel.ch
ZITADEL_ISSUER=http://localhost:50002/oauth/v2
ZITADEL_ACCOUNTS=http://localhost:50003/login
ZITADEL_AUTHORIZE=http://localhost:50002/oauth/v2
ZITADEL_OAUTH=http://localhost:50002/oauth/v2
ZITADEL_CONSOLE=http://localhost:4200
ZITADEL_COOKIE_DOMAIN=localhost
#caching is used in UI's and API's
ZITADEL_CACHE_MAXAGE=12h
ZITADEL_CACHE_SHARED_MAXAGE=168h
ZITADEL_SHORT_CACHE_MAXAGE=5m
ZITADEL_SHORT_CACHE_SHARED_MAXAGE=15m
#console authorization configuration
ZITADEL_CONSOLE_RESPONSE_TYPE=CODE
ZITADEL_CONSOLE_GRANT_TYPE=AUTHORIZATION_CODE
ZITADEL_CONSOLE_DEV_MODE=true
ZITADEL_CONSOLE_ENV_DIR=console/src/assets/

37
build/local/zitadel-started.sh Executable file
View File

@@ -0,0 +1,37 @@
#!/bin/bash
# ------------------------------
# prints a message as soon as
# ZITADEL is ready
# ------------------------------
be_status=""
fe_status=""
while [[ $be_status -ne 200 || $fe_status -ne 200 ]]; do
sleep 5
be_status=$(curl -s -o /dev/null -I -w "%{http_code}" host.docker.internal:${BE_PORT}/clientID)
fe_status=$(curl -s -o /dev/null -I -w "%{http_code}" host.docker.internal:${FE_PORT}/assets/environment.json)
echo "backend (${be_status}) or frontend (${fe_status}) not ready yet"
done
echo -e "++=======================================================================================++
|| ||
|| ZZZZZZZZZZZZ II TTTTTTTTTTTT AAAA DDDDDD EEEEEEEEEE LL ||
|| ZZ II TT AA AA DD DD EE LL ||
|| ZZ II TT AA AA DD DD EE LL ||
|| ZZ II TT AA AA DD DD EEEEEEEE LL ||
|| ZZ II TT AAAAAAAAAAAA DD DD EE LL ||
|| ZZ II TT AA AA DD DD EE LL ||
|| ZZZZZZZZZZZZ II TT AA AA DDDDDD EEEEEEEEEE LLLLLLLLLL ||
|| ||
|| ||
|| SSSSSSSSSS TTTTTTTTTTTT AAAA RRRRRRRR TTTTTTTTTTTT EEEEEEEEEE DDDDDD ||
|| SS TT AA AA RR RR TT EE DD DD ||
|| SS TT AA AA RR RR TT EE DD DD ||
|| SSSSSS TT AA AA RRRRRRRR TT EEEEEEEE DD DD ||
|| SS TT AAAAAAAAAAAA RRRR TT EE DD DD ||
|| SS TT AA AA RR RR TT EE DD DD ||
|| SSSSSSSSSS TT AA AA RR RR TT EEEEEEEEEE DDDDDD ||
|| ||
++=======================================================================================++"

View File

@@ -1,5 +0,0 @@
#! /bin/sh
set -eux
go generate internal/ui/login/statik/generate.go

View File

@@ -1,5 +0,0 @@
#! /bin/sh
set -eux
go generate internal/notification/statik/generate.go

View File

@@ -1,5 +0,0 @@
#! /bin/sh
set -eux
go generate openapi/statik/generate.go

View File

@@ -1,5 +0,0 @@
#! /bin/sh
set -eux
go generate internal/statik/generate.go