From c64d756ea7d3a9f9ca22b67740a2f6fabed46916 Mon Sep 17 00:00:00 2001 From: Casey Marshall Date: Wed, 16 Jun 2021 23:27:40 -0500 Subject: [PATCH] Add k8s deployment, standalone app Dockerfile. Tested with Rancher k3s. See k8s/README.md for site configuration and deployment instructions. Add cert-manager, tls, remote headscale script. --- Dockerfile | 12 +++ k8s/.gitignore | 2 + k8s/README.md | 99 +++++++++++++++++++++++ k8s/base/configmap.yaml | 8 ++ k8s/base/ingress.yaml | 18 +++++ k8s/base/kustomization.yaml | 42 ++++++++++ k8s/base/service.yaml | 13 +++ k8s/headscale.bash | 3 + k8s/init.bash | 22 +++++ k8s/install-cert-manager.bash | 3 + k8s/postgres/deployment.yaml | 78 ++++++++++++++++++ k8s/postgres/kustomization.yaml | 13 +++ k8s/postgres/postgres-service.yaml | 13 +++ k8s/postgres/postgres-statefulset.yaml | 49 +++++++++++ k8s/production-tls/ingress-patch.yaml | 11 +++ k8s/production-tls/kustomization.yaml | 9 +++ k8s/production-tls/production-issuer.yaml | 16 ++++ k8s/sqlite/kustomization.yaml | 5 ++ k8s/sqlite/statefulset.yaml | 79 ++++++++++++++++++ k8s/staging-tls/ingress-patch.yaml | 11 +++ k8s/staging-tls/kustomization.yaml | 9 +++ k8s/staging-tls/staging-issuer.yaml | 16 ++++ 22 files changed, 531 insertions(+) create mode 100644 Dockerfile create mode 100644 k8s/.gitignore create mode 100644 k8s/README.md create mode 100644 k8s/base/configmap.yaml create mode 100644 k8s/base/ingress.yaml create mode 100644 k8s/base/kustomization.yaml create mode 100644 k8s/base/service.yaml create mode 100755 k8s/headscale.bash create mode 100755 k8s/init.bash create mode 100755 k8s/install-cert-manager.bash create mode 100644 k8s/postgres/deployment.yaml create mode 100644 k8s/postgres/kustomization.yaml create mode 100644 k8s/postgres/postgres-service.yaml create mode 100644 k8s/postgres/postgres-statefulset.yaml create mode 100644 k8s/production-tls/ingress-patch.yaml create mode 100644 k8s/production-tls/kustomization.yaml create mode 100644 k8s/production-tls/production-issuer.yaml create mode 100644 k8s/sqlite/kustomization.yaml create mode 100644 k8s/sqlite/statefulset.yaml create mode 100644 k8s/staging-tls/ingress-patch.yaml create mode 100644 k8s/staging-tls/kustomization.yaml create mode 100644 k8s/staging-tls/staging-issuer.yaml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..e3fd8efc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM golang:latest AS build +ENV GOPATH /go +COPY . /go/src/headscale +WORKDIR /go/src/headscale +RUN go install -a -ldflags="-extldflags=-static" -tags netgo,sqlite_omit_load_extension ./cmd/headscale +RUN test -e /go/bin/headscale + +FROM scratch +COPY --from=build /go/bin/headscale /go/bin/headscale +ENV TZ UTC +EXPOSE 8080/tcp +ENTRYPOINT ["/go/bin/headscale"] diff --git a/k8s/.gitignore b/k8s/.gitignore new file mode 100644 index 00000000..229058d2 --- /dev/null +++ b/k8s/.gitignore @@ -0,0 +1,2 @@ +/**/site +/**/secrets diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 00000000..5cd18ce2 --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,99 @@ +# Deploying Headscale on Kubernetes + +This directory contains [Kustomize](https://kustomize.io) templates that deploy +Headscale in various configurations. + +These templates currently support Rancher k3s. Other clusters may require +adaptation, especially around volume claims and ingress. + +Commands below assume this directory is your current working directory. + +# Generate secrets and site configuration + +Run `./init.bash` to generate keys, passwords, and site configuration files. + +Edit `base/site/public.env`, changing `public-hostname` to the public DNS name +that will be used for your headscale deployment. + +Set `public-proto` to "https" if you're planning to use TLS & Let's Encrypt. + +Configure DERP servers by editing `base/site/derp.yaml` if needed. + +# Add the image to the registry + +You'll somehow need to get `headscale:latest` into your cluster image registry. + +An easy way to do this with k3s: +- Reconfigure k3s to use docker instead of containerd (`k3s server --docker`) +- `docker build -t headscale:latest ..` from here + +# Create the namespace + +If it doesn't already exist, `kubectl create ns headscale`. + +# Deploy headscale + +## sqlite + +`kubectl -n headscale apply -k ./sqlite` + +## postgres + +`kubectl -n headscale apply -k ./postgres` + +# TLS & Let's Encrypt + +Test a staging certificate with your configured DNS name and Let's Encrypt. + +`kubectl -n headscale apply -k ./staging-tls` + +Replace with a production certificate. + +`kubectl -n headscale apply -k ./production-tls` + +## Static / custom TLS certificates + +Only Let's Encrypt is supported. If you need other TLS settings, modify or patch the ingress. + +# Administration + +Use the wrapper script to remotely operate headscale to perform administrative +tasks like creating namespaces, authkeys, etc. + +``` +[c@nix-slate:~/Projects/headscale/k8s]$ ./headscale.bash + +headscale is an open source implementation of the Tailscale control server + +Juan Font Alonso - 2021 +https://gitlab.com/juanfont/headscale + +Usage: + headscale [command] + +Available Commands: + help Help about any command + namespace Manage the namespaces of Headscale + node Manage the nodes of Headscale + preauthkey Handle the preauthkeys in Headscale + routes Manage the routes of Headscale + serve Launches the headscale server + version Print the version. + +Flags: + -h, --help help for headscale + -o, --output string Output format. Empty for human-readable, 'json' or 'json-line' + +Use "headscale [command] --help" for more information about a command. + +``` + +# TODO / Ideas + +- Github action to publish the docker image +- Interpolate `email:` option to the ClusterIssuer from site configuration. + This probably needs to be done with a transformer, kustomize vars don't seem to work. +- Add kustomize examples for cloud-native ingress, load balancer +- CockroachDB for the backend +- DERP server deployment +- Tor hidden service diff --git a/k8s/base/configmap.yaml b/k8s/base/configmap.yaml new file mode 100644 index 00000000..2e25e5f9 --- /dev/null +++ b/k8s/base/configmap.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: headscale-config +data: + server_url: $(PUBLIC_PROTO)://$(PUBLIC_HOSTNAME) + listen_addr: "0.0.0.0:8080" + ephemeral_node_inactivity_timeout: "30m" diff --git a/k8s/base/ingress.yaml b/k8s/base/ingress.yaml new file mode 100644 index 00000000..a279bc1c --- /dev/null +++ b/k8s/base/ingress.yaml @@ -0,0 +1,18 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: headscale + annotations: + kubernetes.io/ingress.class: traefik +spec: + rules: + - host: $(PUBLIC_HOSTNAME) + http: + paths: + - backend: + service: + name: headscale + port: + number: 8080 + path: / + pathType: Prefix diff --git a/k8s/base/kustomization.yaml b/k8s/base/kustomization.yaml new file mode 100644 index 00000000..54d66e54 --- /dev/null +++ b/k8s/base/kustomization.yaml @@ -0,0 +1,42 @@ +namespace: headscale +resources: +- configmap.yaml +- ingress.yaml +- service.yaml +generatorOptions: + disableNameSuffixHash: true +configMapGenerator: +- name: headscale-site + files: + - derp.yaml=site/derp.yaml + envs: + - site/public.env +- name: headscale-etc + literals: + - config.json={} +secretGenerator: +- name: headscale + files: + - secrets/private-key +vars: +- name: PUBLIC_PROTO + objRef: + kind: ConfigMap + name: headscale-site + apiVersion: v1 + fieldRef: + fieldPath: data.public-proto +- name: PUBLIC_HOSTNAME + objRef: + kind: ConfigMap + name: headscale-site + apiVersion: v1 + fieldRef: + fieldPath: data.public-hostname +- name: CONTACT_EMAIL + objRef: + kind: ConfigMap + name: headscale-site + apiVersion: v1 + fieldRef: + fieldPath: data.contact-email diff --git a/k8s/base/service.yaml b/k8s/base/service.yaml new file mode 100644 index 00000000..7fdf738f --- /dev/null +++ b/k8s/base/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: headscale + labels: + app: headscale +spec: + selector: + app: headscale + ports: + - name: http + targetPort: http + port: 8080 diff --git a/k8s/headscale.bash b/k8s/headscale.bash new file mode 100755 index 00000000..66bfe92c --- /dev/null +++ b/k8s/headscale.bash @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -eu +exec kubectl -n headscale exec -ti pod/headscale-0 -- /go/bin/headscale "$@" diff --git a/k8s/init.bash b/k8s/init.bash new file mode 100755 index 00000000..e5b7965c --- /dev/null +++ b/k8s/init.bash @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -eux +cd $(dirname $0) + +umask 022 +mkdir -p base/site/ +[ ! -e base/site/public.env ] && ( + cat >base/site/public.env < base/secrets/private-key +) +mkdir -p postgres/secrets/ +[ ! -e postgres/secrets/password ] && (head -c 32 /dev/urandom | base64 -w0 > postgres/secrets/password) diff --git a/k8s/install-cert-manager.bash b/k8s/install-cert-manager.bash new file mode 100755 index 00000000..1a5ecacb --- /dev/null +++ b/k8s/install-cert-manager.bash @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -eux +kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.4.0/cert-manager.yaml diff --git a/k8s/postgres/deployment.yaml b/k8s/postgres/deployment.yaml new file mode 100644 index 00000000..dd45d05b --- /dev/null +++ b/k8s/postgres/deployment.yaml @@ -0,0 +1,78 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: headscale +spec: + replicas: 2 + selector: + matchLabels: + app: headscale + template: + metadata: + labels: + app: headscale + spec: + containers: + - name: headscale + image: "headscale:latest" + imagePullPolicy: IfNotPresent + command: ["/go/bin/headscale", "serve"] + env: + - name: SERVER_URL + value: $(PUBLIC_PROTO)://$(PUBLIC_HOSTNAME) + - name: LISTEN_ADDR + valueFrom: + configMapKeyRef: + name: headscale-config + key: listen_addr + - name: PRIVATE_KEY_PATH + value: /vol/secret/private-key + - name: DERP_MAP_PATH + value: /vol/config/derp.yaml + - name: EPHEMERAL_NODE_INACTIVITY_TIMEOUT + valueFrom: + configMapKeyRef: + name: headscale-config + key: ephemeral_node_inactivity_timeout + - name: DB_TYPE + value: postgres + - name: DB_HOST + value: postgres.headscale.svc.cluster.local + - name: DB_PORT + value: "5432" + - name: DB_USER + value: headscale + - name: DB_PASS + valueFrom: + secretKeyRef: + name: postgresql + key: password + - name: DB_NAME + value: headscale + ports: + - name: http + protocol: TCP + containerPort: 8080 + livenessProbe: + tcpSocket: + port: http + initialDelaySeconds: 30 + timeoutSeconds: 5 + periodSeconds: 15 + volumeMounts: + - name: config + mountPath: /vol/config + - name: secret + mountPath: /vol/secret + - name: etc + mountPath: /etc/headscale + volumes: + - name: config + configMap: + name: headscale-site + - name: etc + configMap: + name: headscale-etc + - name: secret + secret: + secretName: headscale diff --git a/k8s/postgres/kustomization.yaml b/k8s/postgres/kustomization.yaml new file mode 100644 index 00000000..8bd6c40c --- /dev/null +++ b/k8s/postgres/kustomization.yaml @@ -0,0 +1,13 @@ +namespace: headscale +bases: +- ../base +resources: +- deployment.yaml +- postgres-service.yaml +- postgres-statefulset.yaml +generatorOptions: + disableNameSuffixHash: true +secretGenerator: +- name: postgresql + files: + - secrets/password diff --git a/k8s/postgres/postgres-service.yaml b/k8s/postgres/postgres-service.yaml new file mode 100644 index 00000000..e2f486cc --- /dev/null +++ b/k8s/postgres/postgres-service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: postgres + labels: + app: postgres +spec: + selector: + app: postgres + ports: + - name: postgres + targetPort: postgres + port: 5432 diff --git a/k8s/postgres/postgres-statefulset.yaml b/k8s/postgres/postgres-statefulset.yaml new file mode 100644 index 00000000..25285c5e --- /dev/null +++ b/k8s/postgres/postgres-statefulset.yaml @@ -0,0 +1,49 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres +spec: + serviceName: postgres + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: "postgres:13" + imagePullPolicy: IfNotPresent + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgresql + key: password + - name: POSTGRES_USER + value: headscale + ports: + - name: postgres + protocol: TCP + containerPort: 5432 + livenessProbe: + tcpSocket: + port: 5432 + initialDelaySeconds: 30 + timeoutSeconds: 5 + periodSeconds: 15 + volumeMounts: + - name: pgdata + mountPath: /var/lib/postgresql/data + volumeClaimTemplates: + - metadata: + name: pgdata + spec: + storageClassName: local-path + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi diff --git a/k8s/production-tls/ingress-patch.yaml b/k8s/production-tls/ingress-patch.yaml new file mode 100644 index 00000000..387c7364 --- /dev/null +++ b/k8s/production-tls/ingress-patch.yaml @@ -0,0 +1,11 @@ +kind: Ingress +metadata: + name: headscale + annotations: + cert-manager.io/cluster-issuer: letsencrypt-production + traefik.ingress.kubernetes.io/router.tls: "true" +spec: + tls: + - hosts: + - $(PUBLIC_HOSTNAME) + secretName: production-cert diff --git a/k8s/production-tls/kustomization.yaml b/k8s/production-tls/kustomization.yaml new file mode 100644 index 00000000..f57cb540 --- /dev/null +++ b/k8s/production-tls/kustomization.yaml @@ -0,0 +1,9 @@ +namespace: headscale +bases: +- ../base +resources: +- production-issuer.yaml +patches: +- path: ingress-patch.yaml + target: + kind: Ingress diff --git a/k8s/production-tls/production-issuer.yaml b/k8s/production-tls/production-issuer.yaml new file mode 100644 index 00000000..7ae9131a --- /dev/null +++ b/k8s/production-tls/production-issuer.yaml @@ -0,0 +1,16 @@ +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-production +spec: + acme: + # TODO: figure out how to get kustomize to interpolate this, or use a transformer + #email: $(CONTACT_EMAIL) + server: https://acme-v02.api.letsencrypt.org/directory + privateKeySecretRef: + # Secret resource used to store the account's private key. + name: letsencrypt-production-acc-key + solvers: + - http01: + ingress: + class: traefik diff --git a/k8s/sqlite/kustomization.yaml b/k8s/sqlite/kustomization.yaml new file mode 100644 index 00000000..5be451ca --- /dev/null +++ b/k8s/sqlite/kustomization.yaml @@ -0,0 +1,5 @@ +namespace: headscale +bases: +- ../base +resources: +- statefulset.yaml diff --git a/k8s/sqlite/statefulset.yaml b/k8s/sqlite/statefulset.yaml new file mode 100644 index 00000000..9075e007 --- /dev/null +++ b/k8s/sqlite/statefulset.yaml @@ -0,0 +1,79 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: headscale +spec: + serviceName: headscale + replicas: 1 + selector: + matchLabels: + app: headscale + template: + metadata: + labels: + app: headscale + spec: + containers: + - name: headscale + image: "headscale:latest" + imagePullPolicy: IfNotPresent + command: ["/go/bin/headscale", "serve"] + env: + - name: SERVER_URL + value: $(PUBLIC_PROTO)://$(PUBLIC_HOSTNAME) + - name: LISTEN_ADDR + valueFrom: + configMapKeyRef: + name: headscale-config + key: listen_addr + - name: PRIVATE_KEY_PATH + value: /vol/secret/private-key + - name: DERP_MAP_PATH + value: /vol/config/derp.yaml + - name: EPHEMERAL_NODE_INACTIVITY_TIMEOUT + valueFrom: + configMapKeyRef: + name: headscale-config + key: ephemeral_node_inactivity_timeout + - name: DB_TYPE + value: sqlite3 + - name: DB_PATH + value: /vol/data/db.sqlite + ports: + - name: http + protocol: TCP + containerPort: 8080 + livenessProbe: + tcpSocket: + port: http + initialDelaySeconds: 30 + timeoutSeconds: 5 + periodSeconds: 15 + volumeMounts: + - name: config + mountPath: /vol/config + - name: data + mountPath: /vol/data + - name: secret + mountPath: /vol/secret + - name: etc + mountPath: /etc/headscale + volumes: + - name: config + configMap: + name: headscale-site + - name: etc + configMap: + name: headscale-etc + - name: secret + secret: + secretName: headscale + volumeClaimTemplates: + - metadata: + name: data + spec: + storageClassName: local-path + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi diff --git a/k8s/staging-tls/ingress-patch.yaml b/k8s/staging-tls/ingress-patch.yaml new file mode 100644 index 00000000..f97974bd --- /dev/null +++ b/k8s/staging-tls/ingress-patch.yaml @@ -0,0 +1,11 @@ +kind: Ingress +metadata: + name: headscale + annotations: + cert-manager.io/cluster-issuer: letsencrypt-staging + traefik.ingress.kubernetes.io/router.tls: "true" +spec: + tls: + - hosts: + - $(PUBLIC_HOSTNAME) + secretName: staging-cert diff --git a/k8s/staging-tls/kustomization.yaml b/k8s/staging-tls/kustomization.yaml new file mode 100644 index 00000000..931f27d5 --- /dev/null +++ b/k8s/staging-tls/kustomization.yaml @@ -0,0 +1,9 @@ +namespace: headscale +bases: +- ../base +resources: +- staging-issuer.yaml +patches: +- path: ingress-patch.yaml + target: + kind: Ingress diff --git a/k8s/staging-tls/staging-issuer.yaml b/k8s/staging-tls/staging-issuer.yaml new file mode 100644 index 00000000..95325f6e --- /dev/null +++ b/k8s/staging-tls/staging-issuer.yaml @@ -0,0 +1,16 @@ +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-staging +spec: + acme: + # TODO: figure out how to get kustomize to interpolate this, or use a transformer + #email: $(CONTACT_EMAIL) + server: https://acme-staging-v02.api.letsencrypt.org/directory + privateKeySecretRef: + # Secret resource used to store the account's private key. + name: letsencrypt-staging-acc-key + solvers: + - http01: + ingress: + class: traefik