cmd/k8s-operator: generate static kube manifests from the Helm chart. (#10436)

* cmd/k8s-operator: generate static manifests from Helm charts This is done to ensure that there is a single source of truth for the operator kube manifests. Also adds linux node selector to the static manifests as this was added as a default to the Helm chart. Static manifests can now be generated by running `go generate tailscale.com/cmd/k8s-operator`. Updates tailscale/tailscale#9222 Signed-off-by: Irbe Krumina <irbe@tailscale.com>
2024-11-25 11:05:45 +00:00 · 2023-12-04 10:18:07 +00:00 · 2023-12-04 10:18:07 +00:00 · 49fd0a62c9
commit 49fd0a62c9
parent 263e01c47b
10 changed files with 270 additions and 142 deletions
--- a/.github/workflows/kubemanifests.yaml
+++ b/.github/workflows/kubemanifests.yaml
@ -22,3 +22,9 @@ jobs:
        eval `./tool/go run ./cmd/mkversion`
        ./tool/helm package --app-version="${VERSION_SHORT}" --version=${VERSION_SHORT} './cmd/k8s-operator/deploy/chart'
        ./tool/helm lint "tailscale-operator-${VERSION_SHORT}.tgz"
+    - name: Verify that static manifests are up to date
+      run: |
+        ./tool/go generate tailscale.com/cmd/k8s-operator
+        echo
+        echo
+        git diff --name-only --exit-code || (echo "Static manifests for Tailscale Kubernetes operator are out of date. Please run 'go generate tailscale.com/cmd/k8s-operator' and commit the diff."; exit 1)
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@ -429,7 +429,7 @@ jobs:
      uses: actions/checkout@v4
    - name: check that 'go generate' is clean
      run: |
-        pkgs=$(./tool/go list ./... | grep -v dnsfallback)
+        pkgs=$(./tool/go list ./... | grep -Ev 'dnsfallback|k8s-operator')
        ./tool/go generate $pkgs
        echo
        echo
--- a/cmd/k8s-operator/deploy/README.md
+++ b/cmd/k8s-operator/deploy/README.md
@ -0,0 +1,12 @@
+# Tailscale Kubernetes operator deployment manifests
+
+./cmd/k8s-operator/deploy contain various Tailscale Kubernetes operator deployment manifests.
+
+## Helm chart
+
+`./cmd/k8s-operator/deploy/chart` contains Tailscale operator Helm chart templates.
+The chart templates are also used to generate the static manifest, so developers must ensure that any changes applied to the chart have been propagated to the static manifest by running `go generate tailscale.com/cmd/k8s-operator`
+
+## Static manifests
+
+`./cmd/k8s-operator/deploy/manifests/operator.yaml` is a static manifest for the operator generated from the Helm chart templates for the operator.
--- a/cmd/k8s-operator/deploy/manifests/operator.yaml
+++ b/cmd/k8s-operator/deploy/manifests/operator.yaml
@ -7,94 +7,6 @@ metadata:
  name: tailscale
 ---
 apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: proxies
-  namespace: tailscale
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: proxies
-  namespace: tailscale
-rules:
- apiGroups: [""]
-  resources: ["secrets"]
-  verbs: ["*"]
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: proxies
-  namespace: tailscale
-subjects:
- kind: ServiceAccount
-  name: proxies
-  namespace: tailscale
-roleRef:
-  kind: Role
-  name: proxies
-  apiGroup: rbac.authorization.k8s.io
---
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: operator
-  namespace: tailscale
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: tailscale-operator
-rules:
- apiGroups: [""]
-  resources: ["events", "services", "services/status"]
-  verbs: ["*"]
- apiGroups: ["networking.k8s.io"]
-  resources: ["ingresses", "ingresses/status"]
-  verbs: ["*"]
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: tailscale-operator
-subjects:
- kind: ServiceAccount
-  name: operator
-  namespace: tailscale
-roleRef:
-  kind: ClusterRole
-  name: tailscale-operator
-  apiGroup: rbac.authorization.k8s.io
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: operator
-  namespace: tailscale
-rules:
- apiGroups: [""]
-  resources: ["secrets"]
-  verbs: ["*"]
- apiGroups: ["apps"]
-  resources: ["statefulsets"]
-  verbs: ["*"]
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: operator
-  namespace: tailscale
-subjects:
- kind: ServiceAccount
-  name: operator
-  namespace: tailscale
-roleRef:
-  kind: Role
-  name: operator
-  apiGroup: rbac.authorization.k8s.io
---
-apiVersion: v1
 kind: Secret
 metadata:
  name: operator-oauth
@ -103,59 +15,164 @@ stringData:
  client_id: # SET CLIENT ID HERE
  client_secret: # SET CLIENT SECRET HERE
 ---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+    name: operator
+    namespace: tailscale
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+    name: proxies
+    namespace: tailscale
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+    name: tailscale-operator
+rules:
+    - apiGroups:
+        - ""
+      resources:
+        - events
+        - services
+        - services/status
+      verbs:
+        - '*'
+    - apiGroups:
+        - networking.k8s.io
+      resources:
+        - ingresses
+        - ingresses/status
+      verbs:
+        - '*'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+    name: tailscale-operator
+roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: ClusterRole
+    name: tailscale-operator
+subjects:
+    - kind: ServiceAccount
+      name: operator
+      namespace: tailscale
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+    name: operator
+    namespace: tailscale
+rules:
+    - apiGroups:
+        - ""
+      resources:
+        - secrets
+      verbs:
+        - '*'
+    - apiGroups:
+        - apps
+      resources:
+        - statefulsets
+      verbs:
+        - '*'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+    name: proxies
+    namespace: tailscale
+rules:
+    - apiGroups:
+        - ""
+      resources:
+        - secrets
+      verbs:
+        - '*'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+    name: operator
+    namespace: tailscale
+roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: Role
+    name: operator
+subjects:
+    - kind: ServiceAccount
+      name: operator
+      namespace: tailscale
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+    name: proxies
+    namespace: tailscale
+roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: Role
+    name: proxies
+subjects:
+    - kind: ServiceAccount
+      name: proxies
+      namespace: tailscale
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: operator
-  namespace: tailscale
+    name: operator
+    namespace: tailscale
 spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app: operator
-  template:
-    metadata:
-      labels:
-        app: operator
-    spec:
-      serviceAccountName: operator
-      volumes:
-      - name: oauth
-        secret:
-          secretName: operator-oauth
-      containers:
-        - name: operator
-          image: tailscale/k8s-operator:unstable
-          resources:
-            requests:
-              cpu: 500m
-              memory: 100Mi
-          env:
-            - name: OPERATOR_HOSTNAME
-              value: tailscale-operator
-            - name: OPERATOR_SECRET
-              value: operator
-            - name: OPERATOR_LOGGING
-              value: info
-            - name: OPERATOR_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-            - name: CLIENT_ID_FILE
-              value: /oauth/client_id
-            - name: CLIENT_SECRET_FILE
-              value: /oauth/client_secret
-            - name: PROXY_IMAGE
-              value: tailscale/tailscale:unstable
-            - name: PROXY_TAGS
-              value: tag:k8s
-            - name: APISERVER_PROXY
-              value: "false"
-            - name: PROXY_FIREWALL_MODE
-              value: auto
-          volumeMounts:
-          - name: oauth
-            mountPath: /oauth
-            readOnly: true
+    replicas: 1
+    selector:
+        matchLabels:
+            app: operator
+    strategy:
+        type: Recreate
+    template:
+        metadata:
+            labels:
+                app: operator
+        spec:
+            containers:
+                - env:
+                    - name: OPERATOR_HOSTNAME
+                      value: tailscale-operator
+                    - name: OPERATOR_SECRET
+                      value: operator
+                    - name: OPERATOR_LOGGING
+                      value: info
+                    - name: OPERATOR_NAMESPACE
+                      valueFrom:
+                        fieldRef:
+                            fieldPath: metadata.namespace
+                    - name: CLIENT_ID_FILE
+                      value: /oauth/client_id
+                    - name: CLIENT_SECRET_FILE
+                      value: /oauth/client_secret
+                    - name: PROXY_IMAGE
+                      value: tailscale/tailscale:unstable
+                    - name: PROXY_TAGS
+                      value: tag:k8s
+                    - name: APISERVER_PROXY
+                      value: "false"
+                    - name: PROXY_FIREWALL_MODE
+                      value: auto
+                  image: tailscale/k8s-operator:unstable
+                  imagePullPolicy: Always
+                  name: operator
+                  volumeMounts:
+                    - mountPath: /oauth
+                      name: oauth
+                      readOnly: true
+            nodeSelector:
+                kubernetes.io/os: linux
+            serviceAccountName: operator
+            volumes:
+                - name: oauth
+                  secret:
+                    secretName: operator-oauth
--- a/cmd/k8s-operator/deploy/manifests/templates/01-header.yaml
+++ b/cmd/k8s-operator/deploy/manifests/templates/01-header.yaml
@ -0,0 +1,3 @@
+# Copyright (c) Tailscale Inc & AUTHORS
+# SPDX-License-Identifier: BSD-3-Clause
+
--- a/cmd/k8s-operator/deploy/manifests/templates/02-namespace.yaml
+++ b/cmd/k8s-operator/deploy/manifests/templates/02-namespace.yaml
@ -0,0 +1,5 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: tailscale
+---
--- a/cmd/k8s-operator/deploy/manifests/templates/03-secret.yaml
+++ b/cmd/k8s-operator/deploy/manifests/templates/03-secret.yaml
@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: operator-oauth
+  namespace: tailscale
+stringData:
+  client_id: # SET CLIENT ID HERE
+  client_secret: # SET CLIENT SECRET HERE
+---
--- a/cmd/k8s-operator/generate/main.go
+++ b/cmd/k8s-operator/generate/main.go
@ -0,0 +1,74 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+func main() {
+	repoRoot := "../../"
+	cmd := exec.Command("./tool/helm", "template", "operator", "./cmd/k8s-operator/deploy/chart",
+		"--namespace=tailscale")
+	cmd.Dir = repoRoot
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		log.Fatalf("error templating helm manifests: %v", err)
+	}
+
+	var final bytes.Buffer
+
+	templatePath := filepath.Join(repoRoot, "cmd/k8s-operator/deploy/manifests/templates")
+	fileInfos, err := os.ReadDir(templatePath)
+	if err != nil {
+		log.Fatalf("error reading templates: %v", err)
+	}
+	for _, fi := range fileInfos {
+		templateBytes, err := os.ReadFile(filepath.Join(templatePath, fi.Name()))
+		if err != nil {
+			log.Fatalf("error reading template: %v", err)
+		}
+		final.Write(templateBytes)
+	}
+	decoder := yaml.NewDecoder(&out)
+	for {
+		var document any
+		err := decoder.Decode(&document)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			log.Fatalf("failed read from input data: %v", err)
+		}
+
+		bytes, err := yaml.Marshal(document)
+		if err != nil {
+			log.Fatalf("failed to marshal YAML document: %v", err)
+		}
+		if strings.TrimSpace(string(bytes)) == "null" {
+			continue
+		}
+		if _, err = final.Write(bytes); err != nil {
+			log.Fatalf("error marshaling yaml: %v", err)
+		}
+		fmt.Fprint(&final, "---\n")
+	}
+	finalString, _ := strings.CutSuffix(final.String(), "---\n")
+	if err := os.WriteFile(filepath.Join(repoRoot, "cmd/k8s-operator/deploy/manifests/operator.yaml"), []byte(finalString), 0664); err != nil {
+		log.Fatalf("error writing new file: %v", err)
+	}
+}
--- a/cmd/k8s-operator/operator.go
+++ b/cmd/k8s-operator/operator.go
@ -42,6 +42,8 @@
 	"tailscale.com/version"
 )

+//go:generate go run tailscale.com/cmd/k8s-operator/generate
+
 func main() {
 	// Required to use our client API. We're fine with the instability since the
 	// client lives in the same repo as this code.
--- a/go.mod
+++ b/go.mod
@ -358,7 +358,7 @@ require (
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1
 	howett.net/plist v1.0.0 // indirect
 	k8s.io/apiextensions-apiserver v0.28.2 // indirect
 	k8s.io/component-base v0.28.2 // indirect