cmd/k8s-operator: generate static kube manifests from the Helm chart. (#10436)

* cmd/k8s-operator: generate static manifests from Helm charts

This is done to ensure that there is a single source of truth
for the operator kube manifests.
Also adds linux node selector to the static manifests as
this was added as a default to the Helm chart.

Static manifests can now be generated by running
`go generate tailscale.com/cmd/k8s-operator`.

Updates tailscale/tailscale#9222

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

.github/workflows/kubemanifests.yaml

@@ -22,3 +22,9 @@ jobs:
         eval `./tool/go run ./cmd/mkversion`
         ./tool/helm package --app-version="${VERSION_SHORT}" --version=${VERSION_SHORT} './cmd/k8s-operator/deploy/chart'
         ./tool/helm lint "tailscale-operator-${VERSION_SHORT}.tgz"
+    - name: Verify that static manifests are up to date
+      run: |
+        ./tool/go generate tailscale.com/cmd/k8s-operator
+        echo
+        echo
+        git diff --name-only --exit-code || (echo "Static manifests for Tailscale Kubernetes operator are out of date. Please run 'go generate tailscale.com/cmd/k8s-operator' and commit the diff."; exit 1)

.github/workflows/test.yml

@@ -429,7 +429,7 @@ jobs:
       uses: actions/checkout@v4
     - name: check that 'go generate' is clean
       run: |
-        pkgs=$(./tool/go list ./... | grep -v dnsfallback)
+        pkgs=$(./tool/go list ./... | grep -Ev 'dnsfallback|k8s-operator')
         ./tool/go generate $pkgs
         echo
         echo

cmd/k8s-operator/deploy/README.md

@@ -0,0 +1,12 @@
+# Tailscale Kubernetes operator deployment manifests
+
+./cmd/k8s-operator/deploy contain various Tailscale Kubernetes operator deployment manifests.
+
+## Helm chart
+
+`./cmd/k8s-operator/deploy/chart` contains Tailscale operator Helm chart templates.
+The chart templates are also used to generate the static manifest, so developers must ensure that any changes applied to the chart have been propagated to the static manifest by running `go generate tailscale.com/cmd/k8s-operator`
+
+## Static manifests
+
+`./cmd/k8s-operator/deploy/manifests/operator.yaml` is a static manifest for the operator generated from the Helm chart templates for the operator.

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -7,94 +7,6 @@ metadata:
   name: tailscale
 ---
 apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: proxies
-  namespace: tailscale
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: proxies
-  namespace: tailscale
-rules:
-- apiGroups: [""]
-  resources: ["secrets"]
-  verbs: ["*"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: proxies
-  namespace: tailscale
-subjects:
-- kind: ServiceAccount
-  name: proxies
-  namespace: tailscale
-roleRef:
-  kind: Role
-  name: proxies
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: operator
-  namespace: tailscale
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: tailscale-operator
-rules:
-- apiGroups: [""]
-  resources: ["events", "services", "services/status"]
-  verbs: ["*"]
-- apiGroups: ["networking.k8s.io"]
-  resources: ["ingresses", "ingresses/status"]
-  verbs: ["*"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: tailscale-operator
-subjects:
-- kind: ServiceAccount
-  name: operator
-  namespace: tailscale
-roleRef:
-  kind: ClusterRole
-  name: tailscale-operator
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: operator
-  namespace: tailscale
-rules:
-- apiGroups: [""]
-  resources: ["secrets"]
-  verbs: ["*"]
-- apiGroups: ["apps"]
-  resources: ["statefulsets"]
-  verbs: ["*"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: operator
-  namespace: tailscale
-subjects:
-- kind: ServiceAccount
-  name: operator
-  namespace: tailscale
-roleRef:
-  kind: Role
-  name: operator
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: v1
 kind: Secret
 metadata:
   name: operator-oauth
@@ -103,59 +15,164 @@ stringData:
   client_id: # SET CLIENT ID HERE
   client_secret: # SET CLIENT SECRET HERE
 ---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+    name: operator
+    namespace: tailscale
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+    name: proxies
+    namespace: tailscale
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+    name: tailscale-operator
+rules:
+    - apiGroups:
+        - ""
+      resources:
+        - events
+        - services
+        - services/status
+      verbs:
+        - '*'
+    - apiGroups:
+        - networking.k8s.io
+      resources:
+        - ingresses
+        - ingresses/status
+      verbs:
+        - '*'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+    name: tailscale-operator
+roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: ClusterRole
+    name: tailscale-operator
+subjects:
+    - kind: ServiceAccount
+      name: operator
+      namespace: tailscale
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+    name: operator
+    namespace: tailscale
+rules:
+    - apiGroups:
+        - ""
+      resources:
+        - secrets
+      verbs:
+        - '*'
+    - apiGroups:
+        - apps
+      resources:
+        - statefulsets
+      verbs:
+        - '*'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+    name: proxies
+    namespace: tailscale
+rules:
+    - apiGroups:
+        - ""
+      resources:
+        - secrets
+      verbs:
+        - '*'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+    name: operator
+    namespace: tailscale
+roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: Role
+    name: operator
+subjects:
+    - kind: ServiceAccount
+      name: operator
+      namespace: tailscale
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+    name: proxies
+    namespace: tailscale
+roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: Role
+    name: proxies
+subjects:
+    - kind: ServiceAccount
+      name: proxies
+      namespace: tailscale
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: operator
-  namespace: tailscale
+    name: operator
+    namespace: tailscale
 spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app: operator
-  template:
-    metadata:
-      labels:
-        app: operator
-    spec:
-      serviceAccountName: operator
-      volumes:
-      - name: oauth
-        secret:
-          secretName: operator-oauth
-      containers:
-        - name: operator
-          image: tailscale/k8s-operator:unstable
-          resources:
-            requests:
-              cpu: 500m
-              memory: 100Mi
-          env:
-            - name: OPERATOR_HOSTNAME
-              value: tailscale-operator
-            - name: OPERATOR_SECRET
-              value: operator
-            - name: OPERATOR_LOGGING
-              value: info
-            - name: OPERATOR_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-            - name: CLIENT_ID_FILE
-              value: /oauth/client_id
-            - name: CLIENT_SECRET_FILE
-              value: /oauth/client_secret
-            - name: PROXY_IMAGE
-              value: tailscale/tailscale:unstable
-            - name: PROXY_TAGS
-              value: tag:k8s
-            - name: APISERVER_PROXY
-              value: "false"
-            - name: PROXY_FIREWALL_MODE
-              value: auto
-          volumeMounts:
-          - name: oauth
-            mountPath: /oauth
-            readOnly: true
+    replicas: 1
+    selector:
+        matchLabels:
+            app: operator
+    strategy:
+        type: Recreate
+    template:
+        metadata:
+            labels:
+                app: operator
+        spec:
+            containers:
+                - env:
+                    - name: OPERATOR_HOSTNAME
+                      value: tailscale-operator
+                    - name: OPERATOR_SECRET
+                      value: operator
+                    - name: OPERATOR_LOGGING
+                      value: info
+                    - name: OPERATOR_NAMESPACE
+                      valueFrom:
+                        fieldRef:
+                            fieldPath: metadata.namespace
+                    - name: CLIENT_ID_FILE
+                      value: /oauth/client_id
+                    - name: CLIENT_SECRET_FILE
+                      value: /oauth/client_secret
+                    - name: PROXY_IMAGE
+                      value: tailscale/tailscale:unstable
+                    - name: PROXY_TAGS
+                      value: tag:k8s
+                    - name: APISERVER_PROXY
+                      value: "false"
+                    - name: PROXY_FIREWALL_MODE
+                      value: auto
+                  image: tailscale/k8s-operator:unstable
+                  imagePullPolicy: Always
+                  name: operator
+                  volumeMounts:
+                    - mountPath: /oauth
+                      name: oauth
+                      readOnly: true
+            nodeSelector:
+                kubernetes.io/os: linux
+            serviceAccountName: operator
+            volumes:
+                - name: oauth
+                  secret:
+                    secretName: operator-oauth

cmd/k8s-operator/deploy/manifests/templates/01-header.yaml

@@ -0,0 +1,3 @@
+# Copyright (c) Tailscale Inc & AUTHORS
+# SPDX-License-Identifier: BSD-3-Clause
+

cmd/k8s-operator/deploy/manifests/templates/02-namespace.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: tailscale
+---

cmd/k8s-operator/deploy/manifests/templates/03-secret.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: operator-oauth
+  namespace: tailscale
+stringData:
+  client_id: # SET CLIENT ID HERE
+  client_secret: # SET CLIENT SECRET HERE
+---

cmd/k8s-operator/generate/main.go

@@ -0,0 +1,74 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+func main() {
+	repoRoot := "../../"
+	cmd := exec.Command("./tool/helm", "template", "operator", "./cmd/k8s-operator/deploy/chart",
+		"--namespace=tailscale")
+	cmd.Dir = repoRoot
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		log.Fatalf("error templating helm manifests: %v", err)
+	}
+
+	var final bytes.Buffer
+
+	templatePath := filepath.Join(repoRoot, "cmd/k8s-operator/deploy/manifests/templates")
+	fileInfos, err := os.ReadDir(templatePath)
+	if err != nil {
+		log.Fatalf("error reading templates: %v", err)
+	}
+	for _, fi := range fileInfos {
+		templateBytes, err := os.ReadFile(filepath.Join(templatePath, fi.Name()))
+		if err != nil {
+			log.Fatalf("error reading template: %v", err)
+		}
+		final.Write(templateBytes)
+	}
+	decoder := yaml.NewDecoder(&out)
+	for {
+		var document any
+		err := decoder.Decode(&document)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			log.Fatalf("failed read from input data: %v", err)
+		}
+
+		bytes, err := yaml.Marshal(document)
+		if err != nil {
+			log.Fatalf("failed to marshal YAML document: %v", err)
+		}
+		if strings.TrimSpace(string(bytes)) == "null" {
+			continue
+		}
+		if _, err = final.Write(bytes); err != nil {
+			log.Fatalf("error marshaling yaml: %v", err)
+		}
+		fmt.Fprint(&final, "---\n")
+	}
+	finalString, _ := strings.CutSuffix(final.String(), "---\n")
+	if err := os.WriteFile(filepath.Join(repoRoot, "cmd/k8s-operator/deploy/manifests/operator.yaml"), []byte(finalString), 0664); err != nil {
+		log.Fatalf("error writing new file: %v", err)
+	}
+}

cmd/k8s-operator/operator.go

@@ -42,6 +42,8 @@
 	"tailscale.com/version"
 )
 
+//go:generate go run tailscale.com/cmd/k8s-operator/generate
+
 func main() {
 	// Required to use our client API. We're fine with the instability since the
 	// client lives in the same repo as this code.

go.mod

@@ -358,7 +358,7 @@ require (
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1
 	howett.net/plist v1.0.0 // indirect
 	k8s.io/apiextensions-apiserver v0.28.2 // indirect
 	k8s.io/component-base v0.28.2 // indirect