cmd/k8s-operator,k8s-operator,go.{mod,sum}: make individual proxy images/image pull policies configurable (#11928)

cmd/k8s-operator,k8s-operator,go.{mod,sum}: make individual proxy images/image pull policies configurable

Allow to configure images and image pull policies for individual proxies
via ProxyClass.Spec.StatefulSet.Pod.{TailscaleContainer,TailscaleInitContainer}.Image,
and ProxyClass.Spec.StatefulSet.Pod.{TailscaleContainer,TailscaleInitContainer}.ImagePullPolicy
fields.
Document that we have images in ghcr.io on the relevant Helm chart fields.

Updates tailscale/tailscale#11675

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

cmd/k8s-operator/deploy/chart/values.yaml

@@ -23,6 +23,7 @@ operatorConfig:
     - "tag:k8s-operator"
 
   image:
+    # Repository defaults to DockerHub, but images are also synced to ghcr.io/tailscale/k8s-operator.
     repository: tailscale/k8s-operator
     # Digest will be prioritized over tag. If neither are set appVersion will be
     # used.
@@ -57,6 +58,7 @@ operatorConfig:
 # https://tailscale.com/kb/1236/kubernetes-operator#cluster-resource-customization-using-proxyclass-custom-resource
 proxyConfig:
   image:
+    # Repository defaults to DockerHub, but images are also synced to ghcr.io/tailscale/tailscale.
     repository: tailscale/tailscale
     # Digest will be prioritized over tag. If neither are set appVersion will be
     # used.

cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml

@@ -721,6 +721,16 @@ spec:
                                   value:
                                     description: 'Variable references $(VAR_NAME) are expanded using the previously defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to "".'
                                     type: string
+                            image:
+                              description: Container image name. By default images are pulled from docker.io/tailscale/tailscale, but the official images are also available at ghcr.io/tailscale/tailscale. Specifying image name here will override any proxy image values specified via the Kubernetes operator's Helm chart values or PROXY_IMAGE env var in the operator Deployment. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image
+                              type: string
+                            imagePullPolicy:
+                              description: Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image
+                              type: string
+                              enum:
+                                - Always
+                                - Never
+                                - IfNotPresent
                             resources:
                               description: Container resource requirements. By default Tailscale Kubernetes operator does not apply any resource requirements. The amount of resources required wil depend on the amount of resources the operator needs to parse, usage patterns and cluster size. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources
                               type: object
@@ -864,6 +874,16 @@ spec:
                                   value:
                                     description: 'Variable references $(VAR_NAME) are expanded using the previously defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to "".'
                                     type: string
+                            image:
+                              description: Container image name. By default images are pulled from docker.io/tailscale/tailscale, but the official images are also available at ghcr.io/tailscale/tailscale. Specifying image name here will override any proxy image values specified via the Kubernetes operator's Helm chart values or PROXY_IMAGE env var in the operator Deployment. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image
+                              type: string
+                            imagePullPolicy:
+                              description: Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image
+                              type: string
+                              enum:
+                                - Always
+                                - Never
+                                - IfNotPresent
                             resources:
                               description: Container resource requirements. By default Tailscale Kubernetes operator does not apply any resource requirements. The amount of resources required wil depend on the amount of resources the operator needs to parse, usage patterns and cluster size. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources
                               type: object

cmd/k8s-operator/deploy/examples/proxyclass.yaml

@@ -15,3 +15,9 @@ spec:
         kubernetes.io/os: "linux"
       imagePullSecrets:
       - name: "foo"
+      tailscaleContainer:
+        image: "ghcr.io/tailscale/tailscale:v1.64"
+        imagePullPolicy: IfNotPresent
+      tailscaleInitContainer:
+        image: "ghcr.io/tailscale/tailscale:v1.64"
+        imagePullPolicy: IfNotPresent

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -970,6 +970,16 @@ spec:
                                                                 - name
                                                             type: object
                                                         type: array
+                                                    image:
+                                                        description: Container image name. By default images are pulled from docker.io/tailscale/tailscale, but the official images are also available at ghcr.io/tailscale/tailscale. Specifying image name here will override any proxy image values specified via the Kubernetes operator's Helm chart values or PROXY_IMAGE env var in the operator Deployment. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image
+                                                        type: string
+                                                    imagePullPolicy:
+                                                        description: Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image
+                                                        enum:
+                                                            - Always
+                                                            - Never
+                                                            - IfNotPresent
+                                                        type: string
                                                     resources:
                                                         description: Container resource requirements. By default Tailscale Kubernetes operator does not apply any resource requirements. The amount of resources required wil depend on the amount of resources the operator needs to parse, usage patterns and cluster size. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources
                                                         properties:
@@ -1113,6 +1123,16 @@ spec:
                                                                 - name
                                                             type: object
                                                         type: array
+                                                    image:
+                                                        description: Container image name. By default images are pulled from docker.io/tailscale/tailscale, but the official images are also available at ghcr.io/tailscale/tailscale. Specifying image name here will override any proxy image values specified via the Kubernetes operator's Helm chart values or PROXY_IMAGE env var in the operator Deployment. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image
+                                                        type: string
+                                                    imagePullPolicy:
+                                                        description: Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image
+                                                        enum:
+                                                            - Always
+                                                            - Never
+                                                            - IfNotPresent
+                                                        type: string
                                                     resources:
                                                         description: Container resource requirements. By default Tailscale Kubernetes operator does not apply any resource requirements. The amount of resources required wil depend on the amount of resources the operator needs to parse, usage patterns and cluster size. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources
                                                         properties:

cmd/k8s-operator/proxyclass.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"strings"
 
+	dockerref "github.com/distribution/reference"
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
@@ -111,6 +112,20 @@ func (a *ProxyClassReconciler) validate(pc *tsapi.ProxyClass) (violations field.
 						a.recorder.Event(pc, corev1.EventTypeWarning, reasonCustomTSEnvVar, fmt.Sprintf(messageCustomTSEnvVar, string(e.Name), "tailscale"))
 					}
 				}
+				if tc.Image != "" {
+					// Same validation as used by kubelet https://github.com/kubernetes/kubernetes/blob/release-1.30/pkg/kubelet/images/image_manager.go#L212
+					if _, err := dockerref.ParseNormalizedNamed(tc.Image); err != nil {
+						violations = append(violations, field.TypeInvalid(field.NewPath("spec", "statefulSet", "pod", "tailscaleContainer", "image"), tc.Image, err.Error()))
+					}
+				}
+			}
+			if tc := pod.TailscaleInitContainer; tc != nil {
+				if tc.Image != "" {
+					// Same validation as used by kubelet https://github.com/kubernetes/kubernetes/blob/release-1.30/pkg/kubelet/images/image_manager.go#L212
+					if _, err := dockerref.ParseNormalizedNamed(tc.Image); err != nil {
+						violations = append(violations, field.TypeInvalid(field.NewPath("spec", "statefulSet", "pod", "tailscaleInitContainer", "image"), tc.Image, err.Error()))
+					}
+				}
 			}
 		}
 	}

cmd/k8s-operator/proxyclass_test.go

@@ -36,9 +36,13 @@ func TestProxyClass(t *testing.T) {
 				Labels:      map[string]string{"foo": "bar", "xyz1234": "abc567"},
 				Annotations: map[string]string{"foo.io/bar": "{'key': 'val1232'}"},
 				Pod: &tsapi.Pod{
-					Labels:             map[string]string{"foo": "bar", "xyz1234": "abc567"},
-					Annotations:        map[string]string{"foo.io/bar": "{'key': 'val1232'}"},
-					TailscaleContainer: &tsapi.Container{Env: []tsapi.Env{{Name: "FOO", Value: "BAR"}}},
+					Labels:      map[string]string{"foo": "bar", "xyz1234": "abc567"},
+					Annotations: map[string]string{"foo.io/bar": "{'key': 'val1232'}"},
+					TailscaleContainer: &tsapi.Container{
+						Env:             []tsapi.Env{{Name: "FOO", Value: "BAR"}},
+						ImagePullPolicy: "IfNotPresent",
+						Image:           "ghcr.my-repo/tailscale:v0.01testsomething",
+					},
 				},
 			},
 		},
@@ -73,7 +77,7 @@ func TestProxyClass(t *testing.T) {
 
 	expectEqual(t, fc, pc, nil)
 
-	// 2. An invalid ProxyClass resource gets its status updated to Invalid.
+	// 2. A ProxyClass resource with invalid labels gets its status updated to Invalid with an error message.
 	pc.Spec.StatefulSet.Labels["foo"] = "?!someVal"
 	mustUpdate(t, fc, "", "test", func(proxyClass *tsapi.ProxyClass) {
 		proxyClass.Spec.StatefulSet.Labels = pc.Spec.StatefulSet.Labels
@@ -85,9 +89,43 @@ func TestProxyClass(t *testing.T) {
 	expectedEvent := "Warning ProxyClassInvalid ProxyClass is not valid: .spec.statefulSet.labels: Invalid value: \"?!someVal\": a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyValue',  or 'my_value',  or '12345', regex used for validation is '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?')"
 	expectEvents(t, fr, []string{expectedEvent})
 
-	// 2. An valid ProxyClass but with a Tailscale env vars set results in warning events.
+	// 3. A ProxyClass resource with invalid image reference gets it status updated to Invalid with an error message.
+	pc.Spec.StatefulSet.Labels = nil
+	pc.Spec.StatefulSet.Pod.TailscaleContainer.Image = "FOO bar"
 	mustUpdate(t, fc, "", "test", func(proxyClass *tsapi.ProxyClass) {
-		proxyClass.Spec.StatefulSet.Labels = nil // unset invalid labels from the previous test
+		proxyClass.Spec.StatefulSet.Labels = nil
+		proxyClass.Spec.StatefulSet.Pod.TailscaleContainer.Image = pc.Spec.StatefulSet.Pod.TailscaleContainer.Image
+	})
+	expectReconciled(t, pcr, "", "test")
+	msg = `ProxyClass is not valid: spec.statefulSet.pod.tailscaleContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
+	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
+	expectEqual(t, fc, pc, nil)
+	expectedEvent = `Warning ProxyClassInvalid ProxyClass is not valid: spec.statefulSet.pod.tailscaleContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
+	expectEvents(t, fr, []string{expectedEvent})
+
+	// 4. A ProxyClass resource with invalid init container image reference gets it status updated to Invalid with an error message.
+	pc.Spec.StatefulSet.Labels = nil
+	pc.Spec.StatefulSet.Pod.TailscaleContainer.Image = ""
+	pc.Spec.StatefulSet.Pod.TailscaleInitContainer = &tsapi.Container{
+		Image: "FOO bar",
+	}
+	mustUpdate(t, fc, "", "test", func(proxyClass *tsapi.ProxyClass) {
+		proxyClass.Spec.StatefulSet.Pod.TailscaleContainer.Image = pc.Spec.StatefulSet.Pod.TailscaleContainer.Image
+		proxyClass.Spec.StatefulSet.Pod.TailscaleInitContainer = &tsapi.Container{
+			Image: pc.Spec.StatefulSet.Pod.TailscaleInitContainer.Image,
+		}
+	})
+	expectReconciled(t, pcr, "", "test")
+	msg = `ProxyClass is not valid: spec.statefulSet.pod.tailscaleInitContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
+	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
+	expectEqual(t, fc, pc, nil)
+	expectedEvent = `Warning ProxyClassInvalid ProxyClass is not valid: spec.statefulSet.pod.tailscaleInitContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
+	expectEvents(t, fr, []string{expectedEvent})
+
+	// 5. An valid ProxyClass but with a Tailscale env vars set results in warning events.
+	pc.Spec.StatefulSet.Pod.TailscaleInitContainer.Image = "" // unset previous test
+	mustUpdate(t, fc, "", "test", func(proxyClass *tsapi.ProxyClass) {
+		proxyClass.Spec.StatefulSet.Pod.TailscaleInitContainer.Image = pc.Spec.StatefulSet.Pod.TailscaleInitContainer.Image
 		proxyClass.Spec.StatefulSet.Pod.TailscaleContainer.Env = []tsapi.Env{{Name: "TS_USERSPACE", Value: "true"}, {Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH"}, {Name: "EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS"}}
 	})
 	expectedEvents := []string{"Warning CustomTSEnvVar ProxyClass overrides the default value for TS_USERSPACE env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future.",

cmd/k8s-operator/sts.go

@@ -690,6 +690,12 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
 			// in the env var list overrides an earlier one.
 			base.Env = append(base.Env, corev1.EnvVar{Name: string(e.Name), Value: e.Value})
 		}
+		if overlay.Image != "" {
+			base.Image = overlay.Image
+		}
+		if overlay.ImagePullPolicy != "" {
+			base.ImagePullPolicy = overlay.ImagePullPolicy
+		}
 		return base
 	}
 	for i, c := range ss.Spec.Template.Spec.Containers {

cmd/k8s-operator/sts_test.go

@@ -81,7 +81,9 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 							Limits:   corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1000m"), corev1.ResourceMemory: resource.MustParse("128Mi")},
 							Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("64Mi")},
 						},
-						Env: []tsapi.Env{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}},
+						Env:             []tsapi.Env{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}},
+						ImagePullPolicy: "IfNotPresent",
+						Image:           "ghcr.io/my-repo/tailscale:v0.01testsomething",
 					},
 					TailscaleInitContainer: &tsapi.Container{
 						SecurityContext: &corev1.SecurityContext{
@@ -92,7 +94,9 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 							Limits:   corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1000m"), corev1.ResourceMemory: resource.MustParse("128Mi")},
 							Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("64Mi")},
 						},
-						Env: []tsapi.Env{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}},
+						Env:             []tsapi.Env{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}},
+						ImagePullPolicy: "IfNotPresent",
+						Image:           "ghcr.io/my-repo/tailscale:v0.01testsomething",
 					},
 				},
 			},
@@ -135,10 +139,12 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	env := []corev1.EnvVar{{Name: "TS_HOSTNAME", Value: "nginx"}}
 	userspaceProxySS.Labels = labels
 	userspaceProxySS.Annotations = annots
+	userspaceProxySS.Spec.Template.Spec.Containers[0].Image = "tailscale/tailscale:v0.0.1"
 	userspaceProxySS.Spec.Template.Spec.Containers[0].Env = env
 	nonUserspaceProxySS.ObjectMeta.Labels = labels
 	nonUserspaceProxySS.ObjectMeta.Annotations = annots
 	nonUserspaceProxySS.Spec.Template.Spec.Containers[0].Env = env
+	nonUserspaceProxySS.Spec.Template.Spec.InitContainers[0].Image = "tailscale/tailscale:v0.0.1"
 
 	// 1. Test that a ProxyClass with all fields set gets correctly applied
 	// to a Statefulset built from non-userspace proxy template.
@@ -159,6 +165,10 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Spec.InitContainers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleInitContainer.Resources
 	wantSS.Spec.Template.Spec.InitContainers[0].Env = append(wantSS.Spec.Template.Spec.InitContainers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...)
 	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...)
+	wantSS.Spec.Template.Spec.Containers[0].Image = "ghcr.io/my-repo/tailscale:v0.01testsomething"
+	wantSS.Spec.Template.Spec.Containers[0].ImagePullPolicy = "IfNotPresent"
+	wantSS.Spec.Template.Spec.InitContainers[0].Image = "ghcr.io/my-repo/tailscale:v0.01testsomething"
+	wantSS.Spec.Template.Spec.InitContainers[0].ImagePullPolicy = "IfNotPresent"
 
 	gotSS := applyProxyClassToStatefulSet(proxyClassAllOpts, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
@@ -194,9 +204,11 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Spec.Containers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.SecurityContext
 	wantSS.Spec.Template.Spec.Containers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.Resources
 	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...)
+	wantSS.Spec.Template.Spec.Containers[0].ImagePullPolicy = "IfNotPresent"
+	wantSS.Spec.Template.Spec.Containers[0].Image = "ghcr.io/my-repo/tailscale:v0.01testsomething"
 	gotSS = applyProxyClassToStatefulSet(proxyClassAllOpts, userspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
-		t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
+		t.Fatalf("Unexpected result applying ProxyClass with all options to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
 	}
 
 	// 4. Test that a ProxyClass with custom labels and annotations gets correctly applied