From 07c157ee9f1b427437e0deab4f8344f249597991 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Tue, 8 Oct 2024 20:05:08 +0100 Subject: [PATCH] cmd/k8s-operator: base ProxyGroup StatefulSet on common proxy.yaml definition (#13714) As discussed in #13684, base the ProxyGroup's proxy definitions on the same scaffolding as the existing proxies, as defined in proxy.yaml Updates #13406 Signed-off-by: Tom Proctor --- cmd/k8s-operator/proxygroup.go | 5 +- cmd/k8s-operator/proxygroup_specs.go | 296 +++++++++++++-------------- cmd/k8s-operator/proxygroup_test.go | 5 +- 3 files changed, 151 insertions(+), 155 deletions(-) diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index 7ba59586b..b96641d68 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -239,7 +239,10 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro return fmt.Errorf("error provisioning ConfigMap: %w", err) } } - ss := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, cfgHash) + ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, cfgHash) + if err != nil { + return fmt.Errorf("error generating StatefulSet spec: %w", err) + } ss = applyProxyClassToStatefulSet(proxyClass, ss, nil, logger) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) { s.ObjectMeta.Labels = ss.ObjectMeta.Labels diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index 100c0707d..9aa7ac3b0 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -12,6 +12,7 @@ corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/yaml" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/egressservices" "tailscale.com/types/ptr" @@ -19,163 +20,152 @@ // Returns the base StatefulSet definition for a ProxyGroup. A ProxyClass may be // applied over the top after. -func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHash string) *appsv1.StatefulSet { - return &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: pg.Name, - Namespace: namespace, - Labels: pgLabels(pg.Name, nil), - OwnerReferences: pgOwnerReference(pg), - }, - Spec: appsv1.StatefulSetSpec{ - Replicas: ptr.To(pgReplicas(pg)), - Selector: &metav1.LabelSelector{ - MatchLabels: pgLabels(pg.Name, nil), - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: pg.Name, - Namespace: namespace, - Labels: pgLabels(pg.Name, nil), - DeletionGracePeriodSeconds: ptr.To[int64](10), - Annotations: map[string]string{ - podAnnotationLastSetConfigFileHash: cfgHash, - }, - }, - Spec: corev1.PodSpec{ - ServiceAccountName: pg.Name, - InitContainers: []corev1.Container{ - { - Name: "sysctler", - Image: image, - SecurityContext: &corev1.SecurityContext{ - Privileged: ptr.To(true), - }, - Command: []string{ - "/bin/sh", - "-c", - }, - Args: []string{ - "sysctl -w net.ipv4.ip_forward=1 && if sysctl net.ipv6.conf.all.forwarding; then sysctl -w net.ipv6.conf.all.forwarding=1; fi", - }, - }, - }, - Containers: []corev1.Container{ - { - Name: "tailscale", - Image: image, - SecurityContext: &corev1.SecurityContext{ - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{ - "NET_ADMIN", - }, - }, - }, - VolumeMounts: func() []corev1.VolumeMount { - var mounts []corev1.VolumeMount - for i := range pgReplicas(pg) { - mounts = append(mounts, corev1.VolumeMount{ - Name: fmt.Sprintf("tailscaledconfig-%d", i), - ReadOnly: true, - MountPath: fmt.Sprintf("/etc/tsconfig/%s-%d", pg.Name, i), - }) - } +func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHash string) (*appsv1.StatefulSet, error) { + ss := new(appsv1.StatefulSet) + if err := yaml.Unmarshal(proxyYaml, &ss); err != nil { + return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err) + } + // Validate some base assumptions. + if len(ss.Spec.Template.Spec.InitContainers) != 1 { + return nil, fmt.Errorf("[unexpected] base proxy config had %d init containers instead of 1", len(ss.Spec.Template.Spec.InitContainers)) + } + if len(ss.Spec.Template.Spec.Containers) != 1 { + return nil, fmt.Errorf("[unexpected] base proxy config had %d containers instead of 1", len(ss.Spec.Template.Spec.Containers)) + } - if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { - mounts = append(mounts, corev1.VolumeMount{ - Name: pgEgressCMName(pg.Name), - MountPath: "/etc/proxies", - ReadOnly: true, - }) - } - return mounts - }(), - Env: func() []corev1.EnvVar { - envs := []corev1.EnvVar{ - { - // TODO(irbekrm): verify that .status.podIPs are always set, else read in .status.podIP as well. - Name: "POD_IPS", // this will be a comma separate list i.e 10.136.0.6,2600:1900:4011:161:0:e:0:6 - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "status.podIPs", - }, - }, - }, - { - Name: "POD_NAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - // Secret is named after the pod. - FieldPath: "metadata.name", - }, - }, - }, - { - Name: "TS_KUBE_SECRET", - Value: "$(POD_NAME)", - }, - { - Name: "TS_STATE", - Value: "kube:$(POD_NAME)", - }, - { - Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", - Value: "/etc/tsconfig/$(POD_NAME)", - }, - { - Name: "TS_USERSPACE", - Value: "false", - }, - } - if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { - envs = append(envs, corev1.EnvVar{ - Name: "TS_EGRESS_SERVICES_CONFIG_PATH", - Value: fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices), - }) - } + // StatefulSet config. + ss.ObjectMeta = metav1.ObjectMeta{ + Name: pg.Name, + Namespace: namespace, + Labels: pgLabels(pg.Name, nil), + OwnerReferences: pgOwnerReference(pg), + } + ss.Spec.Replicas = ptr.To(pgReplicas(pg)) + ss.Spec.Selector = &metav1.LabelSelector{ + MatchLabels: pgLabels(pg.Name, nil), + } - if tsFirewallMode != "" { - envs = append(envs, corev1.EnvVar{ - Name: "TS_DEBUG_FIREWALL_MODE", - Value: tsFirewallMode, - }) - } - - return envs - }(), - }, - }, - Volumes: func() []corev1.Volume { - var volumes []corev1.Volume - for i := range pgReplicas(pg) { - volumes = append(volumes, corev1.Volume{ - Name: fmt.Sprintf("tailscaledconfig-%d", i), - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: fmt.Sprintf("%s-%d-config", pg.Name, i), - }, - }, - }) - } - if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { - volumes = append(volumes, corev1.Volume{ - Name: pgEgressCMName(pg.Name), - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: pgEgressCMName(pg.Name), - }, - }, - }, - }) - } - - return volumes - }(), - }, - }, + // Template config. + tmpl := &ss.Spec.Template + tmpl.ObjectMeta = metav1.ObjectMeta{ + Name: pg.Name, + Namespace: namespace, + Labels: pgLabels(pg.Name, nil), + DeletionGracePeriodSeconds: ptr.To[int64](10), + Annotations: map[string]string{ + podAnnotationLastSetConfigFileHash: cfgHash, }, } + tmpl.Spec.ServiceAccountName = pg.Name + tmpl.Spec.InitContainers[0].Image = image + tmpl.Spec.Volumes = func() []corev1.Volume { + var volumes []corev1.Volume + for i := range pgReplicas(pg) { + volumes = append(volumes, corev1.Volume{ + Name: fmt.Sprintf("tailscaledconfig-%d", i), + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: fmt.Sprintf("%s-%d-config", pg.Name, i), + }, + }, + }) + } + + if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { + volumes = append(volumes, corev1.Volume{ + Name: pgEgressCMName(pg.Name), + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: pgEgressCMName(pg.Name), + }, + }, + }, + }) + } + + return volumes + }() + + // Main container config. + c := &ss.Spec.Template.Spec.Containers[0] + c.Image = image + c.VolumeMounts = func() []corev1.VolumeMount { + var mounts []corev1.VolumeMount + for i := range pgReplicas(pg) { + mounts = append(mounts, corev1.VolumeMount{ + Name: fmt.Sprintf("tailscaledconfig-%d", i), + ReadOnly: true, + MountPath: fmt.Sprintf("/etc/tsconfig/%s-%d", pg.Name, i), + }) + } + + if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { + mounts = append(mounts, corev1.VolumeMount{ + Name: pgEgressCMName(pg.Name), + MountPath: "/etc/proxies", + ReadOnly: true, + }) + } + + return mounts + }() + c.Env = func() []corev1.EnvVar { + envs := []corev1.EnvVar{ + { + // TODO(irbekrm): verify that .status.podIPs are always set, else read in .status.podIP as well. + Name: "POD_IPS", // this will be a comma separate list i.e 10.136.0.6,2600:1900:4011:161:0:e:0:6 + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIPs", + }, + }, + }, + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + // Secret is named after the pod. + FieldPath: "metadata.name", + }, + }, + }, + { + Name: "TS_KUBE_SECRET", + Value: "$(POD_NAME)", + }, + { + Name: "TS_STATE", + Value: "kube:$(POD_NAME)", + }, + { + Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", + Value: "/etc/tsconfig/$(POD_NAME)", + }, + { + Name: "TS_USERSPACE", + Value: "false", + }, + } + + if tsFirewallMode != "" { + envs = append(envs, corev1.EnvVar{ + Name: "TS_DEBUG_FIREWALL_MODE", + Value: tsFirewallMode, + }) + } + + if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { + envs = append(envs, corev1.EnvVar{ + Name: "TS_EGRESS_SERVICES_CONFIG_PATH", + Value: fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices), + }) + } + + return envs + }() + + return ss, nil } func pgServiceAccount(pg *tsapi.ProxyGroup, namespace string) *corev1.ServiceAccount { diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index b5a6a4d8d..445db7537 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -197,7 +197,10 @@ func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.Prox role := pgRole(pg, tsNamespace) roleBinding := pgRoleBinding(pg, tsNamespace) serviceAccount := pgServiceAccount(pg, tsNamespace) - statefulSet := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", "") + statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", "") + if err != nil { + t.Fatal(err) + } statefulSet.Annotations = defaultProxyClassAnnotations if shouldExist {