cmd/k8s-operator,k8s-operator: support ingress ProxyGroup type (#14548)

Currently this does not yet do anything apart from creating
the ProxyGroup resources like StatefulSet.

Updates tailscale/corp#24795

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml

@@ -20,6 +20,10 @@ spec:
           jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason
           name: Status
           type: string
+        - description: ProxyGroup type.
+          jsonPath: .spec.type
+          name: Type
+          type: string
       name: v1alpha1
       schema:
         openAPIV3Schema:
@@ -84,6 +88,7 @@ spec:
                     Defaults to 2.
                   type: integer
                   format: int32
+                  minimum: 0
                 tags:
                   description: |-
                     Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].
@@ -97,10 +102,16 @@ spec:
                     type: string
                     pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
                 type:
-                  description: Type of the ProxyGroup proxies. Currently the only supported type is egress.
+                  description: |-
+                    Type of the ProxyGroup proxies. Supported types are egress and ingress.
+                    Type is immutable once a ProxyGroup is created.
                   type: string
                   enum:
                     - egress
+                    - ingress
+                  x-kubernetes-validations:
+                    - rule: self == oldSelf
+                      message: ProxyGroup type is immutable
             status:
               description: |-
                 ProxyGroupStatus describes the status of the ProxyGroup resources. This is

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -2721,6 +2721,10 @@ spec:
               jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason
               name: Status
               type: string
+            - description: ProxyGroup type.
+              jsonPath: .spec.type
+              name: Type
+              type: string
           name: v1alpha1
           schema:
             openAPIV3Schema:
@@ -2778,6 +2782,7 @@ spec:
                                     Replicas specifies how many replicas to create the StatefulSet with.
                                     Defaults to 2.
                                 format: int32
+                                minimum: 0
                                 type: integer
                             tags:
                                 description: |-
@@ -2792,10 +2797,16 @@ spec:
                                     type: string
                                 type: array
                             type:
-                                description: Type of the ProxyGroup proxies. Currently the only supported type is egress.
+                                description: |-
+                                    Type of the ProxyGroup proxies. Supported types are egress and ingress.
+                                    Type is immutable once a ProxyGroup is created.
                                 enum:
                                     - egress
+                                    - ingress
                                 type: string
+                                x-kubernetes-validations:
+                                    - message: ProxyGroup type is immutable
+                                      rule: self == oldSelf
                         required:
                             - type
                         type: object

cmd/k8s-operator/egress-services.go

@@ -495,13 +495,6 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s
 		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
 		return false, err
 	}
-	if !tsoperator.ProxyGroupIsReady(pg) {
-		l.Infof("ProxyGroup %s is not ready, waiting...", proxyGroupName)
-		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l)
-		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
-		return false, nil
-	}
-
 	if violations := validateEgressService(svc, pg); len(violations) > 0 {
 		msg := fmt.Sprintf("invalid egress Service: %s", strings.Join(violations, ", "))
 		esr.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg)
@@ -510,6 +503,13 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s
 		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
 		return false, nil
 	}
+	if !tsoperator.ProxyGroupIsReady(pg) {
+		l.Infof("ProxyGroup %s is not ready, waiting...", proxyGroupName)
+		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l)
+		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
+		return false, nil
+	}
+
 	l.Debugf("egress service is valid")
 	tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionTrue, reasonEgressSvcValid, reasonEgressSvcValid, esr.clock, l)
 	return true, nil

cmd/k8s-operator/operator.go

@@ -499,7 +499,7 @@ func runReconcilers(opts reconcilerOpts) {
 		startlog.Fatalf("could not create Recorder reconciler: %v", err)
 	}
 
-	// Recorder reconciler.
+	// ProxyGroup reconciler.
 	ownedByProxyGroupFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.ProxyGroup{})
 	proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog))
 	err = builder.ControllerManagedBy(mgr).

cmd/k8s-operator/proxygroup.go

@@ -51,7 +51,10 @@ const (
 	optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again"
 )
 
-var gaugeProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupEgressCount)
+var (
+	gaugeEgressProxyGroupResources  = clientmetric.NewGauge(kubetypes.MetricProxyGroupEgressCount)
+	gaugeIngressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupIngressCount)
+)
 
 // ProxyGroupReconciler ensures cluster resources for a ProxyGroup definition.
 type ProxyGroupReconciler struct {
@@ -69,7 +72,8 @@ type ProxyGroupReconciler struct {
 	defaultProxyClass string
 
 	mu                 sync.Mutex           // protects following
-	proxyGroups set.Slice[types.UID] // for proxygroups gauge
+	egressProxyGroups  set.Slice[types.UID] // for egress proxygroups gauge
+	ingressProxyGroups set.Slice[types.UID] // for ingress proxygroups gauge
 }
 
 func (r *ProxyGroupReconciler) logger(name string) *zap.SugaredLogger {
@@ -203,8 +207,7 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) error {
 	logger := r.logger(pg.Name)
 	r.mu.Lock()
-	r.proxyGroups.Add(pg.UID)
+	r.ensureAddedToGaugeForProxyGroup(pg)
-	gaugeProxyGroupResources.Set(int64(r.proxyGroups.Len()))
 	r.mu.Unlock()
 
 	cfgHash, err := r.ensureConfigSecretsCreated(ctx, pg, proxyClass)
@@ -358,8 +361,7 @@ func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, pg *tsapi.Proxy
 
 	logger.Infof("cleaned up ProxyGroup resources")
 	r.mu.Lock()
-	r.proxyGroups.Remove(pg.UID)
+	r.ensureRemovedFromGaugeForProxyGroup(pg)
-	gaugeProxyGroupResources.Set(int64(r.proxyGroups.Len()))
 	r.mu.Unlock()
 	return true, nil
 }
@@ -469,6 +471,32 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 	return configSHA256Sum, nil
 }
 
+// ensureAddedToGaugeForProxyGroup ensures the gauge metric for the ProxyGroup resource is updated when the ProxyGroup
+// is created. r.mu must be held.
+func (r *ProxyGroupReconciler) ensureAddedToGaugeForProxyGroup(pg *tsapi.ProxyGroup) {
+	switch pg.Spec.Type {
+	case tsapi.ProxyGroupTypeEgress:
+		r.egressProxyGroups.Add(pg.UID)
+	case tsapi.ProxyGroupTypeIngress:
+		r.ingressProxyGroups.Add(pg.UID)
+	}
+	gaugeEgressProxyGroupResources.Set(int64(r.egressProxyGroups.Len()))
+	gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len()))
+}
+
+// ensureRemovedFromGaugeForProxyGroup ensures the gauge metric for the ProxyGroup resource type is updated when the
+// ProxyGroup is deleted. r.mu must be held.
+func (r *ProxyGroupReconciler) ensureRemovedFromGaugeForProxyGroup(pg *tsapi.ProxyGroup) {
+	switch pg.Spec.Type {
+	case tsapi.ProxyGroupTypeEgress:
+		r.egressProxyGroups.Remove(pg.UID)
+	case tsapi.ProxyGroupTypeIngress:
+		r.ingressProxyGroups.Remove(pg.UID)
+	}
+	gaugeEgressProxyGroupResources.Set(int64(r.egressProxyGroups.Len()))
+	gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len()))
+}
+
 func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32, authKey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) {
 	conf := &ipn.ConfigVAlpha{
 		Version:      "alpha0",

cmd/k8s-operator/proxygroup_specs.go

@@ -138,10 +138,6 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
 				Name:  "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
 				Value: "/etc/tsconfig/$(POD_NAME)",
 			},
-			{
-				Name:  "TS_INTERNAL_APP",
-				Value: kubetypes.AppProxyGroupEgress,
-			},
 		}
 
 		if tsFirewallMode != "" {
@@ -155,9 +151,18 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
 			envs = append(envs, corev1.EnvVar{
 				Name:  "TS_EGRESS_SERVICES_CONFIG_PATH",
 				Value: fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices),
+			},
+				corev1.EnvVar{
+					Name:  "TS_INTERNAL_APP",
+					Value: kubetypes.AppProxyGroupEgress,
+				},
+			)
+		} else {
+			envs = append(envs, corev1.EnvVar{
+				Name:  "TS_INTERNAL_APP",
+				Value: kubetypes.AppProxyGroupIngress,
 			})
 		}
-
 		return append(c.Env, envs...)
 	}()
 

cmd/k8s-operator/proxygroup_test.go

@@ -25,6 +25,8 @@ import (
 	"tailscale.com/client/tailscale"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/egressservices"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tstest"
 	"tailscale.com/types/ptr"
 )
@@ -53,6 +55,9 @@ func TestProxyGroup(t *testing.T) {
 			Name:       "test",
 			Finalizers: []string{"tailscale.com/finalizer"},
 		},
+		Spec: tsapi.ProxyGroupSpec{
+			Type: tsapi.ProxyGroupTypeEgress,
+		},
 	}
 
 	fc := fake.NewClientBuilder().
@@ -112,8 +117,8 @@ func TestProxyGroup(t *testing.T) {
 		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar())
 		expectEqual(t, fc, pg, nil)
 		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
-		if expected := 1; reconciler.proxyGroups.Len() != expected {
+		if expected := 1; reconciler.egressProxyGroups.Len() != expected {
-			t.Fatalf("expected %d recorders, got %d", expected, reconciler.proxyGroups.Len())
+			t.Fatalf("expected %d egress ProxyGroups, got %d", expected, reconciler.egressProxyGroups.Len())
 		}
 		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
 		keyReq := tailscale.KeyCapabilities{
@@ -227,8 +232,8 @@ func TestProxyGroup(t *testing.T) {
 		expectReconciled(t, reconciler, "", pg.Name)
 
 		expectMissing[tsapi.ProxyGroup](t, fc, "", pg.Name)
-		if expected := 0; reconciler.proxyGroups.Len() != expected {
+		if expected := 0; reconciler.egressProxyGroups.Len() != expected {
-			t.Fatalf("expected %d ProxyGroups, got %d", expected, reconciler.proxyGroups.Len())
+			t.Fatalf("expected %d ProxyGroups, got %d", expected, reconciler.egressProxyGroups.Len())
 		}
 		// 2 nodes should get deleted as part of the scale down, and then finally
 		// the first node gets deleted with the ProxyGroup cleanup.
@@ -241,6 +246,131 @@ func TestProxyGroup(t *testing.T) {
 	})
 }
 
+func TestProxyGroupTypes(t *testing.T) {
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		Build()
+
+	zl, _ := zap.NewDevelopment()
+	reconciler := &ProxyGroupReconciler{
+		tsNamespace: tsNamespace,
+		proxyImage:  testProxyImage,
+		Client:      fc,
+		l:           zl.Sugar(),
+		tsClient:    &fakeTSClient{},
+		clock:       tstest.NewClock(tstest.ClockOpts{}),
+	}
+
+	t.Run("egress_type", func(t *testing.T) {
+		pg := &tsapi.ProxyGroup{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-egress",
+				UID:  "test-egress-uid",
+			},
+			Spec: tsapi.ProxyGroupSpec{
+				Type:     tsapi.ProxyGroupTypeEgress,
+				Replicas: ptr.To[int32](0),
+			},
+		}
+		if err := fc.Create(context.Background(), pg); err != nil {
+			t.Fatal(err)
+		}
+
+		expectReconciled(t, reconciler, "", pg.Name)
+		verifyProxyGroupCounts(t, reconciler, 0, 1)
+
+		sts := &appsv1.StatefulSet{}
+		if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil {
+			t.Fatalf("failed to get StatefulSet: %v", err)
+		}
+		verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupEgress)
+		verifyEnvVar(t, sts, "TS_EGRESS_SERVICES_CONFIG_PATH", fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices))
+
+		// Verify that egress configuration has been set up.
+		cm := &corev1.ConfigMap{}
+		cmName := fmt.Sprintf("%s-egress-config", pg.Name)
+		if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: cmName}, cm); err != nil {
+			t.Fatalf("failed to get ConfigMap: %v", err)
+		}
+
+		expectedVolumes := []corev1.Volume{
+			{
+				Name: cmName,
+				VolumeSource: corev1.VolumeSource{
+					ConfigMap: &corev1.ConfigMapVolumeSource{
+						LocalObjectReference: corev1.LocalObjectReference{
+							Name: cmName,
+						},
+					},
+				},
+			},
+		}
+
+		expectedVolumeMounts := []corev1.VolumeMount{
+			{
+				Name:      cmName,
+				MountPath: "/etc/proxies",
+				ReadOnly:  true,
+			},
+		}
+
+		if diff := cmp.Diff(expectedVolumes, sts.Spec.Template.Spec.Volumes); diff != "" {
+			t.Errorf("unexpected volumes (-want +got):\n%s", diff)
+		}
+
+		if diff := cmp.Diff(expectedVolumeMounts, sts.Spec.Template.Spec.Containers[0].VolumeMounts); diff != "" {
+			t.Errorf("unexpected volume mounts (-want +got):\n%s", diff)
+		}
+	})
+
+	t.Run("ingress_type", func(t *testing.T) {
+		pg := &tsapi.ProxyGroup{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-ingress",
+				UID:  "test-ingress-uid",
+			},
+			Spec: tsapi.ProxyGroupSpec{
+				Type: tsapi.ProxyGroupTypeIngress,
+			},
+		}
+		if err := fc.Create(context.Background(), pg); err != nil {
+			t.Fatal(err)
+		}
+
+		expectReconciled(t, reconciler, "", pg.Name)
+		verifyProxyGroupCounts(t, reconciler, 1, 1)
+
+		sts := &appsv1.StatefulSet{}
+		if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil {
+			t.Fatalf("failed to get StatefulSet: %v", err)
+		}
+		verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupIngress)
+	})
+}
+
+func verifyProxyGroupCounts(t *testing.T, r *ProxyGroupReconciler, wantIngress, wantEgress int) {
+	t.Helper()
+	if r.ingressProxyGroups.Len() != wantIngress {
+		t.Errorf("expected %d ingress proxy groups, got %d", wantIngress, r.ingressProxyGroups.Len())
+	}
+	if r.egressProxyGroups.Len() != wantEgress {
+		t.Errorf("expected %d egress proxy groups, got %d", wantEgress, r.egressProxyGroups.Len())
+	}
+}
+
+func verifyEnvVar(t *testing.T, sts *appsv1.StatefulSet, name, expectedValue string) {
+	t.Helper()
+	for _, env := range sts.Spec.Template.Spec.Containers[0].Env {
+		if env.Name == name {
+			if env.Value != expectedValue {
+				t.Errorf("expected %s=%s, got %s", name, expectedValue, env.Value)
+			}
+			return
+		}
+	}
+	t.Errorf("%s environment variable not found", name)
+}
+
 func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool, cfgHash string) {
 	t.Helper()
 

k8s-operator/api.md

@@ -568,9 +568,9 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
-| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Currently the only supported type is egress. |  | Enum: [egress] <br />Type: string <br /> |
+| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Supported types are egress and ingress.<br />Type is immutable once a ProxyGroup is created. |  | Enum: [egress ingress] <br />Type: string <br /> |
 | `tags` _[Tags](#tags)_ | Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].<br />If you specify custom tags here, make sure you also make the operator<br />an owner of these tags.<br />See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.<br />Tags cannot be changed once a ProxyGroup device has been created.<br />Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. |  | Pattern: `^tag:[a-zA-Z][a-zA-Z0-9-]*$` <br />Type: string <br /> |
-| `replicas` _integer_ | Replicas specifies how many replicas to create the StatefulSet with.<br />Defaults to 2. |  |  |
+| `replicas` _integer_ | Replicas specifies how many replicas to create the StatefulSet with.<br />Defaults to 2. |  | Minimum: 0 <br /> |
 | `hostnamePrefix` _[HostnamePrefix](#hostnameprefix)_ | HostnamePrefix is the hostname prefix to use for tailnet devices created<br />by the ProxyGroup. Each device will have the integer number from its<br />StatefulSet pod appended to this prefix to form the full hostname.<br />HostnamePrefix can contain lower case letters, numbers and dashes, it<br />must not start with a dash and must be between 1 and 62 characters long. |  | Pattern: `^[a-z0-9][a-z0-9-]{0,61}$` <br />Type: string <br /> |
 | `proxyClass` _string_ | ProxyClass is the name of the ProxyClass custom resource that contains<br />configuration options that should be applied to the resources created<br />for this ProxyGroup. If unset, and there is no default ProxyClass<br />configured, the operator will create resources with the default<br />configuration. |  |  |
 
@@ -599,7 +599,7 @@ _Underlying type:_ _string_
 
 
 _Validation:_
-- Enum: [egress]
+- Enum: [egress ingress]
 - Type: string
 
 _Appears in:_

k8s-operator/apis/v1alpha1/types_proxygroup.go

@@ -13,6 +13,7 @@ import (
 // +kubebuilder:subresource:status
 // +kubebuilder:resource:scope=Cluster,shortName=pg
 // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "ProxyGroupReady")].reason`,description="Status of the deployed ProxyGroup resources."
+// +kubebuilder:printcolumn:name="Type",type="string",JSONPath=`.spec.type`,description="ProxyGroup type."
 
 // ProxyGroup defines a set of Tailscale devices that will act as proxies.
 // Currently only egress ProxyGroups are supported.
@@ -47,7 +48,9 @@ type ProxyGroupList struct {
 }
 
 type ProxyGroupSpec struct {
-	// Type of the ProxyGroup proxies. Currently the only supported type is egress.
+	// Type of the ProxyGroup proxies. Supported types are egress and ingress.
+	// Type is immutable once a ProxyGroup is created.
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="ProxyGroup type is immutable"
 	Type ProxyGroupType `json:"type"`
 
 	// Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].
@@ -62,6 +65,7 @@ type ProxyGroupSpec struct {
 	// Replicas specifies how many replicas to create the StatefulSet with.
 	// Defaults to 2.
 	// +optional
+	// +kubebuilder:validation:Minimum=0
 	Replicas *int32 `json:"replicas,omitempty"`
 
 	// HostnamePrefix is the hostname prefix to use for tailnet devices created
@@ -109,11 +113,12 @@ type TailnetDevice struct {
 }
 
 // +kubebuilder:validation:Type=string
-// +kubebuilder:validation:Enum=egress
+// +kubebuilder:validation:Enum=egress;ingress
 type ProxyGroupType string
 
 const (
 	ProxyGroupTypeEgress  ProxyGroupType = "egress"
+	ProxyGroupTypeIngress ProxyGroupType = "ingress"
 )
 
 // +kubebuilder:validation:Type=string