VIP

cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml

@@ -85,10 +85,7 @@ spec:
                     type: string
                     pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
                 type:
-                  description: |-
+                  description: Type of the ProxyGroup proxies. Currently the only supported type is egress.
-                    Type of the ProxyGroup, either ingress or egress. Each set of proxies
-                    managed by a single ProxyGroup definition operate as only ingress or
-                    only egress proxies.
                   type: string
                   enum:
                     - egress

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -2497,10 +2497,7 @@ spec:
                                     type: string
                                 type: array
                             type:
-                                description: |-
+                                description: Type of the ProxyGroup proxies. Currently the only supported type is egress.
-                                    Type of the ProxyGroup, either ingress or egress. Each set of proxies
-                                    managed by a single ProxyGroup definition operate as only ingress or
-                                    only egress proxies.
                                 enum:
                                     - egress
                                 type: string

cmd/k8s-operator/egress-eps.go

@@ -58,8 +58,8 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 	// resources are set up for this tailnet service.
 	svc := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      eps.Labels[labelExternalSvcName],
+			Name:      eps.Labels[LabelParentName],
-			Namespace: eps.Labels[labelExternalSvcNamespace],
+			Namespace: eps.Labels[LabelParentNamespace],
 		},
 	}
 	err = er.Get(ctx, client.ObjectKeyFromObject(svc), svc)
@@ -98,7 +98,10 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 	// Check which Pods in ProxyGroup are ready to route traffic to this
 	// egress service.
 	podList := &corev1.PodList{}
-	if err := er.List(ctx, podList, client.MatchingLabels(map[string]string{labelProxyGroup: proxyGroupName})); err != nil {
+	if err := er.List(ctx, podList, client.MatchingLabels(map[string]string{
+		LabelParentName: proxyGroupName,
+		LabelParentType: "proxygroup",
+	})); err != nil {
 		return res, fmt.Errorf("error listing Pods for ProxyGroup %s: %w", proxyGroupName, err)
 	}
 	newEndpoints := make([]discoveryv1.Endpoint, 0)

cmd/k8s-operator/egress-eps_test.go

@@ -75,7 +75,11 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) {
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "foo",
 			Namespace: "operator-ns",
-			Labels:    map[string]string{labelExternalSvcName: "test", labelExternalSvcNamespace: "default", labelProxyGroup: "foo"},
+			Labels: map[string]string{
+				LabelParentName:      "test",
+				LabelParentNamespace: "default",
+				labelSvcType:         typeEgress,
+				labelProxyGroup:      "foo"},
 		},
 		AddressType: discoveryv1.AddressTypeIPv4,
 	}
@@ -173,8 +177,10 @@ func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) {
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      fmt.Sprintf("%s-0", pg),
 			Namespace: "operator-ns",
-			Labels:    map[string]string{labelProxyGroup: pg},
+			Labels: map[string]string{
-			UID:       "foo",
+				LabelParentType: "proxygroup",
+				LabelParentName: pg},
+			UID: "foo",
 		},
 		Status: corev1.PodStatus{
 			PodIP: "10.0.0.1",
@@ -184,7 +190,9 @@ func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) {
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      fmt.Sprintf("%s-0", pg),
 			Namespace: "operator-ns",
-			Labels:    map[string]string{labelProxyGroup: pg},
+			Labels: map[string]string{
+				LabelParentType: "proxygroup",
+				LabelParentName: pg},
 		},
 	}
 	return p, s

cmd/k8s-operator/egress-services.go

@@ -46,10 +46,7 @@
 	reasonEgressSvcCreationFailed = "EgressSvcCreationFailed"
 	reasonProxyGroupNotReady      = "ProxyGroupNotReady"
 
-	labelProxyGroup           = "tailscale.com/proxy-group"
+	labelProxyGroup = "tailscale.com/proxy-group"
-	labelProxyGroupType       = "tailscale.com/proxy-group-type"
-	labelExternalSvcName      = "tailscale.com/external-service-name"
-	labelExternalSvcNamespace = "tailscale.com/external-service-namespace"
 
 	labelSvcType = "tailscale.com/svc-type" // ingress or egress
 	typeEgress   = "egress"
@@ -63,7 +60,7 @@
 
 	indexEgressProxyGroup = ".metadata.annotations.egress-proxy-group"
 
-	egressSvcsCMNameTemplate = "proxy-cfg-%s"
+	egressSvcsCMNameTemplate = "%s-egress-config"
 )
 
 var gaugeEgressServices = clientmetric.NewGauge(kubetypes.MetricEgressServiceCount)
@@ -416,7 +413,7 @@ func (esr *egressSvcsReconciler) usedPortsForPG(ctx context.Context, pg string)
 func (esr *egressSvcsReconciler) clusterIPSvcForEgress(crl map[string]string) *corev1.Service {
 	return &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
-			GenerateName: svcNameBase(crl[labelExternalSvcName]),
+			GenerateName: svcNameBase(crl[LabelParentName]),
 			Namespace:    esr.tsNamespace,
 			Labels:       crl,
 		},
@@ -479,15 +476,18 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s
 	if err := esr.Get(ctx, client.ObjectKeyFromObject(pg), pg); apierrors.IsNotFound(err) {
 		l.Infof("ProxyGroup %q not found, waiting...", proxyGroupName)
 		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l)
+		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
 		return false, nil
 	} else if err != nil {
 		err := fmt.Errorf("unable to retrieve ProxyGroup %s: %w", proxyGroupName, err)
 		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, err.Error(), esr.clock, l)
+		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
 		return false, err
 	}
 	if !tsoperator.ProxyGroupIsReady(pg) {
 		l.Infof("ProxyGroup %s is not ready, waiting...", proxyGroupName)
 		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l)
+		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
 		return false, nil
 	}
 
@@ -496,6 +496,7 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s
 		esr.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg)
 		l.Info(msg)
 		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionFalse, reasonEgressSvcInvalid, msg, esr.clock, l)
+		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
 		return false, nil
 	}
 	l.Debugf("egress service is valid")
@@ -626,11 +627,12 @@ func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, ts
 // should probably validate and truncate (?) the names is they are too long.
 func egressSvcChildResourceLabels(svc *corev1.Service) map[string]string {
 	return map[string]string{
-		LabelManaged:              "true",
+		LabelManaged:         "true",
-		labelProxyGroup:           svc.Annotations[AnnotationProxyGroup],
+		LabelParentType:      "svc",
-		labelExternalSvcName:      svc.Name,
+		LabelParentName:      svc.Name,
-		labelExternalSvcNamespace: svc.Namespace,
+		LabelParentNamespace: svc.Namespace,
-		labelSvcType:              typeEgress,
+		labelProxyGroup:      svc.Annotations[AnnotationProxyGroup],
+		labelSvcType:         typeEgress,
 	}
 }
 

cmd/k8s-operator/operator.go

@@ -378,7 +378,7 @@ func runReconcilers(opts reconcilerOpts) {
 
 	epsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsHandler)
 	podsSecretsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsFromEgressPGChildResources(mgr.GetClient(), opts.log, opts.tailscaleNamespace))
-	epsFromExtNSvcFilter := handler.EnqueueRequestsFromMapFunc(epsFromExternalNameService(mgr.GetClient(), opts.log))
+	epsFromExtNSvcFilter := handler.EnqueueRequestsFromMapFunc(epsFromExternalNameService(mgr.GetClient(), opts.log, opts.tailscaleNamespace))
 
 	err = builder.
 		ControllerManagedBy(mgr).
@@ -844,18 +844,20 @@ func egressEpsHandler(_ context.Context, o client.Object) []reconcile.Request {
 // that ProxyGroup.
 func egressEpsFromEgressPGChildResources(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc {
 	return func(_ context.Context, o client.Object) []reconcile.Request {
-		pg, ok := o.GetLabels()[labelProxyGroup]
+		// TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we
+		// have ingress ProxyGroups.
+		if typ := o.GetLabels()[LabelParentType]; typ != "proxygroup" {
+			return nil
+		}
+		pg, ok := o.GetLabels()[LabelParentName]
 		if !ok {
 			return nil
 		}
-		// TODO(irbekrm): depending on what labels we add to ProxyGroup
+
-		// resources and which resources, this might need some extra
-		// checks.
-		if typ, ok := o.GetLabels()[labelProxyGroupType]; !ok || typ != typeEgress {
-			return nil
-		}
 		epsList := discoveryv1.EndpointSliceList{}
-		if err := cl.List(context.Background(), &epsList, client.InNamespace(ns), client.MatchingLabels(map[string]string{labelProxyGroup: pg})); err != nil {
+		if err := cl.List(context.Background(), &epsList,
+			client.InNamespace(ns),
+			client.MatchingLabels(map[string]string{labelProxyGroup: pg})); err != nil {
 			logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on %s %s", err, o.GetName(), o.GetObjectKind().GroupVersionKind().Kind)
 			return nil
 		}
@@ -872,6 +874,8 @@ func egressEpsFromEgressPGChildResources(cl client.Client, logger *zap.SugaredLo
 	}
 }
 
+// egressSvcsFromEgressProxyGroup is an event handler for egress ProxyGroups. It returns reconcile requests for all
+// user-created ExternalName Services that should be exposed on this ProxyGroup.
 func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
 	return func(_ context.Context, o client.Object) []reconcile.Request {
 		pg, ok := o.(*tsapi.ProxyGroup)
@@ -900,7 +904,9 @@ func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger)
 	}
 }
 
-func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
+// epsFromExternalNameService is an event handler for ExternalName Services that define a Tailscale egress service that
+// should be exposed on a ProxyGroup. It returns reconcile requests for EndpointSlices created for this service.
+func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc {
 	return func(_ context.Context, o client.Object) []reconcile.Request {
 		svc, ok := o.(*corev1.Service)
 		if !ok {
@@ -911,10 +917,11 @@ func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger) han
 			return nil
 		}
 		epsList := &discoveryv1.EndpointSliceList{}
-		if err := cl.List(context.Background(), epsList, client.MatchingLabels(map[string]string{
+		if err := cl.List(context.Background(), epsList, client.InNamespace(ns),
-			labelExternalSvcName:      svc.Name,
+			client.MatchingLabels(map[string]string{
-			labelExternalSvcNamespace: svc.Namespace,
+				LabelParentName:      svc.Name,
-		})); err != nil {
+				LabelParentNamespace: svc.Namespace,
+			})); err != nil {
 			logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on Service %s", err, svc.Name)
 			return nil
 		}
@@ -931,6 +938,8 @@ func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger) han
 	}
 }
 
+// indexEgressServices adds a local index to a cached Tailscale egress Services meant to be exposed on a ProxyGroup. The
+// index is used a list filter.
 func indexEgressServices(o client.Object) []string {
 	if !isEgressSvcForProxyGroup(o) {
 		return nil

cmd/k8s-operator/proxygroup.go

@@ -219,6 +219,15 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
 	}); err != nil {
 		return fmt.Errorf("error provisioning RoleBinding: %w", err)
 	}
+	if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
+		cm := pgEgressCM(pg, r.tsNamespace)
+		if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, cm, func(existing *corev1.ConfigMap) {
+			existing.ObjectMeta.Labels = cm.ObjectMeta.Labels
+			existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences
+		}); err != nil {
+			return fmt.Errorf("error provisioning ConfigMap: %w", err)
+		}
+	}
 	ss := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, cfgHash)
 	ss = applyProxyClassToStatefulSet(proxyClass, ss, nil, logger)
 	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) {

cmd/k8s-operator/proxygroup_specs.go

@@ -13,6 +13,7 @@
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/egressservices"
 	"tailscale.com/types/ptr"
 )
 
@@ -81,6 +82,13 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, cfgHash string) *apps
 									})
 								}
 
+								if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
+									mounts = append(mounts, corev1.VolumeMount{
+										Name:      fmt.Sprintf(egressSvcsCMNameTemplate, pg.Name),
+										MountPath: "/etc/proxies",
+										ReadOnly:  true,
+									})
+								}
 								return mounts
 							}(),
 						},
@@ -97,6 +105,18 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, cfgHash string) *apps
 								},
 							})
 						}
+						if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
+							volumes = append(volumes, corev1.Volume{
+								Name: fmt.Sprintf(egressSvcsCMNameTemplate, pg.Name),
+								VolumeSource: corev1.VolumeSource{
+									ConfigMap: &corev1.ConfigMapVolumeSource{
+										LocalObjectReference: corev1.LocalObjectReference{
+											Name: fmt.Sprintf(egressSvcsCMNameTemplate, pg.Name),
+										},
+									},
+								},
+							})
+						}
 
 						return volumes
 					}(),
@@ -185,6 +205,17 @@ func pgStateSecrets(pg *tsapi.ProxyGroup, namespace string) (secrets []*corev1.S
 	return secrets
 }
 
+func pgEgressCM(pg *tsapi.ProxyGroup, namespace string) *corev1.ConfigMap {
+	return &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            fmt.Sprintf(egressSvcsCMNameTemplate, pg.Name),
+			Namespace:       namespace,
+			Labels:          pgLabels(pg.Name, nil),
+			OwnerReferences: pgOwnerReference(pg),
+		},
+	}
+}
+
 func pgSecretLabels(pgName, typ string) map[string]string {
 	return pgLabels(pgName, map[string]string{
 		labelSecretType: typ, // "config" or "state".
@@ -204,7 +235,7 @@ func pgLabels(pgName string, customLabels map[string]string) map[string]string {
 	return l
 }
 
-func pgEnv(_ *tsapi.ProxyGroup) []corev1.EnvVar {
+func pgEnv(pg *tsapi.ProxyGroup) []corev1.EnvVar {
 	envs := []corev1.EnvVar{
 		{
 			Name: "POD_IP",
@@ -235,6 +266,20 @@ func pgEnv(_ *tsapi.ProxyGroup) []corev1.EnvVar {
 			Name:  "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
 			Value: "/etc/tsconfig/$(POD_NAME)",
 		},
+		{
+			Name:  "TS_USERSPACE",
+			Value: "false",
+		},
+		{
+			Name:  "TS_DEBUG_FIREWALL_MODE",
+			Value: "auto",
+		},
+	}
+	if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
+		envs = append(envs, corev1.EnvVar{
+			Name:  "TS_EGRESS_SERVICES_CONFIG_PATH",
+			Value: fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices),
+		})
 	}
 
 	return envs

cmd/k8s-operator/sts.go

@@ -49,10 +49,9 @@
 	LabelParentNamespace = "tailscale.com/parent-resource-ns"
 	labelSecretType      = "tailscale.com/secret-type" // "config" or "state".
 
-	// LabelProxyClass can be set by users on Connectors, tailscale
+	// LabelProxyClass can be set by users on tailscale Ingresses and Services that define cluster ingress or
-	// Ingresses and Services that define cluster ingress or cluster egress,
+	// cluster egress, to specify that configuration in this ProxyClass should be applied to resources created for
-	// to specify that configuration in this ProxyClass should be applied to
+	// the Ingress or Service.
-	// resources created for the Connector, Ingress or Service.
 	LabelProxyClass = "tailscale.com/proxy-class"
 
 	FinalizerName = "tailscale.com/finalizer"

cmd/k8s-operator/svc.go

@@ -112,6 +112,10 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
 		return reconcile.Result{}, fmt.Errorf("failed to get svc: %w", err)
 	}
 
+	if _, ok := svc.Annotations[AnnotationProxyGroup]; ok {
+		return reconcile.Result{}, nil // this reconciler should not look at Services for ProxyGroup
+	}
+
 	if !svc.DeletionTimestamp.IsZero() || !a.isTailscaleService(svc) {
 		logger.Debugf("service is being deleted or is (no longer) referring to Tailscale ingress/egress, ensuring any created resources are cleaned up")
 		return reconcile.Result{}, a.maybeCleanup(ctx, logger, svc)

k8s-operator/api.md

@@ -522,7 +522,7 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
-| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup, either ingress or egress. Each set of proxies<br />managed by a single ProxyGroup definition operate as only ingress or<br />only egress proxies. |  | Enum: [egress] <br />Type: string <br /> |
+| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Currently the only supported type is egress. |  | Enum: [egress] <br />Type: string <br /> |
 | `tags` _[Tags](#tags)_ | Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].<br />If you specify custom tags here, make sure you also make the operator<br />an owner of these tags.<br />See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.<br />Tags cannot be changed once a ProxyGroup device has been created.<br />Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. |  | Pattern: `^tag:[a-zA-Z][a-zA-Z0-9-]*$` <br />Type: string <br /> |
 | `replicas` _integer_ | Replicas specifies how many replicas to create the StatefulSet with.<br />Defaults to 2. |  |  |
 | `hostnamePrefix` _[HostnamePrefix](#hostnameprefix)_ | HostnamePrefix is the hostname prefix to use for tailnet devices created<br />by the ProxyGroup. Each device will have the integer number from its<br />StatefulSet pod appended to this prefix to form the full hostname.<br />HostnamePrefix can contain lower case letters, numbers and dashes, it<br />must not start with a dash and must be between 1 and 62 characters long. |  | Pattern: `^[a-z0-9][a-z0-9-]{0,61}$` <br />Type: string <br /> |

k8s-operator/apis/v1alpha1/types_proxygroup.go

@@ -37,9 +37,7 @@ type ProxyGroupList struct {
 }
 
 type ProxyGroupSpec struct {
-	// Type of the ProxyGroup, either ingress or egress. Each set of proxies
+	// Type of the ProxyGroup proxies. Currently the only supported type is egress.
-	// managed by a single ProxyGroup definition operate as only ingress or
-	// only egress proxies.
 	Type ProxyGroupType `json:"type"`
 
 	// Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].