diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml index 32e2ab450..035d04786 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml @@ -85,10 +85,7 @@ spec: type: string pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ type: - description: |- - Type of the ProxyGroup, either ingress or egress. Each set of proxies - managed by a single ProxyGroup definition operate as only ingress or - only egress proxies. + description: Type of the ProxyGroup proxies. Currently the only supported type is egress. type: string enum: - egress diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index e6358708b..14166fed9 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -2497,10 +2497,7 @@ spec: type: string type: array type: - description: |- - Type of the ProxyGroup, either ingress or egress. Each set of proxies - managed by a single ProxyGroup definition operate as only ingress or - only egress proxies. + description: Type of the ProxyGroup proxies. Currently the only supported type is egress. enum: - egress type: string diff --git a/cmd/k8s-operator/egress-eps.go b/cmd/k8s-operator/egress-eps.go index 510d58783..0d0f42441 100644 --- a/cmd/k8s-operator/egress-eps.go +++ b/cmd/k8s-operator/egress-eps.go @@ -58,8 +58,8 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ // resources are set up for this tailnet service. svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: eps.Labels[labelExternalSvcName], - Namespace: eps.Labels[labelExternalSvcNamespace], + Name: eps.Labels[LabelParentName], + Namespace: eps.Labels[LabelParentNamespace], }, } err = er.Get(ctx, client.ObjectKeyFromObject(svc), svc) @@ -98,7 +98,10 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ // Check which Pods in ProxyGroup are ready to route traffic to this // egress service. podList := &corev1.PodList{} - if err := er.List(ctx, podList, client.MatchingLabels(map[string]string{labelProxyGroup: proxyGroupName})); err != nil { + if err := er.List(ctx, podList, client.MatchingLabels(map[string]string{ + LabelParentName: proxyGroupName, + LabelParentType: "proxygroup", + })); err != nil { return res, fmt.Errorf("error listing Pods for ProxyGroup %s: %w", proxyGroupName, err) } newEndpoints := make([]discoveryv1.Endpoint, 0) diff --git a/cmd/k8s-operator/egress-eps_test.go b/cmd/k8s-operator/egress-eps_test.go index a2e95e5d3..d7e08261d 100644 --- a/cmd/k8s-operator/egress-eps_test.go +++ b/cmd/k8s-operator/egress-eps_test.go @@ -75,7 +75,11 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "operator-ns", - Labels: map[string]string{labelExternalSvcName: "test", labelExternalSvcNamespace: "default", labelProxyGroup: "foo"}, + Labels: map[string]string{ + LabelParentName: "test", + LabelParentNamespace: "default", + labelSvcType: typeEgress, + labelProxyGroup: "foo"}, }, AddressType: discoveryv1.AddressTypeIPv4, } @@ -173,8 +177,10 @@ func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) { ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-0", pg), Namespace: "operator-ns", - Labels: map[string]string{labelProxyGroup: pg}, - UID: "foo", + Labels: map[string]string{ + LabelParentType: "proxygroup", + LabelParentName: pg}, + UID: "foo", }, Status: corev1.PodStatus{ PodIP: "10.0.0.1", @@ -184,7 +190,9 @@ func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) { ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-0", pg), Namespace: "operator-ns", - Labels: map[string]string{labelProxyGroup: pg}, + Labels: map[string]string{ + LabelParentType: "proxygroup", + LabelParentName: pg}, }, } return p, s diff --git a/cmd/k8s-operator/egress-services.go b/cmd/k8s-operator/egress-services.go index 1c4f70a96..5fed20795 100644 --- a/cmd/k8s-operator/egress-services.go +++ b/cmd/k8s-operator/egress-services.go @@ -46,10 +46,7 @@ reasonEgressSvcCreationFailed = "EgressSvcCreationFailed" reasonProxyGroupNotReady = "ProxyGroupNotReady" - labelProxyGroup = "tailscale.com/proxy-group" - labelProxyGroupType = "tailscale.com/proxy-group-type" - labelExternalSvcName = "tailscale.com/external-service-name" - labelExternalSvcNamespace = "tailscale.com/external-service-namespace" + labelProxyGroup = "tailscale.com/proxy-group" labelSvcType = "tailscale.com/svc-type" // ingress or egress typeEgress = "egress" @@ -63,7 +60,7 @@ indexEgressProxyGroup = ".metadata.annotations.egress-proxy-group" - egressSvcsCMNameTemplate = "proxy-cfg-%s" + egressSvcsCMNameTemplate = "%s-egress-config" ) var gaugeEgressServices = clientmetric.NewGauge(kubetypes.MetricEgressServiceCount) @@ -416,7 +413,7 @@ func (esr *egressSvcsReconciler) usedPortsForPG(ctx context.Context, pg string) func (esr *egressSvcsReconciler) clusterIPSvcForEgress(crl map[string]string) *corev1.Service { return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - GenerateName: svcNameBase(crl[labelExternalSvcName]), + GenerateName: svcNameBase(crl[LabelParentName]), Namespace: esr.tsNamespace, Labels: crl, }, @@ -479,15 +476,18 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s if err := esr.Get(ctx, client.ObjectKeyFromObject(pg), pg); apierrors.IsNotFound(err) { l.Infof("ProxyGroup %q not found, waiting...", proxyGroupName) tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l) + tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) return false, nil } else if err != nil { err := fmt.Errorf("unable to retrieve ProxyGroup %s: %w", proxyGroupName, err) tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, err.Error(), esr.clock, l) + tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) return false, err } if !tsoperator.ProxyGroupIsReady(pg) { l.Infof("ProxyGroup %s is not ready, waiting...", proxyGroupName) tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l) + tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) return false, nil } @@ -496,6 +496,7 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s esr.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg) l.Info(msg) tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionFalse, reasonEgressSvcInvalid, msg, esr.clock, l) + tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) return false, nil } l.Debugf("egress service is valid") @@ -626,11 +627,12 @@ func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, ts // should probably validate and truncate (?) the names is they are too long. func egressSvcChildResourceLabels(svc *corev1.Service) map[string]string { return map[string]string{ - LabelManaged: "true", - labelProxyGroup: svc.Annotations[AnnotationProxyGroup], - labelExternalSvcName: svc.Name, - labelExternalSvcNamespace: svc.Namespace, - labelSvcType: typeEgress, + LabelManaged: "true", + LabelParentType: "svc", + LabelParentName: svc.Name, + LabelParentNamespace: svc.Namespace, + labelProxyGroup: svc.Annotations[AnnotationProxyGroup], + labelSvcType: typeEgress, } } diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index 70247ace7..11e0e8a90 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -378,7 +378,7 @@ func runReconcilers(opts reconcilerOpts) { epsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsHandler) podsSecretsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsFromEgressPGChildResources(mgr.GetClient(), opts.log, opts.tailscaleNamespace)) - epsFromExtNSvcFilter := handler.EnqueueRequestsFromMapFunc(epsFromExternalNameService(mgr.GetClient(), opts.log)) + epsFromExtNSvcFilter := handler.EnqueueRequestsFromMapFunc(epsFromExternalNameService(mgr.GetClient(), opts.log, opts.tailscaleNamespace)) err = builder. ControllerManagedBy(mgr). @@ -844,18 +844,20 @@ func egressEpsHandler(_ context.Context, o client.Object) []reconcile.Request { // that ProxyGroup. func egressEpsFromEgressPGChildResources(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc { return func(_ context.Context, o client.Object) []reconcile.Request { - pg, ok := o.GetLabels()[labelProxyGroup] + // TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we + // have ingress ProxyGroups. + if typ := o.GetLabels()[LabelParentType]; typ != "proxygroup" { + return nil + } + pg, ok := o.GetLabels()[LabelParentName] if !ok { return nil } - // TODO(irbekrm): depending on what labels we add to ProxyGroup - // resources and which resources, this might need some extra - // checks. - if typ, ok := o.GetLabels()[labelProxyGroupType]; !ok || typ != typeEgress { - return nil - } + epsList := discoveryv1.EndpointSliceList{} - if err := cl.List(context.Background(), &epsList, client.InNamespace(ns), client.MatchingLabels(map[string]string{labelProxyGroup: pg})); err != nil { + if err := cl.List(context.Background(), &epsList, + client.InNamespace(ns), + client.MatchingLabels(map[string]string{labelProxyGroup: pg})); err != nil { logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on %s %s", err, o.GetName(), o.GetObjectKind().GroupVersionKind().Kind) return nil } @@ -872,6 +874,8 @@ func egressEpsFromEgressPGChildResources(cl client.Client, logger *zap.SugaredLo } } +// egressSvcsFromEgressProxyGroup is an event handler for egress ProxyGroups. It returns reconcile requests for all +// user-created ExternalName Services that should be exposed on this ProxyGroup. func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { return func(_ context.Context, o client.Object) []reconcile.Request { pg, ok := o.(*tsapi.ProxyGroup) @@ -900,7 +904,9 @@ func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) } } -func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { +// epsFromExternalNameService is an event handler for ExternalName Services that define a Tailscale egress service that +// should be exposed on a ProxyGroup. It returns reconcile requests for EndpointSlices created for this service. +func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc { return func(_ context.Context, o client.Object) []reconcile.Request { svc, ok := o.(*corev1.Service) if !ok { @@ -911,10 +917,11 @@ func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger) han return nil } epsList := &discoveryv1.EndpointSliceList{} - if err := cl.List(context.Background(), epsList, client.MatchingLabels(map[string]string{ - labelExternalSvcName: svc.Name, - labelExternalSvcNamespace: svc.Namespace, - })); err != nil { + if err := cl.List(context.Background(), epsList, client.InNamespace(ns), + client.MatchingLabels(map[string]string{ + LabelParentName: svc.Name, + LabelParentNamespace: svc.Namespace, + })); err != nil { logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on Service %s", err, svc.Name) return nil } @@ -931,6 +938,8 @@ func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger) han } } +// indexEgressServices adds a local index to a cached Tailscale egress Services meant to be exposed on a ProxyGroup. The +// index is used a list filter. func indexEgressServices(o client.Object) []string { if !isEgressSvcForProxyGroup(o) { return nil diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index 6256feb57..1e858db5f 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -219,6 +219,15 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro }); err != nil { return fmt.Errorf("error provisioning RoleBinding: %w", err) } + if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { + cm := pgEgressCM(pg, r.tsNamespace) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, cm, func(existing *corev1.ConfigMap) { + existing.ObjectMeta.Labels = cm.ObjectMeta.Labels + existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences + }); err != nil { + return fmt.Errorf("error provisioning ConfigMap: %w", err) + } + } ss := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, cfgHash) ss = applyProxyClassToStatefulSet(proxyClass, ss, nil, logger) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) { diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index 94a095ff5..ded5da331 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -13,6 +13,7 @@ rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/kube/egressservices" "tailscale.com/types/ptr" ) @@ -81,6 +82,13 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, cfgHash string) *apps }) } + if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { + mounts = append(mounts, corev1.VolumeMount{ + Name: fmt.Sprintf(egressSvcsCMNameTemplate, pg.Name), + MountPath: "/etc/proxies", + ReadOnly: true, + }) + } return mounts }(), }, @@ -97,6 +105,18 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, cfgHash string) *apps }, }) } + if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { + volumes = append(volumes, corev1.Volume{ + Name: fmt.Sprintf(egressSvcsCMNameTemplate, pg.Name), + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: fmt.Sprintf(egressSvcsCMNameTemplate, pg.Name), + }, + }, + }, + }) + } return volumes }(), @@ -185,6 +205,17 @@ func pgStateSecrets(pg *tsapi.ProxyGroup, namespace string) (secrets []*corev1.S return secrets } +func pgEgressCM(pg *tsapi.ProxyGroup, namespace string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(egressSvcsCMNameTemplate, pg.Name), + Namespace: namespace, + Labels: pgLabels(pg.Name, nil), + OwnerReferences: pgOwnerReference(pg), + }, + } +} + func pgSecretLabels(pgName, typ string) map[string]string { return pgLabels(pgName, map[string]string{ labelSecretType: typ, // "config" or "state". @@ -204,7 +235,7 @@ func pgLabels(pgName string, customLabels map[string]string) map[string]string { return l } -func pgEnv(_ *tsapi.ProxyGroup) []corev1.EnvVar { +func pgEnv(pg *tsapi.ProxyGroup) []corev1.EnvVar { envs := []corev1.EnvVar{ { Name: "POD_IP", @@ -235,6 +266,20 @@ func pgEnv(_ *tsapi.ProxyGroup) []corev1.EnvVar { Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig/$(POD_NAME)", }, + { + Name: "TS_USERSPACE", + Value: "false", + }, + { + Name: "TS_DEBUG_FIREWALL_MODE", + Value: "auto", + }, + } + if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { + envs = append(envs, corev1.EnvVar{ + Name: "TS_EGRESS_SERVICES_CONFIG_PATH", + Value: fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices), + }) } return envs diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 19c98100f..6378a8263 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -49,10 +49,9 @@ LabelParentNamespace = "tailscale.com/parent-resource-ns" labelSecretType = "tailscale.com/secret-type" // "config" or "state". - // LabelProxyClass can be set by users on Connectors, tailscale - // Ingresses and Services that define cluster ingress or cluster egress, - // to specify that configuration in this ProxyClass should be applied to - // resources created for the Connector, Ingress or Service. + // LabelProxyClass can be set by users on tailscale Ingresses and Services that define cluster ingress or + // cluster egress, to specify that configuration in this ProxyClass should be applied to resources created for + // the Ingress or Service. LabelProxyClass = "tailscale.com/proxy-class" FinalizerName = "tailscale.com/finalizer" diff --git a/cmd/k8s-operator/svc.go b/cmd/k8s-operator/svc.go index e47fcae7f..22487ee26 100644 --- a/cmd/k8s-operator/svc.go +++ b/cmd/k8s-operator/svc.go @@ -112,6 +112,10 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request return reconcile.Result{}, fmt.Errorf("failed to get svc: %w", err) } + if _, ok := svc.Annotations[AnnotationProxyGroup]; ok { + return reconcile.Result{}, nil // this reconciler should not look at Services for ProxyGroup + } + if !svc.DeletionTimestamp.IsZero() || !a.isTailscaleService(svc) { logger.Debugf("service is being deleted or is (no longer) referring to Tailscale ingress/egress, ensuring any created resources are cleaned up") return reconcile.Result{}, a.maybeCleanup(ctx, logger, svc) diff --git a/k8s-operator/api.md b/k8s-operator/api.md index 82a3476ae..fd0a4e6ce 100644 --- a/k8s-operator/api.md +++ b/k8s-operator/api.md @@ -522,7 +522,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup, either ingress or egress. Each set of proxies
managed by a single ProxyGroup definition operate as only ingress or
only egress proxies. | | Enum: [egress]
Type: string
| +| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Currently the only supported type is egress. | | Enum: [egress]
Type: string
| | `tags` _[Tags](#tags)_ | Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].
If you specify custom tags here, make sure you also make the operator
an owner of these tags.
See https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
Tags cannot be changed once a ProxyGroup device has been created.
Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. | | Pattern: `^tag:[a-zA-Z][a-zA-Z0-9-]*$`
Type: string
| | `replicas` _integer_ | Replicas specifies how many replicas to create the StatefulSet with.
Defaults to 2. | | | | `hostnamePrefix` _[HostnamePrefix](#hostnameprefix)_ | HostnamePrefix is the hostname prefix to use for tailnet devices created
by the ProxyGroup. Each device will have the integer number from its
StatefulSet pod appended to this prefix to form the full hostname.
HostnamePrefix can contain lower case letters, numbers and dashes, it
must not start with a dash and must be between 1 and 62 characters long. | | Pattern: `^[a-z0-9][a-z0-9-]{0,61}$`
Type: string
| diff --git a/k8s-operator/apis/v1alpha1/types_proxygroup.go b/k8s-operator/apis/v1alpha1/types_proxygroup.go index 9b0e4215e..ef1e8c8c1 100644 --- a/k8s-operator/apis/v1alpha1/types_proxygroup.go +++ b/k8s-operator/apis/v1alpha1/types_proxygroup.go @@ -37,9 +37,7 @@ type ProxyGroupList struct { } type ProxyGroupSpec struct { - // Type of the ProxyGroup, either ingress or egress. Each set of proxies - // managed by a single ProxyGroup definition operate as only ingress or - // only egress proxies. + // Type of the ProxyGroup proxies. Currently the only supported type is egress. Type ProxyGroupType `json:"type"` // Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].