TheArchive/tailscale
1
0
Fork 0
mirror of https://github.com/tailscale/tailscale.git synced 2024-11-25 11:05:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

VIP

Browse Source
This commit is contained in:
Irbe Krumina 2024-10-07 15:16:49 +01:00
parent 101bd89efd
commit 1cecc43522
12 changed files with 120 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml
View File

@ -85,10 +85,7 @@ spec:
type: string
pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
type:
description: |-
Type of the ProxyGroup, either ingress or egress. Each set of proxies
managed by a single ProxyGroup definition operate as only ingress or
only egress proxies.
description: Type of the ProxyGroup proxies. Currently the only supported type is egress.
type: string
enum:
- egress

5
cmd/k8s-operator/deploy/manifests/operator.yaml
View File

@ -2497,10 +2497,7 @@ spec:
type: string
type: array
type:
description: |-
Type of the ProxyGroup, either ingress or egress. Each set of proxies
managed by a single ProxyGroup definition operate as only ingress or
only egress proxies.
description: Type of the ProxyGroup proxies. Currently the only supported type is egress.
enum:
- egress
type: string

9
cmd/k8s-operator/egress-eps.go
View File

@ -58,8 +58,8 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ
// resources are set up for this tailnet service.
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: eps.Labels[labelExternalSvcName],
Namespace: eps.Labels[labelExternalSvcNamespace],
Name: eps.Labels[LabelParentName],
Namespace: eps.Labels[LabelParentNamespace],
},
}
err = er.Get(ctx, client.ObjectKeyFromObject(svc), svc)
@ -98,7 +98,10 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ
// Check which Pods in ProxyGroup are ready to route traffic to this
// egress service.
podList := &corev1.PodList{}
if err := er.List(ctx, podList, client.MatchingLabels(map[string]string{labelProxyGroup: proxyGroupName})); err != nil {
if err := er.List(ctx, podList, client.MatchingLabels(map[string]string{
LabelParentName: proxyGroupName,
LabelParentType: "proxygroup",
})); err != nil {
return res, fmt.Errorf("error listing Pods for ProxyGroup %s: %w", proxyGroupName, err)
}
newEndpoints := make([]discoveryv1.Endpoint, 0)

16
cmd/k8s-operator/egress-eps_test.go
View File

@ -75,7 +75,11 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{
Name: "foo",
Namespace: "operator-ns",
Labels: map[string]string{labelExternalSvcName: "test", labelExternalSvcNamespace: "default", labelProxyGroup: "foo"},
Labels: map[string]string{
LabelParentName: "test",
LabelParentNamespace: "default",
labelSvcType: typeEgress,
labelProxyGroup: "foo"},
},
AddressType: discoveryv1.AddressTypeIPv4,
}
@ -173,8 +177,10 @@ func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) {
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-0", pg),
Namespace: "operator-ns",
Labels: map[string]string{labelProxyGroup: pg},
UID: "foo",
Labels: map[string]string{
LabelParentType: "proxygroup",
LabelParentName: pg},
UID: "foo",
},
Status: corev1.PodStatus{
PodIP: "10.0.0.1",
@ -184,7 +190,9 @@ func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) {
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-0", pg),
Namespace: "operator-ns",
Labels: map[string]string{labelProxyGroup: pg},
Labels: map[string]string{
LabelParentType: "proxygroup",
LabelParentName: pg},
},
}
return p, s

24
cmd/k8s-operator/egress-services.go
View File

@ -46,10 +46,7 @@
reasonEgressSvcCreationFailed = "EgressSvcCreationFailed"
reasonProxyGroupNotReady = "ProxyGroupNotReady"
labelProxyGroup = "tailscale.com/proxy-group"
labelProxyGroupType = "tailscale.com/proxy-group-type"
labelExternalSvcName = "tailscale.com/external-service-name"
labelExternalSvcNamespace = "tailscale.com/external-service-namespace"
labelProxyGroup = "tailscale.com/proxy-group"
labelSvcType = "tailscale.com/svc-type" // ingress or egress
typeEgress = "egress"
@ -63,7 +60,7 @@
indexEgressProxyGroup = ".metadata.annotations.egress-proxy-group"
egressSvcsCMNameTemplate = "proxy-cfg-%s"
egressSvcsCMNameTemplate = "%s-egress-config"
)
var gaugeEgressServices = clientmetric.NewGauge(kubetypes.MetricEgressServiceCount)
@ -416,7 +413,7 @@ func (esr *egressSvcsReconciler) usedPortsForPG(ctx context.Context, pg string)
func (esr *egressSvcsReconciler) clusterIPSvcForEgress(crl map[string]string) *corev1.Service {
return &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
GenerateName: svcNameBase(crl[labelExternalSvcName]),
GenerateName: svcNameBase(crl[LabelParentName]),
Namespace: esr.tsNamespace,
Labels: crl,
},
@ -479,15 +476,18 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s
if err := esr.Get(ctx, client.ObjectKeyFromObject(pg), pg); apierrors.IsNotFound(err) {
l.Infof("ProxyGroup %q not found, waiting...", proxyGroupName)
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l)
tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
return false, nil
} else if err != nil {
err := fmt.Errorf("unable to retrieve ProxyGroup %s: %w", proxyGroupName, err)
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, err.Error(), esr.clock, l)
tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
return false, err
}
if !tsoperator.ProxyGroupIsReady(pg) {
l.Infof("ProxyGroup %s is not ready, waiting...", proxyGroupName)
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l)
tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
return false, nil
}
@ -496,6 +496,7 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s
esr.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg)
l.Info(msg)
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionFalse, reasonEgressSvcInvalid, msg, esr.clock, l)
tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
return false, nil
}
l.Debugf("egress service is valid")
@ -626,11 +627,12 @@ func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, ts
// should probably validate and truncate (?) the names is they are too long.
func egressSvcChildResourceLabels(svc *corev1.Service) map[string]string {
return map[string]string{
LabelManaged: "true",
labelProxyGroup: svc.Annotations[AnnotationProxyGroup],
labelExternalSvcName: svc.Name,
labelExternalSvcNamespace: svc.Namespace,
labelSvcType: typeEgress,
LabelManaged: "true",
LabelParentType: "svc",
LabelParentName: svc.Name,
LabelParentNamespace: svc.Namespace,
labelProxyGroup: svc.Annotations[AnnotationProxyGroup],
labelSvcType: typeEgress,
}
}

37
cmd/k8s-operator/operator.go
View File

@ -378,7 +378,7 @@ func runReconcilers(opts reconcilerOpts) {
epsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsHandler)
podsSecretsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsFromEgressPGChildResources(mgr.GetClient(), opts.log, opts.tailscaleNamespace))
epsFromExtNSvcFilter := handler.EnqueueRequestsFromMapFunc(epsFromExternalNameService(mgr.GetClient(), opts.log))
epsFromExtNSvcFilter := handler.EnqueueRequestsFromMapFunc(epsFromExternalNameService(mgr.GetClient(), opts.log, opts.tailscaleNamespace))
err = builder.
ControllerManagedBy(mgr).
@ -844,18 +844,20 @@ func egressEpsHandler(_ context.Context, o client.Object) []reconcile.Request {
// that ProxyGroup.
func egressEpsFromEgressPGChildResources(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc {
return func(_ context.Context, o client.Object) []reconcile.Request {
pg, ok := o.GetLabels()[labelProxyGroup]
// TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we
// have ingress ProxyGroups.
if typ := o.GetLabels()[LabelParentType]; typ != "proxygroup" {
return nil
}
pg, ok := o.GetLabels()[LabelParentName]
if !ok {
return nil
}
// TODO(irbekrm): depending on what labels we add to ProxyGroup
// resources and which resources, this might need some extra
// checks.
if typ, ok := o.GetLabels()[labelProxyGroupType]; !ok || typ != typeEgress {
return nil
}
epsList := discoveryv1.EndpointSliceList{}
if err := cl.List(context.Background(), &epsList, client.InNamespace(ns), client.MatchingLabels(map[string]string{labelProxyGroup: pg})); err != nil {
if err := cl.List(context.Background(), &epsList,
client.InNamespace(ns),
client.MatchingLabels(map[string]string{labelProxyGroup: pg})); err != nil {
logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on %s %s", err, o.GetName(), o.GetObjectKind().GroupVersionKind().Kind)
return nil
}
@ -872,6 +874,8 @@ func egressEpsFromEgressPGChildResources(cl client.Client, logger *zap.SugaredLo
}
}
// egressSvcsFromEgressProxyGroup is an event handler for egress ProxyGroups. It returns reconcile requests for all
// user-created ExternalName Services that should be exposed on this ProxyGroup.
func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
return func(_ context.Context, o client.Object) []reconcile.Request {
pg, ok := o.(*tsapi.ProxyGroup)
@ -900,7 +904,9 @@ func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger)
}
}
func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
// epsFromExternalNameService is an event handler for ExternalName Services that define a Tailscale egress service that
// should be exposed on a ProxyGroup. It returns reconcile requests for EndpointSlices created for this service.
func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc {
return func(_ context.Context, o client.Object) []reconcile.Request {
svc, ok := o.(*corev1.Service)
if !ok {
@ -911,10 +917,11 @@ func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger) han
return nil
}
epsList := &discoveryv1.EndpointSliceList{}
if err := cl.List(context.Background(), epsList, client.MatchingLabels(map[string]string{
labelExternalSvcName: svc.Name,
labelExternalSvcNamespace: svc.Namespace,
})); err != nil {
if err := cl.List(context.Background(), epsList, client.InNamespace(ns),
client.MatchingLabels(map[string]string{
LabelParentName: svc.Name,
LabelParentNamespace: svc.Namespace,
})); err != nil {
logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on Service %s", err, svc.Name)
return nil
}
@ -931,6 +938,8 @@ func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger) han
}
}
// indexEgressServices adds a local index to a cached Tailscale egress Services meant to be exposed on a ProxyGroup. The
// index is used a list filter.
func indexEgressServices(o client.Object) []string {
if !isEgressSvcForProxyGroup(o) {
return nil

9
cmd/k8s-operator/proxygroup.go
View File

@ -219,6 +219,15 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
}); err != nil {
return fmt.Errorf("error provisioning RoleBinding: %w", err)
}
if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
cm := pgEgressCM(pg, r.tsNamespace)
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, cm, func(existing *corev1.ConfigMap) {
existing.ObjectMeta.Labels = cm.ObjectMeta.Labels
existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences
}); err != nil {
return fmt.Errorf("error provisioning ConfigMap: %w", err)
}
}
ss := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, cfgHash)
ss = applyProxyClassToStatefulSet(proxyClass, ss, nil, logger)
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) {

47
cmd/k8s-operator/proxygroup_specs.go
View File

@ -13,6 +13,7 @@
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/egressservices"
"tailscale.com/types/ptr"
)
@ -81,6 +82,13 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, cfgHash string) *apps
})
}
if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
mounts = append(mounts, corev1.VolumeMount{
Name: fmt.Sprintf(egressSvcsCMNameTemplate, pg.Name),
MountPath: "/etc/proxies",
ReadOnly: true,
})
}
return mounts
}(),
},
@ -97,6 +105,18 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, cfgHash string) *apps
},
})
}
if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
volumes = append(volumes, corev1.Volume{
Name: fmt.Sprintf(egressSvcsCMNameTemplate, pg.Name),
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: fmt.Sprintf(egressSvcsCMNameTemplate, pg.Name),
},
},
},
})
}
return volumes
}(),
@ -185,6 +205,17 @@ func pgStateSecrets(pg *tsapi.ProxyGroup, namespace string) (secrets []*corev1.S
return secrets
}
func pgEgressCM(pg *tsapi.ProxyGroup, namespace string) *corev1.ConfigMap {
return &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf(egressSvcsCMNameTemplate, pg.Name),
Namespace: namespace,
Labels: pgLabels(pg.Name, nil),
OwnerReferences: pgOwnerReference(pg),
},
}
}
func pgSecretLabels(pgName, typ string) map[string]string {
return pgLabels(pgName, map[string]string{
labelSecretType: typ, // "config" or "state".
@ -204,7 +235,7 @@ func pgLabels(pgName string, customLabels map[string]string) map[string]string {
return l
}
func pgEnv(_ *tsapi.ProxyGroup) []corev1.EnvVar {
func pgEnv(pg *tsapi.ProxyGroup) []corev1.EnvVar {
envs := []corev1.EnvVar{
{
Name: "POD_IP",
@ -235,6 +266,20 @@ func pgEnv(_ *tsapi.ProxyGroup) []corev1.EnvVar {
Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
Value: "/etc/tsconfig/$(POD_NAME)",
},
{
Name: "TS_USERSPACE",
Value: "false",
},
{
Name: "TS_DEBUG_FIREWALL_MODE",
Value: "auto",
},
}
if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
envs = append(envs, corev1.EnvVar{
Name: "TS_EGRESS_SERVICES_CONFIG_PATH",
Value: fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices),
})
}
return envs

7
cmd/k8s-operator/sts.go
View File

@ -49,10 +49,9 @@
LabelParentNamespace = "tailscale.com/parent-resource-ns"
labelSecretType = "tailscale.com/secret-type" // "config" or "state".
// LabelProxyClass can be set by users on Connectors, tailscale
// Ingresses and Services that define cluster ingress or cluster egress,
// to specify that configuration in this ProxyClass should be applied to
// resources created for the Connector, Ingress or Service.
// LabelProxyClass can be set by users on tailscale Ingresses and Services that define cluster ingress or
// cluster egress, to specify that configuration in this ProxyClass should be applied to resources created for
// the Ingress or Service.
LabelProxyClass = "tailscale.com/proxy-class"
FinalizerName = "tailscale.com/finalizer"

4
cmd/k8s-operator/svc.go
View File

@ -112,6 +112,10 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
return reconcile.Result{}, fmt.Errorf("failed to get svc: %w", err)
}
if _, ok := svc.Annotations[AnnotationProxyGroup]; ok {
return reconcile.Result{}, nil // this reconciler should not look at Services for ProxyGroup
}
if !svc.DeletionTimestamp.IsZero() || !a.isTailscaleService(svc) {
logger.Debugf("service is being deleted or is (no longer) referring to Tailscale ingress/egress, ensuring any created resources are cleaned up")
return reconcile.Result{}, a.maybeCleanup(ctx, logger, svc)

2
k8s-operator/api.md
View File

@ -522,7 +522,7 @@ _Appears in:_
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup, either ingress or egress. Each set of proxies<br />managed by a single ProxyGroup definition operate as only ingress or<br />only egress proxies. | | Enum: [egress] <br />Type: string <br /> |
| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Currently the only supported type is egress. | | Enum: [egress] <br />Type: string <br /> |
| `tags` _[Tags](#tags)_ | Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].<br />If you specify custom tags here, make sure you also make the operator<br />an owner of these tags.<br />See https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.<br />Tags cannot be changed once a ProxyGroup device has been created.<br />Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. | | Pattern: `^tag:[a-zA-Z][a-zA-Z0-9-]*$` <br />Type: string <br /> |
| `replicas` _integer_ | Replicas specifies how many replicas to create the StatefulSet with.<br />Defaults to 2. | | |
| `hostnamePrefix` _[HostnamePrefix](#hostnameprefix)_ | HostnamePrefix is the hostname prefix to use for tailnet devices created<br />by the ProxyGroup. Each device will have the integer number from its<br />StatefulSet pod appended to this prefix to form the full hostname.<br />HostnamePrefix can contain lower case letters, numbers and dashes, it<br />must not start with a dash and must be between 1 and 62 characters long. | | Pattern: `^[a-z0-9][a-z0-9-]{0,61}$` <br />Type: string <br /> |

4
k8s-operator/apis/v1alpha1/types_proxygroup.go
View File

@ -37,9 +37,7 @@ type ProxyGroupList struct {
}
type ProxyGroupSpec struct {
// Type of the ProxyGroup, either ingress or egress. Each set of proxies
// managed by a single ProxyGroup definition operate as only ingress or
// only egress proxies.
// Type of the ProxyGroup proxies. Currently the only supported type is egress.
Type ProxyGroupType `json:"type"`
// Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].