TheArchive/tailscale
1
0
Fork 0
mirror of https://github.com/tailscale/tailscale.git synced 2025-08-22 11:01:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

cmd/k8s-operator: users can configure firewall mode for kube operator proxies (#9769)

Browse Source 
* cmd/k8s-operator: users can configure operator to set firewall mode for proxies

Users can now pass PROXY_FIREWALL_MODE={nftables,auto,iptables} to operator to make it create ingress/egress proxies with that firewall mode

Also makes sure that if an invalid firewall mode gets configured, the operator will not start provisioning proxy resources, but will instead log an error and write an error event to the related Service.

Updates tailscale/tailscale#9310

Signed-off-by: Irbe Krumina <irbe@tailscale.com>
This commit is contained in:
Irbe Krumina
2023-10-17 18:05:02 +01:00
committed by GitHub
parent ddb2a6eb8d
commit cac290da87
5 changed files with 204 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

262
cmd/k8s-operator/operator_test.go
View File

@@ -70,7 +70,12 @@ func TestLoadBalancerClass(t *testing.T) {
expectEqual(t, fc, expectedSecret(fullName))
expectEqual(t, fc, expectedHeadlessService(shortName))
expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", ""))
o := stsOpts{
name: shortName,
secretName: fullName,
hostname: "default-test",
}
expectEqual(t, fc, expectedSTS(o))
// Normally the Tailscale proxy pod would come up here and write its info
// into the secret. Simulate that, then verify reconcile again and verify
@@ -202,7 +207,13 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
expectEqual(t, fc, expectedSecret(fullName))
expectEqual(t, fc, expectedHeadlessService(shortName))
expectEqual(t, fc, expectedEgressSTS(shortName, fullName, tailnetTargetIP, "default-test", ""))
o := stsOpts{
name: shortName,
secretName: fullName,
tailnetTargetIP: tailnetTargetIP,
hostname: "default-test",
}
expectEqual(t, fc, expectedSTS(o))
want := &corev1.Service{
TypeMeta: metav1.TypeMeta{
Kind: "Service",
@@ -226,7 +237,13 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
expectEqual(t, fc, want)
expectEqual(t, fc, expectedSecret(fullName))
expectEqual(t, fc, expectedHeadlessService(shortName))
expectEqual(t, fc, expectedEgressSTS(shortName, fullName, tailnetTargetIP, "default-test", ""))
o = stsOpts{
name: shortName,
secretName: fullName,
tailnetTargetIP: tailnetTargetIP,
hostname: "default-test",
}
expectEqual(t, fc, expectedSTS(o))
// Change the tailscale-target-ip annotation which should update the
// StatefulSet
@@ -305,7 +322,12 @@ func TestAnnotations(t *testing.T) {
expectEqual(t, fc, expectedSecret(fullName))
expectEqual(t, fc, expectedHeadlessService(shortName))
expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", ""))
o := stsOpts{
name: shortName,
secretName: fullName,
hostname: "default-test",
}
expectEqual(t, fc, expectedSTS(o))
want := &corev1.Service{
TypeMeta: metav1.TypeMeta{
Kind: "Service",
@@ -405,7 +427,12 @@ func TestAnnotationIntoLB(t *testing.T) {
expectEqual(t, fc, expectedSecret(fullName))
expectEqual(t, fc, expectedHeadlessService(shortName))
expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", ""))
o := stsOpts{
name: shortName,
secretName: fullName,
hostname: "default-test",
}
expectEqual(t, fc, expectedSTS(o))
// Normally the Tailscale proxy pod would come up here and write its info
// into the secret. Simulate that, since it would have normally happened at
@@ -450,7 +477,12 @@ func TestAnnotationIntoLB(t *testing.T) {
expectReconciled(t, sr, "default", "test")
// None of the proxy machinery should have changed...
expectEqual(t, fc, expectedHeadlessService(shortName))
expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", ""))
o = stsOpts{
name: shortName,
secretName: fullName,
hostname: "default-test",
}
expectEqual(t, fc, expectedSTS(o))
// ... but the service should have a LoadBalancer status.
want = &corev1.Service{
@@ -528,7 +560,12 @@ func TestLBIntoAnnotation(t *testing.T) {
expectEqual(t, fc, expectedSecret(fullName))
expectEqual(t, fc, expectedHeadlessService(shortName))
expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", ""))
o := stsOpts{
name: shortName,
secretName: fullName,
hostname: "default-test",
}
expectEqual(t, fc, expectedSTS(o))
// Normally the Tailscale proxy pod would come up here and write its info
// into the secret. Simulate that, then verify reconcile again and verify
@@ -591,7 +628,12 @@ func TestLBIntoAnnotation(t *testing.T) {
expectReconciled(t, sr, "default", "test")
expectEqual(t, fc, expectedHeadlessService(shortName))
expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", ""))
o = stsOpts{
name: shortName,
secretName: fullName,
hostname: "default-test",
}
expectEqual(t, fc, expectedSTS(o))
want = &corev1.Service{
TypeMeta: metav1.TypeMeta{
@@ -661,7 +703,12 @@ func TestCustomHostname(t *testing.T) {
expectEqual(t, fc, expectedSecret(fullName))
expectEqual(t, fc, expectedHeadlessService(shortName))
expectEqual(t, fc, expectedSTS(shortName, fullName, "reindeer-flotilla", ""))
o := stsOpts{
name: shortName,
secretName: fullName,
hostname: "reindeer-flotilla",
}
expectEqual(t, fc, expectedSTS(o))
want := &corev1.Service{
TypeMeta: metav1.TypeMeta{
Kind: "Service",
@@ -735,7 +782,7 @@ func TestCustomPriorityClassName(t *testing.T) {
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
proxyPriorityClassName: "tailscale-critical",
proxyPriorityClassName: "custom-priority-class-name",
},
logger: zl.Sugar(),
}
@@ -752,7 +799,7 @@ func TestCustomPriorityClassName(t *testing.T) {
UID: types.UID("1234-UID"),
Annotations: map[string]string{
"tailscale.com/expose": "true",
"tailscale.com/hostname": "custom-priority-class-name",
"tailscale.com/hostname": "tailscale-critical",
},
},
Spec: corev1.ServiceSpec{
@@ -764,8 +811,14 @@ func TestCustomPriorityClassName(t *testing.T) {
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test")
o := stsOpts{
name: shortName,
secretName: fullName,
hostname: "tailscale-critical",
priorityClassName: "custom-priority-class-name",
}
expectEqual(t, fc, expectedSTS(shortName, fullName, "custom-priority-class-name", "tailscale-critical"))
expectEqual(t, fc, expectedSTS(o))
}
func TestDefaultLoadBalancer(t *testing.T) {
@@ -811,7 +864,63 @@ func TestDefaultLoadBalancer(t *testing.T) {
expectEqual(t, fc, expectedSecret(fullName))
expectEqual(t, fc, expectedHeadlessService(shortName))
expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", ""))
o := stsOpts{
name: shortName,
secretName: fullName,
hostname: "default-test",
}
expectEqual(t, fc, expectedSTS(o))
}
func TestProxyFirewallMode(t *testing.T) {
fc := fake.NewFakeClient()
ft := &fakeTSClient{}
zl, err := zap.NewDevelopment()
if err != nil {
t.Fatal(err)
}
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
tsClient: ft,
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
tsFirewallMode: "nftables",
},
logger: zl.Sugar(),
isDefaultLoadBalancer: true,
}
// Create a service that we should manage, and check that the initial round
// of objects looks right.
mustCreate(t, fc, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
// The apiserver is supposed to set the UID, but the fake client
// doesn't. So, set it explicitly because other code later depends
// on it being set.
UID: types.UID("1234-UID"),
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeLoadBalancer,
},
})
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test")
o := stsOpts{
name: shortName,
secretName: fullName,
hostname: "default-test",
firewallMode: "nftables",
}
expectEqual(t, fc, expectedSTS(o))
}
func expectedSecret(name string) *corev1.Secret {
@@ -862,83 +971,44 @@ func expectedHeadlessService(name string) *corev1.Service {
}
}
func expectedSTS(stsName, secretName, hostname, priorityClassName string) *appsv1.StatefulSet {
return &appsv1.StatefulSet{
TypeMeta: metav1.TypeMeta{
Kind: "StatefulSet",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: stsName,
Namespace: "operator-ns",
Labels: map[string]string{
"tailscale.com/managed": "true",
"tailscale.com/parent-resource": "test",
"tailscale.com/parent-resource-ns": "default",
"tailscale.com/parent-resource-type": "svc",
},
},
Spec: appsv1.StatefulSetSpec{
Replicas: ptr.To[int32](1),
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"app": "1234-UID"},
},
ServiceName: stsName,
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
"tailscale.com/operator-last-set-hostname": hostname,
"tailscale.com/operator-last-set-cluster-ip": "10.20.30.40",
},
DeletionGracePeriodSeconds: ptr.To[int64](10),
Labels: map[string]string{"app": "1234-UID"},
},
Spec: corev1.PodSpec{
ServiceAccountName: "proxies",
PriorityClassName: priorityClassName,
InitContainers: []corev1.Container{
{
Name: "sysctler",
Image: "tailscale/tailscale",
Command: []string{"/bin/sh"},
Args: []string{"-c", "sysctl -w net.ipv4.ip_forward=1 net.ipv6.conf.all.forwarding=1"},
SecurityContext: &corev1.SecurityContext{
Privileged: ptr.To(true),
},
},
},
Containers: []corev1.Container{
{
Name: "tailscale",
Image: "tailscale/tailscale",
Env: []corev1.EnvVar{
{Name: "TS_USERSPACE", Value: "false"},
{Name: "TS_AUTH_ONCE", Value: "true"},
{Name: "TS_KUBE_SECRET", Value: secretName},
{Name: "TS_HOSTNAME", Value: hostname},
{Name: "TS_DEST_IP", Value: "10.20.30.40"},
},
SecurityContext: &corev1.SecurityContext{
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{"NET_ADMIN"},
},
},
ImagePullPolicy: "Always",
},
},
},
},
},
func expectedSTS(opts stsOpts) *appsv1.StatefulSet {
containerEnv := []corev1.EnvVar{
{Name: "TS_USERSPACE", Value: "false"},
{Name: "TS_AUTH_ONCE", Value: "true"},
{Name: "TS_KUBE_SECRET", Value: opts.secretName},
{Name: "TS_HOSTNAME", Value: opts.hostname},
}
annots := map[string]string{
"tailscale.com/operator-last-set-hostname": opts.hostname,
}
if opts.tailnetTargetIP != "" {
annots["tailscale.com/operator-last-set-ts-tailnet-target-ip"] = opts.tailnetTargetIP
containerEnv = append(containerEnv, corev1.EnvVar{
Name: "TS_TAILNET_TARGET_IP",
Value: opts.tailnetTargetIP,
})
} else {
containerEnv = append(containerEnv, corev1.EnvVar{
Name: "TS_DEST_IP",
Value: "10.20.30.40",
})
annots["tailscale.com/operator-last-set-cluster-ip"] = "10.20.30.40"
}
if opts.firewallMode != "" {
containerEnv = append(containerEnv, corev1.EnvVar{
Name: "TS_DEBUG_FIREWALL_MODE",
Value: opts.firewallMode,
})
}
}
func expectedEgressSTS(stsName, secretName, tailnetTargetIP, hostname, priorityClassName string) *appsv1.StatefulSet {
return &appsv1.StatefulSet{
TypeMeta: metav1.TypeMeta{
Kind: "StatefulSet",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: stsName,
Name: opts.name,
Namespace: "operator-ns",
Labels: map[string]string{
"tailscale.com/managed": "true",
@@ -952,19 +1022,16 @@ func expectedEgressSTS(stsName, secretName, tailnetTargetIP, hostname, priorityC
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"app": "1234-UID"},
},
ServiceName: stsName,
ServiceName: opts.name,
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
"tailscale.com/operator-last-set-hostname": hostname,
"tailscale.com/operator-last-set-ts-tailnet-target-ip": tailnetTargetIP,
},
Annotations: annots,
DeletionGracePeriodSeconds: ptr.To[int64](10),
Labels: map[string]string{"app": "1234-UID"},
},
Spec: corev1.PodSpec{
ServiceAccountName: "proxies",
PriorityClassName: priorityClassName,
PriorityClassName: opts.priorityClassName,
InitContainers: []corev1.Container{
{
Name: "sysctler",
@@ -980,13 +1047,7 @@ func expectedEgressSTS(stsName, secretName, tailnetTargetIP, hostname, priorityC
{
Name: "tailscale",
Image: "tailscale/tailscale",
Env: []corev1.EnvVar{
{Name: "TS_USERSPACE", Value: "false"},
{Name: "TS_AUTH_ONCE", Value: "true"},
{Name: "TS_KUBE_SECRET", Value: secretName},
{Name: "TS_HOSTNAME", Value: hostname},
{Name: "TS_TAILNET_TARGET_IP", Value: tailnetTargetIP},
},
Env: containerEnv,
SecurityContext: &corev1.SecurityContext{
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{"NET_ADMIN"},
@@ -1126,6 +1187,15 @@ func expectRequeue(t *testing.T, sr *ServiceReconciler, ns, name string) {
}
}
type stsOpts struct {
name string
secretName string
hostname string
priorityClassName string
firewallMode string
tailnetTargetIP string
}
type fakeTSClient struct {
sync.Mutex
keyRequests []tailscale.KeyCapabilities