TheArchive/tailscale
1
0
Fork 0
mirror of https://github.com/tailscale/tailscale.git synced 2024-11-29 04:55:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

cmd/k8s-operator: users can configure firewall mode for kube operator proxies (#9769)

Browse Source 
* cmd/k8s-operator: users can configure operator to set firewall mode for proxies

Users can now pass PROXY_FIREWALL_MODE={nftables,auto,iptables} to operator to make it create ingress/egress proxies with that firewall mode

Also makes sure that if an invalid firewall mode gets configured, the operator will not start provisioning proxy resources, but will instead log an error and write an error event to the related Service.

Updates tailscale/tailscale#9310

Signed-off-by: Irbe Krumina <irbe@tailscale.com>
This commit is contained in:
Irbe Krumina 2023-10-17 18:05:02 +01:00 committed by GitHub
parent ddb2a6eb8d
commit cac290da87
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 204 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/k8s-operator/manifests/operator.yaml
View File

@ -153,6 +153,8 @@ spec:
value: tag:k8s
- name: APISERVER_PROXY
value: "false"
- name: PROXY_FIREWALL_MODE
value: auto
volumeMounts:
- name: oauth
mountPath: /oauth

7
cmd/k8s-operator/operator.go
View File

@ -52,6 +52,7 @@ func main() {
image = defaultEnv("PROXY_IMAGE", "tailscale/tailscale:latest")
priorityClassName = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "")
tags = defaultEnv("PROXY_TAGS", "tag:k8s")
tsFirewallMode = defaultEnv("PROXY_FIREWALL_MODE", "")
)
var opts []kzap.Opts
@ -70,7 +71,7 @@ func main() {
defer s.Close()
restConfig := config.GetConfigOrDie()
maybeLaunchAPIServerProxy(zlog, restConfig, s)
runReconcilers(zlog, s, tsNamespace, restConfig, tsClient, image, priorityClassName, tags)
runReconcilers(zlog, s, tsNamespace, restConfig, tsClient, image, priorityClassName, tags, tsFirewallMode)
}
// initTSNet initializes the tsnet.Server and logs in to Tailscale. It uses the
@ -179,7 +180,7 @@ func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, *tailscale.Client) {
// runReconcilers starts the controller-runtime manager and registers the
// ServiceReconciler. It blocks forever.
func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string, restConfig *rest.Config, tsClient *tailscale.Client, image, priorityClassName, tags string) {
func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string, restConfig *rest.Config, tsClient *tailscale.Client, image, priorityClassName, tags, tsFirewallMode string) {
var (
isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false)
)
@ -216,6 +217,7 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
operatorNamespace: tsNamespace,
proxyImage: image,
proxyPriorityClassName: priorityClassName,
tsFirewallMode: tsFirewallMode,
}
err = builder.
ControllerManagedBy(mgr).
@ -228,6 +230,7 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
Client: mgr.GetClient(),
logger: zlog.Named("service-reconciler"),
isDefaultLoadBalancer: isDefaultLoadBalancer,
recorder: eventRecorder,
})
if err != nil {
startlog.Fatalf("could not create controller: %v", err)

262
cmd/k8s-operator/operator_test.go
View File

@ -70,7 +70,12 @@ func TestLoadBalancerClass(t *testing.T) {
expectEqual(t, fc, expectedSecret(fullName))
expectEqual(t, fc, expectedHeadlessService(shortName))
expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", ""))
o := stsOpts{
name: shortName,
secretName: fullName,
hostname: "default-test",
}
expectEqual(t, fc, expectedSTS(o))
// Normally the Tailscale proxy pod would come up here and write its info
// into the secret. Simulate that, then verify reconcile again and verify
@ -202,7 +207,13 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
expectEqual(t, fc, expectedSecret(fullName))
expectEqual(t, fc, expectedHeadlessService(shortName))
expectEqual(t, fc, expectedEgressSTS(shortName, fullName, tailnetTargetIP, "default-test", ""))
o := stsOpts{
name: shortName,
secretName: fullName,
tailnetTargetIP: tailnetTargetIP,
hostname: "default-test",
}
expectEqual(t, fc, expectedSTS(o))
want := &corev1.Service{
TypeMeta: metav1.TypeMeta{
Kind: "Service",
@ -226,7 +237,13 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
expectEqual(t, fc, want)
expectEqual(t, fc, expectedSecret(fullName))
expectEqual(t, fc, expectedHeadlessService(shortName))
expectEqual(t, fc, expectedEgressSTS(shortName, fullName, tailnetTargetIP, "default-test", ""))
o = stsOpts{
name: shortName,
secretName: fullName,
tailnetTargetIP: tailnetTargetIP,
hostname: "default-test",
}
expectEqual(t, fc, expectedSTS(o))
// Change the tailscale-target-ip annotation which should update the
// StatefulSet
@ -305,7 +322,12 @@ func TestAnnotations(t *testing.T) {
expectEqual(t, fc, expectedSecret(fullName))
expectEqual(t, fc, expectedHeadlessService(shortName))
expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", ""))
o := stsOpts{
name: shortName,
secretName: fullName,
hostname: "default-test",
}
expectEqual(t, fc, expectedSTS(o))
want := &corev1.Service{
TypeMeta: metav1.TypeMeta{
Kind: "Service",
@ -405,7 +427,12 @@ func TestAnnotationIntoLB(t *testing.T) {
expectEqual(t, fc, expectedSecret(fullName))
expectEqual(t, fc, expectedHeadlessService(shortName))
expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", ""))
o := stsOpts{
name: shortName,
secretName: fullName,
hostname: "default-test",
}
expectEqual(t, fc, expectedSTS(o))
// Normally the Tailscale proxy pod would come up here and write its info
// into the secret. Simulate that, since it would have normally happened at
@ -450,7 +477,12 @@ func TestAnnotationIntoLB(t *testing.T) {
expectReconciled(t, sr, "default", "test")
// None of the proxy machinery should have changed...
expectEqual(t, fc, expectedHeadlessService(shortName))
expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", ""))
o = stsOpts{
name: shortName,
secretName: fullName,
hostname: "default-test",
}
expectEqual(t, fc, expectedSTS(o))
// ... but the service should have a LoadBalancer status.
want = &corev1.Service{
@ -528,7 +560,12 @@ func TestLBIntoAnnotation(t *testing.T) {
expectEqual(t, fc, expectedSecret(fullName))
expectEqual(t, fc, expectedHeadlessService(shortName))
expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", ""))
o := stsOpts{
name: shortName,
secretName: fullName,
hostname: "default-test",
}
expectEqual(t, fc, expectedSTS(o))
// Normally the Tailscale proxy pod would come up here and write its info
// into the secret. Simulate that, then verify reconcile again and verify
@ -591,7 +628,12 @@ func TestLBIntoAnnotation(t *testing.T) {
expectReconciled(t, sr, "default", "test")
expectEqual(t, fc, expectedHeadlessService(shortName))
expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", ""))
o = stsOpts{
name: shortName,
secretName: fullName,
hostname: "default-test",
}
expectEqual(t, fc, expectedSTS(o))
want = &corev1.Service{
TypeMeta: metav1.TypeMeta{
@ -661,7 +703,12 @@ func TestCustomHostname(t *testing.T) {
expectEqual(t, fc, expectedSecret(fullName))
expectEqual(t, fc, expectedHeadlessService(shortName))
expectEqual(t, fc, expectedSTS(shortName, fullName, "reindeer-flotilla", ""))
o := stsOpts{
name: shortName,
secretName: fullName,
hostname: "reindeer-flotilla",
}
expectEqual(t, fc, expectedSTS(o))
want := &corev1.Service{
TypeMeta: metav1.TypeMeta{
Kind: "Service",
@ -735,7 +782,7 @@ func TestCustomPriorityClassName(t *testing.T) {
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
proxyPriorityClassName: "tailscale-critical",
proxyPriorityClassName: "custom-priority-class-name",
},
logger: zl.Sugar(),
}
@ -752,7 +799,7 @@ func TestCustomPriorityClassName(t *testing.T) {
UID: types.UID("1234-UID"),
Annotations: map[string]string{
"tailscale.com/expose": "true",
"tailscale.com/hostname": "custom-priority-class-name",
"tailscale.com/hostname": "tailscale-critical",
},
},
Spec: corev1.ServiceSpec{
@ -764,8 +811,14 @@ func TestCustomPriorityClassName(t *testing.T) {
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test")
o := stsOpts{
name: shortName,
secretName: fullName,
hostname: "tailscale-critical",
priorityClassName: "custom-priority-class-name",
}
expectEqual(t, fc, expectedSTS(shortName, fullName, "custom-priority-class-name", "tailscale-critical"))
expectEqual(t, fc, expectedSTS(o))
}
func TestDefaultLoadBalancer(t *testing.T) {
@ -811,7 +864,63 @@ func TestDefaultLoadBalancer(t *testing.T) {
expectEqual(t, fc, expectedSecret(fullName))
expectEqual(t, fc, expectedHeadlessService(shortName))
expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", ""))
o := stsOpts{
name: shortName,
secretName: fullName,
hostname: "default-test",
}
expectEqual(t, fc, expectedSTS(o))
}
func TestProxyFirewallMode(t *testing.T) {
fc := fake.NewFakeClient()
ft := &fakeTSClient{}
zl, err := zap.NewDevelopment()
if err != nil {
t.Fatal(err)
}
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
tsClient: ft,
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
tsFirewallMode: "nftables",
},
logger: zl.Sugar(),
isDefaultLoadBalancer: true,
}
// Create a service that we should manage, and check that the initial round
// of objects looks right.
mustCreate(t, fc, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
// The apiserver is supposed to set the UID, but the fake client
// doesn't. So, set it explicitly because other code later depends
// on it being set.
UID: types.UID("1234-UID"),
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeLoadBalancer,
},
})
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test")
o := stsOpts{
name: shortName,
secretName: fullName,
hostname: "default-test",
firewallMode: "nftables",
}
expectEqual(t, fc, expectedSTS(o))
}
func expectedSecret(name string) *corev1.Secret {
@ -862,83 +971,44 @@ func expectedHeadlessService(name string) *corev1.Service {
}
}
func expectedSTS(stsName, secretName, hostname, priorityClassName string) *appsv1.StatefulSet {
return &appsv1.StatefulSet{
TypeMeta: metav1.TypeMeta{
Kind: "StatefulSet",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: stsName,
Namespace: "operator-ns",
Labels: map[string]string{
"tailscale.com/managed": "true",
"tailscale.com/parent-resource": "test",
"tailscale.com/parent-resource-ns": "default",
"tailscale.com/parent-resource-type": "svc",
},
},
Spec: appsv1.StatefulSetSpec{
Replicas: ptr.To[int32](1),
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"app": "1234-UID"},
},
ServiceName: stsName,
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
"tailscale.com/operator-last-set-hostname": hostname,
"tailscale.com/operator-last-set-cluster-ip": "10.20.30.40",
},
DeletionGracePeriodSeconds: ptr.To[int64](10),
Labels: map[string]string{"app": "1234-UID"},
},
Spec: corev1.PodSpec{
ServiceAccountName: "proxies",
PriorityClassName: priorityClassName,
InitContainers: []corev1.Container{
{
Name: "sysctler",
Image: "tailscale/tailscale",
Command: []string{"/bin/sh"},
Args: []string{"-c", "sysctl -w net.ipv4.ip_forward=1 net.ipv6.conf.all.forwarding=1"},
SecurityContext: &corev1.SecurityContext{
Privileged: ptr.To(true),
},
},
},
Containers: []corev1.Container{
{
Name: "tailscale",
Image: "tailscale/tailscale",
Env: []corev1.EnvVar{
{Name: "TS_USERSPACE", Value: "false"},
{Name: "TS_AUTH_ONCE", Value: "true"},
{Name: "TS_KUBE_SECRET", Value: secretName},
{Name: "TS_HOSTNAME", Value: hostname},
{Name: "TS_DEST_IP", Value: "10.20.30.40"},
},
SecurityContext: &corev1.SecurityContext{
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{"NET_ADMIN"},
},
},
ImagePullPolicy: "Always",
},
},
},
},
},
func expectedSTS(opts stsOpts) *appsv1.StatefulSet {
containerEnv := []corev1.EnvVar{
{Name: "TS_USERSPACE", Value: "false"},
{Name: "TS_AUTH_ONCE", Value: "true"},
{Name: "TS_KUBE_SECRET", Value: opts.secretName},
{Name: "TS_HOSTNAME", Value: opts.hostname},
}
annots := map[string]string{
"tailscale.com/operator-last-set-hostname": opts.hostname,
}
if opts.tailnetTargetIP != "" {
annots["tailscale.com/operator-last-set-ts-tailnet-target-ip"] = opts.tailnetTargetIP
containerEnv = append(containerEnv, corev1.EnvVar{
Name: "TS_TAILNET_TARGET_IP",
Value: opts.tailnetTargetIP,
})
} else {
containerEnv = append(containerEnv, corev1.EnvVar{
Name: "TS_DEST_IP",
Value: "10.20.30.40",
})
annots["tailscale.com/operator-last-set-cluster-ip"] = "10.20.30.40"
}
if opts.firewallMode != "" {
containerEnv = append(containerEnv, corev1.EnvVar{
Name: "TS_DEBUG_FIREWALL_MODE",
Value: opts.firewallMode,
})
}
}
func expectedEgressSTS(stsName, secretName, tailnetTargetIP, hostname, priorityClassName string) *appsv1.StatefulSet {
return &appsv1.StatefulSet{
TypeMeta: metav1.TypeMeta{
Kind: "StatefulSet",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: stsName,
Name: opts.name,
Namespace: "operator-ns",
Labels: map[string]string{
"tailscale.com/managed": "true",
@ -952,19 +1022,16 @@ func expectedEgressSTS(stsName, secretName, tailnetTargetIP, hostname, priorityC
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"app": "1234-UID"},
},
ServiceName: stsName,
ServiceName: opts.name,
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
"tailscale.com/operator-last-set-hostname": hostname,
"tailscale.com/operator-last-set-ts-tailnet-target-ip": tailnetTargetIP,
},
Annotations: annots,
DeletionGracePeriodSeconds: ptr.To[int64](10),
Labels: map[string]string{"app": "1234-UID"},
},
Spec: corev1.PodSpec{
ServiceAccountName: "proxies",
PriorityClassName: priorityClassName,
PriorityClassName: opts.priorityClassName,
InitContainers: []corev1.Container{
{
Name: "sysctler",
@ -980,13 +1047,7 @@ func expectedEgressSTS(stsName, secretName, tailnetTargetIP, hostname, priorityC
{
Name: "tailscale",
Image: "tailscale/tailscale",
Env: []corev1.EnvVar{
{Name: "TS_USERSPACE", Value: "false"},
{Name: "TS_AUTH_ONCE", Value: "true"},
{Name: "TS_KUBE_SECRET", Value: secretName},
{Name: "TS_HOSTNAME", Value: hostname},
{Name: "TS_TAILNET_TARGET_IP", Value: tailnetTargetIP},
},
Env: containerEnv,
SecurityContext: &corev1.SecurityContext{
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{"NET_ADMIN"},
@ -1126,6 +1187,15 @@ func expectRequeue(t *testing.T, sr *ServiceReconciler, ns, name string) {
}
}
type stsOpts struct {
name string
secretName string
hostname string
priorityClassName string
firewallMode string
tailnetTargetIP string
}
type fakeTSClient struct {
sync.Mutex
keyRequests []tailscale.KeyCapabilities

19
cmd/k8s-operator/sts.go
View File

@ -79,6 +79,14 @@ type tailscaleSTSReconciler struct {
operatorNamespace string
proxyImage string
proxyPriorityClassName string
tsFirewallMode string
}
func (sts tailscaleSTSReconciler) validate() error {
if sts.tsFirewallMode != "" && !isValidFirewallMode(sts.tsFirewallMode) {
return fmt.Errorf("invalid proxy firewall mode %s, valid modes are iptables, nftables or unset", sts.tsFirewallMode)
}
return nil
}
// IsHTTPSEnabledOnTailnet reports whether HTTPS is enabled on the tailnet.
@ -360,6 +368,13 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
},
})
}
if a.tsFirewallMode != "" {
container.Env = append(container.Env, corev1.EnvVar{
Name: "TS_DEBUG_FIREWALL_MODE",
Value: a.tsFirewallMode,
},
)
}
ss.ObjectMeta = metav1.ObjectMeta{
Name: headlessSvc.Name,
Namespace: a.operatorNamespace,
@ -499,3 +514,7 @@ func nameForService(svc *corev1.Service) (string, error) {
}
return svc.Namespace + "-" + svc.Name, nil
}
func isValidFirewallMode(m string) bool {
return m == "auto" || m == "nftables" || m == "iptables"
}

12
cmd/k8s-operator/svc.go
View File

@ -17,6 +17,7 @@
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"tailscale.com/util/clientmetric"
@ -37,6 +38,8 @@ type ServiceReconciler struct {
// managedEgressProxies is a set of all egress proxies that we're currently
// managing. This is only used for metrics.
managedEgressProxies set.Slice[types.UID]
recorder record.EventRecorder
}
var (
@ -136,6 +139,15 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
// This function adds a finalizer to svc, ensuring that we can handle orderly
// deprovisioning later.
func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) error {
// run for proxy config related validations here as opposed to running
// them earlier. This is to prevent cleanup etc being blocked on a
// misconfigured proxy param
if err := a.ssr.validate(); err != nil {
msg := fmt.Sprintf("unable to provision proxy resources: invalid config: %v", err)
a.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDCONFIG", msg)
a.logger.Error(msg)
return nil
}
hostname, err := nameForService(svc)
if err != nil {
return err