cmd/containerboot,kube,util/linuxfw: configure kube egress proxies to route to 1+ tailnet targets (#13531)

* cmd/containerboot,kube,util/linuxfw: configure kube egress proxies to route to 1+ tailnet targets

This commit is first part of the work to allow running multiple
replicas of the Kubernetes operator egress proxies per tailnet service +
to allow exposing multiple tailnet services via each proxy replica.

This expands the existing iptables/nftables-based proxy configuration
mechanism.

A proxy can now be configured to route to one or more tailnet targets
via a (mounted) config file that, for each tailnet target, specifies:
- the target's tailnet IP or FQDN
- mappings of container ports to which cluster workloads will send traffic to
tailnet target ports where the traffic should be forwarded.

Example configfile contents:
{
  "some-svc": {"tailnetTarget":{"fqdn":"foo.tailnetxyz.ts.net","ports"{"tcp:4006:80":{"protocol":"tcp","matchPort":4006,"targetPort":80},"tcp:4007:443":{"protocol":"tcp","matchPort":4007,"targetPort":443}}}}
}

A proxy that is configured with this config file will configure firewall rules
to route cluster traffic to the tailnet targets. It will then watch the config file
for updates as well as monitor relevant netmap updates and reconfigure firewall
as needed.

This adds a bunch of new iptables/nftables functionality to make it easier to dynamically update
the firewall rules without needing to restart the proxy Pod as well as to make
it easier to debug/understand the rules:

- for iptables, each portmapping is a DNAT rule with a comment pointing
at the 'service',i.e:

-A PREROUTING ! -i tailscale0 -p tcp -m tcp --dport 4006 -m comment --comment "some-svc:tcp:4006 -> tcp:80" -j DNAT --to-destination 100.64.1.18:80
Additionally there is a SNAT rule for each tailnet target, to mask the source address.

- for nftables, a separate prerouting chain is created for each tailnet target
and all the portmapping rules are placed in that chain. This makes it easier
to look up rules and delete services when no longer needed.
(nftables allows hooking a custom chain to a prerouting hook, so no extra work
is needed to ensure that the rules in the service chains are evaluated).

The next steps will be to get the Kubernetes Operator to generate
the configfile and ensure it is mounted to the relevant proxy nodes.

Updates tailscale/tailscale#13406

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

util/linuxfw/iptables_for_svcs.go

@@ -0,0 +1,79 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package linuxfw
+
+import (
+	"fmt"
+	"net/netip"
+)
+
+// This file contains functionality to insert portmapping rules for a 'service'.
+// These are currently only used by the Kubernetes operator proxies.
+// An iptables rule for such a service contains a comment with the service name.
+
+// EnsurePortMapRuleForSvc adds a prerouting rule that forwards traffic received
+// on match port and NOT on the provided interface to target IP and target port.
+// Rule will only be added if it does not already exists.
+func (i *iptablesRunner) EnsurePortMapRuleForSvc(svc, tun string, targetIP netip.Addr, pm PortMap) error {
+	table := i.getIPTByAddr(targetIP)
+	args := argsForPortMapRule(svc, tun, targetIP, pm)
+	exists, err := table.Exists("nat", "PREROUTING", args...)
+	if err != nil {
+		return fmt.Errorf("error checking if rule exists: %w", err)
+	}
+	if !exists {
+		return table.Append("nat", "PREROUTING", args...)
+	}
+	return nil
+}
+
+// DeleteMapRuleForSvc constructs a prerouting rule as would be created by
+// EnsurePortMapRuleForSvc with the provided args and, if such a rule exists,
+// deletes it.
+func (i *iptablesRunner) DeletePortMapRuleForSvc(svc, excludeI string, targetIP netip.Addr, pm PortMap) error {
+	table := i.getIPTByAddr(targetIP)
+	args := argsForPortMapRule(svc, excludeI, targetIP, pm)
+	exists, err := table.Exists("nat", "PREROUTING", args...)
+	if err != nil {
+		return fmt.Errorf("error checking if rule exists: %w", err)
+	}
+	if exists {
+		return table.Delete("nat", "PREROUTING", args...)
+	}
+	return nil
+}
+
+// DeleteSvc constructs all possible rules that would have been created by
+// EnsurePortMapRuleForSvc from the provided args and ensures that each one that
+// exists is deleted.
+func (i *iptablesRunner) DeleteSvc(svc, tun string, targetIPs []netip.Addr, pms []PortMap) error {
+	for _, tip := range targetIPs {
+		for _, pm := range pms {
+			if err := i.DeletePortMapRuleForSvc(svc, tun, tip, pm); err != nil {
+				return fmt.Errorf("error deleting rule: %w", err)
+			}
+		}
+	}
+	return nil
+}
+
+func argsForPortMapRule(svc, excludeI string, targetIP netip.Addr, pm PortMap) []string {
+	c := commentForSvc(svc, pm)
+	return []string{
+		"!", "-i", excludeI,
+		"-p", pm.Protocol,
+		"--dport", fmt.Sprintf("%d", pm.MatchPort),
+		"-m", "comment", "--comment", c,
+		"-j", "DNAT",
+		"--to-destination", fmt.Sprintf("%v:%v", targetIP, pm.TargetPort),
+	}
+}
+
+// commentForSvc generates a comment to be added to an iptables DNAT rule for a
+// service. This is for iptables debugging/readability purposes only.
+func commentForSvc(svc string, pm PortMap) string {
+	return fmt.Sprintf("%s:%s:%d -> %s:%d", svc, pm.Protocol, pm.MatchPort, pm.Protocol, pm.TargetPort)
+}

util/linuxfw/iptables_for_svcs_test.go

@@ -0,0 +1,196 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package linuxfw
+
+import (
+	"net/netip"
+	"testing"
+)
+
+func Test_iptablesRunner_EnsurePortMapRuleForSvc(t *testing.T) {
+	v4Addr := netip.MustParseAddr("10.0.0.4")
+	v6Addr := netip.MustParseAddr("fd7a:115c:a1e0::701:b62a")
+	testPM := PortMap{Protocol: "tcp", MatchPort: 4003, TargetPort: 80}
+	testPM2 := PortMap{Protocol: "udp", MatchPort: 4004, TargetPort: 53}
+	v4Rule := argsForPortMapRule("test-svc", "tailscale0", v4Addr, testPM)
+	tests := []struct {
+		name              string
+		targetIP          netip.Addr
+		svc               string
+		pm                PortMap
+		precreateSvcRules [][]string
+	}{
+		{
+			name:     "pm_for_ipv4",
+			targetIP: v4Addr,
+			svc:      "test-svc",
+			pm:       testPM,
+		},
+		{
+			name:     "pm_for_ipv6",
+			targetIP: v6Addr,
+			svc:      "test-svc-2",
+			pm:       testPM2,
+		},
+		{
+			name:              "add_existing_rule",
+			targetIP:          v4Addr,
+			svc:               "test-svc",
+			pm:                testPM,
+			precreateSvcRules: [][]string{v4Rule},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			iptr := NewFakeIPTablesRunner()
+			table := iptr.getIPTByAddr(tt.targetIP)
+			for _, ruleset := range tt.precreateSvcRules {
+				mustPrecreatePortMapRule(t, ruleset, table)
+			}
+			if err := iptr.EnsurePortMapRuleForSvc(tt.svc, "tailscale0", tt.targetIP, tt.pm); err != nil {
+				t.Errorf("[unexpected error] iptablesRunner.EnsurePortMapRuleForSvc() = %v", err)
+			}
+			args := argsForPortMapRule(tt.svc, "tailscale0", tt.targetIP, tt.pm)
+			exists, err := table.Exists("nat", "PREROUTING", args...)
+			if err != nil {
+				t.Fatalf("error checking if rule exists: %v", err)
+			}
+			if !exists {
+				t.Errorf("expected rule was not created")
+			}
+		})
+	}
+}
+
+func Test_iptablesRunner_DeletePortMapRuleForSvc(t *testing.T) {
+	v4Addr := netip.MustParseAddr("10.0.0.4")
+	v6Addr := netip.MustParseAddr("fd7a:115c:a1e0::701:b62a")
+	testPM := PortMap{Protocol: "tcp", MatchPort: 4003, TargetPort: 80}
+	v4Rule := argsForPortMapRule("test", "tailscale0", v4Addr, testPM)
+	v6Rule := argsForPortMapRule("test", "tailscale0", v6Addr, testPM)
+
+	tests := []struct {
+		name              string
+		targetIP          netip.Addr
+		svc               string
+		pm                PortMap
+		precreateSvcRules [][]string
+	}{
+		{
+			name:              "multiple_rules_ipv4_deleted",
+			targetIP:          v4Addr,
+			svc:               "test",
+			pm:                testPM,
+			precreateSvcRules: [][]string{v4Rule, v6Rule},
+		},
+		{
+			name:              "multiple_rules_ipv6_deleted",
+			targetIP:          v6Addr,
+			svc:               "test",
+			pm:                testPM,
+			precreateSvcRules: [][]string{v4Rule, v6Rule},
+		},
+		{
+			name:              "non-existent_rule_deleted",
+			targetIP:          v4Addr,
+			svc:               "test",
+			pm:                testPM,
+			precreateSvcRules: [][]string{v6Rule},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			iptr := NewFakeIPTablesRunner()
+			table := iptr.getIPTByAddr(tt.targetIP)
+			for _, ruleset := range tt.precreateSvcRules {
+				mustPrecreatePortMapRule(t, ruleset, table)
+			}
+			if err := iptr.DeletePortMapRuleForSvc(tt.svc, "tailscale0", tt.targetIP, tt.pm); err != nil {
+				t.Errorf("iptablesRunner.DeletePortMapRuleForSvc() errored: %v ", err)
+			}
+			deletedRule := argsForPortMapRule(tt.svc, "tailscale0", tt.targetIP, tt.pm)
+			exists, err := table.Exists("nat", "PREROUTING", deletedRule...)
+			if err != nil {
+				t.Fatalf("error verifying that rule does not exist after deletion: %v", err)
+			}
+			if exists {
+				t.Errorf("portmap rule exists after deletion")
+			}
+		})
+	}
+}
+
+func Test_iptablesRunner_DeleteSvc(t *testing.T) {
+	v4Addr := netip.MustParseAddr("10.0.0.4")
+	v6Addr := netip.MustParseAddr("fd7a:115c:a1e0::701:b62a")
+	testPM := PortMap{Protocol: "tcp", MatchPort: 4003, TargetPort: 80}
+	iptr := NewFakeIPTablesRunner()
+
+	// create two rules that will consitute svc1
+	s1R1 := argsForPortMapRule("svc1", "tailscale0", v4Addr, testPM)
+	mustPrecreatePortMapRule(t, s1R1, iptr.getIPTByAddr(v4Addr))
+	s1R2 := argsForPortMapRule("svc1", "tailscale0", v6Addr, testPM)
+	mustPrecreatePortMapRule(t, s1R2, iptr.getIPTByAddr(v6Addr))
+
+	// create two rules that will consitute svc2
+	s2R1 := argsForPortMapRule("svc2", "tailscale0", v4Addr, testPM)
+	mustPrecreatePortMapRule(t, s2R1, iptr.getIPTByAddr(v4Addr))
+	s2R2 := argsForPortMapRule("svc2", "tailscale0", v6Addr, testPM)
+	mustPrecreatePortMapRule(t, s2R2, iptr.getIPTByAddr(v6Addr))
+
+	// delete svc1
+	if err := iptr.DeleteSvc("svc1", "tailscale0", []netip.Addr{v4Addr, v6Addr}, []PortMap{testPM}); err != nil {
+		t.Fatalf("error deleting service: %v", err)
+	}
+
+	// validate that svc1 no longer exists
+	svcMustNotExist(t, "svc1", map[string][]string{v4Addr.String(): s1R1, v6Addr.String(): s1R2}, iptr)
+
+	// validate that svc2 still exists
+	svcMustExist(t, "svc2", map[string][]string{v4Addr.String(): s2R1, v6Addr.String(): s2R2}, iptr)
+}
+
+func svcMustExist(t *testing.T, svcName string, rules map[string][]string, iptr *iptablesRunner) {
+	t.Helper()
+	for dst, ruleset := range rules {
+		tip := netip.MustParseAddr(dst)
+		exists, err := iptr.getIPTByAddr(tip).Exists("nat", "PREROUTING", ruleset...)
+		if err != nil {
+			t.Fatalf("error checking whether %s exists: %v", svcName, err)
+		}
+		if !exists {
+			t.Fatalf("service %s should be deleted,but found rule for %s", svcName, dst)
+		}
+	}
+}
+
+func svcMustNotExist(t *testing.T, svcName string, rules map[string][]string, iptr *iptablesRunner) {
+	t.Helper()
+	for dst, ruleset := range rules {
+		tip := netip.MustParseAddr(dst)
+		exists, err := iptr.getIPTByAddr(tip).Exists("nat", "PREROUTING", ruleset...)
+		if err != nil {
+			t.Fatalf("error checking whether %s exists: %v", svcName, err)
+		}
+		if exists {
+			t.Fatalf("service %s should exist, but rule for %s is missing", svcName, dst)
+		}
+	}
+}
+
+func mustPrecreatePortMapRule(t *testing.T, rules []string, table iptablesInterface) {
+	t.Helper()
+	exists, err := table.Exists("nat", "PREROUTING", rules...)
+	if err != nil {
+		t.Fatalf("error ensuring that nat PREROUTING table exists: %v", err)
+	}
+	if exists {
+		return
+	}
+	if err := table.Append("nat", "PREROUTING", rules...); err != nil {
+		t.Fatalf("error precreating portmap rule: %v", err)
+	}
+}

util/linuxfw/iptables_runner.go

@@ -682,7 +682,7 @@ func delTSHook(ipt iptablesInterface, table, chain string, logf logger.Logf) err
 	return nil
 }
 
-// delChain flushs and deletes a chain. If the chain does not exist, it's a no-op
+// delChain flushes and deletes a chain. If the chain does not exist, it's a no-op
 // since the desired state is already achieved. otherwise, it returns an error.
 func delChain(ipt iptablesInterface, table, chain string) error {
 	if err := ipt.ClearChain(table, chain); err != nil {

util/linuxfw/nftables_for_svcs.go

@@ -0,0 +1,245 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package linuxfw
+
+import (
+	"errors"
+	"fmt"
+	"net/netip"
+	"reflect"
+	"strings"
+
+	"github.com/google/nftables"
+	"github.com/google/nftables/binaryutil"
+	"github.com/google/nftables/expr"
+	"golang.org/x/sys/unix"
+)
+
+// This file contains functionality that is currently (09/2024) used to set up
+// routing for the Tailscale Kubernetes operator egress proxies. A tailnet
+// service (identified by tailnet IP or FQDN) that gets exposed to cluster
+// workloads gets a separate prerouting chain created for it for each IP family
+// of the chain's target addresses. Each service's prerouting chain contains one
+// or more portmapping rules. A portmapping rule DNATs traffic received on a
+// particular port to a port of the tailnet service. Creating a chain per
+// service makes it easier to delete a service when no longer needed and helps
+// with readability.
+
+// EnsurePortMapRuleForSvc:
+// - ensures that nat table exists
+// - ensures that there is a prerouting chain for the given service and IP family of the target address in the nat table
+// - ensures that there is a portmapping rule mathcing the given portmap (only creates the rule if it does not already exist)
+func (n *nftablesRunner) EnsurePortMapRuleForSvc(svc, tun string, targetIP netip.Addr, pm PortMap) error {
+	t, ch, err := n.ensureChainForSvc(svc, targetIP)
+	if err != nil {
+		return fmt.Errorf("error ensuring chain for %s: %w", svc, err)
+	}
+	meta := svcPortMapRuleMeta(svc, targetIP, pm)
+	rule, err := n.findRuleByMetadata(t, ch, meta)
+	if err != nil {
+		return fmt.Errorf("error looking up rule: %w", err)
+	}
+	if rule != nil {
+		return nil
+	}
+	p, err := protoFromString(pm.Protocol)
+	if err != nil {
+		return fmt.Errorf("error converting protocol %s: %w", pm.Protocol, err)
+	}
+
+	rule = portMapRule(t, ch, tun, targetIP, pm.MatchPort, pm.TargetPort, p, meta)
+	n.conn.InsertRule(rule)
+	return n.conn.Flush()
+}
+
+// DeletePortMapRuleForSvc deletes a portmapping rule in the given service/IP family chain.
+// It finds the matching rule using metadata attached to the rule.
+// The caller is expected to call DeleteSvc if the whole service (the chain)
+// needs to be deleted, so we don't deal with the case where this is the only
+// rule in the chain here.
+func (n *nftablesRunner) DeletePortMapRuleForSvc(svc, tun string, targetIP netip.Addr, pm PortMap) error {
+	table, err := n.getNFTByAddr(targetIP)
+	if err != nil {
+		return fmt.Errorf("error setting up nftables for IP family of %s: %w", targetIP, err)
+	}
+	t, err := getTableIfExists(n.conn, table.Proto, "nat")
+	if err != nil {
+		return fmt.Errorf("error checking if nat table exists: %w", err)
+	}
+	if t == nil {
+		return nil
+	}
+	ch, err := getChainFromTable(n.conn, t, svc)
+	if err != nil && !errors.Is(err, errorChainNotFound{t.Name, svc}) {
+		return fmt.Errorf("error checking if chain %s exists: %w", svc, err)
+	}
+	if errors.Is(err, errorChainNotFound{t.Name, svc}) {
+		return nil // service chain does not exist, so neither does the portmapping rule
+	}
+	meta := svcPortMapRuleMeta(svc, targetIP, pm)
+	rule, err := n.findRuleByMetadata(t, ch, meta)
+	if err != nil {
+		return fmt.Errorf("error checking if rule exists: %w", err)
+	}
+	if rule == nil {
+		return nil
+	}
+	if err := n.conn.DelRule(rule); err != nil {
+		return fmt.Errorf("error deleting rule: %w", err)
+	}
+	return n.conn.Flush()
+}
+
+// DeleteSvc deletes the chains for the given service if any exist.
+func (n *nftablesRunner) DeleteSvc(svc, tun string, targetIPs []netip.Addr, pm []PortMap) error {
+	for _, tip := range targetIPs {
+		table, err := n.getNFTByAddr(tip)
+		if err != nil {
+			return fmt.Errorf("error setting up nftables for IP family of %s: %w", tip, err)
+		}
+		t, err := getTableIfExists(n.conn, table.Proto, "nat")
+		if err != nil {
+			return fmt.Errorf("error checking if nat table exists: %w", err)
+		}
+		if t == nil {
+			return nil
+		}
+		ch, err := getChainFromTable(n.conn, t, svc)
+		if err != nil && !errors.Is(err, errorChainNotFound{t.Name, svc}) {
+			return fmt.Errorf("error checking if chain %s exists: %w", svc, err)
+		}
+		if errors.Is(err, errorChainNotFound{t.Name, svc}) {
+			return nil
+		}
+		n.conn.DelChain(ch)
+	}
+	return n.conn.Flush()
+}
+
+func portMapRule(t *nftables.Table, ch *nftables.Chain, tun string, targetIP netip.Addr, matchPort, targetPort uint16, proto uint8, meta []byte) *nftables.Rule {
+	var fam uint32
+	if targetIP.Is4() {
+		fam = unix.NFPROTO_IPV4
+	} else {
+		fam = unix.NFPROTO_IPV6
+	}
+	rule := &nftables.Rule{
+		Table:    t,
+		Chain:    ch,
+		UserData: meta,
+		Exprs: []expr.Any{
+			&expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1},
+			&expr.Cmp{
+				Op:       expr.CmpOpNeq,
+				Register: 1,
+				Data:     []byte(tun),
+			},
+			&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     []byte{proto},
+			},
+			&expr.Payload{
+				DestRegister: 1,
+				Base:         expr.PayloadBaseTransportHeader,
+				Offset:       2,
+				Len:          2,
+			},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     binaryutil.BigEndian.PutUint16(matchPort),
+			},
+			&expr.Immediate{
+				Register: 1,
+				Data:     targetIP.AsSlice(),
+			},
+			&expr.Immediate{
+				Register: 2,
+				Data:     binaryutil.BigEndian.PutUint16(targetPort),
+			},
+			&expr.NAT{
+				Type:        expr.NATTypeDestNAT,
+				Family:      fam,
+				RegAddrMin:  1,
+				RegAddrMax:  1,
+				RegProtoMin: 2,
+				RegProtoMax: 2,
+			},
+		},
+	}
+	return rule
+}
+
+// svcPortMapRuleMeta generates metadata for a rule.
+// This metadata can then be used to find the rule.
+// https://github.com/google/nftables/issues/48
+func svcPortMapRuleMeta(svcName string, targetIP netip.Addr, pm PortMap) []byte {
+	return []byte(fmt.Sprintf("svc:%s,targetIP:%s:matchPort:%v,targetPort:%v,proto:%v", svcName, targetIP.String(), pm.MatchPort, pm.TargetPort, pm.Protocol))
+}
+
+func (n *nftablesRunner) findRuleByMetadata(t *nftables.Table, ch *nftables.Chain, meta []byte) (*nftables.Rule, error) {
+	if n.conn == nil || t == nil || ch == nil || len(meta) == 0 {
+		return nil, nil
+	}
+	rules, err := n.conn.GetRules(t, ch)
+	if err != nil {
+		return nil, fmt.Errorf("error listing rules: %w", err)
+	}
+	for _, rule := range rules {
+		if reflect.DeepEqual(rule.UserData, meta) {
+			return rule, nil
+		}
+	}
+	return nil, nil
+}
+
+func (n *nftablesRunner) ensureChainForSvc(svc string, targetIP netip.Addr) (*nftables.Table, *nftables.Chain, error) {
+	polAccept := nftables.ChainPolicyAccept
+	table, err := n.getNFTByAddr(targetIP)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error setting up nftables for IP family of %v: %w", targetIP, err)
+	}
+	nat, err := createTableIfNotExist(n.conn, table.Proto, "nat")
+	if err != nil {
+		return nil, nil, fmt.Errorf("error ensuring nat table: %w", err)
+	}
+	svcCh, err := getOrCreateChain(n.conn, chainInfo{
+		table:         nat,
+		name:          svc,
+		chainType:     nftables.ChainTypeNAT,
+		chainHook:     nftables.ChainHookPrerouting,
+		chainPriority: nftables.ChainPriorityNATDest,
+		chainPolicy:   &polAccept,
+	})
+	if err != nil {
+		return nil, nil, fmt.Errorf("error ensuring prerouting chain: %w", err)
+	}
+	return nat, svcCh, nil
+}
+
+// // PortMap is the port mapping for a service rule.
+type PortMap struct {
+	// MatchPort is the local port to which the rule should apply.
+	MatchPort uint16
+	// TargetPort is the port to which the traffic should be forwarded.
+	TargetPort uint16
+	// Protocol is the protocol to match packets on. Only TCP and UDP are
+	// supported.
+	Protocol string
+}
+
+func protoFromString(s string) (uint8, error) {
+	switch strings.ToLower(s) {
+	case "tcp":
+		return unix.IPPROTO_TCP, nil
+	case "udp":
+		return unix.IPPROTO_UDP, nil
+	default:
+		return 0, fmt.Errorf("unrecognized protocol: %q", s)
+	}
+}

util/linuxfw/nftables_for_svcs_test.go

@@ -0,0 +1,156 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package linuxfw
+
+import (
+	"net/netip"
+	"testing"
+
+	"github.com/google/nftables"
+)
+
+// This test creates a temporary network namespace for the nftables rules being
+// set up, so it needs to run in a privileged mode. Locally it needs to be run
+// by root, else it will be silently skipped. In CI it runs in a privileged
+// container.
+func Test_nftablesRunner_EnsurePortMapRuleForSvc(t *testing.T) {
+	conn := newSysConn(t)
+	runner := newFakeNftablesRunnerWithConn(t, conn, true)
+	ipv4, ipv6 := netip.MustParseAddr("100.99.99.99"), netip.MustParseAddr("fd7a:115c:a1e0::701:b62a")
+	pmTCP := PortMap{MatchPort: 4003, TargetPort: 80, Protocol: "TCP"}
+	pmTCP1 := PortMap{MatchPort: 4004, TargetPort: 443, Protocol: "TCP"}
+
+	// Create a rule for service 'foo' to forward TCP traffic to IPv4 endpoint
+	runner.EnsurePortMapRuleForSvc("foo", "tailscale0", ipv4, pmTCP)
+	svcChains(t, 1, conn)
+	chainRuleCount(t, "foo", 1, conn, nftables.TableFamilyIPv4)
+	chainRule(t, "foo", ipv4, pmTCP, runner, nftables.TableFamilyIPv4)
+
+	// Create another rule for service 'foo' to forward TCP traffic to the
+	// same IPv4 endpoint, but to a different port.
+	runner.EnsurePortMapRuleForSvc("foo", "tailscale0", ipv4, pmTCP1)
+	svcChains(t, 1, conn)
+	chainRuleCount(t, "foo", 2, conn, nftables.TableFamilyIPv4)
+	chainRule(t, "foo", ipv4, pmTCP1, runner, nftables.TableFamilyIPv4)
+
+	// Create a rule for service 'foo' to forward TCP traffic to an IPv6 endpoint
+	runner.EnsurePortMapRuleForSvc("foo", "tailscale0", ipv6, pmTCP)
+	svcChains(t, 2, conn)
+	chainRuleCount(t, "foo", 1, conn, nftables.TableFamilyIPv6)
+	chainRule(t, "foo", ipv6, pmTCP, runner, nftables.TableFamilyIPv6)
+
+	// Create a rule for service 'bar' to forward TCP traffic to IPv4 endpoint
+	runner.EnsurePortMapRuleForSvc("bar", "tailscale0", ipv4, pmTCP)
+	svcChains(t, 3, conn)
+	chainRuleCount(t, "bar", 1, conn, nftables.TableFamilyIPv4)
+	chainRule(t, "bar", ipv4, pmTCP, runner, nftables.TableFamilyIPv4)
+
+	// Create a rule for service 'bar' to forward TCP traffic to an IPv6 endpoint
+	runner.EnsurePortMapRuleForSvc("bar", "tailscale0", ipv6, pmTCP)
+	svcChains(t, 4, conn)
+	chainRuleCount(t, "bar", 1, conn, nftables.TableFamilyIPv6)
+	chainRule(t, "bar", ipv6, pmTCP, runner, nftables.TableFamilyIPv6)
+
+	// Delete service bar
+	runner.DeleteSvc("bar", "tailscale0", []netip.Addr{ipv4, ipv6}, []PortMap{pmTCP})
+	svcChains(t, 2, conn)
+
+	// Delete a rule from service foo
+	runner.DeletePortMapRuleForSvc("foo", "tailscale0", ipv4, pmTCP)
+	svcChains(t, 2, conn)
+	chainRuleCount(t, "foo", 1, conn, nftables.TableFamilyIPv4)
+
+	// Delete service foo
+	runner.DeleteSvc("foo", "tailscale0", []netip.Addr{ipv4, ipv6}, []PortMap{pmTCP, pmTCP1})
+	svcChains(t, 0, conn)
+}
+
+// svcChains verifies that the expected number of chains exist (for either IP
+// family) and that each of them is configured as NAT prerouting chain.
+func svcChains(t *testing.T, wantCount int, conn *nftables.Conn) {
+	t.Helper()
+	chains, err := conn.ListChains()
+	if err != nil {
+		t.Fatalf("error listing chains: %v", err)
+	}
+	if len(chains) != wantCount {
+		t.Fatalf("wants %d chains, got %d", wantCount, len(chains))
+	}
+	for _, ch := range chains {
+		if *ch.Policy != nftables.ChainPolicyAccept {
+			t.Fatalf("chain %s has unexpected policy %v", ch.Name, *ch.Policy)
+		}
+		if ch.Type != nftables.ChainTypeNAT {
+			t.Fatalf("chain %s has unexpected type %v", ch.Name, ch.Type)
+		}
+		if *ch.Hooknum != *nftables.ChainHookPrerouting {
+			t.Fatalf("chain %s is attached to unexpected hook %v", ch.Name, ch.Hooknum)
+		}
+		if *ch.Priority != *nftables.ChainPriorityNATDest {
+			t.Fatalf("chain %s has unexpected priority %v", ch.Name, ch.Priority)
+		}
+	}
+}
+
+// chainRuleCount returns number of rules in a chain identified by service name and IP family.
+func chainRuleCount(t *testing.T, svc string, count int, conn *nftables.Conn, fam nftables.TableFamily) {
+	t.Helper()
+	chains, err := conn.ListChainsOfTableFamily(fam)
+	if err != nil {
+		t.Fatalf("error listing chains: %v", err)
+	}
+
+	found := false
+	for _, ch := range chains {
+		if ch.Name == svc {
+			found = true
+			rules, err := conn.GetRules(ch.Table, ch)
+			if err != nil {
+				t.Fatalf("error getting rules: %v", err)
+			}
+			if len(rules) != count {
+				t.Fatalf("unexpected number of rules, wants %d got %d", count, len(rules))
+			}
+			break
+		}
+	}
+	if !found {
+		t.Fatalf("chain for service %s does not exist", svc)
+	}
+}
+
+// chainRule verifies that rule for the provided target IP and PortMap exists in
+// a chain identified by service name and IP family.
+func chainRule(t *testing.T, svc string, targetIP netip.Addr, pm PortMap, runner *nftablesRunner, fam nftables.TableFamily) {
+	t.Helper()
+	chains, err := runner.conn.ListChainsOfTableFamily(fam)
+	if err != nil {
+		t.Fatalf("error listing chains: %v", err)
+	}
+	var chain *nftables.Chain
+	for _, ch := range chains {
+		if ch.Name == svc {
+			chain = ch
+			break
+		}
+	}
+	if chain == nil {
+		t.Fatalf("chain for service %s does not exist", svc)
+	}
+	meta := svcPortMapRuleMeta(svc, targetIP, pm)
+	p, err := protoFromString(pm.Protocol)
+	if err != nil {
+		t.Fatalf("error converting protocol: %v", err)
+	}
+	wantsRule := portMapRule(chain.Table, chain, "tailscale0", targetIP, pm.MatchPort, pm.TargetPort, p, meta)
+	gotRule, err := findRule(runner.conn, wantsRule)
+	if err != nil {
+		t.Fatalf("error looking up rule: %v", err)
+	}
+	if gotRule == nil {
+		t.Fatalf("rule not found")
+	}
+}

util/linuxfw/nftables_runner.go

@@ -569,6 +569,12 @@ type NetfilterRunner interface {
 	// the Tailscale interface, as used in the Kubernetes egress proxies.
 	DNATNonTailscaleTraffic(exemptInterface string, dst netip.Addr) error
 
+	EnsurePortMapRuleForSvc(svc, tun string, targetIP netip.Addr, pm PortMap) error
+
+	DeletePortMapRuleForSvc(svc, tun string, targetIP netip.Addr, pm PortMap) error
+
+	DeleteSvc(svc, tun string, targetIPs []netip.Addr, pm []PortMap) error
+
 	// ClampMSSToPMTU adds a rule to the mangle/FORWARD chain to clamp MSS for
 	// traffic destined for the provided tun interface.
 	ClampMSSToPMTU(tun string, addr netip.Addr) error