util/linuxfw: add additional nftable detection logic

We were previously using the netlink API to see if there are chains/rules that
already exist. This works fine in environments where there is either full
nftable support or no support at all. However, we have identified certain
environments which have partial nftable support and the only feasible way of
detecting such an environment is to try to create some of the chains that we
need.

This adds a check to create a dummy postrouting chain which is immediately
deleted. The goal of the check is to ensure we are able to use nftables and
that it won't error out later. This check is only done in the path where we
detected that the system has no preexisting nftable rules.

Updates #5621
Updates #8555
Updates #8762

Signed-off-by: Maisem Ali <maisem@tailscale.com>

util/linuxfw/nftables_runner_test.go

@@ -851,6 +851,26 @@ func (t *testFWDetector) nftDetect() (int, error) {
 	return t.nftRuleCount, t.nftErr
 }
 
+// TestCreateDummyPostroutingChains tests that on a system with nftables
+// available, the function does not return an error and that the dummy
+// postrouting chains are cleaned up.
+func TestCreateDummyPostroutingChains(t *testing.T) {
+	conn := newSysConn(t)
+	runner := newFakeNftablesRunner(t, conn)
+	if err := runner.createDummyPostroutingChains(); err != nil {
+		t.Fatalf("createDummyPostroutingChains() failed: %v", err)
+	}
+	for _, table := range runner.getNATTables() {
+		nt, err := getTableIfExists(conn, table.Proto, tsDummyTableName)
+		if err != nil {
+			t.Fatalf("getTableIfExists() failed: %v", err)
+		}
+		if nt != nil {
+			t.Fatalf("expected table to be nil, got %v", nt)
+		}
+	}
+}
+
 func TestPickFirewallModeFromInstalledRules(t *testing.T) {
 	tests := []struct {
 		name string