wgengine/router: add auto selection heuristic for iptables/nftables

This commit replaces the TS_DEBUG_USE_NETLINK_NFTABLES envknob with
a TS_DEBUG_FIREWALL_MODE that should be set to either 'iptables' or
'nftables' to select firewall mode manually, other wise tailscaled
will automatically choose between iptables and nftables depending on
environment and system availability.

updates: #319
Signed-off-by: KevinLiang10 <kevinliang@tailscale.com>

wgengine/router/router_linux_test.go

@@ -1063,3 +1063,63 @@ func adjustFwmask(t *testing.T, s string) string {
 
 	return fwmaskAdjustRe.ReplaceAllString(s, "$1")
 }
+
+type testFWDetector struct {
+	iptRuleCount, nftRuleCount int
+	iptErr, nftErr             error
+}
+
+func (t *testFWDetector) iptDetect() (int, error) {
+	return t.iptRuleCount, t.iptErr
+}
+
+func (t *testFWDetector) nftDetect() (int, error) {
+	return t.nftRuleCount, t.nftErr
+}
+
+func TestChooseFireWallMode(t *testing.T) {
+	tests := []struct {
+		name string
+		det  *testFWDetector
+		want linuxfw.FirewallMode
+	}{
+		{
+			name: "using iptables legacy",
+			det:  &testFWDetector{iptRuleCount: 1},
+			want: linuxfw.FirewallModeIPTables,
+		},
+		{
+			name: "using nftables",
+			det:  &testFWDetector{nftRuleCount: 1},
+			want: linuxfw.FirewallModeNfTables,
+		},
+		{
+			name: "using both iptables and nftables",
+			det:  &testFWDetector{iptRuleCount: 2, nftRuleCount: 2},
+			want: linuxfw.FirewallModeNfTables,
+		},
+		{
+			name: "not using any firewall, both available",
+			det:  &testFWDetector{},
+			want: linuxfw.FirewallModeNfTables,
+		},
+		{
+			name: "not using any firewall, iptables available only",
+			det:  &testFWDetector{iptRuleCount: 1, nftErr: errors.New("nft error")},
+			want: linuxfw.FirewallModeIPTables,
+		},
+		{
+			name: "not using any firewall, nftables available only",
+			det:  &testFWDetector{iptErr: errors.New("iptables error"), nftRuleCount: 1},
+			want: linuxfw.FirewallModeNfTables,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, _ := chooseFireWallMode(t.Logf, tt.det)
+			if got != tt.want {
+				t.Errorf("chooseFireWallMode() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}