wgengine/router: add auto selection heuristic for iptables/nftables

This commit replaces the TS_DEBUG_USE_NETLINK_NFTABLES envknob with
a TS_DEBUG_FIREWALL_MODE that should be set to either 'iptables' or
'nftables' to select firewall mode manually, other wise tailscaled
will automatically choose between iptables and nftables depending on
environment and system availability.

updates: #319
Signed-off-by: KevinLiang10 <kevinliang@tailscale.com>

util/linuxfw/iptables.go

@@ -7,7 +7,13 @@
 package linuxfw
 
 import (
+	"fmt"
+	"os/exec"
+	"strings"
+	"unicode"
+
 	"tailscale.com/types/logger"
+	"tailscale.com/util/multierr"
 )
 
 // DebugNetfilter prints debug information about iptables rules to the
@@ -21,9 +27,44 @@ func DebugIptables(logf logger.Logf) error {
 // system, ignoring the default "ACCEPT" rule present in the standard iptables
 // chains.
 //
-// It only returns an error when the kernel returns an error (i.e. when a
-// syscall fails); when there are no iptables rules, it is valid for this
-// function to return 0, nil.
+// It only returns an error when there is no iptables binary, or when iptables -S
+// fails. In all other cases, it returns the number of non-default rules.
 func DetectIptables() (int, error) {
-	panic("unused")
+	// run "iptables -S" to get the list of rules using iptables
+	// exec.Command returns an error if the binary is not found
+	cmd := exec.Command("iptables", "-S")
+	output, err := cmd.Output()
+	ip6cmd := exec.Command("ip6tables", "-S")
+	ip6output, ip6err := ip6cmd.Output()
+	var allLines []string
+	outputStr := string(output)
+	lines := strings.Split(outputStr, "\n")
+	ip6outputStr := string(ip6output)
+	ip6lines := strings.Split(ip6outputStr, "\n")
+	switch {
+	case err == nil && ip6err == nil:
+		allLines = append(lines, ip6lines...)
+	case err == nil && ip6err != nil:
+		allLines = lines
+	case err != nil && ip6err == nil:
+		allLines = ip6lines
+	default:
+		return 0, ErrorFWModeNotSupported{
+			Mode: FirewallModeIPTables,
+			Err:  fmt.Errorf("iptables command run fail: %w", multierr.New(err, ip6err)),
+		}
+	}
+
+	// count the number of non-default rules
+	count := 0
+	for _, line := range allLines {
+		trimmedLine := strings.TrimLeftFunc(line, unicode.IsSpace)
+		if line != "" && strings.HasPrefix(trimmedLine, "-A") {
+			// if the line is not empty and starts with "-A", it is a rule appended not default
+			count++
+		}
+	}
+
+	// return the count of non-default rules
+	return count, nil
 }

util/linuxfw/linuxfw.go

@@ -29,6 +29,31 @@ const (
 	Masq
 )
 
+type ErrorFWModeNotSupported struct {
+	Mode FirewallMode
+	Err  error
+}
+
+func (e ErrorFWModeNotSupported) Error() string {
+	return fmt.Sprintf("firewall mode %q not supported: %v", e.Mode, e.Err)
+}
+
+func (e ErrorFWModeNotSupported) Is(target error) bool {
+	_, ok := target.(ErrorFWModeNotSupported)
+	return ok
+}
+
+func (e ErrorFWModeNotSupported) Unwrap() error {
+	return e.Err
+}
+
+type FirewallMode string
+
+const (
+	FirewallModeIPTables FirewallMode = "iptables"
+	FirewallModeNfTables FirewallMode = "nftables"
+)
+
 // The following bits are added to packet marks for Tailscale use.
 //
 // We tried to pick bits sufficiently out of the way that it's

util/linuxfw/nftables.go

@@ -107,12 +107,18 @@ func DebugNetfilter(logf logger.Logf) error {
 func DetectNetfilter() (int, error) {
 	conn, err := nftables.New()
 	if err != nil {
-		return 0, err
+		return 0, ErrorFWModeNotSupported{
+			Mode: FirewallModeNfTables,
+			Err:  err,
+		}
 	}
 
 	chains, err := conn.ListChains()
 	if err != nil {
-		return 0, fmt.Errorf("cannot list chains: %w", err)
+		return 0, ErrorFWModeNotSupported{
+			Mode: FirewallModeNfTables,
+			Err:  fmt.Errorf("cannot list chains: %w", err),
+		}
 	}
 
 	var validRules int