wgengine/router: fix BSD router to support multiple local addrs, IPv6

Fixes #1201

wgengine/router/router_userspace_bsd.go

@@ -7,7 +7,6 @@
 package router
 
 import (
-	"errors"
 	"fmt"
 	"log"
 	"os/exec"
@@ -23,7 +22,7 @@ import (
 type userspaceBSDRouter struct {
 	logf    logger.Logf
 	tunname string
-	local   netaddr.IPPrefix
+	local   []netaddr.IPPrefix
 	routes  map[netaddr.IPPrefix]struct{}
 
 	dns *dns.Manager
@@ -47,6 +46,38 @@ func newUserspaceBSDRouter(logf logger.Logf, _ *device.Device, tundev tun.Device
 	}, nil
 }
 
+func (r *userspaceBSDRouter) addrsToRemove(newLocalAddrs []netaddr.IPPrefix) (remove []netaddr.IPPrefix) {
+	for _, cur := range r.local {
+		found := false
+		for _, v := range newLocalAddrs {
+			found = (v == cur)
+			if found {
+				break
+			}
+		}
+		if !found {
+			remove = append(remove, cur)
+		}
+	}
+	return
+}
+
+func (r *userspaceBSDRouter) addrsToAdd(newLocalAddrs []netaddr.IPPrefix) (add []netaddr.IPPrefix) {
+	for _, cur := range newLocalAddrs {
+		found := false
+		for _, v := range r.local {
+			found = (v == cur)
+			if found {
+				break
+			}
+		}
+		if !found {
+			add = append(add, cur)
+		}
+	}
+	return
+}
+
 func cmd(args ...string) *exec.Cmd {
 	if len(args) == 0 {
 		log.Fatalf("exec.Cmd(%#v) invalid; need argv[0]", args)
@@ -63,46 +94,41 @@ func (r *userspaceBSDRouter) Up() error {
 	return nil
 }
 
-func (r *userspaceBSDRouter) Set(cfg *Config) error {
+func inet(p netaddr.IPPrefix) string {
+	if p.IP.Is6() {
+		return "inet6"
+	}
+	return "inet"
+}
+
+func (r *userspaceBSDRouter) Set(cfg *Config) (reterr error) {
 	if cfg == nil {
 		cfg = &shutdownConfig
 	}
-	if len(cfg.LocalAddrs) == 0 {
-		return nil
-	}
-	// TODO: support configuring multiple local addrs on interface.
-	if len(cfg.LocalAddrs) != 1 {
-		return errors.New("freebsd doesn't support setting multiple local addrs yet")
-	}
-	localAddr := cfg.LocalAddrs[0]
 
 	var errq error
-
+	setErr := func(err error) {
-	// Update the address.
-	if localAddr != r.local {
-		// If the interface is already set, remove it.
-		if !r.local.IsZero() {
-			addrdel := []string{"ifconfig", r.tunname,
-				"inet", r.local.String(), "-alias"}
-			out, err := cmd(addrdel...).CombinedOutput()
-			if err != nil {
-				r.logf("addr del failed: %v: %v\n%s", addrdel, err, out)
 		if errq == nil {
 			errq = err
 		}
 	}
-		}
 
-		// Add the interface.
+	// Update the addresses.
-		addradd := []string{"ifconfig", r.tunname,
+	for _, addr := range r.addrsToRemove(cfg.LocalAddrs) {
-			"inet", localAddr.String(), localAddr.IP.String()}
+		arg := []string{"ifconfig", r.tunname, inet(addr), addr.String(), "-alias"}
-		out, err := cmd(addradd...).CombinedOutput()
+		out, err := cmd(arg...).CombinedOutput()
 		if err != nil {
-			r.logf("addr add failed: %v: %v\n%s", addradd, err, out)
+			r.logf("addr del failed: %v => %v\n%s", arg, err, out)
-			if errq == nil {
+			setErr(err)
-				errq = err
 		}
 	}
+	for _, addr := range r.addrsToAdd(cfg.LocalAddrs) {
+		arg := []string{"ifconfig", r.tunname, inet(addr), addr.String(), addr.IP.String()}
+		out, err := cmd(arg...).CombinedOutput()
+		if err != nil {
+			r.logf("addr add failed: %v => %v\n%s", arg, err, out)
+			setErr(err)
+		}
 	}
 
 	newRoutes := make(map[netaddr.IPPrefix]struct{})
@@ -120,14 +146,12 @@ func (r *userspaceBSDRouter) Set(cfg *Config) error {
 				del = "delete"
 			}
 			routedel := []string{"route", "-q", "-n",
-				del, "-inet", nstr,
+				del, "-" + inet(route), nstr,
 				"-iface", r.tunname}
 			out, err := cmd(routedel...).CombinedOutput()
 			if err != nil {
 				r.logf("route del failed: %v: %v\n%s", routedel, err, out)
-				if errq == nil {
+				setErr(err)
-					errq = err
-				}
 			}
 		}
 	}
@@ -138,24 +162,25 @@ func (r *userspaceBSDRouter) Set(cfg *Config) error {
 			nip := net.IP.Mask(net.Mask)
 			nstr := fmt.Sprintf("%v/%d", nip, route.Bits)
 			routeadd := []string{"route", "-q", "-n",
-				"add", "-inet", nstr,
+				"add", "-" + inet(route), nstr,
 				"-iface", r.tunname}
 			out, err := cmd(routeadd...).CombinedOutput()
 			if err != nil {
 				r.logf("addr add failed: %v: %v\n%s", routeadd, err, out)
-				if errq == nil {
+				setErr(err)
-					errq = err
-				}
 			}
 		}
 	}
 
 	// Store the interface and routes so we know what to change on an update.
-	r.local = localAddr
+	if errq == nil {
+		r.local = append([]netaddr.IPPrefix{}, cfg.LocalAddrs...)
+	}
 	r.routes = newRoutes
 
 	if err := r.dns.Set(cfg.DNS); err != nil {
-		errq = fmt.Errorf("dns set: %v", err)
+		r.logf("DNS set: %v", err)
+		setErr(err)
 	}
 
 	return errq