wgengine/router: split out from wgengine.

The router implementations are logically separate, with their own API.

Signed-off-by: David Anderson <danderson@tailscale.com>

wgengine/router/ifconfig_windows.go

@@ -0,0 +1,415 @@
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2019 WireGuard LLC. All Rights Reserved.
+ */
+
+package router
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"log"
+	"net"
+	"sort"
+	"time"
+	"unsafe"
+
+	ole "github.com/go-ole/go-ole"
+	winipcfg "github.com/tailscale/winipcfg-go"
+	"github.com/tailscale/wireguard-go/device"
+	"github.com/tailscale/wireguard-go/tun"
+	"github.com/tailscale/wireguard-go/wgcfg"
+	"golang.org/x/sys/windows"
+	"golang.org/x/sys/windows/registry"
+	"tailscale.com/wgengine/winnet"
+)
+
+const (
+	sockoptIP_UNICAST_IF   = 31
+	sockoptIPV6_UNICAST_IF = 31
+)
+
+func htonl(val uint32) uint32 {
+	bytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(bytes, val)
+	return *(*uint32)(unsafe.Pointer(&bytes[0]))
+}
+
+func bindSocketRoute(family winipcfg.AddressFamily, device *device.Device, ourLuid uint64, lastLuid *uint64) error {
+	routes, err := winipcfg.GetRoutes(family)
+	if err != nil {
+		return err
+	}
+	lowestMetric := ^uint32(0)
+	index := uint32(0) // Zero is "unspecified", which for IP_UNICAST_IF resets the value, which is what we want.
+	luid := uint64(0)  // Hopefully luid zero is unspecified, but hard to find docs saying so.
+	for _, route := range routes {
+		if route.DestinationPrefix.PrefixLength != 0 || route.InterfaceLuid == ourLuid {
+			continue
+		}
+		if route.Metric < lowestMetric {
+			lowestMetric = route.Metric
+			index = route.InterfaceIndex
+			luid = route.InterfaceLuid
+		}
+	}
+	if luid == *lastLuid {
+		return nil
+	}
+	*lastLuid = luid
+	if false {
+		// TODO(apenwarr): doesn't work with magic socket yet.
+		if family == winipcfg.AF_INET {
+			return device.BindSocketToInterface4(index, false)
+		} else if family == winipcfg.AF_INET6 {
+			return device.BindSocketToInterface6(index, false)
+		}
+	} else {
+		log.Printf("WARNING: skipping windows socket binding.\n")
+	}
+	return nil
+}
+
+func monitorDefaultRoutes(device *device.Device, autoMTU bool, tun *tun.NativeTun) (*winipcfg.RouteChangeCallback, error) {
+	guid := tun.GUID()
+	ourLuid, err := winipcfg.InterfaceGuidToLuid(&guid)
+	lastLuid4 := uint64(0)
+	lastLuid6 := uint64(0)
+	lastMtu := uint32(0)
+	if err != nil {
+		return nil, err
+	}
+	doIt := func() error {
+		err = bindSocketRoute(winipcfg.AF_INET, device, ourLuid, &lastLuid4)
+		if err != nil {
+			return err
+		}
+		err = bindSocketRoute(winipcfg.AF_INET6, device, ourLuid, &lastLuid6)
+		if err != nil {
+			return err
+		}
+		if !autoMTU {
+			return nil
+		}
+		mtu := uint32(0)
+		if lastLuid4 != 0 {
+			iface, err := winipcfg.InterfaceFromLUID(lastLuid4)
+			if err != nil {
+				return err
+			}
+			if iface.Mtu > 0 {
+				mtu = iface.Mtu
+			}
+		}
+		if lastLuid6 != 0 {
+			iface, err := winipcfg.InterfaceFromLUID(lastLuid6)
+			if err != nil {
+				return err
+			}
+			if iface.Mtu > 0 && iface.Mtu < mtu {
+				mtu = iface.Mtu
+			}
+		}
+		if mtu > 0 && (lastMtu == 0 || lastMtu != mtu) {
+			iface, err := winipcfg.GetIpInterface(ourLuid, winipcfg.AF_INET)
+			if err != nil {
+				return err
+			}
+			iface.NlMtu = mtu - 80
+			if iface.NlMtu < 576 {
+				iface.NlMtu = 576
+			}
+			err = iface.Set()
+			if err != nil {
+				return err
+			}
+			tun.ForceMTU(int(iface.NlMtu)) //TODO: it sort of breaks the model with v6 mtu and v4 mtu being different. Just set v4 one for now.
+			iface, err = winipcfg.GetIpInterface(ourLuid, winipcfg.AF_INET6)
+			if err != nil {
+				return err
+			}
+			iface.NlMtu = mtu - 80
+			if iface.NlMtu < 1280 {
+				iface.NlMtu = 1280
+			}
+			err = iface.Set()
+			if err != nil {
+				return err
+			}
+			lastMtu = mtu
+		}
+		return nil
+	}
+	err = doIt()
+	if err != nil {
+		return nil, err
+	}
+	cb, err := winipcfg.RegisterRouteChangeCallback(func(notificationType winipcfg.MibNotificationType, route *winipcfg.Route) {
+		//fmt.Printf("MonitorDefaultRoutes: changed: %v\n", route.DestinationPrefix)
+		if route.DestinationPrefix.PrefixLength == 0 {
+			_ = doIt()
+		}
+	})
+	if err != nil {
+		return nil, err
+	}
+	return cb, nil
+}
+
+func setDNSDomains(g windows.GUID, dnsDomains []string) {
+	gs := g.String()
+	log.Printf("setDNSDomains(%v) guid=%v\n", dnsDomains, gs)
+	p := `SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces\` + gs
+	key, err := registry.OpenKey(registry.LOCAL_MACHINE, p, registry.READ|registry.SET_VALUE)
+	if err != nil {
+		log.Printf("setDNSDomains(%v): open: %v\n", p, err)
+		return
+	}
+	defer key.Close()
+
+	// Windows only supports a single per-interface DNS domain.
+	dom := ""
+	if len(dnsDomains) > 0 {
+		dom = dnsDomains[0]
+	}
+	err = key.SetStringValue("Domain", dom)
+	if err != nil {
+		log.Printf("setDNSDomains(%v): SetStringValue: %v\n", p, err)
+	}
+}
+
+func setFirewall(ifcGUID *windows.GUID) (bool, error) {
+	c := ole.Connection{}
+	err := c.Initialize()
+	if err != nil {
+		return false, fmt.Errorf("c.Initialize: %v", err)
+	}
+	defer c.Uninitialize()
+
+	m, err := winnet.NewNetworkListManager(&c)
+	if err != nil {
+		return false, fmt.Errorf("winnet.NewNetworkListManager: %v", err)
+	}
+	defer m.Release()
+
+	cl, err := m.GetNetworkConnections()
+	if err != nil {
+		return false, fmt.Errorf("m.GetNetworkConnections: %v", err)
+	}
+	defer cl.Release()
+
+	for _, nco := range cl {
+		aid, err := nco.GetAdapterId()
+		if err != nil {
+			return false, fmt.Errorf("nco.GetAdapterId: %v", err)
+		}
+		if aid != ifcGUID.String() {
+			log.Printf("skipping adapter id: %v\n", aid)
+			continue
+		}
+		log.Printf("found! adapter id: %v\n", aid)
+
+		n, err := nco.GetNetwork()
+		if err != nil {
+			return false, fmt.Errorf("GetNetwork: %v", err)
+		}
+		defer n.Release()
+
+		cat, err := n.GetCategory()
+		if err != nil {
+			return false, fmt.Errorf("GetCategory: %v", err)
+		}
+
+		if cat == 0 {
+			err = n.SetCategory(1)
+			if err != nil {
+				return false, fmt.Errorf("SetCategory: %v", err)
+			}
+		} else {
+			log.Printf("setFirewall: already category %v\n", cat)
+		}
+
+		return true, nil
+	}
+
+	return false, nil
+}
+
+func configureInterface(m *wgcfg.Config, tun *tun.NativeTun, dns []wgcfg.IP, dnsDomains []string) error {
+	const mtu = 0
+	guid := tun.GUID()
+	log.Printf("wintun GUID is %v\n", guid)
+	iface, err := winipcfg.InterfaceFromGUID(&guid)
+	if err != nil {
+		return err
+	}
+
+	go func() {
+		// It takes a weirdly long time for Windows to notice the
+		// new interface has come up. Poll periodically until it
+		// does.
+		for i := 0; i < 20; i++ {
+			found, err := setFirewall(&guid)
+			if err != nil {
+				log.Printf("setFirewall: %v\n", err)
+				// fall through anyway, this isn't fatal.
+			}
+			if found {
+				break
+			}
+			time.Sleep(1 * time.Second)
+		}
+	}()
+
+	setDNSDomains(guid, dnsDomains)
+
+	routes := []winipcfg.RouteData{}
+	var firstGateway4 *net.IP
+	var firstGateway6 *net.IP
+	addresses := make([]*net.IPNet, len(m.Addresses))
+	for i, addr := range m.Addresses {
+		ipnet := addr.IPNet()
+		addresses[i] = ipnet
+		gateway := ipnet.IP
+		if addr.IP.Is4() && firstGateway4 == nil {
+			firstGateway4 = &gateway
+		} else if addr.IP.Is6() && firstGateway6 == nil {
+			firstGateway6 = &gateway
+		}
+	}
+
+	foundDefault4 := false
+	foundDefault6 := false
+	for _, peer := range m.Peers {
+		for _, allowedip := range peer.AllowedIPs {
+			if (allowedip.IP.Is4() && firstGateway4 == nil) || (allowedip.IP.Is6() && firstGateway6 == nil) {
+				return errors.New("Due to a Windows limitation, one cannot have interface routes without an interface address")
+			}
+
+			ipn := allowedip.IPNet()
+			var gateway net.IP
+			if allowedip.IP.Is4() {
+				gateway = *firstGateway4
+			} else if allowedip.IP.Is6() {
+				gateway = *firstGateway6
+			}
+			r := winipcfg.RouteData{
+				Destination: net.IPNet{
+					IP:   ipn.IP.Mask(ipn.Mask),
+					Mask: ipn.Mask,
+				},
+				NextHop: gateway,
+				Metric:  0,
+			}
+			if bytes.Compare(r.Destination.IP, gateway) == 0 {
+				// no need to add a route for the interface's
+				// own IP. The kernel does that for us.
+				// If we try to replace it, we'll fail to
+				// add the route unless NextHop is set, but
+				// then the interface's IP won't be pingable.
+				continue
+			}
+			if allowedip.IP.Is4() {
+				if allowedip.Mask == 0 {
+					foundDefault4 = true
+				}
+				r.NextHop = *firstGateway4
+			} else if allowedip.IP.Is6() {
+				if allowedip.Mask == 0 {
+					foundDefault6 = true
+				}
+				r.NextHop = *firstGateway6
+			}
+			routes = append(routes, r)
+		}
+	}
+
+	err = iface.SyncAddresses(addresses)
+	if err != nil {
+		return err
+	}
+
+	sort.Slice(routes, func(i, j int) bool {
+		return (bytes.Compare(routes[i].Destination.IP, routes[j].Destination.IP) == -1 ||
+			// Narrower masks first
+			bytes.Compare(routes[i].Destination.Mask, routes[j].Destination.Mask) == 1 ||
+			// No nexthop before non-empty nexthop
+			bytes.Compare(routes[i].NextHop, routes[j].NextHop) == -1 ||
+			// Lower metrics first
+			routes[i].Metric < routes[j].Metric)
+	})
+
+	deduplicatedRoutes := []*winipcfg.RouteData{}
+	for i := 0; i < len(routes); i++ {
+		// There's only one way to get to a given IP+Mask, so delete
+		// all matches after the first.
+		if i > 0 &&
+			bytes.Equal(routes[i].Destination.IP, routes[i-1].Destination.IP) &&
+			bytes.Equal(routes[i].Destination.Mask, routes[i-1].Destination.Mask) {
+			continue
+		}
+		deduplicatedRoutes = append(deduplicatedRoutes, &routes[i])
+	}
+	log.Printf("routes: %v\n", routes)
+
+	var errAcc error
+	err = iface.SyncRoutes(deduplicatedRoutes)
+	if err != nil && errAcc == nil {
+		log.Printf("setroutes: %v\n", err)
+		errAcc = err
+	}
+
+	var dnsIPs []net.IP
+	for _, ip := range dns {
+		dnsIPs = append(dnsIPs, ip.IP())
+	}
+	err = iface.SetDNS(dnsIPs)
+	if err != nil && errAcc == nil {
+		log.Printf("setdns: %v\n", err)
+		errAcc = err
+	}
+
+	ipif, err := iface.GetIpInterface(winipcfg.AF_INET)
+	if err != nil {
+		log.Printf("getipif: %v\n", err)
+		return err
+	}
+	log.Printf("foundDefault4: %v\n", foundDefault4)
+	if foundDefault4 {
+		ipif.UseAutomaticMetric = false
+		ipif.Metric = 0
+	}
+	if mtu > 0 {
+		ipif.NlMtu = uint32(mtu)
+		tun.ForceMTU(int(ipif.NlMtu))
+	}
+	err = ipif.Set()
+	if err != nil && errAcc == nil {
+		errAcc = err
+	}
+
+	ipif, err = iface.GetIpInterface(winipcfg.AF_INET6)
+	if err != nil {
+		return err
+	}
+	if err != nil && errAcc == nil {
+		errAcc = err
+	}
+	if foundDefault6 {
+		ipif.UseAutomaticMetric = false
+		ipif.Metric = 0
+	}
+	if mtu > 0 {
+		ipif.NlMtu = uint32(mtu)
+	}
+	ipif.DadTransmits = 0
+	ipif.RouterDiscoveryBehavior = winipcfg.RouterDiscoveryDisabled
+	err = ipif.Set()
+	if err != nil && errAcc == nil {
+		errAcc = err
+	}
+
+	return errAcc
+}

wgengine/router/router.go

@@ -0,0 +1,57 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package router presents an interface to manipulate the host network
+// stack's state.
+package router
+
+import (
+	"fmt"
+
+	"github.com/tailscale/wireguard-go/device"
+	"github.com/tailscale/wireguard-go/tun"
+	"github.com/tailscale/wireguard-go/wgcfg"
+	"tailscale.com/types/logger"
+)
+
+// Router is responsible for managing the system network stack.
+//
+// There is typically only one instance of this interface per process.
+type Router interface {
+	// Up brings the router up.
+	Up() error
+
+	// SetRoutes is called regularly on network map updates.
+	// It's how you kernel route table entries are populated for
+	// each peer.
+	SetRoutes(RouteSettings) error
+
+	// Close closes the router.
+	Close() error
+}
+
+// NewUserspaceRouter returns a new Router for the current platform, using the provided tun device.
+func New(logf logger.Logf, wgdev *device.Device, tundev tun.Device) (Router, error) {
+	return newUserspaceRouter(logf, wgdev, tundev)
+}
+
+// RouteSettings is the full WireGuard config data (set of peers keys,
+// IP, etc in wgcfg.Config) plus the things that WireGuard doesn't do
+// itself, like DNS stuff.
+type RouteSettings struct {
+	LocalAddr  wgcfg.CIDR // TODO: why is this here? how does it differ from wgcfg.Config's info?
+	DNS        []wgcfg.IP
+	DNSDomains []string
+	Cfg        *wgcfg.Config
+}
+
+// OnlyRelevantParts returns a string minimally describing the route settings.
+func (rs *RouteSettings) OnlyRelevantParts() string {
+	var peers [][]wgcfg.CIDR
+	for _, p := range rs.Cfg.Peers {
+		peers = append(peers, p.AllowedIPs)
+	}
+	return fmt.Sprintf("%v %v %v %v",
+		rs.LocalAddr, rs.DNS, rs.DNSDomains, peers)
+}

wgengine/router/router_darwin.go

@@ -0,0 +1,38 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package router
+
+import (
+	"github.com/tailscale/wireguard-go/device"
+	"github.com/tailscale/wireguard-go/tun"
+	"tailscale.com/types/logger"
+)
+
+type darwinRouter struct {
+	tunname string
+}
+
+func newUserspaceRouter(logf logger.Logf, _ *device.Device, tundev tun.Device) (Router, error) {
+	tunname, err := tundev.Name()
+	if err != nil {
+		return nil, err
+	}
+	return &darwinRouter{tunname: tunname}, nil
+}
+
+func (r *darwinRouter) Up() error {
+	return nil
+}
+
+func (r *darwinRouter) SetRoutes(rs RouteSettings) error {
+	if SetRoutesFunc != nil {
+		return SetRoutesFunc(rs)
+	}
+	return nil
+}
+
+func (r *darwinRouter) Close() error {
+	return nil
+}

wgengine/router/router_darwin_support.go

@@ -0,0 +1,25 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux darwin
+
+package router
+
+// SetRoutesFunc applies the given route settings to the OS network
+// stack.
+//
+// This is logically part of the router_darwin.go implementation, and
+// should not be used on other platforms.
+//
+// The code to reconfigure the network stack on MacOS and iOS is in
+// the non-open `ipn-go-bridge` package, which bridges between the Go
+// and Swift pieces of the application. The ipn-go-bridge sets
+// SetRoutesFunc at startup.
+//
+// So why isn't this in router_darwin.go? Because in the non-oss
+// repository, we build ipn-go-bridge when developing on Linux as well
+// as MacOS, so that we don't have to wait until the Mac CI to
+// discover that we broke it. So this one definition needs to exist in
+// both the darwin and linux builds. Hence this file and build tag.
+var SetRoutesFunc func(rs RouteSettings) error

wgengine/router/router_default.go

@@ -0,0 +1,17 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !windows,!linux,!darwin,!openbsd,!freebsd
+
+package router
+
+import (
+	"github.com/tailscale/wireguard-go/device"
+	"github.com/tailscale/wireguard-go/tun"
+	"tailscale.com/types/logger"
+)
+
+func newUserspaceRouter(logf logger.Logf, tunname string, dev *device.Device, tuntap tun.Device, netChanged func()) Router {
+	return NewFakeRouter(logf, tunname, dev, tuntap, netChanged)
+}

wgengine/router/router_fake.go

@@ -0,0 +1,36 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package router
+
+import (
+	"github.com/tailscale/wireguard-go/device"
+	"github.com/tailscale/wireguard-go/tun"
+	"tailscale.com/types/logger"
+)
+
+// NewFakeRouter returns a Router that does nothing when called and
+// always returns nil errors.
+func NewFake(logf logger.Logf, _ *device.Device, _ tun.Device) (Router, error) {
+	return fakeRouter{logf: logf}, nil
+}
+
+type fakeRouter struct {
+	logf logger.Logf
+}
+
+func (r fakeRouter) Up() error {
+	r.logf("Warning: fakeRouter.Up: not implemented.")
+	return nil
+}
+
+func (r fakeRouter) SetRoutes(rs RouteSettings) error {
+	r.logf("Warning: fakeRouter.SetRoutes: not implemented.")
+	return nil
+}
+
+func (r fakeRouter) Close() error {
+	r.logf("Warning: fakeRouter.Close: not implemented.")
+	return nil
+}

wgengine/router/router_freebsd.go

@@ -0,0 +1,152 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package router
+
+import (
+	"fmt"
+	"log"
+	"os/exec"
+
+	"github.com/tailscale/wireguard-go/device"
+	"github.com/tailscale/wireguard-go/tun"
+	"github.com/tailscale/wireguard-go/wgcfg"
+	"tailscale.com/types/logger"
+)
+
+// For now this router only supports the userspace WireGuard implementations.
+//
+// Work is currently underway for an in-kernel FreeBSD implementation of wireguard
+// https://svnweb.freebsd.org/base?view=revision&revision=357986
+
+type freebsdRouter struct {
+	logf    logger.Logf
+	tunname string
+	local   wgcfg.CIDR
+	routes  map[wgcfg.CIDR]struct{}
+}
+
+func newUserspaceRouter(logf logger.Logf, _ *device.Device, tundev tun.Device) (Router, error) {
+	tunname, err := tundev.Name()
+	if err != nil {
+		return nil, err
+	}
+	return &freebsdRouter{
+		logf:    logf,
+		tunname: tunname,
+	}, nil
+}
+
+func cmd(args ...string) *exec.Cmd {
+	if len(args) == 0 {
+		log.Fatalf("exec.Cmd(%#v) invalid; need argv[0]\n", args)
+	}
+	return exec.Command(args[0], args[1:]...)
+}
+
+func (r *freebsdRouter) Up() error {
+	ifup := []string{"ifconfig", r.tunname, "up"}
+	if out, err := cmd(ifup...).CombinedOutput(); err != nil {
+		r.logf("running ifconfig failed: %v\n%s", err, out)
+		return err
+	}
+	return nil
+}
+
+func (r *freebsdRouter) SetRoutes(rs RouteSettings) error {
+	if rs.LocalAddr == (wgcfg.CIDR{}) {
+		return nil
+	}
+
+	var errq error
+
+	// Update the address.
+	if rs.LocalAddr != r.local {
+		// If the interface is already set, remove it.
+		if r.local != (wgcfg.CIDR{}) {
+			addrdel := []string{"ifconfig", r.tunname,
+				"inet", r.local.String(), "-alias"}
+			out, err := cmd(addrdel...).CombinedOutput()
+			if err != nil {
+				r.logf("addr del failed: %v: %v\n%s", addrdel, err, out)
+				if errq == nil {
+					errq = err
+				}
+			}
+		}
+
+		// Add the interface.
+		addradd := []string{"ifconfig", r.tunname,
+			"inet", rs.LocalAddr.String(), rs.LocalAddr.IP.String()}
+		out, err := cmd(addradd...).CombinedOutput()
+		if err != nil {
+			r.logf("addr add failed: %v: %v\n%s", addradd, err, out)
+			if errq == nil {
+				errq = err
+			}
+		}
+	}
+
+	newRoutes := make(map[wgcfg.CIDR]struct{})
+	for _, peer := range rs.Cfg.Peers {
+		for _, route := range peer.AllowedIPs {
+			newRoutes[route] = struct{}{}
+		}
+	}
+	// Delete any pre-existing routes.
+	for route := range r.routes {
+		if _, keep := newRoutes[route]; !keep {
+			net := route.IPNet()
+			nip := net.IP.Mask(net.Mask)
+			nstr := fmt.Sprintf("%v/%d", nip, route.Mask)
+			routedel := []string{"route", "-q", "-n",
+				"del", "-inet", nstr,
+				"-iface", r.tunname}
+			out, err := cmd(routedel...).CombinedOutput()
+			if err != nil {
+				r.logf("route del failed: %v: %v\n%s", routedel, err, out)
+				if errq == nil {
+					errq = err
+				}
+			}
+		}
+	}
+	// Add the routes.
+	for route := range newRoutes {
+		if _, exists := r.routes[route]; !exists {
+			net := route.IPNet()
+			nip := net.IP.Mask(net.Mask)
+			nstr := fmt.Sprintf("%v/%d", nip, route.Mask)
+			routeadd := []string{"route", "-q", "-n",
+				"add", "-inet", nstr,
+				"-iface", r.tunname}
+			out, err := cmd(routeadd...).CombinedOutput()
+			if err != nil {
+				r.logf("addr add failed: %v: %v\n%s", routeadd, err, out)
+				if errq == nil {
+					errq = err
+				}
+			}
+		}
+	}
+
+	// Store the interface and routes so we know what to change on an update.
+	r.local = rs.LocalAddr
+	r.routes = newRoutes
+
+	if err := r.replaceResolvConf(rs.DNS, rs.DNSDomains); err != nil {
+		errq = fmt.Errorf("replacing resolv.conf failed: %v", err)
+	}
+
+	return errq
+}
+
+func (r *freebsdRouter) Close() error {
+	return nil
+}
+
+// TODO(mbaillie): these are no-ops for now. They could re-use the Linux funcs
+// (sans systemd parts), but I note Linux DNS is disabled(?) so leaving for now.
+func (r *freebsdRouter) replaceResolvConf(_ []wgcfg.IP, _ []string) error { return nil }
+func (r *freebsdRouter) restoreResolvConf() error                         { return nil }

wgengine/router/router_linux.go

@@ -0,0 +1,291 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package router
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/coreos/go-iptables/iptables"
+	"github.com/tailscale/wireguard-go/device"
+	"github.com/tailscale/wireguard-go/tun"
+	"github.com/tailscale/wireguard-go/wgcfg"
+	"tailscale.com/atomicfile"
+	"tailscale.com/types/logger"
+)
+
+type linuxRouter struct {
+	logf    func(fmt string, args ...interface{})
+	tunname string
+	local   wgcfg.CIDR
+	routes  map[wgcfg.CIDR]struct{}
+
+	ipt4 *iptables.IPTables
+}
+
+func newUserspaceRouter(logf logger.Logf, _ *device.Device, tunDev tun.Device) (Router, error) {
+	tunname, err := tunDev.Name()
+	if err != nil {
+		return nil, err
+	}
+
+	ipt4, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
+	if err != nil {
+		return nil, err
+	}
+
+	return &linuxRouter{
+		logf:    logf,
+		tunname: tunname,
+		ipt4:    ipt4,
+	}, nil
+}
+
+func cmd(args ...string) *exec.Cmd {
+	if len(args) == 0 {
+		log.Fatalf("exec.Cmd(%#v) invalid; need argv[0]\n", args)
+	}
+	return exec.Command(args[0], args[1:]...)
+}
+
+func (r *linuxRouter) Up() error {
+	out, err := cmd("ip", "link", "set", r.tunname, "up").CombinedOutput()
+	if err != nil {
+		// TODO: this should return an error; why is it calling log.Fatalf?
+		// Audit callers to make sure they're handling errors.
+		log.Fatalf("running ip link failed: %v\n%s", err, out)
+	}
+
+	err = r.ipt4.AppendUnique("filter", "FORWARD", r.forwardRule()...)
+	if err != nil {
+		r.logf("iptables forward failed: %v", err)
+	}
+	err = r.ipt4.AppendUnique("nat", "POSTROUTING", r.natRule()...)
+	if err != nil {
+		r.logf("iptables nat failed: %v", err)
+	}
+	return nil
+}
+
+func (r *linuxRouter) SetRoutes(rs RouteSettings) error {
+	var errq error
+
+	if rs.LocalAddr != r.local {
+		if r.local != (wgcfg.CIDR{}) {
+			addrdel := []string{"ip", "addr",
+				"del", r.local.String(),
+				"dev", r.tunname}
+			out, err := cmd(addrdel...).CombinedOutput()
+			if err != nil {
+				r.logf("addr del failed: %v: %v\n%s", addrdel, err, out)
+				if errq == nil {
+					errq = err
+				}
+			}
+		}
+		addradd := []string{"ip", "addr",
+			"add", rs.LocalAddr.String(),
+			"dev", r.tunname}
+		out, err := cmd(addradd...).CombinedOutput()
+		if err != nil {
+			r.logf("addr add failed: %v: %v\n%s", addradd, err, out)
+			if errq == nil {
+				errq = err
+			}
+		}
+	}
+
+	newRoutes := make(map[wgcfg.CIDR]struct{})
+	for _, peer := range rs.Cfg.Peers {
+		for _, route := range peer.AllowedIPs {
+			newRoutes[route] = struct{}{}
+		}
+	}
+	for route := range r.routes {
+		if _, keep := newRoutes[route]; !keep {
+			net := route.IPNet()
+			nip := net.IP.Mask(net.Mask)
+			nstr := fmt.Sprintf("%v/%d", nip, route.Mask)
+			addrdel := []string{"ip", "route",
+				"del", nstr,
+				"via", r.local.IP.String(),
+				"dev", r.tunname}
+			out, err := cmd(addrdel...).CombinedOutput()
+			if err != nil {
+				r.logf("addr del failed: %v: %v\n%s", addrdel, err, out)
+				if errq == nil {
+					errq = err
+				}
+			}
+		}
+	}
+	for route := range newRoutes {
+		if _, exists := r.routes[route]; !exists {
+			net := route.IPNet()
+			nip := net.IP.Mask(net.Mask)
+			nstr := fmt.Sprintf("%v/%d", nip, route.Mask)
+			addradd := []string{"ip", "route",
+				"add", nstr,
+				"via", rs.LocalAddr.IP.String(),
+				"dev", r.tunname}
+			out, err := cmd(addradd...).CombinedOutput()
+			if err != nil {
+				r.logf("addr add failed: %v: %v\n%s", addradd, err, out)
+				if errq == nil {
+					errq = err
+				}
+			}
+		}
+	}
+
+	r.local = rs.LocalAddr
+	r.routes = newRoutes
+
+	// TODO: this:
+	if false {
+		if err := r.replaceResolvConf(rs.DNS, rs.DNSDomains); err != nil {
+			errq = fmt.Errorf("replacing resolv.conf failed: %v", err)
+		}
+	}
+	return errq
+}
+
+func (r *linuxRouter) forwardRule() []string {
+	return []string{
+		"-m", "comment", "--comment", "tailscale",
+		"-i", r.tunname,
+		"-j", "ACCEPT",
+	}
+}
+
+func (r *linuxRouter) natRule() []string {
+	// TODO(apenwarr): hardcoded eth0 interface is obviously not right.
+	return []string{
+		"-m", "comment", "--comment", "tailscale",
+		"-o", "eth0",
+		"-j", "MASQUERADE",
+	}
+}
+
+func (r *linuxRouter) Close() error {
+	var ret error
+	set := func(err error) {
+		if ret == nil && err != nil {
+			ret = err
+		}
+	}
+	if err := r.restoreResolvConf(); err != nil {
+		r.logf("failed to restore system resolv.conf: %v", err)
+		set(err)
+	}
+	set(r.ipt4.Delete("filter", "FORWARD", r.forwardRule()...))
+	set(r.ipt4.Delete("nat", "POSTROUTING", r.natRule()...))
+
+	// TODO(apenwarr): clean up routes etc.
+
+	return ret
+}
+
+const (
+	tsConf     = "/etc/resolv.tailscale.conf"
+	backupConf = "/etc/resolv.pre-tailscale-backup.conf"
+	resolvConf = "/etc/resolv.conf"
+)
+
+func (r *linuxRouter) replaceResolvConf(servers []wgcfg.IP, domains []string) error {
+	if len(servers) == 0 {
+		return r.restoreResolvConf()
+	}
+
+	// First write the tsConf file.
+	buf := new(bytes.Buffer)
+	fmt.Fprintf(buf, "# resolv.conf(5) file generated by tailscale\n")
+	fmt.Fprintf(buf, "#     DO NOT EDIT THIS FILE BY HAND -- CHANGES WILL BE OVERWRITTEN\n\n")
+	for _, ns := range servers {
+		fmt.Fprintf(buf, "nameserver %s\n", ns)
+	}
+	if len(domains) > 0 {
+		fmt.Fprintf(buf, "search "+strings.Join(domains, " ")+"\n")
+	}
+	f, err := ioutil.TempFile(filepath.Dir(tsConf), filepath.Base(tsConf)+".*")
+	if err != nil {
+		return err
+	}
+	f.Close()
+	if err := atomicfile.WriteFile(f.Name(), buf.Bytes(), 0644); err != nil {
+		return err
+	}
+	os.Chmod(f.Name(), 0644) // ioutil.TempFile creates the file with 0600
+	if err := os.Rename(f.Name(), tsConf); err != nil {
+		return err
+	}
+
+	if linkPath, err := os.Readlink(resolvConf); err != nil {
+		// Remove any old backup that may exist.
+		os.Remove(backupConf)
+
+		// Backup the existing /etc/resolv.conf file.
+		contents, err := ioutil.ReadFile(resolvConf)
+		if os.IsNotExist(err) {
+			// No existing /etc/resolv.conf file to backup.
+			// Nothing to do.
+			return nil
+		} else if err != nil {
+			return err
+		}
+		if err := atomicfile.WriteFile(backupConf, contents, 0644); err != nil {
+			return err
+		}
+	} else if linkPath != tsConf {
+		// Backup the existing symlink.
+		os.Remove(backupConf)
+		if err := os.Symlink(linkPath, backupConf); err != nil {
+			return err
+		}
+	} else {
+		// Nothing to do, resolvConf already points to tsConf.
+		return nil
+	}
+
+	os.Remove(resolvConf)
+	if err := os.Symlink(tsConf, resolvConf); err != nil {
+		return nil
+	}
+
+	out, _ := exec.Command("service", "systemd-resolved", "restart").CombinedOutput()
+	if len(out) > 0 {
+		r.logf("service systemd-resolved restart: %s", out)
+	}
+	return nil
+}
+
+func (r *linuxRouter) restoreResolvConf() error {
+	if _, err := os.Stat(backupConf); err != nil {
+		if os.IsNotExist(err) {
+			return nil // no backup resolv.conf to restore
+		}
+		return err
+	}
+	if ln, err := os.Readlink(resolvConf); err != nil {
+		return err
+	} else if ln != tsConf {
+		return fmt.Errorf("resolv.conf is not a symlink to %s", tsConf)
+	}
+	if err := os.Rename(backupConf, resolvConf); err != nil {
+		return err
+	}
+	os.Remove(tsConf) // best effort removal of tsConf file
+	out, _ := exec.Command("service", "systemd-resolved", "restart").CombinedOutput()
+	if len(out) > 0 {
+		r.logf("service systemd-resolved restart: %s", out)
+	}
+	return nil
+}

wgengine/router/router_openbsd.go

@@ -0,0 +1,261 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package router
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/tailscale/wireguard-go/device"
+	"github.com/tailscale/wireguard-go/tun"
+	"github.com/tailscale/wireguard-go/wgcfg"
+	"tailscale.com/atomicfile"
+	"tailscale.com/types/logger"
+)
+
+// For now this router only supports the WireGuard userspace implementation.
+// There is an experimental kernel version in the works for OpenBSD:
+// https://git.zx2c4.com/wireguard-openbsd.
+
+type openbsdRouter struct {
+	logf    logger.Logf
+	tunname string
+	local   wgcfg.CIDR
+	routes  map[wgcfg.CIDR]struct{}
+}
+
+func newUserspaceRouter(logf logger.Logf, _ *device.Device, tundev tun.Device) (Router, error) {
+	tunname, err := tundev.Name()
+	if err != nil {
+		return nil, err
+	}
+	return &openbsdRouter{
+		logf:    logf,
+		tunname: tunname,
+	}, nil
+}
+
+func cmd(args ...string) *exec.Cmd {
+	if len(args) == 0 {
+		log.Fatalf("exec.Cmd(%#v) invalid; need argv[0]\n", args)
+	}
+	return exec.Command(args[0], args[1:]...)
+}
+
+func (r *openbsdRouter) Up() error {
+	ifup := []string{"ifconfig", r.tunname, "up"}
+	if out, err := cmd(ifup...).CombinedOutput(); err != nil {
+		r.logf("running ifconfig failed: %v\n%s", err, out)
+		return err
+	}
+	return nil
+}
+
+func (r *openbsdRouter) SetRoutes(rs RouteSettings) error {
+	var errq error
+
+	if rs.LocalAddr != r.local {
+		if r.local != (wgcfg.CIDR{}) {
+			addrdel := []string{"ifconfig", r.tunname,
+				"inet", r.local.String(), "-alias"}
+			out, err := cmd(addrdel...).CombinedOutput()
+			if err != nil {
+				r.logf("addr del failed: %v: %v\n%s", addrdel, err, out)
+				if errq == nil {
+					errq = err
+				}
+			}
+
+			routedel := []string{"route", "-q", "-n",
+				"del", "-inet", r.local.String(),
+				"-iface", r.local.IP.String()}
+			if out, err := cmd(routedel...).CombinedOutput(); err != nil {
+				r.logf("route del failed: %v: %v\n%s", routedel, err, out)
+				if errq == nil {
+					errq = err
+				}
+			}
+		}
+
+		addradd := []string{"ifconfig", r.tunname,
+			"inet", rs.LocalAddr.String(), "alias"}
+		out, err := cmd(addradd...).CombinedOutput()
+		if err != nil {
+			r.logf("addr add failed: %v: %v\n%s", addradd, err, out)
+			if errq == nil {
+				errq = err
+			}
+		}
+
+		routeadd := []string{"route", "-q", "-n",
+			"add", "-inet", rs.LocalAddr.String(),
+			"-iface", rs.LocalAddr.IP.String()}
+		if out, err := cmd(routeadd...).CombinedOutput(); err != nil {
+			r.logf("route add failed: %v: %v\n%s", routeadd, err, out)
+			if errq == nil {
+				errq = err
+			}
+		}
+	}
+
+	newRoutes := make(map[wgcfg.CIDR]struct{})
+	for _, peer := range rs.Cfg.Peers {
+		for _, route := range peer.AllowedIPs {
+			newRoutes[route] = struct{}{}
+		}
+	}
+	for route := range r.routes {
+		if _, keep := newRoutes[route]; !keep {
+			net := route.IPNet()
+			nip := net.IP.Mask(net.Mask)
+			nstr := fmt.Sprintf("%v/%d", nip, route.Mask)
+			routedel := []string{"route", "-q", "-n",
+				"del", "-inet", nstr,
+				"-iface", rs.LocalAddr.IP.String()}
+			out, err := cmd(routedel...).CombinedOutput()
+			if err != nil {
+				r.logf("route del failed: %v: %v\n%s", routedel, err, out)
+				if errq == nil {
+					errq = err
+				}
+			}
+		}
+	}
+	for route := range newRoutes {
+		if _, exists := r.routes[route]; !exists {
+			net := route.IPNet()
+			nip := net.IP.Mask(net.Mask)
+			nstr := fmt.Sprintf("%v/%d", nip, route.Mask)
+			routeadd := []string{"route", "-q", "-n",
+				"add", "-inet", nstr,
+				"-iface", rs.LocalAddr.IP.String()}
+			out, err := cmd(routeadd...).CombinedOutput()
+			if err != nil {
+				r.logf("addr add failed: %v: %v\n%s", routeadd, err, out)
+				if errq == nil {
+					errq = err
+				}
+			}
+		}
+	}
+
+	r.local = rs.LocalAddr
+	r.routes = newRoutes
+
+	if err := r.replaceResolvConf(rs.DNS, rs.DNSDomains); err != nil {
+		errq = fmt.Errorf("replacing resolv.conf failed: %v", err)
+	}
+
+	return errq
+}
+
+func (r *openbsdRouter) Close() error {
+	out, err := cmd("ifconfig", r.tunname, "down").CombinedOutput()
+	if err != nil {
+		r.logf("running ifconfig failed: %v\n%s", err, out)
+	}
+
+	if err := r.restoreResolvConf(); err != nil {
+		r.logf("failed to restore system resolv.conf: %v", err)
+	}
+
+	return nil
+}
+
+const (
+	tsConf     = "/etc/resolv.tailscale.conf"
+	backupConf = "/etc/resolv.pre-tailscale-backup.conf"
+	resolvConf = "/etc/resolv.conf"
+)
+
+func (r *openbsdRouter) replaceResolvConf(servers []wgcfg.IP, domains []string) error {
+	if len(servers) == 0 {
+		return r.restoreResolvConf()
+	}
+
+	// Write the tsConf file.
+	buf := new(bytes.Buffer)
+	fmt.Fprintf(buf, "# resolv.conf(5) file generated by tailscale\n")
+	fmt.Fprintf(buf, "# DO NOT EDIT THIS FILE BY HAND -- CHANGES WILL BE OVERWRITTEN\n\n")
+	for _, ns := range servers {
+		fmt.Fprintf(buf, "nameserver %s\n", ns)
+	}
+	if len(domains) > 0 {
+		fmt.Fprintf(buf, "search "+strings.Join(domains, " ")+"\n")
+	}
+	tf, err := ioutil.TempFile(filepath.Dir(tsConf), filepath.Base(tsConf)+".*")
+	if err != nil {
+		return err
+	}
+	tempName := tf.Name()
+	tf.Close()
+
+	if err := atomicfile.WriteFile(tempName, buf.Bytes(), 0644); err != nil {
+		return err
+	}
+	if err := os.Rename(tempName, tsConf); err != nil {
+		return err
+	}
+
+	if linkPath, err := os.Readlink(resolvConf); err != nil {
+		// Remove any old backup that may exist.
+		os.Remove(backupConf)
+
+		// Backup the existing /etc/resolv.conf file.
+		contents, err := ioutil.ReadFile(resolvConf)
+		if os.IsNotExist(err) {
+			// No existing /etc/resolv.conf file to backup.
+			// Nothing to do.
+			return nil
+		} else if err != nil {
+			return err
+		}
+		if err := atomicfile.WriteFile(backupConf, contents, 0644); err != nil {
+			return err
+		}
+	} else if linkPath != tsConf {
+		// Backup the existing symlink.
+		os.Remove(backupConf)
+		if err := os.Symlink(linkPath, backupConf); err != nil {
+			return err
+		}
+	} else {
+		// Nothing to do, resolvConf already points to tsConf.
+		return nil
+	}
+
+	os.Remove(resolvConf)
+	if err := os.Symlink(tsConf, resolvConf); err != nil {
+		return nil
+	}
+
+	return nil
+}
+
+func (r *openbsdRouter) restoreResolvConf() error {
+	if _, err := os.Stat(backupConf); err != nil {
+		if os.IsNotExist(err) {
+			return nil // No backup resolv.conf to restore.
+		}
+		return err
+	}
+	if ln, err := os.Readlink(resolvConf); err != nil {
+		return err
+	} else if ln != tsConf {
+		return fmt.Errorf("resolv.conf is not a symlink to %s", tsConf)
+	}
+	if err := os.Rename(backupConf, resolvConf); err != nil {
+		return err
+	}
+	os.Remove(tsConf) // Best effort removal.
+
+	return nil
+}

wgengine/router/router_windows.go

@@ -0,0 +1,62 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package router
+
+import (
+	"log"
+
+	winipcfg "github.com/tailscale/winipcfg-go"
+	"github.com/tailscale/wireguard-go/device"
+	"github.com/tailscale/wireguard-go/tun"
+	"tailscale.com/types/logger"
+)
+
+type winRouter struct {
+	logf                func(fmt string, args ...interface{})
+	tunname             string
+	nativeTun           *tun.NativeTun
+	wgdev               *device.Device
+	routeChangeCallback *winipcfg.RouteChangeCallback
+}
+
+func newUserspaceRouter(logf logger.Logf, wgdev *device.Device, tundev tun.Device) (Router, error) {
+	tunname, err := tundev.Name()
+	if err != nil {
+		return nil, err
+	}
+	return &winRouter{
+		logf:      logf,
+		wgdev:     wgdev,
+		tunname:   tunname,
+		nativeTun: tundev.(*tun.NativeTun),
+	}, nil
+}
+
+func (r *winRouter) Up() error {
+	// MonitorDefaultRoutes handles making sure our wireguard UDP
+	// traffic goes through the old route, not recursively through the VPN.
+	var err error
+	r.routeChangeCallback, err = monitorDefaultRoutes(r.wgdev, true, r.nativeTun)
+	if err != nil {
+		log.Fatalf("MonitorDefaultRoutes: %v\n", err)
+	}
+	return nil
+}
+
+func (r *winRouter) SetRoutes(rs RouteSettings) error {
+	err := configureInterface(rs.Cfg, r.nativeTun, rs.DNS, rs.DNSDomains)
+	if err != nil {
+		r.logf("ConfigureInterface: %v\n", err)
+		return err
+	}
+	return nil
+}
+
+func (r *winRouter) Close() error {
+	if r.routeChangeCallback != nil {
+		r.routeChangeCallback.Unregister()
+	}
+	return nil
+}