// Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause package nat import ( "bytes" "cmp" "context" "errors" "flag" "fmt" "io" "net" "net/http" "net/netip" "os" "os/exec" "path/filepath" "strings" "sync" "testing" "time" "golang.org/x/mod/modfile" "golang.org/x/sync/errgroup" "tailscale.com/client/tailscale" "tailscale.com/ipn/ipnstate" "tailscale.com/syncs" "tailscale.com/tailcfg" "tailscale.com/tstest/natlab/vnet" ) var ( logTailscaled = flag.Bool("log-tailscaled", false, "log tailscaled output") pcapFile = flag.String("pcap", "", "write pcap to file") ) type natTest struct { tb testing.TB base string // base image tempDir string // for qcow2 images vnet *vnet.Server kernel string // linux kernel path gotRoute pingRoute } func newNatTest(tb testing.TB) *natTest { root, err := os.Getwd() if err != nil { tb.Fatal(err) } modRoot := filepath.Join(root, "../../..") nt := &natTest{ tb: tb, tempDir: tb.TempDir(), base: filepath.Join(modRoot, "gokrazy/tsapp.qcow2"), } if _, err := os.Stat(nt.base); err != nil { tb.Skipf("skipping test; base image %q not found", nt.base) } nt.kernel, err = findKernelPath(filepath.Join(modRoot, "gokrazy/tsapp/builddir/github.com/tailscale/gokrazy-kernel/go.mod")) if err != nil { tb.Skipf("skipping test; kernel not found: %v", err) } tb.Logf("found kernel: %v", nt.kernel) return nt } func findKernelPath(goMod string) (string, error) { b, err := os.ReadFile(goMod) if err != nil { return "", err } mf, err := modfile.Parse("go.mod", b, nil) if err != nil { return "", err } goModB, err := exec.Command("go", "env", "GOMODCACHE").CombinedOutput() if err != nil { return "", err } for _, r := range mf.Require { if r.Mod.Path == "github.com/tailscale/gokrazy-kernel" { return strings.TrimSpace(string(goModB)) + "/" + r.Mod.String() + "/vmlinuz", nil } } return "", fmt.Errorf("failed to find kernel in %v", goMod) } type addNodeFunc func(c *vnet.Config) *vnet.Node // returns nil to omit test func easy(c *vnet.Config) *vnet.Node { n := c.NumNodes() + 1 return c.AddNode(c.AddNetwork( fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT)) } // easy + host firewall func easyFW(c *vnet.Config) *vnet.Node { n := c.NumNodes() + 1 return c.AddNode(vnet.HostFirewall, c.AddNetwork( fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT)) } func easyAF(c *vnet.Config) *vnet.Node { n := c.NumNodes() + 1 return c.AddNode(c.AddNetwork( fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyAFNAT)) } func sameLAN(c *vnet.Config) *vnet.Node { nw := c.FirstNetwork() if nw == nil { return nil } if !nw.CanTakeMoreNodes() { return nil } return c.AddNode(nw) } func one2one(c *vnet.Config) *vnet.Node { n := c.NumNodes() + 1 return c.AddNode(c.AddNetwork( fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP fmt.Sprintf("172.16.%d.1/24", n), vnet.One2OneNAT)) } func easyPMP(c *vnet.Config) *vnet.Node { n := c.NumNodes() + 1 return c.AddNode(c.AddNetwork( fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT, vnet.NATPMP)) } // easy + port mapping + host firewall func easyPMPFW(c *vnet.Config) *vnet.Node { n := c.NumNodes() + 1 return c.AddNode(vnet.HostFirewall, c.AddNetwork( fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT, vnet.NATPMP)) } // easy + port mapping + host firewall - BPF func easyPMPFWNoBPF(c *vnet.Config) *vnet.Node { n := c.NumNodes() + 1 return c.AddNode( vnet.HostFirewall, vnet.TailscaledEnv{ Key: "TS_DEBUG_DISABLE_RAW_DISCO", Value: "1", }, c.AddNetwork( fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT, vnet.NATPMP)) } func hard(c *vnet.Config) *vnet.Node { n := c.NumNodes() + 1 return c.AddNode(c.AddNetwork( fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP fmt.Sprintf("10.0.%d.1/24", n), vnet.HardNAT)) } func hardPMP(c *vnet.Config) *vnet.Node { n := c.NumNodes() + 1 return c.AddNode(c.AddNetwork( fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP fmt.Sprintf("10.7.%d.1/24", n), vnet.HardNAT, vnet.NATPMP)) } func (nt *natTest) runTest(node1, node2 addNodeFunc) pingRoute { t := nt.tb var c vnet.Config c.SetPCAPFile(*pcapFile) nodes := []*vnet.Node{ node1(&c), node2(&c), } if nodes[0] == nil || nodes[1] == nil { t.Skip("skipping test; not applicable combination") } if *logTailscaled { nodes[0].SetVerboseSyslog(true) nodes[1].SetVerboseSyslog(true) } var err error nt.vnet, err = vnet.New(&c) if err != nil { t.Fatalf("newServer: %v", err) } nt.tb.Cleanup(func() { nt.vnet.Close() }) var wg sync.WaitGroup // waiting for srv.Accept goroutine defer wg.Wait() sockAddr := filepath.Join(nt.tempDir, "qemu.sock") srv, err := net.Listen("unix", sockAddr) if err != nil { t.Fatalf("Listen: %v", err) } defer srv.Close() wg.Add(1) go func() { defer wg.Done() for { c, err := srv.Accept() if err != nil { return } go nt.vnet.ServeUnixConn(c.(*net.UnixConn), vnet.ProtocolQEMU) } }() for i, node := range nodes { disk := fmt.Sprintf("%s/node-%d.qcow2", nt.tempDir, i) out, err := exec.Command("qemu-img", "create", "-f", "qcow2", "-F", "qcow2", "-b", nt.base, disk).CombinedOutput() if err != nil { t.Fatalf("qemu-img create: %v, %s", err, out) } var envBuf bytes.Buffer for _, e := range node.Env() { fmt.Fprintf(&envBuf, " tailscaled.env=%s=%s", e.Key, e.Value) } envStr := envBuf.String() cmd := exec.Command("qemu-system-x86_64", "-M", "microvm,isa-serial=off", "-m", "384M", "-nodefaults", "-no-user-config", "-nographic", "-kernel", nt.kernel, "-append", "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-dd02023b0001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic pci=off nousb tsc=unstable clocksource=hpet gokrazy.remote_syslog.target=52.52.0.9:995 tailscale-tta=1"+envStr, "-drive", "id=blk0,file="+disk+",format=qcow2", "-device", "virtio-blk-device,drive=blk0", "-netdev", "stream,id=net0,addr.type=unix,addr.path="+sockAddr, "-device", "virtio-serial-device", "-device", "virtio-net-device,netdev=net0,mac="+node.MAC().String(), "-chardev", "stdio,id=virtiocon0,mux=on", "-device", "virtconsole,chardev=virtiocon0", "-mon", "chardev=virtiocon0,mode=readline", "-audio", "none", ) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Start(); err != nil { t.Fatalf("qemu: %v", err) } nt.tb.Cleanup(func() { cmd.Process.Kill() cmd.Wait() }) } ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() lc1 := nt.vnet.NodeAgentClient(nodes[0]) lc2 := nt.vnet.NodeAgentClient(nodes[1]) clients := []*vnet.NodeAgentClient{lc1, lc2} var eg errgroup.Group var sts [2]*ipnstate.Status for i, c := range clients { i, c := i, c eg.Go(func() error { st, err := c.Status(ctx) if err != nil { return fmt.Errorf("node%d status: %w", i, err) } t.Logf("node%d status: %v", i, st) node := nodes[i] if node.HostFirewall() { if err := c.EnableHostFirewall(ctx); err != nil { return fmt.Errorf("node%d firewall: %w", i, err) } t.Logf("node%d firewalled", i) } if err := up(ctx, c); err != nil { return fmt.Errorf("node%d up: %w", i, err) } t.Logf("node%d up!", i) st, err = c.Status(ctx) if err != nil { return fmt.Errorf("node%d status: %w", i, err) } sts[i] = st if st.BackendState != "Running" { return fmt.Errorf("node%d state = %q", i, st.BackendState) } t.Logf("node%d up with %v", i, sts[i].Self.TailscaleIPs) return nil }) } if err := eg.Wait(); err != nil { t.Fatalf("initial setup: %v", err) } defer nt.vnet.Close() pingRes, err := ping(ctx, lc1, sts[1].Self.TailscaleIPs[0]) if err != nil { t.Fatalf("ping failure: %v", err) } nt.gotRoute = classifyPing(pingRes) t.Logf("ping route: %v", nt.gotRoute) return nt.gotRoute } func classifyPing(pr *ipnstate.PingResult) pingRoute { if pr == nil { return routeNil } if pr.Endpoint != "" { ap, err := netip.ParseAddrPort(pr.Endpoint) if err == nil { if ap.Addr().IsPrivate() { return routeLocal } return routeDirect } } return routeDERP // presumably } type pingRoute string const ( routeDERP pingRoute = "derp" routeLocal pingRoute = "local" routeDirect pingRoute = "direct" routeNil pingRoute = "nil" // *ipnstate.PingResult is nil ) func ping(ctx context.Context, c *vnet.NodeAgentClient, target netip.Addr) (*ipnstate.PingResult, error) { n := 0 var res *ipnstate.PingResult anyPong := false for n < 10 { n++ pr, err := c.PingWithOpts(ctx, target, tailcfg.PingDisco, tailscale.PingOpts{}) if err != nil { if anyPong { return res, nil } return nil, err } if pr.Err != "" { return nil, errors.New(pr.Err) } if pr.DERPRegionID == 0 { return pr, nil } res = pr select { case <-ctx.Done(): case <-time.After(time.Second): } } if res == nil { return nil, errors.New("no ping response") } return res, nil } func up(ctx context.Context, c *vnet.NodeAgentClient) error { req, err := http.NewRequestWithContext(ctx, "GET", "http://unused/up", nil) if err != nil { return err } res, err := c.HTTPClient.Do(req) if err != nil { return err } defer res.Body.Close() all, _ := io.ReadAll(res.Body) if res.StatusCode != 200 { return fmt.Errorf("unexpected status code %v: %s", res.Status, all) } return nil } type nodeType struct { name string fn addNodeFunc } var types = []nodeType{ {"easy", easy}, {"easyAF", easyAF}, {"hard", hard}, {"easyPMP", easyPMP}, {"hardPMP", hardPMP}, {"one2one", one2one}, {"sameLAN", sameLAN}, } // want sets the expected ping route for the test. func (nt *natTest) want(r pingRoute) { if nt.gotRoute != r { nt.tb.Errorf("ping route = %v; want %v", nt.gotRoute, r) } } func TestEasyEasy(t *testing.T) { nt := newNatTest(t) nt.runTest(easy, easy) nt.want(routeDirect) } func TestSameLAN(t *testing.T) { nt := newNatTest(t) nt.runTest(easy, sameLAN) nt.want(routeLocal) } // TestBPFDisco tests https://github.com/tailscale/tailscale/issues/3824 ... // * server behind a Hard NAT // * client behind a NAT with UPnP support // * client machine has a stateful host firewall (e.g. ufw) func TestBPFDisco(t *testing.T) { nt := newNatTest(t) nt.runTest(easyPMPFW, hard) nt.want(routeDirect) } func TestHostFWNoBPF(t *testing.T) { nt := newNatTest(t) nt.runTest(easyPMPFWNoBPF, hard) nt.want(routeDERP) } func TestHostFWPair(t *testing.T) { nt := newNatTest(t) nt.runTest(easyFW, easyFW) nt.want(routeDirect) } func TestOneHostFW(t *testing.T) { nt := newNatTest(t) nt.runTest(easy, easyFW) nt.want(routeDirect) } var pair = flag.String("pair", "", "comma-separated pair of types to test (easy, easyAF, hard, easyPMP, hardPMP, one2one, sameLAN)") func TestPair(t *testing.T) { t1, t2, ok := strings.Cut(*pair, ",") if !ok { t.Skipf("skipping test without --pair=type1,type2 set") } find := func(name string) addNodeFunc { for _, nt := range types { if nt.name == name { return nt.fn } } t.Fatalf("unknown type %q", name) return nil } nt := newNatTest(t) nt.runTest(find(t1), find(t2)) } var runGrid = flag.Bool("run-grid", false, "run grid test") func TestGrid(t *testing.T) { if !*runGrid { t.Skip("skipping grid test; set --run-grid to run") } t.Parallel() sem := syncs.NewSemaphore(2) var ( mu sync.Mutex res = make(map[string]pingRoute) ) for _, a := range types { for _, b := range types { key := a.name + "-" + b.name keyBack := b.name + "-" + a.name t.Run(key, func(t *testing.T) { t.Parallel() sem.Acquire() defer sem.Release() filename := key + ".cache" contents, _ := os.ReadFile(filename) if len(contents) == 0 { filename2 := keyBack + ".cache" contents, _ = os.ReadFile(filename2) } route := pingRoute(strings.TrimSpace(string(contents))) if route == "" { nt := newNatTest(t) route = nt.runTest(a.fn, b.fn) if err := os.WriteFile(filename, []byte(string(route)), 0666); err != nil { t.Fatalf("writeFile: %v", err) } } mu.Lock() defer mu.Unlock() res[key] = route t.Logf("results: %v", res) }) } } t.Cleanup(func() { mu.Lock() defer mu.Unlock() var hb bytes.Buffer pf := func(format string, args ...any) { fmt.Fprintf(&hb, format, args...) } rewrite := func(s string) string { return strings.ReplaceAll(s, "PMP", "+pm") } pf("
") for _, a := range types { pf(" | %s | ", rewrite(a.name)) } pf("||
%s | ", rewrite(a.name)) for _, b := range types { key := a.name + "-" + b.name key2 := b.name + "-" + a.name v := cmp.Or(res[key], res[key2], "-") if v == "derp" { pf("%s | ", v)
} else if v == "local" {
pf("%s | ", v)
} else {
pf("%s | ", v) } } pf("