diff --git a/tstest/integration/integration.go b/tstest/integration/integration.go index 36a92759f..2761f807d 100644 --- a/tstest/integration/integration.go +++ b/tstest/integration/integration.go @@ -9,19 +9,24 @@ package integration import ( "bytes" + "context" "crypto/tls" "encoding/json" + "flag" "fmt" "io" "log" "net" "net/http" "net/http/httptest" + "net/netip" "os" "os/exec" "path" "path/filepath" + "regexp" "runtime" + "strconv" "strings" "sync" "testing" @@ -30,16 +35,35 @@ import ( "go4.org/mem" "tailscale.com/derp" "tailscale.com/derp/derphttp" + "tailscale.com/ipn" + "tailscale.com/ipn/ipnlocal" + "tailscale.com/ipn/ipnstate" + "tailscale.com/ipn/store" "tailscale.com/net/stun/stuntest" + "tailscale.com/safesocket" + "tailscale.com/syncs" "tailscale.com/tailcfg" + "tailscale.com/tstest" + "tailscale.com/tstest/integration/testcontrol" "tailscale.com/types/key" "tailscale.com/types/logger" "tailscale.com/types/logid" "tailscale.com/types/nettype" + "tailscale.com/util/rands" "tailscale.com/util/zstdframe" "tailscale.com/version" ) +var ( + verboseTailscaled = flag.Bool("verbose-tailscaled", false, "verbose tailscaled logging") + verboseTailscale = flag.Bool("verbose-tailscale", false, "verbose tailscale CLI logging") +) + +// MainError is an error that's set if an error conditions happens outside of a +// context where a testing.TB is available. The caller can check it in its TestMain +// as a last ditch place to report errors. +var MainError syncs.AtomicValue[error] + // CleanupBinaries cleans up any resources created by calls to BinaryDir, TailscaleBinary, or TailscaledBinary. // It should be called from TestMain after all tests have completed. func CleanupBinaries() { @@ -361,3 +385,583 @@ func (lc *LogCatcher) ServeHTTP(w http.ResponseWriter, r *http.Request) { } w.WriteHeader(200) // must have no content, but not a 204 } + +// testEnv contains the test environment (set of servers) used by one +// or more nodes. +type testEnv struct { + t testing.TB + tunMode bool + cli string + daemon string + loopbackPort *int + + LogCatcher *LogCatcher + LogCatcherServer *httptest.Server + + Control *testcontrol.Server + ControlServer *httptest.Server + + TrafficTrap *trafficTrap + TrafficTrapServer *httptest.Server +} + +// controlURL returns e.ControlServer.URL, panicking if it's the empty string, +// which it should never be in tests. +func (e *testEnv) controlURL() string { + s := e.ControlServer.URL + if s == "" { + panic("control server not set") + } + return s +} + +type testEnvOpt interface { + modifyTestEnv(*testEnv) +} + +type configureControl func(*testcontrol.Server) + +func (f configureControl) modifyTestEnv(te *testEnv) { + f(te.Control) +} + +// newTestEnv starts a bunch of services and returns a new test environment. +// newTestEnv arranges for the environment's resources to be cleaned up on exit. +func newTestEnv(t testing.TB, opts ...testEnvOpt) *testEnv { + if runtime.GOOS == "windows" { + t.Skip("not tested/working on Windows yet") + } + derpMap := RunDERPAndSTUN(t, logger.Discard, "127.0.0.1") + logc := new(LogCatcher) + control := &testcontrol.Server{ + DERPMap: derpMap, + } + control.HTTPTestServer = httptest.NewUnstartedServer(control) + trafficTrap := new(trafficTrap) + e := &testEnv{ + t: t, + cli: TailscaleBinary(t), + daemon: TailscaledBinary(t), + LogCatcher: logc, + LogCatcherServer: httptest.NewServer(logc), + Control: control, + ControlServer: control.HTTPTestServer, + TrafficTrap: trafficTrap, + TrafficTrapServer: httptest.NewServer(trafficTrap), + } + for _, o := range opts { + o.modifyTestEnv(e) + } + control.HTTPTestServer.Start() + t.Cleanup(func() { + // Shut down e. + if err := e.TrafficTrap.Err(); err != nil { + e.t.Errorf("traffic trap: %v", err) + e.t.Logf("logs: %s", e.LogCatcher.logsString()) + } + e.LogCatcherServer.Close() + e.TrafficTrapServer.Close() + e.ControlServer.Close() + }) + t.Logf("control URL: %v", e.controlURL()) + return e +} + +// testNode is a machine with a tailscale & tailscaled. +// Currently, the test is simplistic and user==node==machine. +// That may grow complexity later to test more. +type testNode struct { + env *testEnv + tailscaledParser *nodeOutputParser + + dir string // temp dir for sock & state + configFile string // or empty for none + sockFile string + stateFile string + upFlagGOOS string // if non-empty, sets TS_DEBUG_UP_FLAG_GOOS for cmd/tailscale CLI + + mu sync.Mutex + onLogLine []func([]byte) +} + +// newTestNode allocates a temp directory for a new test node. +// The node is not started automatically. +func newTestNode(t *testing.T, env *testEnv) *testNode { + dir := t.TempDir() + sockFile := filepath.Join(dir, "tailscale.sock") + if len(sockFile) >= 104 { + // Maximum length for a unix socket on darwin. Try something else. + sockFile = filepath.Join(os.TempDir(), rands.HexString(8)+".sock") + t.Cleanup(func() { os.Remove(sockFile) }) + } + n := &testNode{ + env: env, + dir: dir, + sockFile: sockFile, + stateFile: filepath.Join(dir, "tailscale.state"), + } + + // Look for a data race. Once we see the start marker, start logging the rest. + var sawRace bool + var sawPanic bool + n.addLogLineHook(func(line []byte) { + lineB := mem.B(line) + if mem.Contains(lineB, mem.S("WARNING: DATA RACE")) { + sawRace = true + } + if mem.HasPrefix(lineB, mem.S("panic: ")) { + sawPanic = true + } + if sawRace || sawPanic { + t.Logf("%s", line) + } + }) + + return n +} + +func (n *testNode) diskPrefs() *ipn.Prefs { + t := n.env.t + t.Helper() + if _, err := os.ReadFile(n.stateFile); err != nil { + t.Fatalf("reading prefs: %v", err) + } + fs, err := store.NewFileStore(nil, n.stateFile) + if err != nil { + t.Fatalf("reading prefs, NewFileStore: %v", err) + } + p, err := ipnlocal.ReadStartupPrefsForTest(t.Logf, fs) + if err != nil { + t.Fatalf("reading prefs, ReadDiskPrefsForTest: %v", err) + } + return p.AsStruct() +} + +// AwaitResponding waits for n's tailscaled to be up enough to be +// responding, but doesn't wait for any particular state. +func (n *testNode) AwaitResponding() { + t := n.env.t + t.Helper() + n.AwaitListening() + + st := n.MustStatus() + t.Logf("Status: %s", st.BackendState) + + if err := tstest.WaitFor(20*time.Second, func() error { + const sub = `Program starting: ` + if !n.env.LogCatcher.logsContains(mem.S(sub)) { + return fmt.Errorf("log catcher didn't see %#q; got %s", sub, n.env.LogCatcher.logsString()) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// addLogLineHook registers a hook f to be called on each tailscaled +// log line output. +func (n *testNode) addLogLineHook(f func([]byte)) { + n.mu.Lock() + defer n.mu.Unlock() + n.onLogLine = append(n.onLogLine, f) +} + +// socks5AddrChan returns a channel that receives the address (e.g. "localhost:23874") +// of the node's SOCKS5 listener, once started. +func (n *testNode) socks5AddrChan() <-chan string { + ch := make(chan string, 1) + n.addLogLineHook(func(line []byte) { + const sub = "SOCKS5 listening on " + i := mem.Index(mem.B(line), mem.S(sub)) + if i == -1 { + return + } + addr := strings.TrimSpace(string(line)[i+len(sub):]) + select { + case ch <- addr: + default: + } + }) + return ch +} + +func (n *testNode) AwaitSocksAddr(ch <-chan string) string { + t := n.env.t + t.Helper() + timer := time.NewTimer(10 * time.Second) + defer timer.Stop() + select { + case v := <-ch: + return v + case <-timer.C: + t.Fatal("timeout waiting for node to log its SOCK5 listening address") + panic("unreachable") + } +} + +// nodeOutputParser parses stderr of tailscaled processes, calling the +// per-line callbacks previously registered via +// testNode.addLogLineHook. +type nodeOutputParser struct { + allBuf bytes.Buffer + pendLineBuf bytes.Buffer + n *testNode +} + +func (op *nodeOutputParser) Write(p []byte) (n int, err error) { + tn := op.n + tn.mu.Lock() + defer tn.mu.Unlock() + + op.allBuf.Write(p) + n, err = op.pendLineBuf.Write(p) + op.parseLinesLocked() + return +} + +func (op *nodeOutputParser) parseLinesLocked() { + n := op.n + buf := op.pendLineBuf.Bytes() + for len(buf) > 0 { + nl := bytes.IndexByte(buf, '\n') + if nl == -1 { + break + } + line := buf[:nl+1] + buf = buf[nl+1:] + + for _, f := range n.onLogLine { + f(line) + } + } + if len(buf) == 0 { + op.pendLineBuf.Reset() + } else { + io.CopyN(io.Discard, &op.pendLineBuf, int64(op.pendLineBuf.Len()-len(buf))) + } +} + +type Daemon struct { + Process *os.Process +} + +func (d *Daemon) MustCleanShutdown(t testing.TB) { + d.Process.Signal(os.Interrupt) + ps, err := d.Process.Wait() + if err != nil { + t.Fatalf("tailscaled Wait: %v", err) + } + if ps.ExitCode() != 0 { + t.Errorf("tailscaled ExitCode = %d; want 0", ps.ExitCode()) + } +} + +// StartDaemon starts the node's tailscaled, failing if it fails to start. +// StartDaemon ensures that the process will exit when the test completes. +func (n *testNode) StartDaemon() *Daemon { + return n.StartDaemonAsIPNGOOS(runtime.GOOS) +} + +func (n *testNode) StartDaemonAsIPNGOOS(ipnGOOS string) *Daemon { + t := n.env.t + cmd := exec.Command(n.env.daemon) + cmd.Args = append(cmd.Args, + "--state="+n.stateFile, + "--socket="+n.sockFile, + "--socks5-server=localhost:0", + ) + if *verboseTailscaled { + cmd.Args = append(cmd.Args, "-verbose=2") + } + if !n.env.tunMode { + cmd.Args = append(cmd.Args, + "--tun=userspace-networking", + ) + } + if n.configFile != "" { + cmd.Args = append(cmd.Args, "--config="+n.configFile) + } + cmd.Env = append(os.Environ(), + "TS_CONTROL_IS_PLAINTEXT_HTTP=1", + "TS_DEBUG_PERMIT_HTTP_C2N=1", + "TS_LOG_TARGET="+n.env.LogCatcherServer.URL, + "HTTP_PROXY="+n.env.TrafficTrapServer.URL, + "HTTPS_PROXY="+n.env.TrafficTrapServer.URL, + "TS_DEBUG_FAKE_GOOS="+ipnGOOS, + "TS_LOGS_DIR="+t.TempDir(), + "TS_NETCHECK_GENERATE_204_URL="+n.env.ControlServer.URL+"/generate_204", + "TS_ASSUME_NETWORK_UP_FOR_TEST=1", // don't pause control client in airplane mode (no wifi, etc) + "TS_PANIC_IF_HIT_MAIN_CONTROL=1", + "TS_DISABLE_PORTMAPPER=1", // shouldn't be needed; test is all localhost + "TS_DEBUG_LOG_RATE=all", + ) + if n.env.loopbackPort != nil { + cmd.Env = append(cmd.Env, "TS_DEBUG_NETSTACK_LOOPBACK_PORT="+strconv.Itoa(*n.env.loopbackPort)) + } + if version.IsRace() { + cmd.Env = append(cmd.Env, "GORACE=halt_on_error=1") + } + n.tailscaledParser = &nodeOutputParser{n: n} + cmd.Stderr = n.tailscaledParser + if *verboseTailscaled { + cmd.Stdout = os.Stdout + cmd.Stderr = io.MultiWriter(cmd.Stderr, os.Stderr) + } + if runtime.GOOS != "windows" { + pr, pw, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { pw.Close() }) + cmd.ExtraFiles = append(cmd.ExtraFiles, pr) + cmd.Env = append(cmd.Env, "TS_PARENT_DEATH_FD=3") + } + if err := cmd.Start(); err != nil { + t.Fatalf("starting tailscaled: %v", err) + } + t.Cleanup(func() { cmd.Process.Kill() }) + return &Daemon{ + Process: cmd.Process, + } +} + +func (n *testNode) MustUp(extraArgs ...string) { + t := n.env.t + t.Helper() + args := []string{ + "up", + "--login-server=" + n.env.controlURL(), + "--reset", + } + args = append(args, extraArgs...) + cmd := n.Tailscale(args...) + t.Logf("Running %v ...", cmd) + cmd.Stdout = nil // in case --verbose-tailscale was set + cmd.Stderr = nil // in case --verbose-tailscale was set + if b, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("up: %v, %v", string(b), err) + } +} + +func (n *testNode) MustDown() { + t := n.env.t + t.Logf("Running down ...") + if err := n.Tailscale("down", "--accept-risk=all").Run(); err != nil { + t.Fatalf("down: %v", err) + } +} + +func (n *testNode) MustLogOut() { + t := n.env.t + t.Logf("Running logout ...") + if err := n.Tailscale("logout").Run(); err != nil { + t.Fatalf("logout: %v", err) + } +} + +func (n *testNode) Ping(otherNode *testNode) error { + t := n.env.t + ip := otherNode.AwaitIP4().String() + t.Logf("Running ping %v (from %v)...", ip, n.AwaitIP4()) + return n.Tailscale("ping", ip).Run() +} + +// AwaitListening waits for the tailscaled to be serving local clients +// over its localhost IPC mechanism. (Unix socket, etc) +func (n *testNode) AwaitListening() { + t := n.env.t + if err := tstest.WaitFor(20*time.Second, func() (err error) { + c, err := safesocket.ConnectContext(context.Background(), n.sockFile) + if err == nil { + c.Close() + } + return err + }); err != nil { + t.Fatal(err) + } +} + +func (n *testNode) AwaitIPs() []netip.Addr { + t := n.env.t + t.Helper() + var addrs []netip.Addr + if err := tstest.WaitFor(20*time.Second, func() error { + cmd := n.Tailscale("ip") + cmd.Stdout = nil // in case --verbose-tailscale was set + cmd.Stderr = nil // in case --verbose-tailscale was set + out, err := cmd.Output() + if err != nil { + return err + } + ips := string(out) + ipslice := strings.Fields(ips) + addrs = make([]netip.Addr, len(ipslice)) + + for i, ip := range ipslice { + netIP, err := netip.ParseAddr(ip) + if err != nil { + t.Fatal(err) + } + addrs[i] = netIP + } + return nil + }); err != nil { + t.Fatalf("awaiting an IP address: %v", err) + } + if len(addrs) == 0 { + t.Fatalf("returned IP address was blank") + } + return addrs +} + +// AwaitIP4 returns the IPv4 address of n. +func (n *testNode) AwaitIP4() netip.Addr { + t := n.env.t + t.Helper() + ips := n.AwaitIPs() + return ips[0] +} + +// AwaitIP6 returns the IPv6 address of n. +func (n *testNode) AwaitIP6() netip.Addr { + t := n.env.t + t.Helper() + ips := n.AwaitIPs() + return ips[1] +} + +// AwaitRunning waits for n to reach the IPN state "Running". +func (n *testNode) AwaitRunning() { + t := n.env.t + t.Helper() + n.AwaitBackendState("Running") +} + +func (n *testNode) AwaitBackendState(state string) { + t := n.env.t + t.Helper() + if err := tstest.WaitFor(20*time.Second, func() error { + st, err := n.Status() + if err != nil { + return err + } + if st.BackendState != state { + return fmt.Errorf("in state %q; want %q", st.BackendState, state) + } + return nil + }); err != nil { + t.Fatalf("failure/timeout waiting for transition to Running status: %v", err) + } +} + +// AwaitNeedsLogin waits for n to reach the IPN state "NeedsLogin". +func (n *testNode) AwaitNeedsLogin() { + t := n.env.t + t.Helper() + if err := tstest.WaitFor(20*time.Second, func() error { + st, err := n.Status() + if err != nil { + return err + } + if st.BackendState != "NeedsLogin" { + return fmt.Errorf("in state %q", st.BackendState) + } + return nil + }); err != nil { + t.Fatalf("failure/timeout waiting for transition to NeedsLogin status: %v", err) + } +} + +func (n *testNode) TailscaleForOutput(arg ...string) *exec.Cmd { + cmd := n.Tailscale(arg...) + cmd.Stdout = nil + cmd.Stderr = nil + return cmd +} + +// Tailscale returns a command that runs the tailscale CLI with the provided arguments. +// It does not start the process. +func (n *testNode) Tailscale(arg ...string) *exec.Cmd { + cmd := exec.Command(n.env.cli) + cmd.Args = append(cmd.Args, "--socket="+n.sockFile) + cmd.Args = append(cmd.Args, arg...) + cmd.Dir = n.dir + cmd.Env = append(os.Environ(), + "TS_DEBUG_UP_FLAG_GOOS="+n.upFlagGOOS, + "TS_LOGS_DIR="+n.env.t.TempDir(), + ) + if *verboseTailscale { + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + } + return cmd +} + +func (n *testNode) Status() (*ipnstate.Status, error) { + cmd := n.Tailscale("status", "--json") + cmd.Stdout = nil // in case --verbose-tailscale was set + cmd.Stderr = nil // in case --verbose-tailscale was set + out, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("running tailscale status: %v, %s", err, out) + } + st := new(ipnstate.Status) + if err := json.Unmarshal(out, st); err != nil { + return nil, fmt.Errorf("decoding tailscale status JSON: %w\njson:\n%s", err, out) + } + return st, nil +} + +func (n *testNode) MustStatus() *ipnstate.Status { + tb := n.env.t + tb.Helper() + st, err := n.Status() + if err != nil { + tb.Fatal(err) + } + return st +} + +// trafficTrap is an HTTP proxy handler to note whether any +// HTTP traffic tries to leave localhost from tailscaled. We don't +// expect any, so any request triggers a failure. +type trafficTrap struct { + atomicErr syncs.AtomicValue[error] +} + +func (tt *trafficTrap) Err() error { + return tt.atomicErr.Load() +} + +func (tt *trafficTrap) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var got bytes.Buffer + r.Write(&got) + err := fmt.Errorf("unexpected HTTP request via proxy: %s", got.Bytes()) + MainError.Store(err) + if tt.Err() == nil { + // Best effort at remembering the first request. + tt.atomicErr.Store(err) + } + log.Printf("Error: %v", err) + w.WriteHeader(403) +} + +type authURLParserWriter struct { + buf bytes.Buffer + fn func(urlStr string) error +} + +var authURLRx = regexp.MustCompile(`(https?://\S+/auth/\S+)`) + +func (w *authURLParserWriter) Write(p []byte) (n int, err error) { + n, err = w.buf.Write(p) + m := authURLRx.FindSubmatch(w.buf.Bytes()) + if m != nil { + urlStr := string(m[1]) + w.buf.Reset() // so it's not matched again + if err := w.fn(urlStr); err != nil { + return 0, err + } + } + return n, err +} diff --git a/tstest/integration/integration_test.go b/tstest/integration/integration_test.go index 20d8908da..d2f054361 100644 --- a/tstest/integration/integration_test.go +++ b/tstest/integration/integration_test.go @@ -13,7 +13,6 @@ import ( "flag" "fmt" "io" - "log" "net" "net/http" "net/http/httptest" @@ -22,10 +21,7 @@ import ( "os/exec" "path/filepath" "regexp" - "runtime" "strconv" - "strings" - "sync" "sync/atomic" "testing" "time" @@ -37,32 +33,17 @@ import ( "tailscale.com/clientupdate" "tailscale.com/cmd/testwrapper/flakytest" "tailscale.com/ipn" - "tailscale.com/ipn/ipnlocal" - "tailscale.com/ipn/ipnstate" - "tailscale.com/ipn/store" "tailscale.com/net/tsaddr" "tailscale.com/net/tstun" - "tailscale.com/safesocket" - "tailscale.com/syncs" "tailscale.com/tailcfg" "tailscale.com/tstest" "tailscale.com/tstest/integration/testcontrol" "tailscale.com/types/key" - "tailscale.com/types/logger" "tailscale.com/types/opt" "tailscale.com/types/ptr" "tailscale.com/util/must" - "tailscale.com/util/rands" - "tailscale.com/version" ) -var ( - verboseTailscaled = flag.Bool("verbose-tailscaled", false, "verbose tailscaled logging") - verboseTailscale = flag.Bool("verbose-tailscale", false, "verbose tailscale CLI logging") -) - -var mainError syncs.AtomicValue[error] - func TestMain(m *testing.M) { // Have to disable UPnP which hits the network, otherwise it fails due to HTTP proxy. os.Setenv("TS_DISABLE_UPNP", "true") @@ -72,7 +53,7 @@ func TestMain(m *testing.M) { if v != 0 { os.Exit(v) } - if err := mainError.Load(); err != nil { + if err := MainError.Load(); err != nil { fmt.Fprintf(os.Stderr, "FAIL: %v\n", err) os.Exit(1) } @@ -1485,583 +1466,3 @@ func TestNetstackUDPLoopback(t *testing.T) { d1.MustCleanShutdown(t) } - -// testEnv contains the test environment (set of servers) used by one -// or more nodes. -type testEnv struct { - t testing.TB - tunMode bool - cli string - daemon string - loopbackPort *int - - LogCatcher *LogCatcher - LogCatcherServer *httptest.Server - - Control *testcontrol.Server - ControlServer *httptest.Server - - TrafficTrap *trafficTrap - TrafficTrapServer *httptest.Server -} - -// controlURL returns e.ControlServer.URL, panicking if it's the empty string, -// which it should never be in tests. -func (e *testEnv) controlURL() string { - s := e.ControlServer.URL - if s == "" { - panic("control server not set") - } - return s -} - -type testEnvOpt interface { - modifyTestEnv(*testEnv) -} - -type configureControl func(*testcontrol.Server) - -func (f configureControl) modifyTestEnv(te *testEnv) { - f(te.Control) -} - -// newTestEnv starts a bunch of services and returns a new test environment. -// newTestEnv arranges for the environment's resources to be cleaned up on exit. -func newTestEnv(t testing.TB, opts ...testEnvOpt) *testEnv { - if runtime.GOOS == "windows" { - t.Skip("not tested/working on Windows yet") - } - derpMap := RunDERPAndSTUN(t, logger.Discard, "127.0.0.1") - logc := new(LogCatcher) - control := &testcontrol.Server{ - DERPMap: derpMap, - } - control.HTTPTestServer = httptest.NewUnstartedServer(control) - trafficTrap := new(trafficTrap) - e := &testEnv{ - t: t, - cli: TailscaleBinary(t), - daemon: TailscaledBinary(t), - LogCatcher: logc, - LogCatcherServer: httptest.NewServer(logc), - Control: control, - ControlServer: control.HTTPTestServer, - TrafficTrap: trafficTrap, - TrafficTrapServer: httptest.NewServer(trafficTrap), - } - for _, o := range opts { - o.modifyTestEnv(e) - } - control.HTTPTestServer.Start() - t.Cleanup(func() { - // Shut down e. - if err := e.TrafficTrap.Err(); err != nil { - e.t.Errorf("traffic trap: %v", err) - e.t.Logf("logs: %s", e.LogCatcher.logsString()) - } - e.LogCatcherServer.Close() - e.TrafficTrapServer.Close() - e.ControlServer.Close() - }) - t.Logf("control URL: %v", e.controlURL()) - return e -} - -// testNode is a machine with a tailscale & tailscaled. -// Currently, the test is simplistic and user==node==machine. -// That may grow complexity later to test more. -type testNode struct { - env *testEnv - tailscaledParser *nodeOutputParser - - dir string // temp dir for sock & state - configFile string // or empty for none - sockFile string - stateFile string - upFlagGOOS string // if non-empty, sets TS_DEBUG_UP_FLAG_GOOS for cmd/tailscale CLI - - mu sync.Mutex - onLogLine []func([]byte) -} - -// newTestNode allocates a temp directory for a new test node. -// The node is not started automatically. -func newTestNode(t *testing.T, env *testEnv) *testNode { - dir := t.TempDir() - sockFile := filepath.Join(dir, "tailscale.sock") - if len(sockFile) >= 104 { - // Maximum length for a unix socket on darwin. Try something else. - sockFile = filepath.Join(os.TempDir(), rands.HexString(8)+".sock") - t.Cleanup(func() { os.Remove(sockFile) }) - } - n := &testNode{ - env: env, - dir: dir, - sockFile: sockFile, - stateFile: filepath.Join(dir, "tailscale.state"), - } - - // Look for a data race. Once we see the start marker, start logging the rest. - var sawRace bool - var sawPanic bool - n.addLogLineHook(func(line []byte) { - lineB := mem.B(line) - if mem.Contains(lineB, mem.S("WARNING: DATA RACE")) { - sawRace = true - } - if mem.HasPrefix(lineB, mem.S("panic: ")) { - sawPanic = true - } - if sawRace || sawPanic { - t.Logf("%s", line) - } - }) - - return n -} - -func (n *testNode) diskPrefs() *ipn.Prefs { - t := n.env.t - t.Helper() - if _, err := os.ReadFile(n.stateFile); err != nil { - t.Fatalf("reading prefs: %v", err) - } - fs, err := store.NewFileStore(nil, n.stateFile) - if err != nil { - t.Fatalf("reading prefs, NewFileStore: %v", err) - } - p, err := ipnlocal.ReadStartupPrefsForTest(t.Logf, fs) - if err != nil { - t.Fatalf("reading prefs, ReadDiskPrefsForTest: %v", err) - } - return p.AsStruct() -} - -// AwaitResponding waits for n's tailscaled to be up enough to be -// responding, but doesn't wait for any particular state. -func (n *testNode) AwaitResponding() { - t := n.env.t - t.Helper() - n.AwaitListening() - - st := n.MustStatus() - t.Logf("Status: %s", st.BackendState) - - if err := tstest.WaitFor(20*time.Second, func() error { - const sub = `Program starting: ` - if !n.env.LogCatcher.logsContains(mem.S(sub)) { - return fmt.Errorf("log catcher didn't see %#q; got %s", sub, n.env.LogCatcher.logsString()) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// addLogLineHook registers a hook f to be called on each tailscaled -// log line output. -func (n *testNode) addLogLineHook(f func([]byte)) { - n.mu.Lock() - defer n.mu.Unlock() - n.onLogLine = append(n.onLogLine, f) -} - -// socks5AddrChan returns a channel that receives the address (e.g. "localhost:23874") -// of the node's SOCKS5 listener, once started. -func (n *testNode) socks5AddrChan() <-chan string { - ch := make(chan string, 1) - n.addLogLineHook(func(line []byte) { - const sub = "SOCKS5 listening on " - i := mem.Index(mem.B(line), mem.S(sub)) - if i == -1 { - return - } - addr := strings.TrimSpace(string(line)[i+len(sub):]) - select { - case ch <- addr: - default: - } - }) - return ch -} - -func (n *testNode) AwaitSocksAddr(ch <-chan string) string { - t := n.env.t - t.Helper() - timer := time.NewTimer(10 * time.Second) - defer timer.Stop() - select { - case v := <-ch: - return v - case <-timer.C: - t.Fatal("timeout waiting for node to log its SOCK5 listening address") - panic("unreachable") - } -} - -// nodeOutputParser parses stderr of tailscaled processes, calling the -// per-line callbacks previously registered via -// testNode.addLogLineHook. -type nodeOutputParser struct { - allBuf bytes.Buffer - pendLineBuf bytes.Buffer - n *testNode -} - -func (op *nodeOutputParser) Write(p []byte) (n int, err error) { - tn := op.n - tn.mu.Lock() - defer tn.mu.Unlock() - - op.allBuf.Write(p) - n, err = op.pendLineBuf.Write(p) - op.parseLinesLocked() - return -} - -func (op *nodeOutputParser) parseLinesLocked() { - n := op.n - buf := op.pendLineBuf.Bytes() - for len(buf) > 0 { - nl := bytes.IndexByte(buf, '\n') - if nl == -1 { - break - } - line := buf[:nl+1] - buf = buf[nl+1:] - - for _, f := range n.onLogLine { - f(line) - } - } - if len(buf) == 0 { - op.pendLineBuf.Reset() - } else { - io.CopyN(io.Discard, &op.pendLineBuf, int64(op.pendLineBuf.Len()-len(buf))) - } -} - -type Daemon struct { - Process *os.Process -} - -func (d *Daemon) MustCleanShutdown(t testing.TB) { - d.Process.Signal(os.Interrupt) - ps, err := d.Process.Wait() - if err != nil { - t.Fatalf("tailscaled Wait: %v", err) - } - if ps.ExitCode() != 0 { - t.Errorf("tailscaled ExitCode = %d; want 0", ps.ExitCode()) - } -} - -// StartDaemon starts the node's tailscaled, failing if it fails to start. -// StartDaemon ensures that the process will exit when the test completes. -func (n *testNode) StartDaemon() *Daemon { - return n.StartDaemonAsIPNGOOS(runtime.GOOS) -} - -func (n *testNode) StartDaemonAsIPNGOOS(ipnGOOS string) *Daemon { - t := n.env.t - cmd := exec.Command(n.env.daemon) - cmd.Args = append(cmd.Args, - "--state="+n.stateFile, - "--socket="+n.sockFile, - "--socks5-server=localhost:0", - ) - if *verboseTailscaled { - cmd.Args = append(cmd.Args, "-verbose=2") - } - if !n.env.tunMode { - cmd.Args = append(cmd.Args, - "--tun=userspace-networking", - ) - } - if n.configFile != "" { - cmd.Args = append(cmd.Args, "--config="+n.configFile) - } - cmd.Env = append(os.Environ(), - "TS_CONTROL_IS_PLAINTEXT_HTTP=1", - "TS_DEBUG_PERMIT_HTTP_C2N=1", - "TS_LOG_TARGET="+n.env.LogCatcherServer.URL, - "HTTP_PROXY="+n.env.TrafficTrapServer.URL, - "HTTPS_PROXY="+n.env.TrafficTrapServer.URL, - "TS_DEBUG_FAKE_GOOS="+ipnGOOS, - "TS_LOGS_DIR="+t.TempDir(), - "TS_NETCHECK_GENERATE_204_URL="+n.env.ControlServer.URL+"/generate_204", - "TS_ASSUME_NETWORK_UP_FOR_TEST=1", // don't pause control client in airplane mode (no wifi, etc) - "TS_PANIC_IF_HIT_MAIN_CONTROL=1", - "TS_DISABLE_PORTMAPPER=1", // shouldn't be needed; test is all localhost - "TS_DEBUG_LOG_RATE=all", - ) - if n.env.loopbackPort != nil { - cmd.Env = append(cmd.Env, "TS_DEBUG_NETSTACK_LOOPBACK_PORT="+strconv.Itoa(*n.env.loopbackPort)) - } - if version.IsRace() { - cmd.Env = append(cmd.Env, "GORACE=halt_on_error=1") - } - n.tailscaledParser = &nodeOutputParser{n: n} - cmd.Stderr = n.tailscaledParser - if *verboseTailscaled { - cmd.Stdout = os.Stdout - cmd.Stderr = io.MultiWriter(cmd.Stderr, os.Stderr) - } - if runtime.GOOS != "windows" { - pr, pw, err := os.Pipe() - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { pw.Close() }) - cmd.ExtraFiles = append(cmd.ExtraFiles, pr) - cmd.Env = append(cmd.Env, "TS_PARENT_DEATH_FD=3") - } - if err := cmd.Start(); err != nil { - t.Fatalf("starting tailscaled: %v", err) - } - t.Cleanup(func() { cmd.Process.Kill() }) - return &Daemon{ - Process: cmd.Process, - } -} - -func (n *testNode) MustUp(extraArgs ...string) { - t := n.env.t - t.Helper() - args := []string{ - "up", - "--login-server=" + n.env.controlURL(), - "--reset", - } - args = append(args, extraArgs...) - cmd := n.Tailscale(args...) - t.Logf("Running %v ...", cmd) - cmd.Stdout = nil // in case --verbose-tailscale was set - cmd.Stderr = nil // in case --verbose-tailscale was set - if b, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("up: %v, %v", string(b), err) - } -} - -func (n *testNode) MustDown() { - t := n.env.t - t.Logf("Running down ...") - if err := n.Tailscale("down", "--accept-risk=all").Run(); err != nil { - t.Fatalf("down: %v", err) - } -} - -func (n *testNode) MustLogOut() { - t := n.env.t - t.Logf("Running logout ...") - if err := n.Tailscale("logout").Run(); err != nil { - t.Fatalf("logout: %v", err) - } -} - -func (n *testNode) Ping(otherNode *testNode) error { - t := n.env.t - ip := otherNode.AwaitIP4().String() - t.Logf("Running ping %v (from %v)...", ip, n.AwaitIP4()) - return n.Tailscale("ping", ip).Run() -} - -// AwaitListening waits for the tailscaled to be serving local clients -// over its localhost IPC mechanism. (Unix socket, etc) -func (n *testNode) AwaitListening() { - t := n.env.t - if err := tstest.WaitFor(20*time.Second, func() (err error) { - c, err := safesocket.ConnectContext(context.Background(), n.sockFile) - if err == nil { - c.Close() - } - return err - }); err != nil { - t.Fatal(err) - } -} - -func (n *testNode) AwaitIPs() []netip.Addr { - t := n.env.t - t.Helper() - var addrs []netip.Addr - if err := tstest.WaitFor(20*time.Second, func() error { - cmd := n.Tailscale("ip") - cmd.Stdout = nil // in case --verbose-tailscale was set - cmd.Stderr = nil // in case --verbose-tailscale was set - out, err := cmd.Output() - if err != nil { - return err - } - ips := string(out) - ipslice := strings.Fields(ips) - addrs = make([]netip.Addr, len(ipslice)) - - for i, ip := range ipslice { - netIP, err := netip.ParseAddr(ip) - if err != nil { - t.Fatal(err) - } - addrs[i] = netIP - } - return nil - }); err != nil { - t.Fatalf("awaiting an IP address: %v", err) - } - if len(addrs) == 0 { - t.Fatalf("returned IP address was blank") - } - return addrs -} - -// AwaitIP4 returns the IPv4 address of n. -func (n *testNode) AwaitIP4() netip.Addr { - t := n.env.t - t.Helper() - ips := n.AwaitIPs() - return ips[0] -} - -// AwaitIP6 returns the IPv6 address of n. -func (n *testNode) AwaitIP6() netip.Addr { - t := n.env.t - t.Helper() - ips := n.AwaitIPs() - return ips[1] -} - -// AwaitRunning waits for n to reach the IPN state "Running". -func (n *testNode) AwaitRunning() { - t := n.env.t - t.Helper() - n.AwaitBackendState("Running") -} - -func (n *testNode) AwaitBackendState(state string) { - t := n.env.t - t.Helper() - if err := tstest.WaitFor(20*time.Second, func() error { - st, err := n.Status() - if err != nil { - return err - } - if st.BackendState != state { - return fmt.Errorf("in state %q; want %q", st.BackendState, state) - } - return nil - }); err != nil { - t.Fatalf("failure/timeout waiting for transition to Running status: %v", err) - } -} - -// AwaitNeedsLogin waits for n to reach the IPN state "NeedsLogin". -func (n *testNode) AwaitNeedsLogin() { - t := n.env.t - t.Helper() - if err := tstest.WaitFor(20*time.Second, func() error { - st, err := n.Status() - if err != nil { - return err - } - if st.BackendState != "NeedsLogin" { - return fmt.Errorf("in state %q", st.BackendState) - } - return nil - }); err != nil { - t.Fatalf("failure/timeout waiting for transition to NeedsLogin status: %v", err) - } -} - -func (n *testNode) TailscaleForOutput(arg ...string) *exec.Cmd { - cmd := n.Tailscale(arg...) - cmd.Stdout = nil - cmd.Stderr = nil - return cmd -} - -// Tailscale returns a command that runs the tailscale CLI with the provided arguments. -// It does not start the process. -func (n *testNode) Tailscale(arg ...string) *exec.Cmd { - cmd := exec.Command(n.env.cli) - cmd.Args = append(cmd.Args, "--socket="+n.sockFile) - cmd.Args = append(cmd.Args, arg...) - cmd.Dir = n.dir - cmd.Env = append(os.Environ(), - "TS_DEBUG_UP_FLAG_GOOS="+n.upFlagGOOS, - "TS_LOGS_DIR="+n.env.t.TempDir(), - ) - if *verboseTailscale { - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - } - return cmd -} - -func (n *testNode) Status() (*ipnstate.Status, error) { - cmd := n.Tailscale("status", "--json") - cmd.Stdout = nil // in case --verbose-tailscale was set - cmd.Stderr = nil // in case --verbose-tailscale was set - out, err := cmd.CombinedOutput() - if err != nil { - return nil, fmt.Errorf("running tailscale status: %v, %s", err, out) - } - st := new(ipnstate.Status) - if err := json.Unmarshal(out, st); err != nil { - return nil, fmt.Errorf("decoding tailscale status JSON: %w\njson:\n%s", err, out) - } - return st, nil -} - -func (n *testNode) MustStatus() *ipnstate.Status { - tb := n.env.t - tb.Helper() - st, err := n.Status() - if err != nil { - tb.Fatal(err) - } - return st -} - -// trafficTrap is an HTTP proxy handler to note whether any -// HTTP traffic tries to leave localhost from tailscaled. We don't -// expect any, so any request triggers a failure. -type trafficTrap struct { - atomicErr syncs.AtomicValue[error] -} - -func (tt *trafficTrap) Err() error { - return tt.atomicErr.Load() -} - -func (tt *trafficTrap) ServeHTTP(w http.ResponseWriter, r *http.Request) { - var got bytes.Buffer - r.Write(&got) - err := fmt.Errorf("unexpected HTTP request via proxy: %s", got.Bytes()) - mainError.Store(err) - if tt.Err() == nil { - // Best effort at remembering the first request. - tt.atomicErr.Store(err) - } - log.Printf("Error: %v", err) - w.WriteHeader(403) -} - -type authURLParserWriter struct { - buf bytes.Buffer - fn func(urlStr string) error -} - -var authURLRx = regexp.MustCompile(`(https?://\S+/auth/\S+)`) - -func (w *authURLParserWriter) Write(p []byte) (n int, err error) { - n, err = w.buf.Write(p) - m := authURLRx.FindSubmatch(w.buf.Bytes()) - if m != nil { - urlStr := string(m[1]) - w.buf.Reset() // so it's not matched again - if err := w.fn(urlStr); err != nil { - return 0, err - } - } - return n, err -}