diff --git a/cmd/tta/fw_linux.go b/cmd/tta/fw_linux.go new file mode 100644 index 000000000..a4ceabad8 --- /dev/null +++ b/cmd/tta/fw_linux.go @@ -0,0 +1,128 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "encoding/binary" + + "github.com/google/nftables" + "github.com/google/nftables/expr" + "tailscale.com/types/ptr" +) + +func init() { + addFirewall = addFirewallLinux +} + +func addFirewallLinux() error { + c, err := nftables.New() + if err != nil { + return err + } + + // Create a new table + table := &nftables.Table{ + Family: nftables.TableFamilyIPv4, // TableFamilyINet doesn't work (why?. oh well.) + Name: "filter", + } + c.AddTable(table) + + // Create a new chain for incoming traffic + inputChain := &nftables.Chain{ + Name: "input", + Table: table, + Type: nftables.ChainTypeFilter, + Hooknum: nftables.ChainHookInput, + Priority: nftables.ChainPriorityFilter, + Policy: ptr.To(nftables.ChainPolicyDrop), + } + c.AddChain(inputChain) + + // Allow traffic from the loopback interface + c.AddRule(&nftables.Rule{ + Table: table, + Chain: inputChain, + Exprs: []expr.Any{ + &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte("lo"), + }, + &expr.Verdict{ + Kind: expr.VerdictAccept, + }, + }, + }) + + // Accept established and related connections + c.AddRule(&nftables.Rule{ + Table: table, + Chain: inputChain, + Exprs: []expr.Any{ + &expr.Ct{ + Register: 1, + Key: expr.CtKeySTATE, + }, + &expr.Bitwise{ + SourceRegister: 1, + DestRegister: 1, + Len: 4, + Mask: binary.NativeEndian.AppendUint32(nil, 0x06), // CT_STATE_BIT_ESTABLISHED | CT_STATE_BIT_RELATED + Xor: binary.NativeEndian.AppendUint32(nil, 0), + }, + &expr.Cmp{ + Op: expr.CmpOpNeq, + Register: 1, + Data: binary.NativeEndian.AppendUint32(nil, 0x00), + }, + &expr.Verdict{ + Kind: expr.VerdictAccept, + }, + }, + }) + + // Allow TCP packets in that don't have the SYN bit set, even if they're not + // ESTABLISHED or RELATED. This is because the test suite gets TCP + // connections up & idle (for HTTP) before it conditionally installs these + // firewall rules. But because conntrack wasn't previously active, existing + // TCP flows aren't ESTABLISHED and get dropped. So this rule allows + // previously established TCP connections that predates the firewall rules + // to continue working, as they don't have conntrack state. + c.AddRule(&nftables.Rule{ + Table: table, + Chain: inputChain, + Exprs: []expr.Any{ + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{0x06}, // TCP + }, + &expr.Payload{ // get TCP flags + DestRegister: 1, + Base: 2, + Offset: 13, // flags + Len: 1, + }, + &expr.Bitwise{ + SourceRegister: 1, + DestRegister: 1, + Len: 1, + Mask: []byte{2}, // TCP_SYN + Xor: []byte{0}, + }, + &expr.Cmp{ + Op: expr.CmpOpNeq, + Register: 1, + Data: []byte{2}, // TCP_SYN + }, + &expr.Verdict{ + Kind: expr.VerdictAccept, + }, + }, + }) + + return c.Flush() +} diff --git a/cmd/tta/tta.go b/cmd/tta/tta.go index 9c4bdf212..6a676b0d2 100644 --- a/cmd/tta/tta.go +++ b/cmd/tta/tta.go @@ -130,13 +130,14 @@ func main() { hs.ConnState = func(c net.Conn, s http.ConnState) { stMu.Lock() defer stMu.Unlock() + oldLen := len(newSet) switch s { case http.StateNew: newSet.Add(c) default: newSet.Delete(c) } - if len(newSet) == 0 { + if oldLen != 0 && len(newSet) == 0 { select { case needConnCh <- true: default: @@ -147,7 +148,12 @@ func main() { lcRP := httputil.NewSingleHostReverseProxy(must.Get(url.Parse("http://local-tailscaled.sock"))) lcRP.Transport = new(localClientRoundTripper) - ttaMux.Handle("/localapi/", lcRP) + ttaMux.HandleFunc("/localapi/", func(w http.ResponseWriter, r *http.Request) { + log.Printf("Got localapi request: %v", r.URL) + t0 := time.Now() + lcRP.ServeHTTP(w, r) + log.Printf("Did localapi request in %v: %v", time.Since(t0).Round(time.Millisecond), r.URL) + }) ttaMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { io.WriteString(w, "TTA\n") @@ -156,8 +162,19 @@ func main() { ttaMux.HandleFunc("/up", func(w http.ResponseWriter, r *http.Request) { serveCmd(w, "tailscale", "up", "--login-server=http://control.tailscale") }) + ttaMux.HandleFunc("/fw", addFirewallHandler) + go hs.Serve(chanListener(conns)) + // For doing agent operations locally from gokrazy: + // (e.g. with "wget -O - localhost:8123/fw") + go func() { + err := http.ListenAndServe("127.0.0.1:8123", &ttaMux) + if err != nil { + log.Fatalf("ListenAndServe: %v", err) + } + }() + var lastErr string needConnCh <- true for { @@ -204,3 +221,18 @@ func (cl chanListener) Addr() net.Addr { Port: 123, } } + +func addFirewallHandler(w http.ResponseWriter, r *http.Request) { + if addFirewall == nil { + http.Error(w, "firewall not supported", 500) + return + } + err := addFirewall() + if err != nil { + http.Error(w, err.Error(), 500) + return + } + io.WriteString(w, "OK\n") +} + +var addFirewall func() error // set by fw_linux.go diff --git a/cmd/vnet/run-krazy.sh b/cmd/vnet/run-krazy.sh index 6cf608b62..d824a0b1d 100755 --- a/cmd/vnet/run-krazy.sh +++ b/cmd/vnet/run-krazy.sh @@ -7,7 +7,7 @@ qemu-system-x86_64 -M microvm,isa-serial=off \ -m 1G \ -nodefaults -no-user-config -nographic \ -kernel $HOME/src/github.com/tailscale/gokrazy-kernel/vmlinuz \ - -append "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-dd02023b0001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic pci=off nousb tsc=unstable clocksource=hpet tailscale-tta=1" \ + -append "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-dd02023b0001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic pci=off nousb tsc=unstable clocksource=hpet tailscale-tta=1 tailscaled.env=TS_DEBUG_RAW_DISCO=1" \ -drive id=blk0,file=$HOME/src/tailscale.com/gokrazy/tsapp.img,format=raw \ -device virtio-blk-device,drive=blk0 \ -netdev stream,id=net0,addr.type=unix,addr.path=/tmp/qemu.sock \ diff --git a/logpolicy/logpolicy.go b/logpolicy/logpolicy.go index 74c480e11..f40ede86a 100644 --- a/logpolicy/logpolicy.go +++ b/logpolicy/logpolicy.go @@ -464,6 +464,11 @@ func New(collection string, netMon *netmon.Monitor, health *health.Tracker, logf // The netMon parameter is optional. It should be specified in environments where // Tailscaled is manipulating the routing table. func NewWithConfigPath(collection, dir, cmdName string, netMon *netmon.Monitor, health *health.Tracker, logf logger.Logf) *Policy { + if hostinfo.IsNATLabGuestVM() { + // In NATLab Gokrazy instances, tailscaled comes up concurently with + // DHCP and the doesn't have DNS for a while. Wait for DHCP first. + awaitGokrazyNetwork() + } var lflags int if term.IsTerminal(2) || runtime.GOOS == "windows" { lflags = 0 @@ -567,7 +572,7 @@ func NewWithConfigPath(collection, dir, cmdName string, netMon *netmon.Monitor, conf.IncludeProcSequence = true } - if envknob.NoLogsNoSupport() || testenv.InTest() || hostinfo.IsNATLabGuestVM() { + if envknob.NoLogsNoSupport() || testenv.InTest() { logf("You have disabled logging. Tailscale will not be able to provide support.") conf.HTTPC = &http.Client{Transport: noopPretendSuccessTransport{}} } else if val := getLogTarget(); val != "" { @@ -817,3 +822,25 @@ func (noopPretendSuccessTransport) RoundTrip(req *http.Request) (*http.Response, Status: "200 OK", }, nil } + +func awaitGokrazyNetwork() { + if runtime.GOOS != "linux" || distro.Get() != distro.Gokrazy { + return + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + for { + // Before DHCP finishes, the /etc/resolv.conf file has just "#MANUAL". + all, _ := os.ReadFile("/etc/resolv.conf") + if bytes.Contains(all, []byte("nameserver ")) { + return + } + select { + case <-ctx.Done(): + return + case <-time.After(500 * time.Millisecond): + } + } +} diff --git a/net/tlsdial/tlsdial.go b/net/tlsdial/tlsdial.go index 2bf88bd0e..a49e7f0f7 100644 --- a/net/tlsdial/tlsdial.go +++ b/net/tlsdial/tlsdial.go @@ -26,6 +26,7 @@ import ( "tailscale.com/envknob" "tailscale.com/health" + "tailscale.com/hostinfo" ) var counterFallbackOK int32 // atomic @@ -77,6 +78,12 @@ func Config(host string, ht *health.Tracker, base *tls.Config) *tls.Config { // (with the baked-in fallback root) in the VerifyConnection hook. conf.InsecureSkipVerify = true conf.VerifyConnection = func(cs tls.ConnectionState) (retErr error) { + if host == "log.tailscale.io" && hostinfo.IsNATLabGuestVM() { + // Allow log.tailscale.io TLS MITM for integration tests when + // the client's running within a NATLab VM. + return nil + } + // Perform some health checks on this certificate before we do // any verification. var selfSignedIssuer string diff --git a/tstest/integration/nat/nat_test.go b/tstest/integration/nat/nat_test.go index 052bab74b..51fa0abbd 100644 --- a/tstest/integration/nat/nat_test.go +++ b/tstest/integration/nat/nat_test.go @@ -102,6 +102,14 @@ func easy(c *vnet.Config) *vnet.Node { fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT)) } +// easy + host firewall +func easyFW(c *vnet.Config) *vnet.Node { + n := c.NumNodes() + 1 + return c.AddNode(vnet.HostFirewall, c.AddNetwork( + fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP + fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT)) +} + func easyAF(c *vnet.Config) *vnet.Node { n := c.NumNodes() + 1 return c.AddNode(c.AddNetwork( @@ -134,6 +142,29 @@ func easyPMP(c *vnet.Config) *vnet.Node { fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT, vnet.NATPMP)) } +// easy + port mapping + host firewall +func easyPMPFW(c *vnet.Config) *vnet.Node { + n := c.NumNodes() + 1 + return c.AddNode(vnet.HostFirewall, + c.AddNetwork( + fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP + fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT, vnet.NATPMP)) +} + +// easy + port mapping + host firewall - BPF +func easyPMPFWNoBPF(c *vnet.Config) *vnet.Node { + n := c.NumNodes() + 1 + return c.AddNode( + vnet.HostFirewall, + vnet.TailscaledEnv{ + Key: "TS_DEBUG_DISABLE_RAW_DISCO", + Value: "1", + }, + c.AddNetwork( + fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP + fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT, vnet.NATPMP)) +} + func hard(c *vnet.Config) *vnet.Node { n := c.NumNodes() + 1 return c.AddNode(c.AddNetwork( @@ -203,12 +234,18 @@ func (nt *natTest) runTest(node1, node2 addNodeFunc) pingRoute { t.Fatalf("qemu-img create: %v, %s", err, out) } + var envBuf bytes.Buffer + for _, e := range node.Env() { + fmt.Fprintf(&envBuf, " tailscaled.env=%s=%s", e.Key, e.Value) + } + envStr := envBuf.String() + cmd := exec.Command("qemu-system-x86_64", "-M", "microvm,isa-serial=off", "-m", "384M", "-nodefaults", "-no-user-config", "-nographic", "-kernel", nt.kernel, - "-append", "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-dd02023b0001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic pci=off nousb tsc=unstable clocksource=hpet tailscale-tta=1", + "-append", "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-dd02023b0001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic pci=off nousb tsc=unstable clocksource=hpet tailscale-tta=1"+envStr, "-drive", "id=blk0,file="+disk+",format=qcow2", "-device", "virtio-blk-device,drive=blk0", "-netdev", "stream,id=net0,addr.type=unix,addr.path="+sockAddr, @@ -254,10 +291,20 @@ func (nt *natTest) runTest(node1, node2 addNodeFunc) pingRoute { return fmt.Errorf("node%d status: %w", i, err) } t.Logf("node%d status: %v", i, st) + + node := nodes[i] + if node.HostFirewall() { + if err := c.EnableHostFirewall(ctx); err != nil { + return fmt.Errorf("node%d firewall: %w", i, err) + } + t.Logf("node%d firewalled", i) + } + if err := up(ctx, c); err != nil { return fmt.Errorf("node%d up: %w", i, err) } t.Logf("node%d up!", i) + st, err = c.Status(ctx) if err != nil { return fmt.Errorf("node%d status: %w", i, err) @@ -408,6 +455,31 @@ func TestEasyEasy(t *testing.T) { nt.runTest(easy, easy) } +// Tests https://github.com/tailscale/tailscale/issues/3824 ... +// * server behind a Hard NAT +// * client behind a NAT with UPnP support +// * client machine has a stateful host firewall (e.g. ufw) + +func TestBPFDisco(t *testing.T) { + nt := newNatTest(t) + nt.runTest(easyPMPFW, hard) +} + +func TestHostFWNoBPF(t *testing.T) { + nt := newNatTest(t) + nt.runTest(easyPMPFWNoBPF, hard) +} + +func TestHostFWPair(t *testing.T) { + nt := newNatTest(t) + nt.runTest(easyFW, easyFW) +} + +func TestOneHostFW(t *testing.T) { + nt := newNatTest(t) + nt.runTest(easy, easyFW) +} + var pair = flag.String("pair", "", "comma-separated pair of types to test (easy, easyAF, hard, easyPMP, hardPMP, one2one, sameLAN)") func TestPair(t *testing.T) { diff --git a/tstest/natlab/vnet/conf.go b/tstest/natlab/vnet/conf.go index 16f9645f9..c5859787e 100644 --- a/tstest/natlab/vnet/conf.go +++ b/tstest/natlab/vnet/conf.go @@ -70,6 +70,16 @@ func (c *Config) AddNode(opts ...any) *Node { o.nodes = append(o.nodes, n) } n.nets = append(n.nets, o) + case TailscaledEnv: + n.env = append(n.env, o) + case NodeOption: + if o == HostFirewall { + n.hostFW = true + } else { + if n.err == nil { + n.err = fmt.Errorf("unknown NodeOption %q", o) + } + } default: if n.err == nil { n.err = fmt.Errorf("unknown AddNode option type %T", o) @@ -79,6 +89,19 @@ func (c *Config) AddNode(opts ...any) *Node { return n } +// NodeOption is an option that can be passed to Config.AddNode. +type NodeOption string + +const ( + HostFirewall NodeOption = "HostFirewall" +) + +// TailscaledEnv is а option that can be passed to Config.AddNode +// to set an environment variable for tailscaled. +type TailscaledEnv struct { + Key, Value string +} + // AddNetwork add a new network. // // The opts may be of the following types: @@ -125,6 +148,9 @@ type Node struct { err error n *node // nil until NewServer called + env []TailscaledEnv + hostFW bool + // TODO(bradfitz): this is halfway converted to supporting multiple NICs // but not done. We need a MAC-per-Network. @@ -137,6 +163,14 @@ func (n *Node) MAC() MAC { return n.mac } +func (n *Node) Env() []TailscaledEnv { + return n.env +} + +func (n *Node) HostFirewall() bool { + return n.hostFW +} + // Network returns the first network this node is connected to, // or nil if none. func (n *Node) Network() *Network { diff --git a/tstest/natlab/vnet/vnet.go b/tstest/natlab/vnet/vnet.go index 42a39b9e4..1f3da988d 100644 --- a/tstest/natlab/vnet/vnet.go +++ b/tstest/natlab/vnet/vnet.go @@ -15,6 +15,7 @@ package vnet import ( "bufio" + "bytes" "context" "crypto/tls" "encoding/binary" @@ -61,6 +62,7 @@ import ( "tailscale.com/util/mak" "tailscale.com/util/must" "tailscale.com/util/set" + "tailscale.com/util/zstdframe" ) const nicID = 1 @@ -325,6 +327,13 @@ func (n *network) acceptTCP(r *tcp.ForwarderRequest) { go hs.Serve(netutil.NewOneConnListener(tc, nil)) return } + if destPort == 443 && destIP == fakeLogCatcherIP { + + r.Complete(false) + tc := gonet.NewTCPConn(&wq, ep) + go n.serveLogCatcherConn(clientRemoteIP, tc) + return + } log.Printf("vnet-AcceptTCP: %v", stringifyTEI(reqDetails)) @@ -354,6 +363,51 @@ func (n *network) acceptTCP(r *tcp.ForwarderRequest) { } } +// serveLogCatchConn serves a TCP connection to "log.tailscale.io", speaking the +// logtail/logcatcher protocol. +// +// We terminate TLS with an arbitrary cert; the client is configured to not +// validate TLS certs for this hostname when running under these integration +// tests. +func (n *network) serveLogCatcherConn(clientRemoteIP netip.Addr, c net.Conn) { + tlsConfig := n.s.derps[0].tlsConfig // self-signed (stealing DERP's); test client configure to not check + tlsConn := tls.Server(c, tlsConfig) + var handler http.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + all, _ := io.ReadAll(r.Body) + if r.Header.Get("Content-Encoding") == "zstd" { + var err error + all, err = zstdframe.AppendDecode(nil, all) + if err != nil { + log.Printf("LOGS DECODE ERROR zstd decode: %v", err) + http.Error(w, "zstd decode error", http.StatusBadRequest) + return + } + } + var logs []struct { + Logtail struct { + Client_Time time.Time + } + Text string + } + if err := json.Unmarshal(all, &logs); err != nil { + log.Printf("Logs decode error: %v", err) + return + } + node := n.nodesByIP[clientRemoteIP] + if node != nil { + node.logMu.Lock() + defer node.logMu.Unlock() + for _, lg := range logs { + tStr := lg.Logtail.Client_Time.Round(time.Millisecond).Format(time.RFC3339Nano) + fmt.Fprintf(&node.logBuf, "[%v] %s\n", tStr, lg.Text) + log.Printf("LOG %v [%v] %s\n", clientRemoteIP, tStr, lg.Text) + } + } + }) + hs := &http.Server{Handler: handler} + hs.Serve(netutil.NewOneConnListener(tlsConn, nil)) +} + var ( fakeDNSIP = netip.AddrFrom4([4]byte{4, 11, 4, 11}) fakeProxyControlplaneIP = netip.AddrFrom4([4]byte{52, 52, 0, 1}) // real controlplane.tailscale.com proxy @@ -451,6 +505,12 @@ type node struct { interfaceID int net *network lanIP netip.Addr // must be in net.lanIP prefix + unique in net + + // logMu guards logBuf. + // TODO(bradfitz): conditionally write these out to separate files at the end? + // Currently they only hold logcatcher logs. + logMu sync.Mutex + logBuf bytes.Buffer } type derpServer struct { @@ -1153,7 +1213,7 @@ func (s *Server) shouldInterceptTCP(pkt gopacket.Packet) bool { dstIP, _ := netip.AddrFromSlice(ipv4.DstIP.To4()) if tcp.DstPort == 80 || tcp.DstPort == 443 { switch dstIP { - case fakeControlIP, fakeDERP1IP, fakeDERP2IP: + case fakeControlIP, fakeDERP1IP, fakeDERP2IP, fakeLogCatcherIP: return true } if dstIP == fakeProxyControlplaneIP { @@ -1613,7 +1673,9 @@ func (s *Server) NodeAgentClient(n *Node) *NodeAgentClient { d := s.NodeAgentDialer(n) return &NodeAgentClient{ LocalClient: &tailscale.LocalClient{ - Dial: d, + UseSocketOnly: true, + OmitAuth: true, + Dial: d, }, HTTPClient: &http.Client{ Transport: &http.Transport{ @@ -1622,3 +1684,21 @@ func (s *Server) NodeAgentClient(n *Node) *NodeAgentClient { }, } } + +// EnableHostFirewall enables the host's stateful firewall. +func (c *NodeAgentClient) EnableHostFirewall(ctx context.Context) error { + req, err := http.NewRequestWithContext(ctx, "GET", "http://unused/fw", nil) + if err != nil { + return err + } + res, err := c.HTTPClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + all, _ := io.ReadAll(res.Body) + if res.StatusCode != 200 { + return fmt.Errorf("unexpected status code %v: %s", res.Status, all) + } + return nil +} diff --git a/wgengine/magicsock/magicsock_linux.go b/wgengine/magicsock/magicsock_linux.go index 69074fd72..a647c90d2 100644 --- a/wgengine/magicsock/magicsock_linux.go +++ b/wgengine/magicsock/magicsock_linux.go @@ -32,6 +32,9 @@ const ( // Enable/disable using raw sockets to receive disco traffic. var debugDisableRawDisco = envknob.RegisterBool("TS_DEBUG_DISABLE_RAW_DISCO") +// debugRawDiscoReads enables logging of raw disco reads. +var debugRawDiscoReads = envknob.RegisterBool("TS_DEBUG_RAW_DISCO") + // These are our BPF filters that we use for testing packets. var ( magicsockFilterV4 = []bpf.Instruction{ @@ -211,6 +214,9 @@ func (c *Conn) receiveDisco(pc net.PacketConn, isIPV6 bool) { var buf [1500]byte for { n, src, err := pc.ReadFrom(buf[:]) + if debugRawDiscoReads() { + c.logf("raw disco read from %v = (%v, %v)", src, n, err) + } if errors.Is(err, net.ErrClosed) { return } else if err != nil {