tstest/integration/vms: smoke test derphttp through mitm proxies

Updates #4377

Very smoky/high-level test to ensure that derphttp internals play well
with an agressive (stare + bump) meddler-in-the-middle proxy.

Signed-off-by: Tom DNetto <tom@tailscale.com>

tstest/integration/vms/nixos_test.go

@@ -124,7 +124,7 @@
   systemd.services.cloud-final.path = with pkgs; [ curl ];
 
   # Curl is needed for one of the integration tests
-  environment.systemPackages = with pkgs; [ curl ];
+  environment.systemPackages = with pkgs; [ curl nix bash squid openssl daemonize ];
 
   # yolo, this vm can sudo freely.
   security.sudo.wheelNeedsPassword = false;

tstest/integration/vms/squid.conf

@@ -0,0 +1,39 @@
+pid_filename    /run/squid.pid
+cache_dir ufs /tmp/squid/cache 500 16 256
+maximum_object_size 4096 KB
+coredump_dir /tmp/squid/core
+visible_hostname localhost
+cache_access_log /tmp/squid/access.log
+cache_log /tmp/squid/cache.log
+
+# Access Control lists
+acl localhost src 127.0.0.1 ::1
+acl manager proto cache_object
+acl SSL_ports port 443
+acl Safe_ports port 80		# http
+acl Safe_ports port 21		# ftp
+acl Safe_ports port 443		# https
+acl Safe_ports port 70		# gopher
+acl Safe_ports port 210		# wais
+acl Safe_ports port 1025-65535	# unregistered ports
+acl Safe_ports port 280		# http-mgmt
+acl Safe_ports port 488		# gss-http
+acl Safe_ports port 591		# filemaker
+acl Safe_ports port 777		# multiling http
+acl CONNECT method CONNECT
+
+http_access allow localhost
+http_access deny all
+forwarded_for on
+
+# sslcrtd_program /nix/store/nqlqk1f6qlxdirlrl1aijgb6vbzxs0gs-squid-4.17/libexec/security_file_certgen -s /tmp/squid/ssl_db -M 4MB
+sslcrtd_children 5
+
+http_port 127.0.0.1:3128 \
+  ssl-bump \
+  generate-host-certificates=on \
+  dynamic_cert_mem_cache_size=4MB \
+  cert=/tmp/squid/myca-mitm.pem
+
+ssl_bump stare all      # mimic the Client Hello, drop unsupported extensions
+ssl_bump bump all       # terminate and establish new TLS connection

tstest/integration/vms/top_level_test.go

@@ -7,20 +7,120 @@
 
 package vms
 
-import "testing"
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/pkg/sftp"
+	expect "github.com/tailscale/goexpect"
+)
 
 func TestRunUbuntu1804(t *testing.T) {
-	setupTests(t)
 	testOneDistribution(t, 0, Distros[0])
 }
 
 func TestRunUbuntu2004(t *testing.T) {
-	setupTests(t)
 	testOneDistribution(t, 1, Distros[1])
 }
 
 func TestRunNixos2111(t *testing.T) {
 	t.Parallel()
-	setupTests(t)
 	testOneDistribution(t, 2, Distros[2])
-}
+}
+
+// TestMITMProxy is a smoke test for derphttp through a MITM proxy.
+// Encountering such proxies is unfortunately commonplace in more
+// traditional enterprise networks.
+//
+// We invoke tailscale netcheck because the networking check is done
+// by tailscale rather than tailscaled, making it easier to configure
+// the proxy.
+//
+// To provide the actual MITM server, we use squid.
+func TestMITMProxy(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	distro := Distros[2] // nixos-21.11
+
+	if distroRex.Unwrap().MatchString(distro.Name) {
+		t.Logf("%s matches %s", distro.Name, distroRex.Unwrap())
+	} else {
+		t.Skip("regex not matched")
+	}
+
+	ctx, done := context.WithCancel(context.Background())
+	t.Cleanup(done)
+
+	h := newHarness(t)
+
+	err := ramsem.sem.Acquire(ctx, int64(distro.MemoryMegs))
+	if err != nil {
+		t.Fatalf("can't acquire ram semaphore: %v", err)
+	}
+	t.Cleanup(func() { ramsem.sem.Release(int64(distro.MemoryMegs)) })
+
+	vm := h.mkVM(t, 2, distro, h.pubKey, h.loginServerURL, t.TempDir())
+	vm.waitStartup(t)
+
+	ipm := h.waitForIPMap(t, vm, distro)
+	_, cli := h.setupSSHShell(t, distro, ipm)
+
+	sftpCli, err := sftp.NewClient(cli)
+	if err != nil {
+		t.Fatalf("can't connect over sftp to copy binaries: %v", err)
+	}
+	defer sftpCli.Close()
+
+	// Initialize a squid installation.
+	//
+	// A few things of note here:
+	// - The first thing we do is append the nsslcrtd_program stanza to the config.
+	//   This must be an absolute path and is based on the nix path of the squid derivation,
+	//   so we compute and write it out here.
+	// - Squid expects a pre-initalized directory layout, so we create that in /tmp/squid then
+	//   invoke squid with -z to have it fill in the rest.
+	// - Doing a meddler-in-the-middle attack requires using some fake keys, so we create
+	//   them using openssl and then use the security_file_certgen tool to setup squids' ssl_db.
+	// - There were some perms issues, so i yeeted 0777. Its only a test anyway
+	copyFile(t, sftpCli, "squid.conf", "/tmp/squid.conf")
+	runTestCommands(t, 30*time.Second, cli, []expect.Batcher{
+		&expect.BSnd{S: "echo -e \"\\nsslcrtd_program $(nix eval --raw nixpkgs.squid)/libexec/security_file_certgen -s /tmp/squid/ssl_db -M 4MB\\n\" >> /tmp/squid.conf\n"},
+		&expect.BSnd{S: "mkdir -p /tmp/squid/{cache,core}\n"},
+		&expect.BSnd{S: "openssl req -batch -new -newkey rsa:4096 -sha256 -days 3650 -nodes -x509 -keyout /tmp/squid/myca-mitm.pem -out /tmp/squid/myca-mitm.pem\n"},
+		&expect.BExp{R: `writing new private key to '/tmp/squid/myca-mitm.pem'`},
+		&expect.BSnd{S: "$(nix eval --raw nixpkgs.squid)/libexec/security_file_certgen -c -s /tmp/squid/ssl_db -M 4MB\n"},
+		&expect.BExp{R: `Done`},
+		&expect.BSnd{S: "sudo chmod -R 0777 /tmp/squid\n"},
+		&expect.BSnd{S: "squid --foreground -YCs -z -f /tmp/squid.conf\n"},
+		&expect.BSnd{S: "echo Success.\n"},
+		&expect.BExp{R: `Success.`},
+	})
+
+	// Start the squid server.
+	runTestCommands(t, 10*time.Second, cli, []expect.Batcher{
+		&expect.BSnd{S: "daemonize -v -c /tmp/squid $(nix eval --raw nixpkgs.squid)/bin/squid --foreground -YCs -f /tmp/squid.conf\n"}, // start daemon
+		// NOTE(tom): Writing to /dev/tcp/* is bash magic, not a file. This
+		//            eldritchian incantation lets us wait till squid is up.
+		&expect.BSnd{S: "while ! timeout 5 bash -c 'echo > /dev/tcp/localhost/3128'; do sleep 1; done\n"},
+		&expect.BSnd{S: "echo Success.\n"},
+		&expect.BExp{R: `Success.`},
+	})
+
+	// Uncomment to help debugging this test if it fails.
+	//
+	// runTestCommands(t, 30 * time.Second, cli, []expect.Batcher{
+	// 	&expect.BSnd{S: "sudo ifconfig\n"},
+	// 	&expect.BSnd{S: "sudo ip link\n"},
+	// 	&expect.BSnd{S: "sudo ip route\n"},
+	// 	&expect.BSnd{S: "ps -aux\n"},
+	// 	&expect.BSnd{S: "netstat -a\n"},
+	// 	&expect.BSnd{S: "cat /tmp/squid/access.log && cat /tmp/squid/cache.log && cat /tmp/squid.conf && echo Success.\n"},
+	// 	&expect.BExp{R: `Success.`},
+	// })
+
+	runTestCommands(t, 30*time.Second, cli, []expect.Batcher{
+		&expect.BSnd{S: "SSL_CERT_FILE=/tmp/squid/myca-mitm.pem HTTPS_PROXY=http://127.0.0.1:3128 tailscale netcheck\n"},
+		&expect.BExp{R: `IPv4: yes`},
+	})
+}

tstest/integration/vms/vm_setup_test.go

@@ -23,6 +23,7 @@
 	"strconv"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
@@ -49,6 +50,19 @@ func (vm *vmInstance) running() bool {
 	}
 }
 
+func (vm *vmInstance) waitStartup(t *testing.T) {
+	t.Helper()
+	for i := 0; i < 100; i++ {
+		if vm.running() {
+			break
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	if !vm.running() {
+		t.Fatal("vm not running")
+	}
+}
+
 func (h *Harness) makeImage(t *testing.T, d Distro, cdir string) string {
 	if !strings.HasPrefix(d.Name, "nixos") {
 		t.Fatal("image generation for non-nixos is not implemented")

tstest/integration/vms/vms_test.go

@@ -276,17 +276,14 @@ func testOneDistribution(t *testing.T, n int, distro Distro) {
 	t.Cleanup(func() { ramsem.sem.Release(int64(distro.MemoryMegs)) })
 
 	vm := h.mkVM(t, n, distro, h.pubKey, h.loginServerURL, dir)
-	var ipm ipMapping
+	vm.waitStartup(t)
 
-	for i := 0; i < 100; i++ {
+	h.testDistro(t, distro, h.waitForIPMap(t, vm, distro))
-		if vm.running() {
+}
-			break
+
-		}
+func (h *Harness) waitForIPMap(t *testing.T, vm *vmInstance, distro Distro) ipMapping {
-		time.Sleep(100 * time.Millisecond)
+	t.Helper()
-	}
+	var ipm ipMapping
-	if !vm.running() {
-		t.Fatal("vm not running")
-	}
 
 	waiter := time.NewTicker(time.Second)
 	defer waiter.Stop()
@@ -305,13 +302,11 @@ func testOneDistribution(t *testing.T, n int, distro Distro) {
 		}
 		<-waiter.C
 	}
-
+	return ipm
-	h.testDistro(t, distro, ipm)
 }
 
-func (h *Harness) testDistro(t *testing.T, d Distro, ipm ipMapping) {
+func (h *Harness) setupSSHShell(t *testing.T, d Distro, ipm ipMapping) (*ssh.ClientConfig, *ssh.Client) {
 	signer := h.signer
-	loginServer := h.loginServerURL
 
 	t.Helper()
 	port := ipm.port
@@ -350,6 +345,13 @@ func (h *Harness) testDistro(t *testing.T, d Distro, ipm ipMapping) {
 	}
 	h.copyBinaries(t, d, cli)
 
+	return ccfg, cli
+}
+
+func (h *Harness) testDistro(t *testing.T, d Distro, ipm ipMapping) {
+	loginServer := h.loginServerURL
+	ccfg, cli := h.setupSSHShell(t, d, ipm)
+
 	timeout := 30 * time.Second
 
 	t.Run("start-tailscale", func(t *testing.T) {