net/tlsdial: bake in LetsEncrypt's ISRG Root X1 root

We still try the host's x509 roots first, but if that fails (like if
the host is old), we fall back to using LetsEncrypt's root and
retrying with that.

tlsdial was used in the three main places: logs, control, DERP. But it
was missing in dnsfallback. So added it there too, so we can run fine
now on a machine with no DNS config and no root CAs configured.

Also, move SSLKEYLOGFILE support out of DERP. tlsdial is the logical place
for that support.

Fixes #1609

Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

derp/derphttp/derphttp_client.go

@@ -19,11 +19,9 @@
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"net"
 	"net/http"
 	"net/url"
-	"os"
 	"strings"
 	"sync"
 	"time"
@@ -430,19 +428,14 @@ func (c *Client) dialRegion(ctx context.Context, reg *tailcfg.DERPRegion) (net.C
 func (c *Client) tlsClient(nc net.Conn, node *tailcfg.DERPNode) *tls.Conn {
 	tlsConf := tlsdial.Config(c.tlsServerName(node), c.TLSConfig)
 	if node != nil {
-		tlsConf.InsecureSkipVerify = node.InsecureForTests
+		if node.InsecureForTests {
+			tlsConf.InsecureSkipVerify = true
+			tlsConf.VerifyConnection = nil
+		}
 		if node.CertName != "" {
 			tlsdial.SetConfigExpectedCert(tlsConf, node.CertName)
 		}
 	}
-	if n := os.Getenv("SSLKEYLOGFILE"); n != "" {
-		f, err := os.OpenFile(n, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
-		if err != nil {
-			log.Fatal(err)
-		}
-		log.Printf("WARNING: writing to SSLKEYLOGFILE %v", n)
-		tlsConf.KeyLogWriter = f
-	}
 	return tls.Client(nc, tlsConf)
 }
 

go.mod

@@ -3,6 +3,7 @@ module tailscale.com
 go 1.17
 
 require (
+	filippo.io/mkcert v1.4.3
 	github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
 	github.com/aws/aws-sdk-go v1.38.52
@@ -188,8 +189,10 @@ require (
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
+	howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect
 	mvdan.cc/gofumpt v0.0.0-20201129102820-5c11c50e9475 // indirect
 	mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect
 	mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b // indirect
 	mvdan.cc/unparam v0.0.0-20200501210554-b37ab49443f7 // indirect
+	software.sslmate.com/src/go-pkcs12 v0.0.0-20180114231543-2291e8f0f237 // indirect
 )

go.sum

@@ -14,6 +14,8 @@ cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqCl
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+filippo.io/mkcert v1.4.3 h1:axpnmtrZMM8u5Hf4N3UXxboGemMOV+Tn+e+pkHM6E3o=
+filippo.io/mkcert v1.4.3/go.mod h1:64ke566uBwAQcdK3vRDABgsgVHqrfORPTw6YytZCTxk=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@@ -876,6 +878,7 @@ golang.org/x/tools v0.0.0-20201011145850-ed2f50202694/go.mod h1:z6u4i615ZeAfBE4X
 golang.org/x/tools v0.0.0-20201013201025-64a9e34f3752/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
 golang.org/x/tools v0.0.0-20201118003311-bd56c0adb394/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201121010211-780cb80bd7fb/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201124202034-299f270db459/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
@@ -957,6 +960,8 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
 honnef.co/go/tools v0.0.1-2020.1.6/go.mod h1:pyyisuGw24ruLjrr1ddx39WE0y9OooInRzEYLhQB2YY=
 honnef.co/go/tools v0.2.1 h1:/EPr//+UMMXwMTkXvCCoaJDq8cpjMO80Ou+L4PDo2mY=
 honnef.co/go/tools v0.2.1/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
+howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M=
+howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0=
 inet.af/netaddr v0.0.0-20210515010201-ad03edc7c841/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
 inet.af/netaddr v0.0.0-20210721214506-ce7a8ad02cc1 h1:mxmfTV6kjXTlFqqFETnG9FQZzNFc6AKunZVAgQ3b7WA=
 inet.af/netaddr v0.0.0-20210721214506-ce7a8ad02cc1/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
@@ -976,3 +981,5 @@ mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jC
 mvdan.cc/unparam v0.0.0-20200501210554-b37ab49443f7 h1:kAREL6MPwpsk1/PQPFD3Eg7WAQR5mPTWZJaBiG5LDbY=
 mvdan.cc/unparam v0.0.0-20200501210554-b37ab49443f7/go.mod h1:HGC5lll35J70Y5v7vCGb9oLhHoScFwkHDJm/05RdSTc=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+software.sslmate.com/src/go-pkcs12 v0.0.0-20180114231543-2291e8f0f237 h1:iAEkCBPbRaflBgZ7o9gjVUuWuvWeV4sytFWg9o+Pj2k=
+software.sslmate.com/src/go-pkcs12 v0.0.0-20180114231543-2291e8f0f237/go.mod h1:/xvNRWUqm0+/ZMiF4EX00vrSCMsE4/NHb+Pt3freEeQ=

net/dnsfallback/dnsfallback.go

@@ -23,6 +23,7 @@
 
 	"inet.af/netaddr"
 	"tailscale.com/net/netns"
+	"tailscale.com/net/tlsdial"
 	"tailscale.com/net/tshttpproxy"
 	"tailscale.com/tailcfg"
 )
@@ -95,6 +96,7 @@ func bootstrapDNSMap(ctx context.Context, serverName string, serverIP netaddr.IP
 	tr.DialContext = func(ctx context.Context, netw, addr string) (net.Conn, error) {
 		return dialer.DialContext(ctx, "tcp", net.JoinHostPort(serverIP.String(), "443"))
 	}
+	tr.TLSClientConfig = tlsdial.Config(serverName, tr.TLSClientConfig)
 	c := &http.Client{Transport: tr}
 	req, err := http.NewRequestWithContext(ctx, "GET", "https://"+serverName+"/bootstrap-dns?q="+url.QueryEscape(queryName), nil)
 	if err != nil {

net/tlsdial/tlsdial.go

@@ -15,9 +15,24 @@
 	"crypto/tls"
 	"crypto/x509"
 	"errors"
+	"log"
+	"os"
+	"strconv"
+	"sync"
+	"sync/atomic"
 	"time"
 )
 
+var counterFallbackOK int32 // atomic
+
+// If SSLKEYLOGFILE is set, it's a file to which we write our TLS private keys
+// in a way that WireShark can read.
+//
+// See https://developer.mozilla.org/en-US/docs/Mozilla/Projects/NSS/Key_Log_Format
+var sslKeyLogFile = os.Getenv("SSLKEYLOGFILE")
+
+var debug, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_TLS_DIAL"))
+
 // Config returns a tls.Config for connecting to a server.
 // If base is non-nil, it's cloned as the base config before
 // being configured and returned.
@@ -30,11 +45,65 @@ func Config(host string, base *tls.Config) *tls.Config {
 	}
 	conf.ServerName = host
 
+	if n := sslKeyLogFile; n != "" {
+		f, err := os.OpenFile(n, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
+		if err != nil {
+			log.Fatal(err)
+		}
+		log.Printf("WARNING: writing to SSLKEYLOGFILE %v", n)
+		conf.KeyLogWriter = f
+	}
+
+	if conf.InsecureSkipVerify {
+		panic("unexpected base.InsecureSkipVerify")
+	}
+	if conf.VerifyConnection != nil {
+		panic("unexpected base.VerifyConnection")
+	}
+
+	// Set InsecureSkipVerify to prevent crypto/tls from doing its
+	// own cert verification, as do the same work that it'd do
+	// (with the baked-in fallback root) in the VerifyConnection hook.
+	conf.InsecureSkipVerify = true
+	conf.VerifyConnection = func(cs tls.ConnectionState) error {
+		// First try doing x509 verification with the system's
+		// root CA pool.
+		opts := x509.VerifyOptions{
+			DNSName:       cs.ServerName,
+			Intermediates: x509.NewCertPool(),
+		}
+		for _, cert := range cs.PeerCertificates[1:] {
+			opts.Intermediates.AddCert(cert)
+		}
+		_, errSys := cs.PeerCertificates[0].Verify(opts)
+		if debug {
+			log.Printf("tlsdial(sys %q): %v", host, errSys)
+		}
+		if errSys == nil {
+			return nil
+		}
+
+		// If that failed, because the system's CA roots are old
+		// or broken, fall back to trying LetsEncrypt at least.
+		opts.Roots = bakedInRoots()
+		_, err := cs.PeerCertificates[0].Verify(opts)
+		if debug {
+			log.Printf("tlsdial(bake %q): %v", host, err)
+		}
+		if err == nil {
+			atomic.AddInt32(&counterFallbackOK, 1)
+			return nil
+		}
+		return errSys
+	}
 	return conf
 }
 
 // SetConfigExpectedCert modifies c to expect and verify that the server returns
 // a certificate for the provided certDNSName.
+//
+// This is for user-configurable client-side domain fronting support,
+// where we send one SNI value but validate a different cert.
 func SetConfigExpectedCert(c *tls.Config, certDNSName string) {
 	if c.ServerName == certDNSName {
 		return
@@ -50,6 +119,7 @@ func SetConfigExpectedCert(c *tls.Config, certDNSName string) {
 	// own cert verification, but do the same work that it'd do
 	// (but using certDNSName) in the VerifyPeerCertificate hook.
 	c.InsecureSkipVerify = true
+	c.VerifyConnection = nil
 	c.VerifyPeerCertificate = func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
 		if len(rawCerts) == 0 {
 			return errors.New("no certs presented")
@@ -70,7 +140,102 @@ func SetConfigExpectedCert(c *tls.Config, certDNSName string) {
 		for _, cert := range certs[1:] {
 			opts.Intermediates.AddCert(cert)
 		}
+		_, errSys := certs[0].Verify(opts)
+		if debug {
+			log.Printf("tlsdial(sys %q/%q): %v", c.ServerName, certDNSName, errSys)
+		}
+		if errSys == nil {
+			return nil
+		}
+		opts.Roots = bakedInRoots()
 		_, err := certs[0].Verify(opts)
-		return err
+		if debug {
+			log.Printf("tlsdial(bake %q/%q): %v", c.ServerName, certDNSName, err)
+		}
+		if err == nil {
+			return nil
+		}
+		return errSys
 	}
 }
+
+/*
+letsEncryptX1 is the LetsEncrypt X1 root:
+
+Certificate:
+    Data:
+        Version: 3 (0x2)
+        Serial Number:
+            82:10:cf:b0:d2:40:e3:59:44:63:e0:bb:63:82:8b:00
+        Signature Algorithm: sha256WithRSAEncryption
+        Issuer: C = US, O = Internet Security Research Group, CN = ISRG Root X1
+        Validity
+            Not Before: Jun  4 11:04:38 2015 GMT
+            Not After : Jun  4 11:04:38 2035 GMT
+        Subject: C = US, O = Internet Security Research Group, CN = ISRG Root X1
+        Subject Public Key Info:
+            Public Key Algorithm: rsaEncryption
+                RSA Public-Key: (4096 bit)
+
+We bake it into the binary as a fallback verification root,
+in case the system we're running on doesn't have it.
+(Tailscale runs on some ancient devices.)
+
+To test that this code is working on Debian/Ubuntu:
+
+$ sudo mv /usr/share/ca-certificates/mozilla/ISRG_Root_X1.crt{,.old}
+$ sudo update-ca-certificates
+
+Then restart tailscaled. To also test dnsfallback's use of it, nuke
+your /etc/resolv.conf and it should still start & run fine.
+
+*/
+const letsEncryptX1 = `
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----
+`
+
+var bakedInRootsOnce struct {
+	sync.Once
+	p *x509.CertPool
+}
+
+func bakedInRoots() *x509.CertPool {
+	bakedInRootsOnce.Do(func() {
+		p := x509.NewCertPool()
+		if !p.AppendCertsFromPEM([]byte(letsEncryptX1)) {
+			panic("bogus PEM")
+		}
+		bakedInRootsOnce.p = p
+	})
+	return bakedInRootsOnce.p
+}

net/tlsdial/tlsdial_test.go

@@ -0,0 +1,131 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package tlsdial
+
+import (
+	"crypto/x509"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"reflect"
+	"runtime"
+	"sync/atomic"
+	"testing"
+)
+
+func resetOnce() {
+	rv := reflect.ValueOf(&bakedInRootsOnce).Elem()
+	rv.Set(reflect.Zero(rv.Type()))
+}
+
+func TestBakedInRoots(t *testing.T) {
+	resetOnce()
+	p := bakedInRoots()
+	got := p.Subjects()
+	if len(got) != 1 {
+		t.Errorf("subjects = %v; want 1", len(got))
+	}
+}
+
+func TestFallbackRootWorks(t *testing.T) {
+	defer resetOnce()
+
+	const debug = false
+	if runtime.GOOS != "linux" {
+		t.Skip("test assumes Linux")
+	}
+	d := t.TempDir()
+	crtFile := filepath.Join(d, "tlsdial.test.crt")
+	keyFile := filepath.Join(d, "tlsdial.test.key")
+	caFile := filepath.Join(d, "rootCA.pem")
+	cmd := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"),
+		"run", "filippo.io/mkcert",
+		"--cert-file="+crtFile,
+		"--key-file="+keyFile,
+		"tlsdial.test")
+	cmd.Env = append(os.Environ(), "CAROOT="+d)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("mkcert: %v, %s", err, out)
+	}
+	if debug {
+		t.Logf("Ran: %s", out)
+		dents, err := os.ReadDir(d)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for _, de := range dents {
+			t.Logf(" - %v", de)
+		}
+	}
+
+	caPEM, err := os.ReadFile(caFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resetOnce()
+	bakedInRootsOnce.Do(func() {
+		p := x509.NewCertPool()
+		if !p.AppendCertsFromPEM(caPEM) {
+			t.Fatal("failed to add")
+		}
+		bakedInRootsOnce.p = p
+	})
+
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ln.Close()
+	if debug {
+		t.Logf("listener running at %v", ln.Addr())
+	}
+	done := make(chan struct{})
+	defer close(done)
+
+	errc := make(chan error, 1)
+	go func() {
+		err := http.ServeTLS(ln, http.HandlerFunc(sayHi), crtFile, keyFile)
+		select {
+		case <-done:
+			return
+		default:
+			t.Logf("ServeTLS: %v", err)
+			errc <- err
+		}
+	}()
+
+	tr := &http.Transport{
+		Dial: func(network, addr string) (net.Conn, error) {
+			return net.Dial("tcp", ln.Addr().String())
+		},
+		DisableKeepAlives: true, // for test cleanup ease
+	}
+	tr.TLSClientConfig = Config("tlsdial.test", tr.TLSClientConfig)
+	c := &http.Client{Transport: tr}
+
+	ctr0 := atomic.LoadInt32(&counterFallbackOK)
+
+	res, err := c.Get("https://tlsdial.test/")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer res.Body.Close()
+	if res.StatusCode != 200 {
+		t.Fatal(res.Status)
+	}
+
+	ctrDelta := atomic.LoadInt32(&counterFallbackOK) - ctr0
+	if ctrDelta != 1 {
+		t.Errorf("fallback root success count = %d; want 1", ctrDelta)
+	}
+}
+
+func sayHi(w http.ResponseWriter, r *http.Request) {
+	io.WriteString(w, "hi")
+}