ipn/ipnlocal: also use LetsEncrypt-baked-in roots for cert validation

We previously baked in the LetsEncrypt x509 root CA for our tlsdial
package.

This moves that out into a new "bakedroots" package and is now also
shared by ipn/ipnlocal's cert validation code (validCertPEM) that
decides whether it's time to fetch a new cert.

Otherwise, a machine without LetsEncrypt roots locally in its system
roots is unable to use tailscale cert/serve and fetch certs.

Fixes #14690

Change-Id: Ic88b3bdaabe25d56b9ff07ada56a27e3f11d7159
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

cmd/derper/depaware.txt

@@ -99,6 +99,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/ipn/ipnstate                                   from tailscale.com/client/tailscale+
         tailscale.com/kube/kubetypes                                 from tailscale.com/envknob
         tailscale.com/metrics                                        from tailscale.com/cmd/derper+
+        tailscale.com/net/bakedroots                                 from tailscale.com/net/tlsdial
         tailscale.com/net/dnscache                                   from tailscale.com/derp/derphttp
         tailscale.com/net/ktimeout                                   from tailscale.com/cmd/derper
         tailscale.com/net/netaddr                                    from tailscale.com/ipn+

cmd/k8s-operator/depaware.txt

@@ -835,6 +835,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/logtail/backoff                                from tailscale.com/control/controlclient+
         tailscale.com/logtail/filch                                  from tailscale.com/log/sockstatlog+
         tailscale.com/metrics                                        from tailscale.com/derp+
+        tailscale.com/net/bakedroots                                 from tailscale.com/net/tlsdial+
         tailscale.com/net/captivedetection                           from tailscale.com/ipn/ipnlocal+
         tailscale.com/net/connstats                                  from tailscale.com/net/tstun+
         tailscale.com/net/dns                                        from tailscale.com/ipn/ipnlocal+

cmd/tailscale/depaware.txt

@@ -97,6 +97,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/kube/kubetypes                                 from tailscale.com/envknob
         tailscale.com/licenses                                       from tailscale.com/client/web+
         tailscale.com/metrics                                        from tailscale.com/derp+
+        tailscale.com/net/bakedroots                                 from tailscale.com/net/tlsdial
         tailscale.com/net/captivedetection                           from tailscale.com/net/netcheck
         tailscale.com/net/dns/recursive                              from tailscale.com/net/dnsfallback
         tailscale.com/net/dnscache                                   from tailscale.com/control/controlhttp+

cmd/tailscaled/depaware.txt

@@ -286,6 +286,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/logtail/backoff                                from tailscale.com/cmd/tailscaled+
         tailscale.com/logtail/filch                                  from tailscale.com/log/sockstatlog+
         tailscale.com/metrics                                        from tailscale.com/derp+
+        tailscale.com/net/bakedroots                                 from tailscale.com/net/tlsdial+
         tailscale.com/net/captivedetection                           from tailscale.com/ipn/ipnlocal+
         tailscale.com/net/connstats                                  from tailscale.com/net/tstun+
         tailscale.com/net/dns                                        from tailscale.com/cmd/tailscaled+

ipn/ipnlocal/cert.go

@@ -40,6 +40,7 @@ import (
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/ipn/store"
 	"tailscale.com/ipn/store/mem"
+	"tailscale.com/net/bakedroots"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/testenv"
 	"tailscale.com/version"
@@ -665,7 +666,7 @@ func acmeClient(cs certStore) (*acme.Client, error) {
 // validCertPEM reports whether the given certificate is valid for domain at now.
 //
 // If roots != nil, it is used instead of the system root pool. This is meant
-// to support testing, and production code should pass roots == nil.
+// to support testing; production code should pass roots == nil.
 func validCertPEM(domain string, keyPEM, certPEM []byte, roots *x509.CertPool, now time.Time) bool {
 	if len(keyPEM) == 0 || len(certPEM) == 0 {
 		return false
@@ -688,15 +689,29 @@ func validCertPEM(domain string, keyPEM, certPEM []byte, roots *x509.CertPool, n
 			intermediates.AddCert(cert)
 		}
 	}
+	return validateLeaf(leaf, intermediates, domain, now, roots)
+}
+
+// validateLeaf is a helper for [validCertPEM].
+//
+// If called with roots == nil, it will use the system root pool as well as the
+// baked-in roots. If non-nil, only those roots are used.
+func validateLeaf(leaf *x509.Certificate, intermediates *x509.CertPool, domain string, now time.Time, roots *x509.CertPool) bool {
 	if leaf == nil {
 		return false
 	}
-	_, err = leaf.Verify(x509.VerifyOptions{
+	_, err := leaf.Verify(x509.VerifyOptions{
 		DNSName:       domain,
 		CurrentTime:   now,
 		Roots:         roots,
 		Intermediates: intermediates,
 	})
+	if err != nil && roots == nil {
+		// If validation failed and they specified nil for roots (meaning to use
+		// the system roots), then give it another chance to validate using the
+		// binary's baked-in roots (LetsEncrypt). See tailscale/tailscale#14690.
+		return validateLeaf(leaf, intermediates, domain, now, bakedroots.Get())
+	}
 	return err == nil
 }
 

net/bakedroots/bakedroots.go

@@ -0,0 +1,122 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Package bakedroots contains WebPKI CA roots we bake into the tailscaled binary,
+// lest the system's CA roots be missing them (or entirely empty).
+package bakedroots
+
+import (
+	"crypto/x509"
+	"sync"
+
+	"tailscale.com/util/testenv"
+)
+
+// Get returns the baked-in roots.
+//
+// As of 2025-01-21, this includes only the LetsEncrypt ISRG Root X1 root.
+func Get() *x509.CertPool {
+	roots.once.Do(func() { roots.parsePEM([]byte(letsEncryptX1)) })
+	return roots.p
+}
+
+// testingTB is a subset of testing.TB needed
+// to verify the caller isn't in a parallel test.
+type testingTB interface {
+	// Setenv panics if it's in a parallel test.
+	Setenv(k, v string)
+}
+
+// ResetForTest resets the cached roots for testing,
+// optionally setting them to caPEM if non-nil.
+func ResetForTest(tb testingTB, caPEM []byte) {
+	if !testenv.InTest() {
+		panic("not in test")
+	}
+	tb.Setenv("ASSERT_NOT_PARALLEL_TEST", "1") // panics if tb's Parallel was called
+
+	roots = rootsOnce{}
+	if caPEM != nil {
+		roots.once.Do(func() { roots.parsePEM(caPEM) })
+	}
+}
+
+var roots rootsOnce
+
+type rootsOnce struct {
+	once sync.Once
+	p    *x509.CertPool
+}
+
+func (r *rootsOnce) parsePEM(caPEM []byte) {
+	p := x509.NewCertPool()
+	if !p.AppendCertsFromPEM(caPEM) {
+		panic("bogus PEM")
+	}
+	r.p = p
+}
+
+/*
+letsEncryptX1 is the LetsEncrypt X1 root:
+
+Certificate:
+
+	Data:
+	    Version: 3 (0x2)
+	    Serial Number:
+	        82:10:cf:b0:d2:40:e3:59:44:63:e0:bb:63:82:8b:00
+	    Signature Algorithm: sha256WithRSAEncryption
+	    Issuer: C = US, O = Internet Security Research Group, CN = ISRG Root X1
+	    Validity
+	        Not Before: Jun  4 11:04:38 2015 GMT
+	        Not After : Jun  4 11:04:38 2035 GMT
+	    Subject: C = US, O = Internet Security Research Group, CN = ISRG Root X1
+	    Subject Public Key Info:
+	        Public Key Algorithm: rsaEncryption
+	            RSA Public-Key: (4096 bit)
+
+We bake it into the binary as a fallback verification root,
+in case the system we're running on doesn't have it.
+(Tailscale runs on some ancient devices.)
+
+To test that this code is working on Debian/Ubuntu:
+
+$ sudo mv /usr/share/ca-certificates/mozilla/ISRG_Root_X1.crt{,.old}
+$ sudo update-ca-certificates
+
+Then restart tailscaled. To also test dnsfallback's use of it, nuke
+your /etc/resolv.conf and it should still start & run fine.
+*/
+const letsEncryptX1 = `
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----
+`

net/bakedroots/bakedroots_test.go

@@ -0,0 +1,15 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package bakedroots
+
+import "testing"
+
+func TestBakedInRoots(t *testing.T) {
+	ResetForTest(t, nil)
+	p := Get()
+	got := p.Subjects()
+	if len(got) != 1 {
+		t.Errorf("subjects = %v; want 1", len(got))
+	}
+}

net/tlsdial/tlsdial.go

@@ -27,6 +27,7 @@ import (
 	"tailscale.com/envknob"
 	"tailscale.com/health"
 	"tailscale.com/hostinfo"
+	"tailscale.com/net/bakedroots"
 	"tailscale.com/net/tlsdial/blockblame"
 )
 
@@ -154,7 +155,7 @@ func Config(host string, ht *health.Tracker, base *tls.Config) *tls.Config {
 		// Always verify with our baked-in Let's Encrypt certificate,
 		// so we can log an informational message. This is useful for
 		// detecting SSL MiTM.
-		opts.Roots = bakedInRoots()
+		opts.Roots = bakedroots.Get()
 		_, bakedErr := cs.PeerCertificates[0].Verify(opts)
 		if debug() {
 			log.Printf("tlsdial(bake %q): %v", host, bakedErr)
@@ -233,7 +234,7 @@ func SetConfigExpectedCert(c *tls.Config, certDNSName string) {
 		if errSys == nil {
 			return nil
 		}
-		opts.Roots = bakedInRoots()
+		opts.Roots = bakedroots.Get()
 		_, err := certs[0].Verify(opts)
 		if debug() {
 			log.Printf("tlsdial(bake %q/%q): %v", c.ServerName, certDNSName, err)
@@ -260,84 +261,3 @@ func NewTransport() *http.Transport {
 		},
 	}
 }
-
-/*
-letsEncryptX1 is the LetsEncrypt X1 root:
-
-Certificate:
-
-	Data:
-	    Version: 3 (0x2)
-	    Serial Number:
-	        82:10:cf:b0:d2:40:e3:59:44:63:e0:bb:63:82:8b:00
-	    Signature Algorithm: sha256WithRSAEncryption
-	    Issuer: C = US, O = Internet Security Research Group, CN = ISRG Root X1
-	    Validity
-	        Not Before: Jun  4 11:04:38 2015 GMT
-	        Not After : Jun  4 11:04:38 2035 GMT
-	    Subject: C = US, O = Internet Security Research Group, CN = ISRG Root X1
-	    Subject Public Key Info:
-	        Public Key Algorithm: rsaEncryption
-	            RSA Public-Key: (4096 bit)
-
-We bake it into the binary as a fallback verification root,
-in case the system we're running on doesn't have it.
-(Tailscale runs on some ancient devices.)
-
-To test that this code is working on Debian/Ubuntu:
-
-$ sudo mv /usr/share/ca-certificates/mozilla/ISRG_Root_X1.crt{,.old}
-$ sudo update-ca-certificates
-
-Then restart tailscaled. To also test dnsfallback's use of it, nuke
-your /etc/resolv.conf and it should still start & run fine.
-*/
-const letsEncryptX1 = `
------BEGIN CERTIFICATE-----
-MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
-TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
-cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
-WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
-ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
-MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
-h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
-0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
-A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
-T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
-B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
-B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
-KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
-OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
-jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
-qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
-rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
-HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
-hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
-ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
-3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
-NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
-ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
-TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
-jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
-oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
-4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
-mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
-emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
------END CERTIFICATE-----
-`
-
-var bakedInRootsOnce struct {
-	sync.Once
-	p *x509.CertPool
-}
-
-func bakedInRoots() *x509.CertPool {
-	bakedInRootsOnce.Do(func() {
-		p := x509.NewCertPool()
-		if !p.AppendCertsFromPEM([]byte(letsEncryptX1)) {
-			panic("bogus PEM")
-		}
-		bakedInRootsOnce.p = p
-	})
-	return bakedInRootsOnce.p
-}

net/tlsdial/tlsdial_test.go

@@ -4,37 +4,22 @@
 package tlsdial
 
 import (
-	"crypto/x509"
 	"io"
 	"net"
 	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
-	"reflect"
 	"runtime"
 	"sync/atomic"
 	"testing"
 
 	"tailscale.com/health"
+	"tailscale.com/net/bakedroots"
 )
 
-func resetOnce() {
-	rv := reflect.ValueOf(&bakedInRootsOnce).Elem()
-	rv.Set(reflect.Zero(rv.Type()))
-}
-
-func TestBakedInRoots(t *testing.T) {
-	resetOnce()
-	p := bakedInRoots()
-	got := p.Subjects()
-	if len(got) != 1 {
-		t.Errorf("subjects = %v; want 1", len(got))
-	}
-}
-
 func TestFallbackRootWorks(t *testing.T) {
-	defer resetOnce()
+	defer bakedroots.ResetForTest(t, nil)
 
 	const debug = false
 	if runtime.GOOS != "linux" {
@@ -69,14 +54,7 @@ func TestFallbackRootWorks(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	resetOnce()
+	bakedroots.ResetForTest(t, caPEM)
-	bakedInRootsOnce.Do(func() {
-		p := x509.NewCertPool()
-		if !p.AppendCertsFromPEM(caPEM) {
-			t.Fatal("failed to add")
-		}
-		bakedInRootsOnce.p = p
-	})
 
 	ln, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {