ipn/ipnlocal: also use LetsEncrypt-baked-in roots for cert validation

We previously baked in the LetsEncrypt x509 root CA for our tlsdial
package.

This moves that out into a new "bakedroots" package and is now also
shared by ipn/ipnlocal's cert validation code (validCertPEM) that
decides whether it's time to fetch a new cert.

Otherwise, a machine without LetsEncrypt roots locally in its system
roots is unable to use tailscale cert/serve and fetch certs.

Fixes #14690

Change-Id: Ic88b3bdaabe25d56b9ff07ada56a27e3f11d7159
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>
This commit is contained in:
Brad Fitzpatrick 2025-01-21 15:30:55 -08:00 committed by Brad Fitzpatrick
parent e12b2a7267
commit 150cd30b1d
9 changed files with 164 additions and 110 deletions

View File

@ -99,6 +99,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
tailscale.com/ipn/ipnstate from tailscale.com/client/tailscale+
tailscale.com/kube/kubetypes from tailscale.com/envknob
tailscale.com/metrics from tailscale.com/cmd/derper+
tailscale.com/net/bakedroots from tailscale.com/net/tlsdial
tailscale.com/net/dnscache from tailscale.com/derp/derphttp
tailscale.com/net/ktimeout from tailscale.com/cmd/derper
tailscale.com/net/netaddr from tailscale.com/ipn+

View File

@ -835,6 +835,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
tailscale.com/logtail/backoff from tailscale.com/control/controlclient+
tailscale.com/logtail/filch from tailscale.com/log/sockstatlog+
tailscale.com/metrics from tailscale.com/derp+
tailscale.com/net/bakedroots from tailscale.com/net/tlsdial+
tailscale.com/net/captivedetection from tailscale.com/ipn/ipnlocal+
tailscale.com/net/connstats from tailscale.com/net/tstun+
tailscale.com/net/dns from tailscale.com/ipn/ipnlocal+

View File

@ -97,6 +97,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
tailscale.com/kube/kubetypes from tailscale.com/envknob
tailscale.com/licenses from tailscale.com/client/web+
tailscale.com/metrics from tailscale.com/derp+
tailscale.com/net/bakedroots from tailscale.com/net/tlsdial
tailscale.com/net/captivedetection from tailscale.com/net/netcheck
tailscale.com/net/dns/recursive from tailscale.com/net/dnsfallback
tailscale.com/net/dnscache from tailscale.com/control/controlhttp+

View File

@ -286,6 +286,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
tailscale.com/logtail/backoff from tailscale.com/cmd/tailscaled+
tailscale.com/logtail/filch from tailscale.com/log/sockstatlog+
tailscale.com/metrics from tailscale.com/derp+
tailscale.com/net/bakedroots from tailscale.com/net/tlsdial+
tailscale.com/net/captivedetection from tailscale.com/ipn/ipnlocal+
tailscale.com/net/connstats from tailscale.com/net/tstun+
tailscale.com/net/dns from tailscale.com/cmd/tailscaled+

View File

@ -40,6 +40,7 @@ import (
"tailscale.com/ipn/ipnstate"
"tailscale.com/ipn/store"
"tailscale.com/ipn/store/mem"
"tailscale.com/net/bakedroots"
"tailscale.com/types/logger"
"tailscale.com/util/testenv"
"tailscale.com/version"
@ -665,7 +666,7 @@ func acmeClient(cs certStore) (*acme.Client, error) {
// validCertPEM reports whether the given certificate is valid for domain at now.
//
// If roots != nil, it is used instead of the system root pool. This is meant
// to support testing, and production code should pass roots == nil.
// to support testing; production code should pass roots == nil.
func validCertPEM(domain string, keyPEM, certPEM []byte, roots *x509.CertPool, now time.Time) bool {
if len(keyPEM) == 0 || len(certPEM) == 0 {
return false
@ -688,15 +689,29 @@ func validCertPEM(domain string, keyPEM, certPEM []byte, roots *x509.CertPool, n
intermediates.AddCert(cert)
}
}
return validateLeaf(leaf, intermediates, domain, now, roots)
}
// validateLeaf is a helper for [validCertPEM].
//
// If called with roots == nil, it will use the system root pool as well as the
// baked-in roots. If non-nil, only those roots are used.
func validateLeaf(leaf *x509.Certificate, intermediates *x509.CertPool, domain string, now time.Time, roots *x509.CertPool) bool {
if leaf == nil {
return false
}
_, err = leaf.Verify(x509.VerifyOptions{
_, err := leaf.Verify(x509.VerifyOptions{
DNSName: domain,
CurrentTime: now,
Roots: roots,
Intermediates: intermediates,
})
if err != nil && roots == nil {
// If validation failed and they specified nil for roots (meaning to use
// the system roots), then give it another chance to validate using the
// binary's baked-in roots (LetsEncrypt). See tailscale/tailscale#14690.
return validateLeaf(leaf, intermediates, domain, now, bakedroots.Get())
}
return err == nil
}

View File

@ -0,0 +1,122 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
// Package bakedroots contains WebPKI CA roots we bake into the tailscaled binary,
// lest the system's CA roots be missing them (or entirely empty).
package bakedroots
import (
"crypto/x509"
"sync"
"tailscale.com/util/testenv"
)
// Get returns the baked-in roots.
//
// As of 2025-01-21, this includes only the LetsEncrypt ISRG Root X1 root.
func Get() *x509.CertPool {
roots.once.Do(func() { roots.parsePEM([]byte(letsEncryptX1)) })
return roots.p
}
// testingTB is a subset of testing.TB needed
// to verify the caller isn't in a parallel test.
type testingTB interface {
// Setenv panics if it's in a parallel test.
Setenv(k, v string)
}
// ResetForTest resets the cached roots for testing,
// optionally setting them to caPEM if non-nil.
func ResetForTest(tb testingTB, caPEM []byte) {
if !testenv.InTest() {
panic("not in test")
}
tb.Setenv("ASSERT_NOT_PARALLEL_TEST", "1") // panics if tb's Parallel was called
roots = rootsOnce{}
if caPEM != nil {
roots.once.Do(func() { roots.parsePEM(caPEM) })
}
}
var roots rootsOnce
type rootsOnce struct {
once sync.Once
p *x509.CertPool
}
func (r *rootsOnce) parsePEM(caPEM []byte) {
p := x509.NewCertPool()
if !p.AppendCertsFromPEM(caPEM) {
panic("bogus PEM")
}
r.p = p
}
/*
letsEncryptX1 is the LetsEncrypt X1 root:
Certificate:
Data:
Version: 3 (0x2)
Serial Number:
82:10:cf:b0:d2:40:e3:59:44:63:e0:bb:63:82:8b:00
Signature Algorithm: sha256WithRSAEncryption
Issuer: C = US, O = Internet Security Research Group, CN = ISRG Root X1
Validity
Not Before: Jun 4 11:04:38 2015 GMT
Not After : Jun 4 11:04:38 2035 GMT
Subject: C = US, O = Internet Security Research Group, CN = ISRG Root X1
Subject Public Key Info:
Public Key Algorithm: rsaEncryption
RSA Public-Key: (4096 bit)
We bake it into the binary as a fallback verification root,
in case the system we're running on doesn't have it.
(Tailscale runs on some ancient devices.)
To test that this code is working on Debian/Ubuntu:
$ sudo mv /usr/share/ca-certificates/mozilla/ISRG_Root_X1.crt{,.old}
$ sudo update-ca-certificates
Then restart tailscaled. To also test dnsfallback's use of it, nuke
your /etc/resolv.conf and it should still start & run fine.
*/
const letsEncryptX1 = `
-----BEGIN CERTIFICATE-----
MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
-----END CERTIFICATE-----
`

View File

@ -0,0 +1,15 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
package bakedroots
import "testing"
func TestBakedInRoots(t *testing.T) {
ResetForTest(t, nil)
p := Get()
got := p.Subjects()
if len(got) != 1 {
t.Errorf("subjects = %v; want 1", len(got))
}
}

View File

@ -27,6 +27,7 @@ import (
"tailscale.com/envknob"
"tailscale.com/health"
"tailscale.com/hostinfo"
"tailscale.com/net/bakedroots"
"tailscale.com/net/tlsdial/blockblame"
)
@ -154,7 +155,7 @@ func Config(host string, ht *health.Tracker, base *tls.Config) *tls.Config {
// Always verify with our baked-in Let's Encrypt certificate,
// so we can log an informational message. This is useful for
// detecting SSL MiTM.
opts.Roots = bakedInRoots()
opts.Roots = bakedroots.Get()
_, bakedErr := cs.PeerCertificates[0].Verify(opts)
if debug() {
log.Printf("tlsdial(bake %q): %v", host, bakedErr)
@ -233,7 +234,7 @@ func SetConfigExpectedCert(c *tls.Config, certDNSName string) {
if errSys == nil {
return nil
}
opts.Roots = bakedInRoots()
opts.Roots = bakedroots.Get()
_, err := certs[0].Verify(opts)
if debug() {
log.Printf("tlsdial(bake %q/%q): %v", c.ServerName, certDNSName, err)
@ -260,84 +261,3 @@ func NewTransport() *http.Transport {
},
}
}
/*
letsEncryptX1 is the LetsEncrypt X1 root:
Certificate:
Data:
Version: 3 (0x2)
Serial Number:
82:10:cf:b0:d2:40:e3:59:44:63:e0:bb:63:82:8b:00
Signature Algorithm: sha256WithRSAEncryption
Issuer: C = US, O = Internet Security Research Group, CN = ISRG Root X1
Validity
Not Before: Jun 4 11:04:38 2015 GMT
Not After : Jun 4 11:04:38 2035 GMT
Subject: C = US, O = Internet Security Research Group, CN = ISRG Root X1
Subject Public Key Info:
Public Key Algorithm: rsaEncryption
RSA Public-Key: (4096 bit)
We bake it into the binary as a fallback verification root,
in case the system we're running on doesn't have it.
(Tailscale runs on some ancient devices.)
To test that this code is working on Debian/Ubuntu:
$ sudo mv /usr/share/ca-certificates/mozilla/ISRG_Root_X1.crt{,.old}
$ sudo update-ca-certificates
Then restart tailscaled. To also test dnsfallback's use of it, nuke
your /etc/resolv.conf and it should still start & run fine.
*/
const letsEncryptX1 = `
-----BEGIN CERTIFICATE-----
MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
-----END CERTIFICATE-----
`
var bakedInRootsOnce struct {
sync.Once
p *x509.CertPool
}
func bakedInRoots() *x509.CertPool {
bakedInRootsOnce.Do(func() {
p := x509.NewCertPool()
if !p.AppendCertsFromPEM([]byte(letsEncryptX1)) {
panic("bogus PEM")
}
bakedInRootsOnce.p = p
})
return bakedInRootsOnce.p
}

View File

@ -4,37 +4,22 @@
package tlsdial
import (
"crypto/x509"
"io"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"reflect"
"runtime"
"sync/atomic"
"testing"
"tailscale.com/health"
"tailscale.com/net/bakedroots"
)
func resetOnce() {
rv := reflect.ValueOf(&bakedInRootsOnce).Elem()
rv.Set(reflect.Zero(rv.Type()))
}
func TestBakedInRoots(t *testing.T) {
resetOnce()
p := bakedInRoots()
got := p.Subjects()
if len(got) != 1 {
t.Errorf("subjects = %v; want 1", len(got))
}
}
func TestFallbackRootWorks(t *testing.T) {
defer resetOnce()
defer bakedroots.ResetForTest(t, nil)
const debug = false
if runtime.GOOS != "linux" {
@ -69,14 +54,7 @@ func TestFallbackRootWorks(t *testing.T) {
if err != nil {
t.Fatal(err)
}
resetOnce()
bakedInRootsOnce.Do(func() {
p := x509.NewCertPool()
if !p.AppendCertsFromPEM(caPEM) {
t.Fatal("failed to add")
}
bakedInRootsOnce.p = p
})
bakedroots.ResetForTest(t, caPEM)
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {