tempfork/x509: store certs for iOS compressed in binary, parse lazily

tempfork/x509/cert_pool.go

@@ -143,12 +143,9 @@ func (s *CertPool) AddCert(cert *Certificate) {
 	if cert == nil {
 		panic("adding nil Certificate to CertPool")
 	}
-	err := s.AddCertFunc(sha256.Sum224(cert.Raw), string(cert.RawSubject), string(cert.SubjectKeyId), func() (*Certificate, error) {
+	s.AddCertFunc(sha256.Sum224(cert.Raw), string(cert.RawSubject), string(cert.SubjectKeyId), func() (*Certificate, error) {
 		return cert, nil
 	})
-	if err != nil {
-		panic(err.Error())
-	}
 }
 
 // AddCertFunc adds metadata about a certificate to a pool, along with
@@ -157,17 +154,19 @@ func (s *CertPool) AddCert(cert *Certificate) {
 // The rawSubject is Certificate.RawSubject and must be non-empty.
 // The subjectKeyID is Certificate.SubjectKeyId and may be empty.
 // The getCert func may be called 0 or more times.
-func (s *CertPool) AddCertFunc(rawSum224 sum224, rawSubject, subjectKeyID string, getCert func() (*Certificate, error)) error {
+func (s *CertPool) AddCertFunc(rawSum224 sum224, rawSubject, subjectKeyID string, getCert func() (*Certificate, error)) {
+	// Check that the certificate isn't being added twice.
+	if s.haveSum[rawSum224] {
+		return
+	}
+	s.haveSum[rawSum224] = true
+	s.addCertFuncNotDup(rawSubject, subjectKeyID, getCert)
+}
+
+func (s *CertPool) addCertFuncNotDup(rawSubject, subjectKeyID string, getCert func() (*Certificate, error)) {
 	if getCert == nil {
 		panic("getCert can't be nil")
 	}
-
-	// Check that the certificate isn't being added twice.
-	if s.haveSum[rawSum224] {
-		return nil
-	}
-	s.haveSum[rawSum224] = true
-
 	n := len(s.getCert)
 	s.getCert = append(s.getCert, getCert)
 
@@ -176,7 +175,6 @@ func (s *CertPool) AddCertFunc(rawSum224 sum224, rawSubject, subjectKeyID string
 	}
 	s.byName[rawSubject] = append(s.byName[rawSubject], n)
 	s.rawSubjects = append(s.rawSubjects, []byte(rawSubject))
-	return nil
 }
 
 // AppendCertsFromPEM attempts to parse a series of PEM encoded certificates.

tempfork/x509/pool_darwin_arm64.go

@@ -0,0 +1,36 @@
+// Copyright 2020 Tailscale Inc. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !x509omitbundledroots
+
+package x509
+
+import (
+	"compress/gzip"
+	"io/ioutil"
+	"strings"
+	"sync"
+)
+
+func certUncompressor(zcertBytes string) func() (*Certificate, error) {
+	var once sync.Once
+	var c *Certificate
+	var err error
+	return func() (*Certificate, error) {
+		once.Do(func() {
+			var certBytes []byte
+			var zr *gzip.Reader
+			zr, err = gzip.NewReader(strings.NewReader(zcertBytes))
+			if err != nil {
+				return
+			}
+			certBytes, err = ioutil.ReadAll(zr)
+			if err != nil {
+				return
+			}
+			c, err = ParseCertificate(certBytes)
+		})
+		return c, err
+	}
+}

tempfork/x509/root_cgo_darwin.go

@@ -306,7 +306,10 @@ func loadSystemRoots() (*CertPool, error) {
 
 	trustedRoots := NewCertPool()
 	for i := 0; i < roots.len(); i++ {
-		c := roots.mustCert(i)
+		c, err := roots.cert(i)
+		if err != nil {
+			return nil, err
+		}
 		contains, err := untrustedRoots.contains(c)
 		if err != nil {
 			return nil, err

tempfork/x509/root_darwin_arm_gen.go

@@ -18,19 +18,14 @@
 
 import (
 	"bytes"
-	"crypto/sha256"
+	"compress/gzip"
 	"crypto/x509"
-	"encoding/hex"
 	"encoding/pem"
 	"flag"
 	"fmt"
 	"go/format"
 	"io/ioutil"
 	"log"
-	"net/http"
-	"os/exec"
-	"regexp"
-	"strings"
 )
 
 var output = flag.String("output", "root_darwin_arm64.go", "file name to write")
@@ -45,18 +40,24 @@ func main() {
 
 	fmt.Fprintf(buf, "// Code generated by root_darwin_arm_gen --output %s; DO NOT EDIT.\n", *output)
 	fmt.Fprintf(buf, "%s", header)
-
-	fmt.Fprintf(buf, "const systemRootsPEM = `\n")
 	for _, cert := range certs {
-		b := &pem.Block{
+		gzbuf := new(bytes.Buffer)
-			Type:  "CERTIFICATE",
+		zw, err := gzip.NewWriterLevel(gzbuf, gzip.BestCompression)
-			Bytes: cert.Raw,
+		if err != nil {
-		}
-		if err := pem.Encode(buf, b); err != nil {
 			log.Fatal(err)
 		}
+		if _, err := zw.Write(cert.Raw); err != nil {
+			log.Fatal(err)
 		}
-	fmt.Fprintf(buf, "`")
+		if err := zw.Close(); err != nil {
+			log.Fatal(err)
+		}
+		fmt.Fprintf(buf, "p.addCertFuncNotDup(%q, %q, certUncompressor(%q))\n",
+			cert.RawSubject,
+			cert.SubjectKeyId,
+			gzbuf.Bytes())
+	}
+	fmt.Fprintf(buf, "%s", footer)
 
 	source, err := format.Source(buf.Bytes())
 	if err != nil {
@@ -67,38 +68,14 @@ func main() {
 	}
 }
 
-func selectCerts() ([]*x509.Certificate, error) {
+func selectCerts() (certs []*x509.Certificate, err error) {
-	ids, err := fetchCertIDs()
+	pemCerts, err := ioutil.ReadFile("certs.pem")
 	if err != nil {
 		return nil, err
 	}
-
+	for len(pemCerts) > 0 {
-	scerts, err := sysCerts()
-	if err != nil {
-		return nil, err
-	}
-
-	var certs []*x509.Certificate
-	for _, id := range ids {
-		if c, ok := scerts[id.fingerprint]; ok {
-			certs = append(certs, c)
-		} else {
-			fmt.Printf("WARNING: cannot find certificate: %s (fingerprint: %s)\n", id.name, id.fingerprint)
-		}
-	}
-	return certs, nil
-}
-
-func sysCerts() (certs map[string]*x509.Certificate, err error) {
-	cmd := exec.Command("/usr/bin/security", "find-certificate", "-a", "-p", "/System/Library/Keychains/SystemRootCertificates.keychain")
-	data, err := cmd.Output()
-	if err != nil {
-		return nil, err
-	}
-	certs = make(map[string]*x509.Certificate)
-	for len(data) > 0 {
 		var block *pem.Block
-		block, data = pem.Decode(data)
+		block, pemCerts = pem.Decode(pemCerts)
 		if block == nil {
 			break
 		}
@@ -108,65 +85,13 @@ func sysCerts() (certs map[string]*x509.Certificate, err error) {
 
 		cert, err := x509.ParseCertificate(block.Bytes)
 		if err != nil {
-			continue
+			return nil, err
 		}
-
+		certs = append(certs, cert)
-		fingerprint := sha256.Sum256(cert.Raw)
-		certs[hex.EncodeToString(fingerprint[:])] = cert
 	}
 	return certs, nil
 }
 
-type certID struct {
-	name        string
-	fingerprint string
-}
-
-// fetchCertIDs fetches IDs of iOS X509 certificates from apple.com.
-func fetchCertIDs() ([]certID, error) {
-	// Download the iOS 11 support page. The index for all iOS versions is here:
-	// https://support.apple.com/en-us/HT204132
-	resp, err := http.Get("https://support.apple.com/en-us/HT208125")
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return nil, err
-	}
-	text := string(body)
-	text = text[strings.Index(text, "<div id=trusted"):]
-	text = text[:strings.Index(text, "</div>")]
-
-	var ids []certID
-	cols := make(map[string]int)
-	for i, rowmatch := range regexp.MustCompile("(?s)<tr>(.*?)</tr>").FindAllStringSubmatch(text, -1) {
-		row := rowmatch[1]
-		if i == 0 {
-			// Parse table header row to extract column names
-			for i, match := range regexp.MustCompile("(?s)<th>(.*?)</th>").FindAllStringSubmatch(row, -1) {
-				cols[match[1]] = i
-			}
-			continue
-		}
-
-		values := regexp.MustCompile("(?s)<td>(.*?)</td>").FindAllStringSubmatch(row, -1)
-		name := values[cols["Certificate name"]][1]
-		fingerprint := values[cols["Fingerprint (SHA-256)"]][1]
-		fingerprint = strings.ReplaceAll(fingerprint, "<br>", "")
-		fingerprint = strings.ReplaceAll(fingerprint, "\n", "")
-		fingerprint = strings.ReplaceAll(fingerprint, " ", "")
-		fingerprint = strings.ToLower(fingerprint)
-
-		ids = append(ids, certID{
-			name:        name,
-			fingerprint: fingerprint,
-		})
-	}
-	return ids, nil
-}
-
 const header = `
 // Copyright 2015 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
@@ -178,7 +103,9 @@ func fetchCertIDs() ([]certID, error) {
 
 func loadSystemRoots() (*CertPool, error) {
 	p := NewCertPool()
-	p.AppendCertsFromPEM([]byte(systemRootsPEM))
+`
+
+const footer = `
 	return p, nil
 }
 `

tempfork/x509/root_nocgo_darwin.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !cgo
+// +build !cgo,!arm64
 
 package x509