client/web: add support for zst precomppressed assets

This will enable us to reduce the size of these embedded assets.

Updates tailscale/corp#20099

Signed-off-by: James Tucker <james@tailscale.com>

client/web/assets.go

@@ -4,6 +4,7 @@
 package web
 
 import (
+	"fmt"
 	"io"
 	"io/fs"
 	"log"
@@ -16,7 +17,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/klauspost/compress/zstd"
 	prebuilt "github.com/tailscale/web-client-prebuilt"
+	"tailscale.com/tsweb/tswebutil"
 )
 
 var start = time.Now()
@@ -63,7 +66,64 @@ func assetsHandler(devMode bool) (_ http.Handler, cleanup func()) {
 	}), nil
 }
 
-func openPrecompressedFile(w http.ResponseWriter, r *http.Request, path string, fs fs.FS) (fs.File, error) {
+type zstFile struct {
+	f fs.File
+	*zstd.Decoder
+}
+
+func newZSTFile(f fs.File) (*zstFile, error) {
+	zr, err := zstd.NewReader(f)
+	if err != nil {
+		return nil, err
+	}
+	return &zstFile{f: f, Decoder: zr}, nil
+}
+
+func (z *zstFile) Seek(offset int64, whence int) (int64, error) {
+	reset := func() error {
+		if seeker, ok := z.f.(io.Seeker); ok {
+			seeker.Seek(0, io.SeekStart)
+		} else {
+			return fmt.Errorf("not seekable: %w", os.ErrInvalid)
+		}
+		return z.Decoder.Reset(z.f)
+	}
+
+	switch whence {
+	case io.SeekStart:
+		if err := reset(); err != nil {
+			return 0, err
+		}
+		return io.CopyN(io.Discard, z, offset)
+	case io.SeekCurrent:
+		if offset >= 0 {
+			io.CopyN(io.Discard, z, offset)
+		} else {
+			return 0, fmt.Errorf("unsupported negative seek: %w", os.ErrInvalid)
+		}
+	case io.SeekEnd:
+		if offset != 0 {
+			return 0, fmt.Errorf("unsupported non-zero offset for SeekEnd: %w", os.ErrInvalid)
+		}
+		return io.Copy(io.Discard, z)
+	}
+	return 0, os.ErrInvalid
+}
+
+func (z *zstFile) Close() error {
+	z.Decoder.Close()
+	return z.f.Close()
+}
+
+func openPrecompressedFile(w http.ResponseWriter, r *http.Request, path string, fs fs.FS) (io.ReadCloser, error) {
+	if f, err := fs.Open(path + ".zst"); err == nil {
+		if tswebutil.AcceptsEncoding(r, "zstd") {
+			w.Header().Set("Content-Encoding", "zstd")
+			return f, nil
+		}
+		return newZSTFile(f)
+	}
+	// TODO(raggi): remove this code path when no longer used
 	if f, err := fs.Open(path + ".gz"); err == nil {
 		w.Header().Set("Content-Encoding", "gzip")
 		return f, nil

cmd/build-webclient/build-webclient.go

@@ -73,10 +73,14 @@ func build(toolDir, appDir string) error {
 		if err := os.Remove(f); err != nil {
 			log.Printf("Failed to cleanup %q: %v", f, err)
 		}
-		// Removing intermediate ".br" version, we use ".gz" asset.
+		// Removing ".br" version, we use the ".zst" asset.
 		if err := os.Remove(f + ".br"); err != nil {
 			log.Printf("Failed to cleanup %q: %v", f+".gz", err)
 		}
+		// Removing ".gz" version, we use the ".zst" asset.
+		if err := os.Remove(f + ".gz"); err != nil {
+			log.Printf("Failed to cleanup %q: %v", f+".gz", err)
+		}
 	}
 
 	return nil

cmd/derper/depaware.txt

@@ -122,6 +122,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/tstime/rate                                    from tailscale.com/derp+
         tailscale.com/tsweb                                          from tailscale.com/cmd/derper
         tailscale.com/tsweb/promvarz                                 from tailscale.com/tsweb
+        tailscale.com/tsweb/tswebutil                                from tailscale.com/tsweb
         tailscale.com/tsweb/varz                                     from tailscale.com/tsweb+
         tailscale.com/types/dnstype                                  from tailscale.com/tailcfg
         tailscale.com/types/empty                                    from tailscale.com/ipn

cmd/stund/depaware.txt

@@ -52,6 +52,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         tailscale.com/tailcfg                                        from tailscale.com/version
         tailscale.com/tsweb                                          from tailscale.com/cmd/stund
         tailscale.com/tsweb/promvarz                                 from tailscale.com/tsweb
+        tailscale.com/tsweb/tswebutil                                from tailscale.com/tsweb
         tailscale.com/tsweb/varz                                     from tailscale.com/tsweb+
         tailscale.com/types/dnstype                                  from tailscale.com/tailcfg
         tailscale.com/types/ipproto                                  from tailscale.com/tailcfg

cmd/tailscale/depaware.txt

@@ -24,6 +24,13 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
    L 💣 github.com/jsimonetti/rtnetlink                              from tailscale.com/net/netmon
    L    github.com/jsimonetti/rtnetlink/internal/unix                from github.com/jsimonetti/rtnetlink
         github.com/kballard/go-shellquote                            from tailscale.com/cmd/tailscale/cli
+        github.com/klauspost/compress                                from github.com/klauspost/compress/zstd
+        github.com/klauspost/compress/fse                            from github.com/klauspost/compress/huff0
+        github.com/klauspost/compress/huff0                          from github.com/klauspost/compress/zstd
+        github.com/klauspost/compress/internal/cpuinfo               from github.com/klauspost/compress/huff0+
+        github.com/klauspost/compress/internal/snapref               from github.com/klauspost/compress/zstd
+        github.com/klauspost/compress/zstd                           from tailscale.com/client/web
+        github.com/klauspost/compress/zstd/internal/xxhash           from github.com/klauspost/compress/zstd
      💣 github.com/mattn/go-colorable                                from tailscale.com/cmd/tailscale/cli
      💣 github.com/mattn/go-isatty                                   from tailscale.com/cmd/tailscale/cli+
    L 💣 github.com/mdlayher/netlink                                  from github.com/google/nftables+
@@ -124,6 +131,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/tstime                                         from tailscale.com/control/controlhttp+
         tailscale.com/tstime/mono                                    from tailscale.com/tstime/rate
         tailscale.com/tstime/rate                                    from tailscale.com/cmd/tailscale/cli+
+        tailscale.com/tsweb/tswebutil                                from tailscale.com/client/web
         tailscale.com/types/dnstype                                  from tailscale.com/tailcfg
         tailscale.com/types/empty                                    from tailscale.com/ipn
         tailscale.com/types/ipproto                                  from tailscale.com/net/flowtrack+

cmd/tailscaled/depaware.txt

@@ -125,7 +125,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         github.com/klauspost/compress/huff0                          from github.com/klauspost/compress/zstd
         github.com/klauspost/compress/internal/cpuinfo               from github.com/klauspost/compress/huff0+
         github.com/klauspost/compress/internal/snapref               from github.com/klauspost/compress/zstd
-        github.com/klauspost/compress/zstd                           from tailscale.com/util/zstdframe
+        github.com/klauspost/compress/zstd                           from tailscale.com/util/zstdframe+
         github.com/klauspost/compress/zstd/internal/xxhash           from github.com/klauspost/compress/zstd
         github.com/kortschak/wol                                     from tailscale.com/ipn/ipnlocal
   LD    github.com/kr/fs                                             from github.com/pkg/sftp
@@ -337,6 +337,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/tstime                                         from tailscale.com/control/controlclient+
         tailscale.com/tstime/mono                                    from tailscale.com/net/tstun+
         tailscale.com/tstime/rate                                    from tailscale.com/derp+
+        tailscale.com/tsweb/tswebutil                                from tailscale.com/client/web
         tailscale.com/tsweb/varz                                     from tailscale.com/cmd/tailscaled
         tailscale.com/types/appctype                                 from tailscale.com/ipn/ipnlocal
         tailscale.com/types/dnstype                                  from tailscale.com/ipn/ipnlocal+

tsweb/tsweb.go

@@ -25,10 +25,10 @@ import (
 	"sync"
 	"time"
 
-	"go4.org/mem"
 	"tailscale.com/envknob"
 	"tailscale.com/metrics"
 	"tailscale.com/net/tsaddr"
+	"tailscale.com/tsweb/tswebutil"
 	"tailscale.com/tsweb/varz"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/vizerror"
@@ -92,25 +92,9 @@ func allowDebugAccessWithKey(r *http.Request) bool {
 
 // AcceptsEncoding reports whether r accepts the named encoding
 // ("gzip", "br", etc).
+// deprecated: use tswebutil.AcceptsEncoding instead.
 func AcceptsEncoding(r *http.Request, enc string) bool {
-	h := r.Header.Get("Accept-Encoding")
-	if h == "" {
-		return false
-	}
-	if !strings.Contains(h, enc) && !mem.ContainsFold(mem.S(h), mem.S(enc)) {
-		return false
-	}
-	remain := h
-	for len(remain) > 0 {
-		var part string
-		part, remain, _ = strings.Cut(remain, ",")
-		part = strings.TrimSpace(part)
-		part, _, _ = strings.Cut(part, ";")
-		if part == enc {
-			return true
-		}
-	}
-	return false
+	return tswebutil.AcceptsEncoding(r, enc)
 }
 
 // Protected wraps a provided debug handler, h, returning a Handler

tsweb/tswebutil/tswebutil.go

@@ -0,0 +1,35 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Package tswebutil contains helper code used in various Tailscale webservers, without the tsweb kitchen sink.
+package tswebutil
+
+import (
+	"net/http"
+	"strings"
+
+	"go4.org/mem"
+)
+
+// AcceptsEncoding reports whether r accepts the named encoding
+// ("gzip", "br", etc).
+func AcceptsEncoding(r *http.Request, enc string) bool {
+	h := r.Header.Get("Accept-Encoding")
+	if h == "" {
+		return false
+	}
+	if !strings.Contains(h, enc) && !mem.ContainsFold(mem.S(h), mem.S(enc)) {
+		return false
+	}
+	remain := h
+	for len(remain) > 0 {
+		var part string
+		part, remain, _ = strings.Cut(remain, ",")
+		part = strings.TrimSpace(part)
+		part, _, _ = strings.Cut(part, ";")
+		if part == enc {
+			return true
+		}
+	}
+	return false
+}

tsweb/tswebutil/tswebutil_test.go

@@ -0,0 +1,37 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package tswebutil
+
+import (
+	"net/http"
+	"testing"
+)
+
+func TestAcceptsEncoding(t *testing.T) {
+	tests := []struct {
+		in, enc string
+		want    bool
+	}{
+		{"", "gzip", false},
+		{"gzip", "gzip", true},
+		{"foo,gzip", "gzip", true},
+		{"foo, gzip", "gzip", true},
+		{"foo, gzip ", "gzip", true},
+		{"gzip, foo ", "gzip", true},
+		{"gzip, foo ", "br", false},
+		{"gzip, foo ", "fo", false},
+		{"gzip;q=1.2, foo ", "gzip", true},
+		{" gzip;q=1.2, foo ", "gzip", true},
+	}
+	for i, tt := range tests {
+		h := make(http.Header)
+		if tt.in != "" {
+			h.Set("Accept-Encoding", tt.in)
+		}
+		got := AcceptsEncoding(&http.Request{Header: h}, tt.enc)
+		if got != tt.want {
+			t.Errorf("%d. got %v; want %v", i, got, tt.want)
+		}
+	}
+}

util/precompress/precompress.go

@@ -17,8 +17,9 @@ import (
 	"path/filepath"
 
 	"github.com/andybalholm/brotli"
+	"github.com/klauspost/compress/zstd"
 	"golang.org/x/sync/errgroup"
-	"tailscale.com/tsweb"
+	"tailscale.com/tsweb/tswebutil"
 )
 
 // PrecompressDir compresses static assets in dirPath using Gzip and Brotli, so
@@ -63,13 +64,19 @@ type Options struct {
 // OpenPrecompressedFile opens a file from fs, preferring compressed versions
 // generated by PrecompressDir if possible.
 func OpenPrecompressedFile(w http.ResponseWriter, r *http.Request, path string, fs fs.FS) (fs.File, error) {
-	if tsweb.AcceptsEncoding(r, "br") {
+	if tswebutil.AcceptsEncoding(r, "zstd") {
+		if f, err := fs.Open(path + ".zst"); err == nil {
+			w.Header().Set("Content-Encoding", "zstd")
+			return f, nil
+		}
+	}
+	if tswebutil.AcceptsEncoding(r, "br") {
 		if f, err := fs.Open(path + ".br"); err == nil {
 			w.Header().Set("Content-Encoding", "br")
 			return f, nil
 		}
 	}
-	if tsweb.AcceptsEncoding(r, "gzip") {
+	if tswebutil.AcceptsEncoding(r, "gzip") {
 		if f, err := fs.Open(path + ".gz"); err == nil {
 			w.Header().Set("Content-Encoding", "gzip")
 			return f, nil
@@ -104,6 +111,17 @@ func Precompress(path string, options Options) error {
 	if err != nil {
 		return err
 	}
+	zstdLevel := zstd.WithEncoderLevel(zstd.SpeedBestCompression)
+	if options.FastCompression {
+		zstdLevel = zstd.WithEncoderLevel(zstd.SpeedFastest)
+	}
+	err = writeCompressed(contents, func(w io.Writer) (io.WriteCloser, error) {
+		// Per RFC 8878, encoders should avoid window sizes larger than 8MB, which is the max that Chrome acccepts.
+		return zstd.NewWriter(w, zstdLevel, zstd.WithWindowSize(8<<20))
+	}, path+".zst", fi.Mode())
+	if err != nil {
+		return err
+	}
 	brotliLevel := brotli.BestCompression
 	if options.FastCompression {
 		brotliLevel = brotli.BestSpeed