From f39ac7d49c1e45852cc28d585651b4386a08d597 Mon Sep 17 00:00:00 2001 From: James Tucker Date: Fri, 17 May 2024 13:47:27 -0700 Subject: [PATCH] client/web: add support for zst precomppressed assets This will enable us to reduce the size of these embedded assets. Updates tailscale/corp#20099 Signed-off-by: James Tucker --- client/web/assets.go | 62 +++++++++++++++++++++++++- cmd/build-webclient/build-webclient.go | 6 ++- cmd/derper/depaware.txt | 1 + cmd/stund/depaware.txt | 1 + cmd/tailscale/depaware.txt | 8 ++++ cmd/tailscaled/depaware.txt | 3 +- tsweb/tsweb.go | 22 ++------- tsweb/tswebutil/tswebutil.go | 35 +++++++++++++++ tsweb/tswebutil/tswebutil_test.go | 37 +++++++++++++++ util/precompress/precompress.go | 24 ++++++++-- 10 files changed, 174 insertions(+), 25 deletions(-) create mode 100644 tsweb/tswebutil/tswebutil.go create mode 100644 tsweb/tswebutil/tswebutil_test.go diff --git a/client/web/assets.go b/client/web/assets.go index c4f4e9e3b..7908f6703 100644 --- a/client/web/assets.go +++ b/client/web/assets.go @@ -4,6 +4,7 @@ package web import ( + "fmt" "io" "io/fs" "log" @@ -16,7 +17,9 @@ import ( "strings" "time" + "github.com/klauspost/compress/zstd" prebuilt "github.com/tailscale/web-client-prebuilt" + "tailscale.com/tsweb/tswebutil" ) var start = time.Now() @@ -63,7 +66,64 @@ func assetsHandler(devMode bool) (_ http.Handler, cleanup func()) { }), nil } -func openPrecompressedFile(w http.ResponseWriter, r *http.Request, path string, fs fs.FS) (fs.File, error) { +type zstFile struct { + f fs.File + *zstd.Decoder +} + +func newZSTFile(f fs.File) (*zstFile, error) { + zr, err := zstd.NewReader(f) + if err != nil { + return nil, err + } + return &zstFile{f: f, Decoder: zr}, nil +} + +func (z *zstFile) Seek(offset int64, whence int) (int64, error) { + reset := func() error { + if seeker, ok := z.f.(io.Seeker); ok { + seeker.Seek(0, io.SeekStart) + } else { + return fmt.Errorf("not seekable: %w", os.ErrInvalid) + } + return z.Decoder.Reset(z.f) + } + + switch whence { + case io.SeekStart: + if err := reset(); err != nil { + return 0, err + } + return io.CopyN(io.Discard, z, offset) + case io.SeekCurrent: + if offset >= 0 { + io.CopyN(io.Discard, z, offset) + } else { + return 0, fmt.Errorf("unsupported negative seek: %w", os.ErrInvalid) + } + case io.SeekEnd: + if offset != 0 { + return 0, fmt.Errorf("unsupported non-zero offset for SeekEnd: %w", os.ErrInvalid) + } + return io.Copy(io.Discard, z) + } + return 0, os.ErrInvalid +} + +func (z *zstFile) Close() error { + z.Decoder.Close() + return z.f.Close() +} + +func openPrecompressedFile(w http.ResponseWriter, r *http.Request, path string, fs fs.FS) (io.ReadCloser, error) { + if f, err := fs.Open(path + ".zst"); err == nil { + if tswebutil.AcceptsEncoding(r, "zstd") { + w.Header().Set("Content-Encoding", "zstd") + return f, nil + } + return newZSTFile(f) + } + // TODO(raggi): remove this code path when no longer used if f, err := fs.Open(path + ".gz"); err == nil { w.Header().Set("Content-Encoding", "gzip") return f, nil diff --git a/cmd/build-webclient/build-webclient.go b/cmd/build-webclient/build-webclient.go index f92c0858f..c0dc6ea41 100644 --- a/cmd/build-webclient/build-webclient.go +++ b/cmd/build-webclient/build-webclient.go @@ -73,10 +73,14 @@ func build(toolDir, appDir string) error { if err := os.Remove(f); err != nil { log.Printf("Failed to cleanup %q: %v", f, err) } - // Removing intermediate ".br" version, we use ".gz" asset. + // Removing ".br" version, we use the ".zst" asset. if err := os.Remove(f + ".br"); err != nil { log.Printf("Failed to cleanup %q: %v", f+".gz", err) } + // Removing ".gz" version, we use the ".zst" asset. + if err := os.Remove(f + ".gz"); err != nil { + log.Printf("Failed to cleanup %q: %v", f+".gz", err) + } } return nil diff --git a/cmd/derper/depaware.txt b/cmd/derper/depaware.txt index 22e910179..54cbcea66 100644 --- a/cmd/derper/depaware.txt +++ b/cmd/derper/depaware.txt @@ -122,6 +122,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa tailscale.com/tstime/rate from tailscale.com/derp+ tailscale.com/tsweb from tailscale.com/cmd/derper tailscale.com/tsweb/promvarz from tailscale.com/tsweb + tailscale.com/tsweb/tswebutil from tailscale.com/tsweb tailscale.com/tsweb/varz from tailscale.com/tsweb+ tailscale.com/types/dnstype from tailscale.com/tailcfg tailscale.com/types/empty from tailscale.com/ipn diff --git a/cmd/stund/depaware.txt b/cmd/stund/depaware.txt index b78465002..f8b8c453b 100644 --- a/cmd/stund/depaware.txt +++ b/cmd/stund/depaware.txt @@ -52,6 +52,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar tailscale.com/tailcfg from tailscale.com/version tailscale.com/tsweb from tailscale.com/cmd/stund tailscale.com/tsweb/promvarz from tailscale.com/tsweb + tailscale.com/tsweb/tswebutil from tailscale.com/tsweb tailscale.com/tsweb/varz from tailscale.com/tsweb+ tailscale.com/types/dnstype from tailscale.com/tailcfg tailscale.com/types/ipproto from tailscale.com/tailcfg diff --git a/cmd/tailscale/depaware.txt b/cmd/tailscale/depaware.txt index 5883507f0..c86d1dd2c 100644 --- a/cmd/tailscale/depaware.txt +++ b/cmd/tailscale/depaware.txt @@ -24,6 +24,13 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep L 💣 github.com/jsimonetti/rtnetlink from tailscale.com/net/netmon L github.com/jsimonetti/rtnetlink/internal/unix from github.com/jsimonetti/rtnetlink github.com/kballard/go-shellquote from tailscale.com/cmd/tailscale/cli + github.com/klauspost/compress from github.com/klauspost/compress/zstd + github.com/klauspost/compress/fse from github.com/klauspost/compress/huff0 + github.com/klauspost/compress/huff0 from github.com/klauspost/compress/zstd + github.com/klauspost/compress/internal/cpuinfo from github.com/klauspost/compress/huff0+ + github.com/klauspost/compress/internal/snapref from github.com/klauspost/compress/zstd + github.com/klauspost/compress/zstd from tailscale.com/client/web + github.com/klauspost/compress/zstd/internal/xxhash from github.com/klauspost/compress/zstd 💣 github.com/mattn/go-colorable from tailscale.com/cmd/tailscale/cli 💣 github.com/mattn/go-isatty from tailscale.com/cmd/tailscale/cli+ L 💣 github.com/mdlayher/netlink from github.com/google/nftables+ @@ -124,6 +131,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep tailscale.com/tstime from tailscale.com/control/controlhttp+ tailscale.com/tstime/mono from tailscale.com/tstime/rate tailscale.com/tstime/rate from tailscale.com/cmd/tailscale/cli+ + tailscale.com/tsweb/tswebutil from tailscale.com/client/web tailscale.com/types/dnstype from tailscale.com/tailcfg tailscale.com/types/empty from tailscale.com/ipn tailscale.com/types/ipproto from tailscale.com/net/flowtrack+ diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index b97160e22..1891a7772 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -125,7 +125,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de github.com/klauspost/compress/huff0 from github.com/klauspost/compress/zstd github.com/klauspost/compress/internal/cpuinfo from github.com/klauspost/compress/huff0+ github.com/klauspost/compress/internal/snapref from github.com/klauspost/compress/zstd - github.com/klauspost/compress/zstd from tailscale.com/util/zstdframe + github.com/klauspost/compress/zstd from tailscale.com/util/zstdframe+ github.com/klauspost/compress/zstd/internal/xxhash from github.com/klauspost/compress/zstd github.com/kortschak/wol from tailscale.com/ipn/ipnlocal LD github.com/kr/fs from github.com/pkg/sftp @@ -337,6 +337,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/tstime from tailscale.com/control/controlclient+ tailscale.com/tstime/mono from tailscale.com/net/tstun+ tailscale.com/tstime/rate from tailscale.com/derp+ + tailscale.com/tsweb/tswebutil from tailscale.com/client/web tailscale.com/tsweb/varz from tailscale.com/cmd/tailscaled tailscale.com/types/appctype from tailscale.com/ipn/ipnlocal tailscale.com/types/dnstype from tailscale.com/ipn/ipnlocal+ diff --git a/tsweb/tsweb.go b/tsweb/tsweb.go index ad9f90eb3..3426b5268 100644 --- a/tsweb/tsweb.go +++ b/tsweb/tsweb.go @@ -25,10 +25,10 @@ import ( "sync" "time" - "go4.org/mem" "tailscale.com/envknob" "tailscale.com/metrics" "tailscale.com/net/tsaddr" + "tailscale.com/tsweb/tswebutil" "tailscale.com/tsweb/varz" "tailscale.com/types/logger" "tailscale.com/util/vizerror" @@ -92,25 +92,9 @@ func allowDebugAccessWithKey(r *http.Request) bool { // AcceptsEncoding reports whether r accepts the named encoding // ("gzip", "br", etc). +// deprecated: use tswebutil.AcceptsEncoding instead. func AcceptsEncoding(r *http.Request, enc string) bool { - h := r.Header.Get("Accept-Encoding") - if h == "" { - return false - } - if !strings.Contains(h, enc) && !mem.ContainsFold(mem.S(h), mem.S(enc)) { - return false - } - remain := h - for len(remain) > 0 { - var part string - part, remain, _ = strings.Cut(remain, ",") - part = strings.TrimSpace(part) - part, _, _ = strings.Cut(part, ";") - if part == enc { - return true - } - } - return false + return tswebutil.AcceptsEncoding(r, enc) } // Protected wraps a provided debug handler, h, returning a Handler diff --git a/tsweb/tswebutil/tswebutil.go b/tsweb/tswebutil/tswebutil.go new file mode 100644 index 000000000..1644cdab4 --- /dev/null +++ b/tsweb/tswebutil/tswebutil.go @@ -0,0 +1,35 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package tswebutil contains helper code used in various Tailscale webservers, without the tsweb kitchen sink. +package tswebutil + +import ( + "net/http" + "strings" + + "go4.org/mem" +) + +// AcceptsEncoding reports whether r accepts the named encoding +// ("gzip", "br", etc). +func AcceptsEncoding(r *http.Request, enc string) bool { + h := r.Header.Get("Accept-Encoding") + if h == "" { + return false + } + if !strings.Contains(h, enc) && !mem.ContainsFold(mem.S(h), mem.S(enc)) { + return false + } + remain := h + for len(remain) > 0 { + var part string + part, remain, _ = strings.Cut(remain, ",") + part = strings.TrimSpace(part) + part, _, _ = strings.Cut(part, ";") + if part == enc { + return true + } + } + return false +} diff --git a/tsweb/tswebutil/tswebutil_test.go b/tsweb/tswebutil/tswebutil_test.go new file mode 100644 index 000000000..f07743114 --- /dev/null +++ b/tsweb/tswebutil/tswebutil_test.go @@ -0,0 +1,37 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package tswebutil + +import ( + "net/http" + "testing" +) + +func TestAcceptsEncoding(t *testing.T) { + tests := []struct { + in, enc string + want bool + }{ + {"", "gzip", false}, + {"gzip", "gzip", true}, + {"foo,gzip", "gzip", true}, + {"foo, gzip", "gzip", true}, + {"foo, gzip ", "gzip", true}, + {"gzip, foo ", "gzip", true}, + {"gzip, foo ", "br", false}, + {"gzip, foo ", "fo", false}, + {"gzip;q=1.2, foo ", "gzip", true}, + {" gzip;q=1.2, foo ", "gzip", true}, + } + for i, tt := range tests { + h := make(http.Header) + if tt.in != "" { + h.Set("Accept-Encoding", tt.in) + } + got := AcceptsEncoding(&http.Request{Header: h}, tt.enc) + if got != tt.want { + t.Errorf("%d. got %v; want %v", i, got, tt.want) + } + } +} diff --git a/util/precompress/precompress.go b/util/precompress/precompress.go index 6d1a26efd..87f986d17 100644 --- a/util/precompress/precompress.go +++ b/util/precompress/precompress.go @@ -17,8 +17,9 @@ import ( "path/filepath" "github.com/andybalholm/brotli" + "github.com/klauspost/compress/zstd" "golang.org/x/sync/errgroup" - "tailscale.com/tsweb" + "tailscale.com/tsweb/tswebutil" ) // PrecompressDir compresses static assets in dirPath using Gzip and Brotli, so @@ -63,13 +64,19 @@ type Options struct { // OpenPrecompressedFile opens a file from fs, preferring compressed versions // generated by PrecompressDir if possible. func OpenPrecompressedFile(w http.ResponseWriter, r *http.Request, path string, fs fs.FS) (fs.File, error) { - if tsweb.AcceptsEncoding(r, "br") { + if tswebutil.AcceptsEncoding(r, "zstd") { + if f, err := fs.Open(path + ".zst"); err == nil { + w.Header().Set("Content-Encoding", "zstd") + return f, nil + } + } + if tswebutil.AcceptsEncoding(r, "br") { if f, err := fs.Open(path + ".br"); err == nil { w.Header().Set("Content-Encoding", "br") return f, nil } } - if tsweb.AcceptsEncoding(r, "gzip") { + if tswebutil.AcceptsEncoding(r, "gzip") { if f, err := fs.Open(path + ".gz"); err == nil { w.Header().Set("Content-Encoding", "gzip") return f, nil @@ -104,6 +111,17 @@ func Precompress(path string, options Options) error { if err != nil { return err } + zstdLevel := zstd.WithEncoderLevel(zstd.SpeedBestCompression) + if options.FastCompression { + zstdLevel = zstd.WithEncoderLevel(zstd.SpeedFastest) + } + err = writeCompressed(contents, func(w io.Writer) (io.WriteCloser, error) { + // Per RFC 8878, encoders should avoid window sizes larger than 8MB, which is the max that Chrome acccepts. + return zstd.NewWriter(w, zstdLevel, zstd.WithWindowSize(8<<20)) + }, path+".zst", fi.Mode()) + if err != nil { + return err + } brotliLevel := brotli.BestCompression if options.FastCompression { brotliLevel = brotli.BestSpeed