From 9cd4e6519108f3f4f34bf6e9ee497d900e5b73aa Mon Sep 17 00:00:00 2001 From: David Anderson Date: Thu, 2 Jul 2020 18:26:33 +0000 Subject: [PATCH] smallzstd: new package that constructs zstd small encoders/decoders. It's just a config wrapper that passes "use less memory at the expense of compression" parameters by default, so that we don't accidentally construct resource-hungry (de)compressors. Also includes a benchmark that measures the memory cost of the small variants vs. the stock variants. The savings are significant on both compressors (~8x less memory) and decompressors (~1.4x less, not including the savings from the significantly smaller window on the compression side - with those savings included it's more like ~140x smaller). BenchmarkSmallEncoder-8 56174 19354 ns/op 31 B/op 0 allocs/op BenchmarkSmallEncoderWithBuild-8 2900 382940 ns/op 1746547 B/op 36 allocs/op BenchmarkStockEncoder-8 48921 25761 ns/op 286 B/op 0 allocs/op BenchmarkStockEncoderWithBuild-8 426 2630241 ns/op 13843842 B/op 124 allocs/op BenchmarkSmallDecoder-8 123814 9344 ns/op 0 B/op 0 allocs/op BenchmarkSmallDecoderWithBuild-8 41547 27455 ns/op 27694 B/op 31 allocs/op BenchmarkStockDecoder-8 129832 9417 ns/op 1 B/op 0 allocs/op BenchmarkStockDecoderWithBuild-8 25561 51751 ns/op 39607 B/op 92 allocs/op Signed-off-by: David Anderson --- go.mod | 3 +- go.sum | 22 +------ ipn/ipnserver/server.go | 7 +-- logpolicy/logpolicy.go | 8 +-- smallzstd/testdata | 14 +++++ smallzstd/zstd.go | 79 ++++++++++++++++++++++++ smallzstd/zstd_test.go | 131 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 231 insertions(+), 33 deletions(-) create mode 100644 smallzstd/testdata create mode 100644 smallzstd/zstd.go create mode 100644 smallzstd/zstd_test.go diff --git a/go.mod b/go.mod index cc95531aa..cb87ea527 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e github.com/google/go-cmp v0.4.0 github.com/goreleaser/nfpm v1.1.10 - github.com/klauspost/compress v1.9.8 + github.com/klauspost/compress v1.10.10 github.com/kr/pty v1.1.1 github.com/mdlayher/netlink v1.1.0 github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3 @@ -28,7 +28,6 @@ require ( golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e golang.org/x/sys v0.0.0-20200501052902-10377860bb8e golang.org/x/time v0.0.0-20191024005414-555d28b269f0 - honnef.co/go/tools v0.0.1-2020.1.4 // indirect inet.af/netaddr v0.0.0-20200702150737-4591d218f82c rsc.io/goversion v1.2.0 ) diff --git a/go.sum b/go.sum index 48184b89b..96a884a85 100644 --- a/go.sum +++ b/go.sum @@ -38,7 +38,6 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/rpmpack v0.0.0-20191226140753-aa36bfddb3a0 h1:BW6OvS3kpT5UEPbCZ+KyX/OB4Ks9/MNMhWjqPPkZxsE= github.com/google/rpmpack v0.0.0-20191226140753-aa36bfddb3a0/go.mod h1:RaTPr0KUf2K7fnZYLNDrr8rxAamWs3iNywJLtQ2AzBg= github.com/goreleaser/nfpm v1.1.10 h1:0nwzKUJTcygNxTzVKq2Dh9wpVP1W2biUH6SNKmoxR3w= @@ -48,9 +47,8 @@ github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw= github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4 h1:nwOc1YaOrYJ37sEBrtWZrdqzK22hiJs3GpDmP3sR2Yw= github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA= -github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I= +github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= @@ -74,7 +72,6 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/sassoftware/go-rpmutils v0.0.0-20190420191620-a8f1baeba37b h1:+gCnWOZV8Z/8jehJ2CdqB47Z3S+SREmQcuXkRFLNsiI= github.com/sassoftware/go-rpmutils v0.0.0-20190420191620-a8f1baeba37b/go.mod h1:am+Fp8Bt506lA3Rk3QCmSqmYmLMnPDhdDUcosQCAx+I= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -82,10 +79,6 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tailscale/winipcfg-go v0.0.0-20200413171540-609dcf2df55f h1:uFj5bslHsMzxIM8UTjAhq4VXeo6GfNW91rpoh/WMJaY= github.com/tailscale/winipcfg-go v0.0.0-20200413171540-609dcf2df55f/go.mod h1:x880GWw5fvrl2DVTQ04ttXQD4DuppTt1Yz6wLibbjNE= -github.com/tailscale/wireguard-go v0.0.0-20200515231107-62868271d710 h1:I6aq3tOYbZob9uwhGpr7R266qTeU9PFqS6NnpfCqEzo= -github.com/tailscale/wireguard-go v0.0.0-20200515231107-62868271d710/go.mod h1:JPm5cTfu1K+qDFRbiHy0sOlHUylYQbpl356sdYFD8V4= -github.com/tailscale/wireguard-go v0.0.0-20200615180905-687c10194779 h1:zg0rgvhBZGA4nvh17nDKcqkEXw6Nbc/Ma2VBvLaW7LU= -github.com/tailscale/wireguard-go v0.0.0-20200615180905-687c10194779/go.mod h1:JPm5cTfu1K+qDFRbiHy0sOlHUylYQbpl356sdYFD8V4= github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0= github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8= github.com/toqueteos/webbrowser v1.2.0 h1:tVP/gpK69Fx+qMJKsLE7TD8LuGWPnEV71wBN9rrstGQ= @@ -97,16 +90,13 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMx go4.org/mem v0.0.0-20200601023850-d8ee1dfa5518 h1:AA3bSGklCgkrqIGnvL4894oa/2K9ltE0RejXh8CgyvA= go4.org/mem v0.0.0-20200601023850-d8ee1dfa5518/go.mod h1:NEYvpHWemiG/E5UWfaN5QAIGZeT1sa0Z2UNk6oeMb/k= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6 h1:TjszyFsQsyZNHwdVdZ5m7bjmreu0znc2kRYsEml9/Ww= golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191003171128-d98b1b443823/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -118,7 +108,6 @@ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BG golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -140,24 +129,17 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d h1:/iIZNFGxc/a7C3yWjGcnboV+Tkc7mxr+p6fDztwoxuM= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= inet.af/netaddr v0.0.0-20200513162223-787f13e36cbe h1:WjJ6wZhXEWQA3FFSwOjG8tO2q1NDFSqrUwNcTvxwMEQ= inet.af/netaddr v0.0.0-20200513162223-787f13e36cbe/go.mod h1:qqYzz/2whtrbWJvt+DNWQyvekNN4ePQZcg2xc2/Yjww= inet.af/netaddr v0.0.0-20200629220211-f44a6d25c536 h1:XFVw2MVOtmHBidx70M+I6vIw2F6f55UyXvkiKfIrE38= diff --git a/ipn/ipnserver/server.go b/ipn/ipnserver/server.go index 17efde19d..776d690f8 100644 --- a/ipn/ipnserver/server.go +++ b/ipn/ipnserver/server.go @@ -18,11 +18,11 @@ "syscall" "time" - "github.com/klauspost/compress/zstd" "tailscale.com/control/controlclient" "tailscale.com/ipn" "tailscale.com/logtail/backoff" "tailscale.com/safesocket" + "tailscale.com/smallzstd" "tailscale.com/types/logger" "tailscale.com/version" "tailscale.com/wgengine" @@ -113,10 +113,7 @@ func Run(rctx context.Context, logf logger.Logf, logid string, opts Options, e w return fmt.Errorf("NewLocalBackend: %v", err) } b.SetDecompressor(func() (controlclient.Decompressor, error) { - return zstd.NewReader(nil, - zstd.WithDecoderLowmem(true), - zstd.WithDecoderConcurrency(1), - ) + return smallzstd.NewDecoder(nil) }) if opts.DebugMux != nil { diff --git a/logpolicy/logpolicy.go b/logpolicy/logpolicy.go index 370eeca02..90d9e864f 100644 --- a/logpolicy/logpolicy.go +++ b/logpolicy/logpolicy.go @@ -25,13 +25,13 @@ "strings" "time" - "github.com/klauspost/compress/zstd" "golang.org/x/crypto/ssh/terminal" "tailscale.com/atomicfile" "tailscale.com/logtail" "tailscale.com/logtail/filch" "tailscale.com/net/netns" "tailscale.com/net/tlsdial" + "tailscale.com/smallzstd" "tailscale.com/version" ) @@ -350,11 +350,7 @@ func New(collection string) *Policy { PrivateID: newc.PrivateID, Stderr: logWriter{console}, NewZstdEncoder: func() logtail.Encoder { - w, err := zstd.NewWriter(nil, - zstd.WithEncoderLevel(zstd.SpeedFastest), - zstd.WithEncoderConcurrency(1), - zstd.WithWindowSize(8192), - ) + w, err := smallzstd.NewEncoder(nil) if err != nil { panic(err) } diff --git a/smallzstd/testdata b/smallzstd/testdata new file mode 100644 index 000000000..76640fdc5 --- /dev/null +++ b/smallzstd/testdata @@ -0,0 +1,14 @@ +{"logtail":{"client_time":"2020-07-01T14:49:40.196597018-07:00","server_time":"2020-07-01T21:49:40.198371511Z"},"text":"9.8M/25.6M magicsock: starting endpoint update (periodic)\n"} +{"logtail":{"client_time":"2020-07-01T14:49:40.345925455-07:00","server_time":"2020-07-01T21:49:40.347904717Z"},"text":"9.9M/25.6M netcheck: udp=true v6=false mapvarydest=false hair=false v4a=202.188.7.1:41641 derp=2 derpdist=1v4:7ms,2v4:3ms,4v4:18ms\n"} +{"logtail":{"client_time":"2020-07-01T14:49:43.347155742-07:00","server_time":"2020-07-01T21:49:43.34828658Z"},"text":"9.9M/25.6M control: map response long-poll timed out!\n"} +{"logtail":{"client_time":"2020-07-01T14:49:43.347539333-07:00","server_time":"2020-07-01T21:49:43.358809354Z"},"text":"9.9M/25.6M control: PollNetMap: context canceled\n"} +{"logtail":{"client_time":"2020-07-01T14:49:43.347767812-07:00","server_time":"2020-07-01T21:49:43.358809354Z"},"text":"10.0M/25.6M control: sendStatus: mapRoutine1: state:authenticated\n"} +{"logtail":{"client_time":"2020-07-01T14:49:43.347817165-07:00","server_time":"2020-07-01T21:49:43.358809354Z"},"text":"10.0M/25.6M blockEngineUpdates(false)\n"} +{"logtail":{"client_time":"2020-07-01T14:49:43.347989028-07:00","server_time":"2020-07-01T21:49:43.358809354Z"},"text":"10.0M/25.6M wgcfg: [SViTM] skipping subnet route\n"} +{"logtail":{"client_time":"2020-07-01T14:49:43.349997554-07:00","server_time":"2020-07-01T21:49:43.358809354Z"},"text":"9.3M/25.6M Received error: PollNetMap: context canceled\n"} +{"logtail":{"client_time":"2020-07-01T14:49:43.350072606-07:00","server_time":"2020-07-01T21:49:43.358809354Z"},"text":"9.3M/25.6M control: mapRoutine: backoff: 30136 msec\n"} +{"logtail":{"client_time":"2020-07-01T14:49:47.998364646-07:00","server_time":"2020-07-01T21:49:47.999333754Z"},"text":"9.5M/25.6M [W1NbE] - [UcppE] Send handshake init [127.3.3.40:1, 6.1.1.6:37388*, 10.3.2.6:41641]\n"} +{"logtail":{"client_time":"2020-07-01T14:49:47.99881914-07:00","server_time":"2020-07-01T21:49:48.009859543Z"},"text":"9.6M/25.6M magicsock: adding connection to derp-1 for [W1NbE]\n"} +{"logtail":{"client_time":"2020-07-01T14:49:47.998904932-07:00","server_time":"2020-07-01T21:49:48.009859543Z"},"text":"9.6M/25.6M magicsock: 2 active derp conns: derp-1=cr0s,wr0s derp-2=cr16h0m0s,wr14h38m0s\n"} +{"logtail":{"client_time":"2020-07-01T14:49:47.999045606-07:00","server_time":"2020-07-01T21:49:48.009859543Z"},"text":"9.6M/25.6M derphttp.Client.Recv: connecting to derp-1 (nyc)\n"} +{"logtail":{"client_time":"2020-07-01T14:49:48.091104119-07:00","server_time":"2020-07-01T21:49:48.09280535Z"},"text":"9.6M/25.6M magicsock: rx [W1NbE] from 6.1.1.6:37388 (1/3), set as new priority\n"} diff --git a/smallzstd/zstd.go b/smallzstd/zstd.go new file mode 100644 index 000000000..3e49e804e --- /dev/null +++ b/smallzstd/zstd.go @@ -0,0 +1,79 @@ +// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package smallzstd produces zstd encoders and decoders optimized for +// low memory usage, at the expense of compression efficiency. +// +// This package is optimized primarily for the memory cost of +// compressing and decompressing data. We reduce this cost in two +// major ways: disable parallelism within the library (i.e. don't use +// multiple CPU cores to decompress), and drop the compression window +// down from the defaults of 4-16MiB, to 8kiB. +// +// Decompressors cost 2x the window size in RAM to run, so by using an +// 8kiB window, we can run ~1000 more decompressors per unit of memory +// than with the defaults. +// +// Depending on context, the benefit is either being able to run more +// decoders (e.g. in our logs processing system), or having a lower +// memory footprint when using compression in network protocols +// (e.g. in tailscaled, which should have a minimal RAM cost). +package smallzstd + +import ( + "io" + + "github.com/klauspost/compress/zstd" +) + +// WindowSize is the window size used for zstd compression. Decoder +// memory usage scales linearly with WindowSize. +const WindowSize = 8 << 10 // 8kiB + +// NewDecoder returns a zstd.Decoder configured for low memory usage, +// at the expense of decompression performance. +func NewDecoder(r io.Reader, options ...zstd.DOption) (*zstd.Decoder, error) { + defaults := []zstd.DOption{ + // Default is GOMAXPROCS, which costs many KiB in stacks. + zstd.WithDecoderConcurrency(1), + // Default is to allocate more upfront for performance. We + // prefer lower memory use and a bit of GC load. + zstd.WithDecoderLowmem(true), + // You might expect to see zstd.WithDecoderMaxMemory + // here. However, it's not terribly safe to use if you're + // doing stateless decoding, because it sets the maximum + // amount of memory the decompressed data can occupy, rather + // than the window size of the zstd stream. This means a very + // compressible piece of data might violate the max memory + // limit here, even if the window size (and thus total memory + // required to decompress the data) is small. + // + // As a result, we don't set a decoder limit here, and rely on + // the encoder below producing "cheap" streams. Callers are + // welcome to set their own max memory setting, if + // contextually there is a clearly correct value (e.g. it's + // known from the upper layer protocol that the decoded data + // can never be more than 1MiB). + } + + return zstd.NewReader(r, append(defaults, options...)...) +} + +// NewEncoder returns a zstd.Encoder configured for low memory usage, +// both during compression and at decompression time, at the expense +// of performance and compression efficiency. +func NewEncoder(w io.Writer, options ...zstd.EOption) (*zstd.Encoder, error) { + defaults := []zstd.EOption{ + // Default is GOMAXPROCS, which costs many KiB in stacks. + zstd.WithEncoderConcurrency(1), + // Default is several MiB, which bloats both encoders and + // their corresponding decoders. + zstd.WithWindowSize(WindowSize), + // Encode zero-length inputs in a way that the `zstd` utility + // can read, because interoperability is handy. + zstd.WithZeroFrames(true), + } + + return zstd.NewWriter(w, append(defaults, options...)...) +} diff --git a/smallzstd/zstd_test.go b/smallzstd/zstd_test.go new file mode 100644 index 000000000..2cdd47f6a --- /dev/null +++ b/smallzstd/zstd_test.go @@ -0,0 +1,131 @@ +// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package smallzstd + +import ( + "io/ioutil" + "testing" + + "github.com/klauspost/compress/zstd" +) + +func BenchmarkSmallEncoder(b *testing.B) { + benchEncoder(b, func() (*zstd.Encoder, error) { return NewEncoder(nil) }) +} + +func BenchmarkSmallEncoderWithBuild(b *testing.B) { + benchEncoderWithConstruction(b, func() (*zstd.Encoder, error) { return NewEncoder(nil) }) +} + +func BenchmarkStockEncoder(b *testing.B) { + benchEncoder(b, func() (*zstd.Encoder, error) { return zstd.NewWriter(nil) }) +} + +func BenchmarkStockEncoderWithBuild(b *testing.B) { + benchEncoderWithConstruction(b, func() (*zstd.Encoder, error) { return zstd.NewWriter(nil) }) +} + +func BenchmarkSmallDecoder(b *testing.B) { + benchDecoder(b, func() (*zstd.Decoder, error) { return NewDecoder(nil) }) +} + +func BenchmarkSmallDecoderWithBuild(b *testing.B) { + benchDecoderWithConstruction(b, func() (*zstd.Decoder, error) { return NewDecoder(nil) }) +} + +func BenchmarkStockDecoder(b *testing.B) { + benchDecoder(b, func() (*zstd.Decoder, error) { return zstd.NewReader(nil) }) +} + +func BenchmarkStockDecoderWithBuild(b *testing.B) { + benchDecoderWithConstruction(b, func() (*zstd.Decoder, error) { return zstd.NewReader(nil) }) +} + +func benchEncoder(b *testing.B, mk func() (*zstd.Encoder, error)) { + b.ReportAllocs() + + in := testdata(b) + out := make([]byte, 0, 10<<10) // 10kiB + + e, err := mk() + if err != nil { + b.Fatalf("making encoder: %v", err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + e.EncodeAll(in, out) + } +} + +func benchEncoderWithConstruction(b *testing.B, mk func() (*zstd.Encoder, error)) { + b.ReportAllocs() + + in := testdata(b) + out := make([]byte, 0, 10<<10) // 10kiB + + b.ResetTimer() + for i := 0; i < b.N; i++ { + e, err := mk() + if err != nil { + b.Fatalf("making encoder: %v", err) + } + + e.EncodeAll(in, out) + } +} + +func benchDecoder(b *testing.B, mk func() (*zstd.Decoder, error)) { + b.ReportAllocs() + + in := compressedTestdata(b) + out := make([]byte, 0, 10<<10) + + d, err := mk() + if err != nil { + b.Fatalf("creating decoder: %v", err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + d.DecodeAll(in, out) + } +} + +func benchDecoderWithConstruction(b *testing.B, mk func() (*zstd.Decoder, error)) { + b.ReportAllocs() + + in := compressedTestdata(b) + out := make([]byte, 0, 10<<10) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + d, err := mk() + if err != nil { + b.Fatalf("creating decoder: %v", err) + } + + d.DecodeAll(in, out) + } +} + +func testdata(b *testing.B) []byte { + b.Helper() + in, err := ioutil.ReadFile("testdata") + if err != nil { + b.Fatalf("reading testdata: %v", err) + } + return in +} + +func compressedTestdata(b *testing.B) []byte { + b.Helper() + uncomp := testdata(b) + e, err := NewEncoder(nil) + if err != nil { + b.Fatalf("creating encoder: %v", err) + } + return e.EncodeAll(uncomp, nil) +}