mirror of
https://github.com/tailscale/tailscale.git
synced 2024-11-25 11:05:45 +00:00
smallzstd: new package that constructs zstd small encoders/decoders.
It's just a config wrapper that passes "use less memory at the expense of compression" parameters by default, so that we don't accidentally construct resource-hungry (de)compressors. Also includes a benchmark that measures the memory cost of the small variants vs. the stock variants. The savings are significant on both compressors (~8x less memory) and decompressors (~1.4x less, not including the savings from the significantly smaller window on the compression side - with those savings included it's more like ~140x smaller). BenchmarkSmallEncoder-8 56174 19354 ns/op 31 B/op 0 allocs/op BenchmarkSmallEncoderWithBuild-8 2900 382940 ns/op 1746547 B/op 36 allocs/op BenchmarkStockEncoder-8 48921 25761 ns/op 286 B/op 0 allocs/op BenchmarkStockEncoderWithBuild-8 426 2630241 ns/op 13843842 B/op 124 allocs/op BenchmarkSmallDecoder-8 123814 9344 ns/op 0 B/op 0 allocs/op BenchmarkSmallDecoderWithBuild-8 41547 27455 ns/op 27694 B/op 31 allocs/op BenchmarkStockDecoder-8 129832 9417 ns/op 1 B/op 0 allocs/op BenchmarkStockDecoderWithBuild-8 25561 51751 ns/op 39607 B/op 92 allocs/op Signed-off-by: David Anderson <danderson@tailscale.com>
This commit is contained in:
David Anderson
Dave Anderson
parent
97910ce712
commit
9cd4e65191
3
go.mod
3
go.mod
@ -12,7 +12,7 @@ require (
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e
|
||||
github.com/google/go-cmp v0.4.0
|
||||
github.com/goreleaser/nfpm v1.1.10
|
||||
github.com/klauspost/compress v1.9.8
|
||||
github.com/klauspost/compress v1.10.10
|
||||
github.com/kr/pty v1.1.1
|
||||
github.com/mdlayher/netlink v1.1.0
|
||||
github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3
|
||||
@ -28,7 +28,6 @@ require (
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
|
||||
golang.org/x/sys v0.0.0-20200501052902-10377860bb8e
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0
|
||||
honnef.co/go/tools v0.0.1-2020.1.4 // indirect
|
||||
inet.af/netaddr v0.0.0-20200702150737-4591d218f82c
|
||||
rsc.io/goversion v1.2.0
|
||||
)
|
||||
|
||||
22
go.sum
22
go.sum
@ -38,7 +38,6 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/rpmpack v0.0.0-20191226140753-aa36bfddb3a0 h1:BW6OvS3kpT5UEPbCZ+KyX/OB4Ks9/MNMhWjqPPkZxsE=
|
||||
github.com/google/rpmpack v0.0.0-20191226140753-aa36bfddb3a0/go.mod h1:RaTPr0KUf2K7fnZYLNDrr8rxAamWs3iNywJLtQ2AzBg=
|
||||
github.com/goreleaser/nfpm v1.1.10 h1:0nwzKUJTcygNxTzVKq2Dh9wpVP1W2biUH6SNKmoxR3w=
|
||||
@ -48,9 +47,8 @@ github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ
|
||||
github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
|
||||
github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4 h1:nwOc1YaOrYJ37sEBrtWZrdqzK22hiJs3GpDmP3sR2Yw=
|
||||
github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA=
|
||||
github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I=
|
||||
github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw=
|
||||
@ -74,7 +72,6 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/sassoftware/go-rpmutils v0.0.0-20190420191620-a8f1baeba37b h1:+gCnWOZV8Z/8jehJ2CdqB47Z3S+SREmQcuXkRFLNsiI=
|
||||
github.com/sassoftware/go-rpmutils v0.0.0-20190420191620-a8f1baeba37b/go.mod h1:am+Fp8Bt506lA3Rk3QCmSqmYmLMnPDhdDUcosQCAx+I=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
@ -82,10 +79,6 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/tailscale/winipcfg-go v0.0.0-20200413171540-609dcf2df55f h1:uFj5bslHsMzxIM8UTjAhq4VXeo6GfNW91rpoh/WMJaY=
|
||||
github.com/tailscale/winipcfg-go v0.0.0-20200413171540-609dcf2df55f/go.mod h1:x880GWw5fvrl2DVTQ04ttXQD4DuppTt1Yz6wLibbjNE=
|
||||
github.com/tailscale/wireguard-go v0.0.0-20200515231107-62868271d710 h1:I6aq3tOYbZob9uwhGpr7R266qTeU9PFqS6NnpfCqEzo=
|
||||
github.com/tailscale/wireguard-go v0.0.0-20200515231107-62868271d710/go.mod h1:JPm5cTfu1K+qDFRbiHy0sOlHUylYQbpl356sdYFD8V4=
|
||||
github.com/tailscale/wireguard-go v0.0.0-20200615180905-687c10194779 h1:zg0rgvhBZGA4nvh17nDKcqkEXw6Nbc/Ma2VBvLaW7LU=
|
||||
github.com/tailscale/wireguard-go v0.0.0-20200615180905-687c10194779/go.mod h1:JPm5cTfu1K+qDFRbiHy0sOlHUylYQbpl356sdYFD8V4=
|
||||
github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
|
||||
github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
|
||||
github.com/toqueteos/webbrowser v1.2.0 h1:tVP/gpK69Fx+qMJKsLE7TD8LuGWPnEV71wBN9rrstGQ=
|
||||
@ -97,16 +90,13 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMx
|
||||
go4.org/mem v0.0.0-20200601023850-d8ee1dfa5518 h1:AA3bSGklCgkrqIGnvL4894oa/2K9ltE0RejXh8CgyvA=
|
||||
go4.org/mem v0.0.0-20200601023850-d8ee1dfa5518/go.mod h1:NEYvpHWemiG/E5UWfaN5QAIGZeT1sa0Z2UNk6oeMb/k=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6 h1:TjszyFsQsyZNHwdVdZ5m7bjmreu0znc2kRYsEml9/Ww=
|
||||
golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20191003171128-d98b1b443823/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
@ -118,7 +108,6 @@ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BG
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@ -140,24 +129,17 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d h1:/iIZNFGxc/a7C3yWjGcnboV+Tkc7mxr+p6fDztwoxuM=
|
||||
golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
|
||||
gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8=
|
||||
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||
inet.af/netaddr v0.0.0-20200513162223-787f13e36cbe h1:WjJ6wZhXEWQA3FFSwOjG8tO2q1NDFSqrUwNcTvxwMEQ=
|
||||
inet.af/netaddr v0.0.0-20200513162223-787f13e36cbe/go.mod h1:qqYzz/2whtrbWJvt+DNWQyvekNN4ePQZcg2xc2/Yjww=
|
||||
inet.af/netaddr v0.0.0-20200629220211-f44a6d25c536 h1:XFVw2MVOtmHBidx70M+I6vIw2F6f55UyXvkiKfIrE38=
|
||||
|
||||
@ -18,11 +18,11 @@
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"tailscale.com/control/controlclient"
|
||||
"tailscale.com/ipn"
|
||||
"tailscale.com/logtail/backoff"
|
||||
"tailscale.com/safesocket"
|
||||
"tailscale.com/smallzstd"
|
||||
"tailscale.com/types/logger"
|
||||
"tailscale.com/version"
|
||||
"tailscale.com/wgengine"
|
||||
@ -113,10 +113,7 @@ func Run(rctx context.Context, logf logger.Logf, logid string, opts Options, e w
|
||||
return fmt.Errorf("NewLocalBackend: %v", err)
|
||||
}
|
||||
b.SetDecompressor(func() (controlclient.Decompressor, error) {
|
||||
return zstd.NewReader(nil,
|
||||
zstd.WithDecoderLowmem(true),
|
||||
zstd.WithDecoderConcurrency(1),
|
||||
)
|
||||
return smallzstd.NewDecoder(nil)
|
||||
})
|
||||
|
||||
if opts.DebugMux != nil {
|
||||
|
||||
@ -25,13 +25,13 @@
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"golang.org/x/crypto/ssh/terminal"
|
||||
"tailscale.com/atomicfile"
|
||||
"tailscale.com/logtail"
|
||||
"tailscale.com/logtail/filch"
|
||||
"tailscale.com/net/netns"
|
||||
"tailscale.com/net/tlsdial"
|
||||
"tailscale.com/smallzstd"
|
||||
"tailscale.com/version"
|
||||
)
|
||||
|
||||
@ -350,11 +350,7 @@ func New(collection string) *Policy {
|
||||
PrivateID: newc.PrivateID,
|
||||
Stderr: logWriter{console},
|
||||
NewZstdEncoder: func() logtail.Encoder {
|
||||
w, err := zstd.NewWriter(nil,
|
||||
zstd.WithEncoderLevel(zstd.SpeedFastest),
|
||||
zstd.WithEncoderConcurrency(1),
|
||||
zstd.WithWindowSize(8192),
|
||||
)
|
||||
w, err := smallzstd.NewEncoder(nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
14
smallzstd/testdata
Normal file
14
smallzstd/testdata
Normal file
@ -0,0 +1,14 @@
|
||||
{"logtail":{"client_time":"2020-07-01T14:49:40.196597018-07:00","server_time":"2020-07-01T21:49:40.198371511Z"},"text":"9.8M/25.6M magicsock: starting endpoint update (periodic)\n"}
|
||||
{"logtail":{"client_time":"2020-07-01T14:49:40.345925455-07:00","server_time":"2020-07-01T21:49:40.347904717Z"},"text":"9.9M/25.6M netcheck: udp=true v6=false mapvarydest=false hair=false v4a=202.188.7.1:41641 derp=2 derpdist=1v4:7ms,2v4:3ms,4v4:18ms\n"}
|
||||
{"logtail":{"client_time":"2020-07-01T14:49:43.347155742-07:00","server_time":"2020-07-01T21:49:43.34828658Z"},"text":"9.9M/25.6M control: map response long-poll timed out!\n"}
|
||||
{"logtail":{"client_time":"2020-07-01T14:49:43.347539333-07:00","server_time":"2020-07-01T21:49:43.358809354Z"},"text":"9.9M/25.6M control: PollNetMap: context canceled\n"}
|
||||
{"logtail":{"client_time":"2020-07-01T14:49:43.347767812-07:00","server_time":"2020-07-01T21:49:43.358809354Z"},"text":"10.0M/25.6M control: sendStatus: mapRoutine1: state:authenticated\n"}
|
||||
{"logtail":{"client_time":"2020-07-01T14:49:43.347817165-07:00","server_time":"2020-07-01T21:49:43.358809354Z"},"text":"10.0M/25.6M blockEngineUpdates(false)\n"}
|
||||
{"logtail":{"client_time":"2020-07-01T14:49:43.347989028-07:00","server_time":"2020-07-01T21:49:43.358809354Z"},"text":"10.0M/25.6M wgcfg: [SViTM] skipping subnet route\n"}
|
||||
{"logtail":{"client_time":"2020-07-01T14:49:43.349997554-07:00","server_time":"2020-07-01T21:49:43.358809354Z"},"text":"9.3M/25.6M Received error: PollNetMap: context canceled\n"}
|
||||
{"logtail":{"client_time":"2020-07-01T14:49:43.350072606-07:00","server_time":"2020-07-01T21:49:43.358809354Z"},"text":"9.3M/25.6M control: mapRoutine: backoff: 30136 msec\n"}
|
||||
{"logtail":{"client_time":"2020-07-01T14:49:47.998364646-07:00","server_time":"2020-07-01T21:49:47.999333754Z"},"text":"9.5M/25.6M [W1NbE] - [UcppE] Send handshake init [127.3.3.40:1, 6.1.1.6:37388*, 10.3.2.6:41641]\n"}
|
||||
{"logtail":{"client_time":"2020-07-01T14:49:47.99881914-07:00","server_time":"2020-07-01T21:49:48.009859543Z"},"text":"9.6M/25.6M magicsock: adding connection to derp-1 for [W1NbE]\n"}
|
||||
{"logtail":{"client_time":"2020-07-01T14:49:47.998904932-07:00","server_time":"2020-07-01T21:49:48.009859543Z"},"text":"9.6M/25.6M magicsock: 2 active derp conns: derp-1=cr0s,wr0s derp-2=cr16h0m0s,wr14h38m0s\n"}
|
||||
{"logtail":{"client_time":"2020-07-01T14:49:47.999045606-07:00","server_time":"2020-07-01T21:49:48.009859543Z"},"text":"9.6M/25.6M derphttp.Client.Recv: connecting to derp-1 (nyc)\n"}
|
||||
{"logtail":{"client_time":"2020-07-01T14:49:48.091104119-07:00","server_time":"2020-07-01T21:49:48.09280535Z"},"text":"9.6M/25.6M magicsock: rx [W1NbE] from 6.1.1.6:37388 (1/3), set as new priority\n"}
|
||||
79
smallzstd/zstd.go
Normal file
79
smallzstd/zstd.go
Normal file
@ -0,0 +1,79 @@
|
||||
// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package smallzstd produces zstd encoders and decoders optimized for
|
||||
// low memory usage, at the expense of compression efficiency.
|
||||
//
|
||||
// This package is optimized primarily for the memory cost of
|
||||
// compressing and decompressing data. We reduce this cost in two
|
||||
// major ways: disable parallelism within the library (i.e. don't use
|
||||
// multiple CPU cores to decompress), and drop the compression window
|
||||
// down from the defaults of 4-16MiB, to 8kiB.
|
||||
//
|
||||
// Decompressors cost 2x the window size in RAM to run, so by using an
|
||||
// 8kiB window, we can run ~1000 more decompressors per unit of memory
|
||||
// than with the defaults.
|
||||
//
|
||||
// Depending on context, the benefit is either being able to run more
|
||||
// decoders (e.g. in our logs processing system), or having a lower
|
||||
// memory footprint when using compression in network protocols
|
||||
// (e.g. in tailscaled, which should have a minimal RAM cost).
|
||||
package smallzstd
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
// WindowSize is the window size used for zstd compression. Decoder
|
||||
// memory usage scales linearly with WindowSize.
|
||||
const WindowSize = 8 << 10 // 8kiB
|
||||
|
||||
// NewDecoder returns a zstd.Decoder configured for low memory usage,
|
||||
// at the expense of decompression performance.
|
||||
func NewDecoder(r io.Reader, options ...zstd.DOption) (*zstd.Decoder, error) {
|
||||
defaults := []zstd.DOption{
|
||||
// Default is GOMAXPROCS, which costs many KiB in stacks.
|
||||
zstd.WithDecoderConcurrency(1),
|
||||
// Default is to allocate more upfront for performance. We
|
||||
// prefer lower memory use and a bit of GC load.
|
||||
zstd.WithDecoderLowmem(true),
|
||||
// You might expect to see zstd.WithDecoderMaxMemory
|
||||
// here. However, it's not terribly safe to use if you're
|
||||
// doing stateless decoding, because it sets the maximum
|
||||
// amount of memory the decompressed data can occupy, rather
|
||||
// than the window size of the zstd stream. This means a very
|
||||
// compressible piece of data might violate the max memory
|
||||
// limit here, even if the window size (and thus total memory
|
||||
// required to decompress the data) is small.
|
||||
//
|
||||
// As a result, we don't set a decoder limit here, and rely on
|
||||
// the encoder below producing "cheap" streams. Callers are
|
||||
// welcome to set their own max memory setting, if
|
||||
// contextually there is a clearly correct value (e.g. it's
|
||||
// known from the upper layer protocol that the decoded data
|
||||
// can never be more than 1MiB).
|
||||
}
|
||||
|
||||
return zstd.NewReader(r, append(defaults, options...)...)
|
||||
}
|
||||
|
||||
// NewEncoder returns a zstd.Encoder configured for low memory usage,
|
||||
// both during compression and at decompression time, at the expense
|
||||
// of performance and compression efficiency.
|
||||
func NewEncoder(w io.Writer, options ...zstd.EOption) (*zstd.Encoder, error) {
|
||||
defaults := []zstd.EOption{
|
||||
// Default is GOMAXPROCS, which costs many KiB in stacks.
|
||||
zstd.WithEncoderConcurrency(1),
|
||||
// Default is several MiB, which bloats both encoders and
|
||||
// their corresponding decoders.
|
||||
zstd.WithWindowSize(WindowSize),
|
||||
// Encode zero-length inputs in a way that the `zstd` utility
|
||||
// can read, because interoperability is handy.
|
||||
zstd.WithZeroFrames(true),
|
||||
}
|
||||
|
||||
return zstd.NewWriter(w, append(defaults, options...)...)
|
||||
}
|
||||
131
smallzstd/zstd_test.go
Normal file
131
smallzstd/zstd_test.go
Normal file
@ -0,0 +1,131 @@
|
||||
// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package smallzstd
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"testing"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
func BenchmarkSmallEncoder(b *testing.B) {
|
||||
benchEncoder(b, func() (*zstd.Encoder, error) { return NewEncoder(nil) })
|
||||
}
|
||||
|
||||
func BenchmarkSmallEncoderWithBuild(b *testing.B) {
|
||||
benchEncoderWithConstruction(b, func() (*zstd.Encoder, error) { return NewEncoder(nil) })
|
||||
}
|
||||
|
||||
func BenchmarkStockEncoder(b *testing.B) {
|
||||
benchEncoder(b, func() (*zstd.Encoder, error) { return zstd.NewWriter(nil) })
|
||||
}
|
||||
|
||||
func BenchmarkStockEncoderWithBuild(b *testing.B) {
|
||||
benchEncoderWithConstruction(b, func() (*zstd.Encoder, error) { return zstd.NewWriter(nil) })
|
||||
}
|
||||
|
||||
func BenchmarkSmallDecoder(b *testing.B) {
|
||||
benchDecoder(b, func() (*zstd.Decoder, error) { return NewDecoder(nil) })
|
||||
}
|
||||
|
||||
func BenchmarkSmallDecoderWithBuild(b *testing.B) {
|
||||
benchDecoderWithConstruction(b, func() (*zstd.Decoder, error) { return NewDecoder(nil) })
|
||||
}
|
||||
|
||||
func BenchmarkStockDecoder(b *testing.B) {
|
||||
benchDecoder(b, func() (*zstd.Decoder, error) { return zstd.NewReader(nil) })
|
||||
}
|
||||
|
||||
func BenchmarkStockDecoderWithBuild(b *testing.B) {
|
||||
benchDecoderWithConstruction(b, func() (*zstd.Decoder, error) { return zstd.NewReader(nil) })
|
||||
}
|
||||
|
||||
func benchEncoder(b *testing.B, mk func() (*zstd.Encoder, error)) {
|
||||
b.ReportAllocs()
|
||||
|
||||
in := testdata(b)
|
||||
out := make([]byte, 0, 10<<10) // 10kiB
|
||||
|
||||
e, err := mk()
|
||||
if err != nil {
|
||||
b.Fatalf("making encoder: %v", err)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
e.EncodeAll(in, out)
|
||||
}
|
||||
}
|
||||
|
||||
func benchEncoderWithConstruction(b *testing.B, mk func() (*zstd.Encoder, error)) {
|
||||
b.ReportAllocs()
|
||||
|
||||
in := testdata(b)
|
||||
out := make([]byte, 0, 10<<10) // 10kiB
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
e, err := mk()
|
||||
if err != nil {
|
||||
b.Fatalf("making encoder: %v", err)
|
||||
}
|
||||
|
||||
e.EncodeAll(in, out)
|
||||
}
|
||||
}
|
||||
|
||||
func benchDecoder(b *testing.B, mk func() (*zstd.Decoder, error)) {
|
||||
b.ReportAllocs()
|
||||
|
||||
in := compressedTestdata(b)
|
||||
out := make([]byte, 0, 10<<10)
|
||||
|
||||
d, err := mk()
|
||||
if err != nil {
|
||||
b.Fatalf("creating decoder: %v", err)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
d.DecodeAll(in, out)
|
||||
}
|
||||
}
|
||||
|
||||
func benchDecoderWithConstruction(b *testing.B, mk func() (*zstd.Decoder, error)) {
|
||||
b.ReportAllocs()
|
||||
|
||||
in := compressedTestdata(b)
|
||||
out := make([]byte, 0, 10<<10)
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
d, err := mk()
|
||||
if err != nil {
|
||||
b.Fatalf("creating decoder: %v", err)
|
||||
}
|
||||
|
||||
d.DecodeAll(in, out)
|
||||
}
|
||||
}
|
||||
|
||||
func testdata(b *testing.B) []byte {
|
||||
b.Helper()
|
||||
in, err := ioutil.ReadFile("testdata")
|
||||
if err != nil {
|
||||
b.Fatalf("reading testdata: %v", err)
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
func compressedTestdata(b *testing.B) []byte {
|
||||
b.Helper()
|
||||
uncomp := testdata(b)
|
||||
e, err := NewEncoder(nil)
|
||||
if err != nil {
|
||||
b.Fatalf("creating encoder: %v", err)
|
||||
}
|
||||
return e.EncodeAll(uncomp, nil)
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user