From 696020227c4cb1e76c4e86255f4b33aaf00843b0 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Fri, 7 Aug 2020 20:44:04 -0700 Subject: [PATCH] tailcfg, control/controlclient: support delta-encoded netmaps Should greatly reduce bandwidth for large networks (including our hello.ipn.dev node). Signed-off-by: Brad Fitzpatrick --- control/controlclient/direct.go | 102 +++++++++++++++++++++++++++ control/controlclient/direct_test.go | 93 ++++++++++++++++++++++++ tailcfg/tailcfg.go | 29 ++++++-- 3 files changed, 217 insertions(+), 7 deletions(-) create mode 100644 control/controlclient/direct_test.go diff --git a/control/controlclient/direct.go b/control/controlclient/direct.go index ffe0bb1f3..a5b5ca203 100644 --- a/control/controlclient/direct.go +++ b/control/controlclient/direct.go @@ -22,6 +22,7 @@ "os" "reflect" "runtime" + "sort" "strconv" "strings" "sync" @@ -489,6 +490,7 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkM request := tailcfg.MapRequest{ Version: 4, IncludeIPv6: true, + DeltaPeers: true, KeepAlive: c.keepAlive, NodeKey: tailcfg.NodeKey(persist.PrivateNodeKey.Public()), DiscoKey: c.discoPubKey, @@ -573,6 +575,7 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkM // the same format before just closing the connection. // We can use this same read loop either way. var msg []byte + var previousPeers []*tailcfg.Node // for delta-purposes for i := 0; i < maxPolls || maxPolls < 0; i++ { vlogf("netmap: starting size read after %v (poll %v)", time.Since(t0).Round(time.Millisecond), i) var siz [4]byte @@ -594,6 +597,7 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkM vlogf("netmap: decode error: %v") return err } + if resp.KeepAlive { vlogf("netmap: got keep-alive") } else { @@ -609,6 +613,10 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkM if resp.KeepAlive { continue } + + undeltaPeers(&resp, previousPeers) + previousPeers = cloneNodes(resp.Peers) // defensive/lazy clone, since this escapes to who knows where + if resp.DERPMap != nil { vlogf("netmap: new map contains DERP map") lastDERPMap = resp.DERPMap @@ -851,3 +859,97 @@ func envBool(k string) bool { } return v } + +// undeltaPeers updates mapRes.Peers to be complete based on the provided previous peer list +// and the PeersRemoved and PeersChanged fields in mapRes. +// It then also nils out the delta fields. +func undeltaPeers(mapRes *tailcfg.MapResponse, prev []*tailcfg.Node) { + if len(mapRes.Peers) > 0 { + // Not delta encoded. + if !nodesSorted(mapRes.Peers) { + log.Printf("netmap: undeltaPeers: MapResponse.Peers not sorted; sorting") + sortNodes(mapRes.Peers) + } + return + } + + var removed map[tailcfg.NodeID]bool + if pr := mapRes.PeersRemoved; len(pr) > 0 { + removed = make(map[tailcfg.NodeID]bool, len(pr)) + for _, id := range pr { + removed[id] = true + } + } + changed := mapRes.PeersChanged + + if len(removed) == 0 && len(changed) == 0 { + // No changes fast path. + mapRes.Peers = prev + return + } + + if !nodesSorted(changed) { + log.Printf("netmap: undeltaPeers: MapResponse.PeersChanged not sorted; sorting") + sortNodes(changed) + } + if !nodesSorted(prev) { + // Internal error (unrelated to the network) if we get here. + log.Printf("netmap: undeltaPeers: [unexpected] prev not sorted; sorting") + sortNodes(prev) + } + + newFull := make([]*tailcfg.Node, 0, len(prev)-len(removed)) + for len(prev) > 0 && len(changed) > 0 { + pID := prev[0].ID + cID := changed[0].ID + if removed[pID] { + prev = prev[1:] + continue + } + switch { + case pID < cID: + newFull = append(newFull, prev[0]) + prev = prev[1:] + case pID == cID: + newFull = append(newFull, changed[0]) + prev, changed = prev[1:], changed[1:] + case cID < pID: + newFull = append(newFull, changed[0]) + changed = changed[1:] + } + } + newFull = append(newFull, changed...) + for _, n := range prev { + if !removed[n.ID] { + newFull = append(newFull, n) + } + } + sortNodes(newFull) + mapRes.Peers = newFull + mapRes.PeersChanged = nil + mapRes.PeersRemoved = nil +} + +func nodesSorted(v []*tailcfg.Node) bool { + for i, n := range v { + if i > 0 && n.ID <= v[i-1].ID { + return false + } + } + return true +} + +func sortNodes(v []*tailcfg.Node) { + sort.Slice(v, func(i, j int) bool { return v[i].ID < v[j].ID }) +} + +func cloneNodes(v1 []*tailcfg.Node) []*tailcfg.Node { + if v1 == nil { + return nil + } + v2 := make([]*tailcfg.Node, len(v1)) + for i, n := range v1 { + v2[i] = n.Clone() + } + return v2 +} diff --git a/control/controlclient/direct_test.go b/control/controlclient/direct_test.go new file mode 100644 index 000000000..e50a14df4 --- /dev/null +++ b/control/controlclient/direct_test.go @@ -0,0 +1,93 @@ +// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package controlclient + +import ( + "fmt" + "reflect" + "strings" + "testing" + + "tailscale.com/tailcfg" +) + +func TestUndeltaPeers(t *testing.T) { + n := func(id tailcfg.NodeID, name string) *tailcfg.Node { + return &tailcfg.Node{ID: id, Name: name} + } + peers := func(nv ...*tailcfg.Node) []*tailcfg.Node { return nv } + tests := []struct { + name string + mapRes *tailcfg.MapResponse + prev []*tailcfg.Node + want []*tailcfg.Node + }{ + { + name: "full_peers", + mapRes: &tailcfg.MapResponse{ + Peers: peers(n(1, "foo"), n(2, "bar")), + }, + want: peers(n(1, "foo"), n(2, "bar")), + }, + { + name: "full_peers_ignores_deltas", + mapRes: &tailcfg.MapResponse{ + Peers: peers(n(1, "foo"), n(2, "bar")), + PeersRemoved: []tailcfg.NodeID{2}, + }, + want: peers(n(1, "foo"), n(2, "bar")), + }, + { + name: "add_and_update", + prev: peers(n(1, "foo"), n(2, "bar")), + mapRes: &tailcfg.MapResponse{ + PeersChanged: peers(n(0, "zero"), n(2, "bar2"), n(3, "three")), + }, + want: peers(n(0, "zero"), n(1, "foo"), n(2, "bar2"), n(3, "three")), + }, + { + name: "remove", + prev: peers(n(1, "foo"), n(2, "bar")), + mapRes: &tailcfg.MapResponse{ + PeersRemoved: []tailcfg.NodeID{1}, + }, + want: peers(n(2, "bar")), + }, + { + name: "add_and_remove", + prev: peers(n(1, "foo"), n(2, "bar")), + mapRes: &tailcfg.MapResponse{ + PeersChanged: peers(n(1, "foo2")), + PeersRemoved: []tailcfg.NodeID{2}, + }, + want: peers(n(1, "foo2")), + }, + { + name: "unchanged", + prev: peers(n(1, "foo"), n(2, "bar")), + mapRes: &tailcfg.MapResponse{}, + want: peers(n(1, "foo"), n(2, "bar")), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + undeltaPeers(tt.mapRes, tt.prev) + if !reflect.DeepEqual(tt.mapRes.Peers, tt.want) { + t.Errorf("wrong results\n got: %s\nwant: %s", formatNodes(tt.mapRes.Peers), formatNodes(tt.want)) + } + }) + } +} + +func formatNodes(nodes []*tailcfg.Node) string { + var sb strings.Builder + for i, n := range nodes { + if i > 0 { + sb.WriteString(", ") + } + fmt.Fprintf(&sb, "(%d, %q)", n.ID, n.Name) + } + return sb.String() +} diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index bb138ff27..f441c8719 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -443,11 +443,12 @@ type RegisterResponse struct { type MapRequest struct { Version int // current version is 4 Compress string // "zstd" or "" (no compression) - KeepAlive bool // server sends keep-alives + KeepAlive bool // whether server should send keep-alives back to us NodeKey NodeKey DiscoKey DiscoKey Endpoints []string // caller's endpoints (IPv4 or IPv6) IncludeIPv6 bool // include IPv6 endpoints in returned Node Endpoints + DeltaPeers bool // whether the 2nd+ network map in response should be deltas, using PeersChanged, PeersRemoved Stream bool // if true, multiple MapResponse objects are returned Hostinfo *Hostinfo @@ -502,22 +503,36 @@ type DNSConfig struct { } type MapResponse struct { - KeepAlive bool // if set, all other fields are ignored + KeepAlive bool `json:",omitempty"` // if set, all other fields are ignored // Networking Node *Node - Peers []*Node - DERPMap *DERPMap + DERPMap *DERPMap `json:",omitempty"` // if non-empty, a change in the DERP map. + + // Peers, if non-empty, is the complete list of peers. + // It will be set in the first MapResponse for a long-polled request/response. + // Subsequent responses will be delta-encoded if DeltaPeers was set in the request. + // If Peers is non-empty, PeersChanged and PeersRemoved should + // be ignored (and should be empty). + // Peers is always returned sorted by Node.ID. + Peers []*Node `json:",omitempty"` + // PeersChanged are the Nodes (identified by their ID) that + // have changed or been added since the past update on the + // HTTP response. It's only set if MapRequest.DeltaPeers was true. + // PeersChanged is always returned sorted by Node.ID. + PeersChanged []*Node `json:",omitempty"` + // PeersRemoved are the NodeIDs that are no longer in the peer list. + PeersRemoved []NodeID `json:",omitempty"` // DNS is the same as DNSConfig.Nameservers. // // TODO(dmytro): should be sent in DNSConfig.Nameservers once clients have updated. - DNS []wgcfg.IP + DNS []wgcfg.IP `json:",omitempty"` // SearchPaths are the same as DNSConfig.Domains. // // TODO(dmytro): should be sent in DNSConfig.Domains once clients have updated. - SearchPaths []string - DNSConfig DNSConfig + SearchPaths []string `json:",omitempty"` + DNSConfig DNSConfig `json:",omitempty"` // ACLs Domain string