all: add extra information to serialized endpoints

magicsock.Conn.ParseEndpoint requires a peer's public key,
disco key, and legacy ip/ports in order to do its job.
We currently accomplish that by:

* adding the public key in our wireguard-go fork
* encoding the disco key as magic hostname
* using a bespoke comma-separated encoding

It's a bit messy.

Instead, switch to something simpler: use a json-encoded struct
containing exactly the information we need, in the form we use it.

Our wireguard-go fork still adds the public key to the
address when it passes it to ParseEndpoint, but now the code
compensating for that is just a couple of simple, well-commented lines.
Once this commit is in, we can remove that part of the fork
and remove the compensating code.

Signed-off-by: Josh Bleecher Snyder <josharian@gmail.com>

internal/deephash/deephash_test.go

@@ -36,7 +36,9 @@ func getVal() []interface{} {
 			Addresses: []netaddr.IPPrefix{{Bits: 5, IP: netaddr.IPFrom16([16]byte{3: 3})}},
 			Peers: []wgcfg.Peer{
 				{
-					Endpoints: "foo:5",
+					Endpoints: wgcfg.Endpoints{
+						IPPorts: wgcfg.NewIPPortSet(netaddr.MustParseIPPort("42.42.42.42:5")),
+					},
 				},
 			},
 		},

wgengine/bench/wg.go

@@ -9,7 +9,6 @@
 	"io"
 	"log"
 	"os"
-	"strings"
 	"sync"
 	"testing"
 
@@ -99,8 +98,14 @@ func setupWGTest(b *testing.B, logf logger.Logf, traf *TrafficGen, a1, a2 netadd
 		logf("e1 status: %v", *st)
 
 		var eps []string
+		var ipps []netaddr.IPPort
 		for _, ep := range st.LocalAddrs {
 			eps = append(eps, ep.Addr.String())
+			ipps = append(ipps, ep.Addr)
+		}
+		endpoint := wgcfg.Endpoints{
+			PublicKey: c1.PrivateKey.Public(),
+			IPPorts:   wgcfg.NewIPPortSet(ipps...),
 		}
 
 		n := tailcfg.Node{
@@ -119,7 +124,7 @@ func setupWGTest(b *testing.B, logf logger.Logf, traf *TrafficGen, a1, a2 netadd
 		p := wgcfg.Peer{
 			PublicKey:  c1.PrivateKey.Public(),
 			AllowedIPs: []netaddr.IPPrefix{a1},
-			Endpoints:  strings.Join(eps, ","),
+			Endpoints:  endpoint,
 		}
 		c2.Peers = []wgcfg.Peer{p}
 		e2.Reconfig(&c2, &router.Config{}, new(dns.Config))
@@ -137,8 +142,14 @@ func setupWGTest(b *testing.B, logf logger.Logf, traf *TrafficGen, a1, a2 netadd
 		logf("e2 status: %v", *st)
 
 		var eps []string
+		var ipps []netaddr.IPPort
 		for _, ep := range st.LocalAddrs {
 			eps = append(eps, ep.Addr.String())
+			ipps = append(ipps, ep.Addr)
+		}
+		endpoint := wgcfg.Endpoints{
+			PublicKey: c2.PrivateKey.Public(),
+			IPPorts:   wgcfg.NewIPPortSet(ipps...),
 		}
 
 		n := tailcfg.Node{
@@ -157,7 +168,7 @@ func setupWGTest(b *testing.B, logf logger.Logf, traf *TrafficGen, a1, a2 netadd
 		p := wgcfg.Peer{
 			PublicKey:  c2.PrivateKey.Public(),
 			AllowedIPs: []netaddr.IPPrefix{a2},
-			Endpoints:  strings.Join(eps, ","),
+			Endpoints:  endpoint,
 		}
 		c1.Peers = []wgcfg.Peer{p}
 		e1.Reconfig(&c1, &router.Config{}, new(dns.Config))

wgengine/magicsock/legacy.go

@@ -10,7 +10,6 @@
 	"crypto/subtle"
 	"encoding/binary"
 	"errors"
-	"fmt"
 	"hash"
 	"net"
 	"strings"
@@ -27,6 +26,7 @@
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/wgkey"
+	"tailscale.com/wgengine/wgcfg"
 )
 
 var (
@@ -34,7 +34,11 @@
 	errDisabled       = errors.New("magicsock: legacy networking disabled")
 )
 
-func (c *Conn) createLegacyEndpointLocked(pk key.Public, addrs string) (conn.Endpoint, error) {
+// createLegacyEndpointLocked creates a new wireguard-go endpoint for a legacy connection.
+// pk is the public key of the remote peer. addrs is the ordered set of addresses for the remote peer.
+// rawDest is the encoded wireguard-go endpoint string. It should be treated as a black box.
+// It is provided so that addrSet.DstToString can return it when requested by wireguard-go.
+func (c *Conn) createLegacyEndpointLocked(pk key.Public, addrs wgcfg.IPPortSet, rawDest string) (conn.Endpoint, error) {
 	if c.disableLegacy {
 		return nil, errDisabled
 	}
@@ -43,18 +47,9 @@ func (c *Conn) createLegacyEndpointLocked(pk key.Public, addrs string) (conn.End
 		Logf:      c.logf,
 		publicKey: pk,
 		curAddr:   -1,
-		rawdst:    addrs,
+		rawdst:    rawDest,
-	}
-
-	if addrs != "" {
-		for _, ep := range strings.Split(addrs, ",") {
-			ipp, err := netaddr.ParseIPPort(ep)
-			if err != nil {
-				return nil, fmt.Errorf("bogus address %q", ep)
-			}
-			a.ipPorts = append(a.ipPorts, ipp)
-		}
 	}
+	a.ipPorts = append(a.ipPorts, addrs.IPPorts()...)
 
 	// If this endpoint is being updated, remember its old set of
 	// endpoints so we can remove any (from c.addrsByUDP) that are

wgengine/magicsock/magicsock.go

@@ -11,6 +11,7 @@
 	"context"
 	crand "crypto/rand"
 	"encoding/binary"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"hash/fnv"
@@ -27,7 +28,6 @@
 	"time"
 
 	"github.com/tailscale/wireguard-go/conn"
-	"go4.org/mem"
 	"golang.org/x/crypto/nacl/box"
 	"golang.org/x/time/rate"
 	"inet.af/netaddr"
@@ -2736,42 +2736,36 @@ func packIPPort(ua netaddr.IPPort) []byte {
 }
 
 // ParseEndpoint is called by WireGuard to connect to an endpoint.
-//
+// endpointStr is a json-serialized wgcfg.Endpoints struct.
-// keyAddrs is the 32 byte public key of the peer followed by addrs.
+// (It it currently prefixed by 32 bytes of junk, but that will change shortly.)
-// Addrs is either:
+// If those Endpoints contain an active discovery key, ParseEndpoint returns a discoEndpoint.
-//
+// Otherwise it returns a legacy endpoint.
-//  1) a comma-separated list of UDP ip:ports (the peer doesn't have a discovery key)
+func (c *Conn) ParseEndpoint(endpointStr string) (conn.Endpoint, error) {
-//  2) "<hex-discovery-key>.disco.tailscale:12345", a magic value that means the peer
+	// Our wireguard-go fork prepends the public key to endpointStr.
-//     is running code that supports active discovery, so ParseEndpoint returns
+	// We don't need it; strip it off.
-//     a discoEndpoint.
+	// This code will be deleted soon.
-func (c *Conn) ParseEndpoint(keyAddrs string) (conn.Endpoint, error) {
+	endpointStr = endpointStr[32:]
-	if len(keyAddrs) < 32 {
+
-		c.logf("[unexpected] ParseEndpoint keyAddrs too short: %q", keyAddrs)
+	var endpoints wgcfg.Endpoints
-		return nil, errors.New("endpoint string too short")
+	err := json.Unmarshal([]byte(endpointStr), &endpoints)
+	if err != nil {
+		return nil, fmt.Errorf("magicsock: ParseEndpoint: json.Unmarshal failed on %q: %w", endpointStr, err)
 	}
-	var pk key.Public
+	pk := key.Public(endpoints.PublicKey)
-	copy(pk[:], keyAddrs)
+	discoKey := endpoints.DiscoKey
-	addrs := keyAddrs[len(pk):]
+	c.logf("magicsock: ParseEndpoint: key=%s: disco=%s ipps=%s", pk.ShortString(), discoKey.ShortString(), derpStr(endpoints.IPPorts.String()))
+
 	c.mu.Lock()
 	defer c.mu.Unlock()
-
+	if discoKey.IsZero() {
-	c.logf("magicsock: ParseEndpoint: key=%s: %s", pk.ShortString(), derpStr(addrs))
+		return c.createLegacyEndpointLocked(pk, endpoints.IPPorts, endpointStr)
-
-	if !strings.HasSuffix(addrs, wgcfg.EndpointDiscoSuffix) {
-		return c.createLegacyEndpointLocked(pk, addrs)
-	}
-
-	discoHex := strings.TrimSuffix(addrs, wgcfg.EndpointDiscoSuffix)
-	discoKey, err := key.NewPublicFromHexMem(mem.S(discoHex))
-	if err != nil {
-		return nil, fmt.Errorf("magicsock: invalid discokey endpoint %q for %v: %w", addrs, pk.ShortString(), err)
 	}
 	de := &discoEndpoint{
 		c:             c,
 		publicKey:     tailcfg.NodeKey(pk),        // peer public key (for WireGuard + DERP)
 		discoKey:      tailcfg.DiscoKey(discoKey), // for discovery mesages
 		discoShort:    tailcfg.DiscoKey(discoKey).ShortString(),
-		wgEndpoint:    addrs,
+		wgEndpoint:    endpointStr,
 		sentPing:      map[stun.TxID]sentPing{},
 		endpointState: map[netaddr.IPPort]*endpointState{},
 	}
@@ -3115,7 +3109,7 @@ type discoEndpoint struct {
 	discoKey   tailcfg.DiscoKey // for discovery mesages
 	discoShort string           // ShortString of discoKey
 	fakeWGAddr netaddr.IPPort   // the UDP address we tell wireguard-go we're using
-	wgEndpoint string           // string from ParseEndpoint: "<hex-discovery-key>.disco.tailscale:12345"
+	wgEndpoint string           // string from ParseEndpoint, holds a JSON-serialized wgcfg.Endpoints
 
 	// Owned by Conn.mu:
 	lastPingFrom netaddr.IPPort

wgengine/magicsock/magicsock_test.go

@@ -10,6 +10,7 @@
 	crand "crypto/rand"
 	"crypto/tls"
 	"encoding/binary"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -468,10 +469,14 @@ func makeConfigs(t *testing.T, addrs []netaddr.IPPort) []wgcfg.Config {
 			if peerNum == i {
 				continue
 			}
+			publicKey := privKeys[peerNum].Public()
 			peer := wgcfg.Peer{
-				PublicKey:           privKeys[peerNum].Public(),
+				PublicKey:  publicKey,
-				AllowedIPs:          addresses[peerNum],
+				AllowedIPs: addresses[peerNum],
-				Endpoints:           addr.String(),
+				Endpoints: wgcfg.Endpoints{
+					PublicKey: publicKey,
+					IPPorts:   wgcfg.NewIPPortSet(addr),
+				},
 				PersistentKeepalive: 25,
 			}
 			cfg.Peers = append(cfg.Peers, peer)
@@ -1242,6 +1247,23 @@ func newNonLegacyTestConn(t testing.TB) *Conn {
 	return conn
 }
 
+func makeEndpoint(tb testing.TB, public tailcfg.NodeKey, disco tailcfg.DiscoKey) string {
+	tb.Helper()
+	ep := wgcfg.Endpoints{
+		PublicKey: wgkey.Key(public),
+		DiscoKey:  disco,
+	}
+	buf, err := json.Marshal(ep)
+	if err != nil {
+		tb.Fatal(err)
+	}
+	// Our wireguard-go fork prepends the public key when calling ParseEndpoint.
+	// We no longer use it; add some junk there to emulate wireguard-go.
+	// This will go away soon.
+	junk := strings.Repeat(" ", 32)
+	return junk + string(buf)
+}
+
 // addTestEndpoint sets conn's network map to a single peer expected
 // to receive packets from sendConn (or DERP), and returns that peer's
 // nodekey and discokey.
@@ -1261,7 +1283,7 @@ func addTestEndpoint(tb testing.TB, conn *Conn, sendConn net.PacketConn) (tailcf
 		},
 	})
 	conn.SetPrivateKey(wgkey.Private{0: 1})
-	_, err := conn.ParseEndpoint(string(nodeKey[:]) + "0000000000000000000000000000000000000000000000000000000000000001.disco.tailscale:12345")
+	_, err := conn.ParseEndpoint(makeEndpoint(tb, nodeKey, discoKey))
 	if err != nil {
 		tb.Fatal(err)
 	}
@@ -1435,7 +1457,7 @@ func TestSetNetworkMapChangingNodeKey(t *testing.T) {
 			},
 		},
 	})
-	_, err := conn.ParseEndpoint(string(nodeKey1[:]) + "0000000000000000000000000000000000000000000000000000000000000001.disco.tailscale:12345")
+	_, err := conn.ParseEndpoint(makeEndpoint(t, nodeKey1, discoKey))
 	if err != nil {
 		t.Fatal(err)
 	}

wgengine/userspace.go

@@ -11,7 +11,6 @@
 	"errors"
 	"fmt"
 	"io"
-	"net"
 	"os"
 	"reflect"
 	"runtime"
@@ -502,15 +501,7 @@ func isTrimmablePeer(p *wgcfg.Peer, numPeers int) bool {
 	if forceFullWireguardConfig(numPeers) {
 		return false
 	}
-	if !isSingleEndpoint(p.Endpoints) {
+	if p.Endpoints.DiscoKey.IsZero() {
-		return false
-	}
-
-	host, _, err := net.SplitHostPort(p.Endpoints)
-	if err != nil {
-		return false
-	}
-	if !strings.HasSuffix(host, ".disco.tailscale") {
 		return false
 	}
 
@@ -580,26 +571,6 @@ func (e *userspaceEngine) isActiveSince(dk tailcfg.DiscoKey, ip netaddr.IP, t ti
 	return unixTime >= t.Unix()
 }
 
-// discoKeyFromPeer returns the DiscoKey for a wireguard config's Peer.
-//
-// Invariant: isTrimmablePeer(p) == true, so it should have 1 endpoint with
-// Host of form "<64-hex-digits>.disco.tailscale". If invariant is violated,
-// we return the zero value.
-func discoKeyFromPeer(p *wgcfg.Peer) tailcfg.DiscoKey {
-	if len(p.Endpoints) < 64 {
-		return tailcfg.DiscoKey{}
-	}
-	host, rest := p.Endpoints[:64], p.Endpoints[64:]
-	if !strings.HasPrefix(rest, ".disco.tailscale") {
-		return tailcfg.DiscoKey{}
-	}
-	k, err := key.NewPublicFromHexMem(mem.S(host))
-	if err != nil {
-		return tailcfg.DiscoKey{}
-	}
-	return tailcfg.DiscoKey(k)
-}
-
 // discoChanged are the set of peers whose disco keys have changed, implying they've restarted.
 // If a peer is in this set and was previously in the live wireguard config,
 // it needs to be first removed and then re-added to flush out its wireguard session key.
@@ -647,7 +618,7 @@ func (e *userspaceEngine) maybeReconfigWireguardLocked(discoChanged map[key.Publ
 			}
 			continue
 		}
-		dk := discoKeyFromPeer(p)
+		dk := p.Endpoints.DiscoKey
 		trackDisco = append(trackDisco, dk)
 		recentlyActive := false
 		for _, cidr := range p.AllowedIPs {
@@ -797,19 +768,19 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config,
 	// and a second time with it.
 	discoChanged := make(map[key.Public]bool)
 	{
-		prevEP := make(map[key.Public]string)
+		prevEP := make(map[key.Public]tailcfg.DiscoKey)
 		for i := range e.lastCfgFull.Peers {
-			if p := &e.lastCfgFull.Peers[i]; isSingleEndpoint(p.Endpoints) {
+			if p := &e.lastCfgFull.Peers[i]; !p.Endpoints.DiscoKey.IsZero() {
-				prevEP[key.Public(p.PublicKey)] = p.Endpoints
+				prevEP[key.Public(p.PublicKey)] = p.Endpoints.DiscoKey
 			}
 		}
 		for i := range cfg.Peers {
 			p := &cfg.Peers[i]
-			if !isSingleEndpoint(p.Endpoints) {
+			if p.Endpoints.DiscoKey.IsZero() {
 				continue
 			}
 			pub := key.Public(p.PublicKey)
-			if old, ok := prevEP[pub]; ok && old != p.Endpoints {
+			if old, ok := prevEP[pub]; ok && old != p.Endpoints.DiscoKey {
 				discoChanged[pub] = true
 				e.logf("wgengine: Reconfig: %s changed from %q to %q", pub.ShortString(), old, p.Endpoints)
 			}
@@ -853,11 +824,6 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config,
 	return nil
 }
 
-// isSingleEndpoint reports whether endpoints contains exactly one host:port pair.
-func isSingleEndpoint(s string) bool {
-	return s != "" && !strings.Contains(s, ",")
-}
-
 func (e *userspaceEngine) GetFilter() *filter.Filter {
 	return e.tundev.GetFilter()
 }

wgengine/userspace_test.go

@@ -104,7 +104,7 @@ func TestUserspaceEngineReconfig(t *testing.T) {
 					AllowedIPs: []netaddr.IPPrefix{
 						{IP: netaddr.IPv4(100, 100, 99, 1), Bits: 32},
 					},
-					Endpoints: discoHex + ".disco.tailscale:12345",
+					Endpoints: wgcfg.Endpoints{DiscoKey: dkFromHex(discoHex)},
 				},
 			},
 		}

wgengine/wgcfg/clone.go

@@ -2,12 +2,13 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Code generated by tailscale.com/cmd/cloner -type Config,Peer; DO NOT EDIT.
+// Code generated by tailscale.com/cmd/cloner -type Config,Peer,Endpoints,IPPortSet; DO NOT EDIT.
 
 package wgcfg
 
 import (
 	"inet.af/netaddr"
+	"tailscale.com/tailcfg"
 	"tailscale.com/types/wgkey"
 )
 
@@ -29,7 +30,7 @@ func (src *Config) Clone() *Config {
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type Config,Peer
+//   tailscale.com/cmd/cloner -type Config,Peer,Endpoints,IPPortSet
 var _ConfigNeedsRegeneration = Config(struct {
 	Name       string
 	PrivateKey wgkey.Private
@@ -48,14 +49,53 @@ func (src *Peer) Clone() *Peer {
 	dst := new(Peer)
 	*dst = *src
 	dst.AllowedIPs = append(src.AllowedIPs[:0:0], src.AllowedIPs...)
+	dst.Endpoints = *src.Endpoints.Clone()
 	return dst
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type Config,Peer
+//   tailscale.com/cmd/cloner -type Config,Peer,Endpoints,IPPortSet
 var _PeerNeedsRegeneration = Peer(struct {
 	PublicKey           wgkey.Key
 	AllowedIPs          []netaddr.IPPrefix
-	Endpoints           string
+	Endpoints           Endpoints
 	PersistentKeepalive uint16
 }{})
+
+// Clone makes a deep copy of Endpoints.
+// The result aliases no memory with the original.
+func (src *Endpoints) Clone() *Endpoints {
+	if src == nil {
+		return nil
+	}
+	dst := new(Endpoints)
+	*dst = *src
+	dst.IPPorts = *src.IPPorts.Clone()
+	return dst
+}
+
+// A compilation failure here means this code must be regenerated, with command:
+//   tailscale.com/cmd/cloner -type Config,Peer,Endpoints,IPPortSet
+var _EndpointsNeedsRegeneration = Endpoints(struct {
+	PublicKey wgkey.Key
+	DiscoKey  tailcfg.DiscoKey
+	IPPorts   IPPortSet
+}{})
+
+// Clone makes a deep copy of IPPortSet.
+// The result aliases no memory with the original.
+func (src *IPPortSet) Clone() *IPPortSet {
+	if src == nil {
+		return nil
+	}
+	dst := new(IPPortSet)
+	*dst = *src
+	dst.ipp = append(src.ipp[:0:0], src.ipp...)
+	return dst
+}
+
+// A compilation failure here means this code must be regenerated, with command:
+//   tailscale.com/cmd/cloner -type Config,Peer,Endpoints,IPPortSet
+var _IPPortSetNeedsRegeneration = IPPortSet(struct {
+	ipp []netaddr.IPPort
+}{})

wgengine/wgcfg/config.go

@@ -6,16 +6,15 @@
 package wgcfg
 
 import (
+	"encoding/json"
+	"strings"
+
 	"inet.af/netaddr"
+	"tailscale.com/tailcfg"
 	"tailscale.com/types/wgkey"
 )
 
-//go:generate go run tailscale.com/cmd/cloner -type=Config,Peer -output=clone.go
+//go:generate go run tailscale.com/cmd/cloner -type=Config,Peer,Endpoints,IPPortSet -output=clone.go
-
-// EndpointDiscoSuffix is appended to the hex representation of a peer's discovery key
-// and is then the sole wireguard endpoint for peers with a non-zero discovery key.
-// This form is then recognize by magicsock's ParseEndpoint.
-const EndpointDiscoSuffix = ".disco.tailscale:12345"
 
 // Config is a WireGuard configuration.
 // It only supports the set of things Tailscale uses.
@@ -31,10 +30,92 @@ type Config struct {
 type Peer struct {
 	PublicKey           wgkey.Key
 	AllowedIPs          []netaddr.IPPrefix
-	Endpoints           string // comma-separated host/port pairs: "1.2.3.4:56,[::]:80"
+	Endpoints           Endpoints
 	PersistentKeepalive uint16
 }
 
+// Endpoints represents the routes to reach a remote node.
+// It is serialized and provided to wireguard-go as a conn.Endpoint.
+type Endpoints struct {
+	// PublicKey is the public key for the remote node.
+	PublicKey wgkey.Key `json:"pk"`
+	// DiscoKey is the disco key associated with the remote node.
+	DiscoKey tailcfg.DiscoKey `json:"dk,omitempty"`
+	// IPPorts is a set of possible ip+ports the remote node can be reached at.
+	// This is used only for legacy connections to pre-disco (pre-0.100) peers.
+	IPPorts IPPortSet `json:"ipp,omitempty"`
+}
+
+func (e Endpoints) Equal(f Endpoints) bool {
+	if e.PublicKey != f.PublicKey {
+		return false
+	}
+	if e.DiscoKey != f.DiscoKey {
+		return false
+	}
+	return e.IPPorts.EqualUnordered(f.IPPorts)
+}
+
+// IPPortSet is an immutable slice of netaddr.IPPorts.
+type IPPortSet struct {
+	ipp []netaddr.IPPort
+}
+
+// NewIPPortSet returns an IPPortSet containing the ports in ipp.
+func NewIPPortSet(ipps ...netaddr.IPPort) IPPortSet {
+	return IPPortSet{ipp: append(ipps[:0:0], ipps...)}
+}
+
+// String returns a comma-separated list of all IPPorts in s.
+func (s IPPortSet) String() string {
+	buf := new(strings.Builder)
+	for i, ipp := range s.ipp {
+		if i > 0 {
+			buf.WriteByte(',')
+		}
+		buf.WriteString(ipp.String())
+	}
+	return buf.String()
+}
+
+// IPPorts returns a slice of netaddr.IPPorts containing the IPPorts in s.
+func (s IPPortSet) IPPorts() []netaddr.IPPort {
+	return append(s.ipp[:0:0], s.ipp...)
+}
+
+// EqualUnordered reports whether s and t contain the same IPPorts, regardless of order.
+func (s IPPortSet) EqualUnordered(t IPPortSet) bool {
+	if len(s.ipp) != len(t.ipp) {
+		return false
+	}
+	// Check whether the endpoints are the same, regardless of order.
+	ipps := make(map[netaddr.IPPort]int, len(s.ipp))
+	for _, ipp := range s.ipp {
+		ipps[ipp]++
+	}
+	for _, ipp := range t.ipp {
+		ipps[ipp]--
+	}
+	for _, n := range ipps {
+		if n != 0 {
+			return false
+		}
+	}
+	return true
+}
+
+// MarshalJSON marshals s into JSON.
+// It is necessary so that IPPortSet's fields can be unexported, to guarantee immutability.
+func (s IPPortSet) MarshalJSON() ([]byte, error) {
+	return json.Marshal(s.ipp)
+}
+
+// UnmarshalJSON unmarshals s from JSON.
+// It is necessary so that IPPortSet's fields can be unexported, to guarantee immutability.
+func (s *IPPortSet) UnmarshalJSON(b []byte) error {
+	return json.Unmarshal(b, &s.ipp)
+}
+
 // PeerWithKey returns the Peer with key k and reports whether it was found.
 func (config Config) PeerWithKey(k wgkey.Key) (Peer, bool) {
 	for _, p := range config.Peers {

wgengine/wgcfg/device_test.go

@@ -10,6 +10,7 @@
 	"io"
 	"net"
 	"os"
+	"reflect"
 	"sort"
 	"strings"
 	"sync"
@@ -90,7 +91,7 @@ func TestDeviceConfig(t *testing.T) {
 				t.Errorf("on error, could not IpcGetOperation: %v", err)
 			}
 			w.Flush()
-			t.Errorf("cfg:\n%s\n---- want:\n%s\n---- uapi:\n%s", gotStr, wantStr, buf.String())
+			t.Errorf("config mismatch:\n---- got:\n%s\n---- want:\n%s\n---- uapi:\n%s", gotStr, wantStr, buf.String())
 		}
 	}
 
@@ -127,7 +128,7 @@ func TestDeviceConfig(t *testing.T) {
 	})
 
 	t.Run("device1 modify peer", func(t *testing.T) {
-		cfg1.Peers[0].Endpoints = "1.2.3.4:12345"
+		cfg1.Peers[0].Endpoints.IPPorts = NewIPPortSet(netaddr.MustParseIPPort("1.2.3.4:12345"))
 		if err := ReconfigDevice(device1, cfg1, t.Logf); err != nil {
 			t.Fatal(err)
 		}
@@ -135,7 +136,7 @@ func TestDeviceConfig(t *testing.T) {
 	})
 
 	t.Run("device1 replace endpoint", func(t *testing.T) {
-		cfg1.Peers[0].Endpoints = "1.1.1.1:123"
+		cfg1.Peers[0].Endpoints.IPPorts = NewIPPortSet(netaddr.MustParseIPPort("1.1.1.1:123"))
 		if err := ReconfigDevice(device1, cfg1, t.Logf); err != nil {
 			t.Fatal(err)
 		}
@@ -176,7 +177,7 @@ func TestDeviceConfig(t *testing.T) {
 		}
 		peersEqual := func(p, q Peer) bool {
 			return p.PublicKey == q.PublicKey && p.PersistentKeepalive == q.PersistentKeepalive &&
-				p.Endpoints == q.Endpoints && cidrsEqual(p.AllowedIPs, q.AllowedIPs)
+				reflect.DeepEqual(p.Endpoints, q.Endpoints) && cidrsEqual(p.AllowedIPs, q.AllowedIPs)
 		}
 		if !peersEqual(peer0(origCfg), peer0(newCfg)) {
 			t.Error("reconfig modified old peer")

wgengine/wgcfg/nmcfg/nmcfg.go

@@ -8,8 +8,6 @@
 import (
 	"bytes"
 	"fmt"
-	"net"
-	"strconv"
 	"strings"
 
 	"inet.af/netaddr"
@@ -79,17 +77,19 @@ func WGCfg(nm *netmap.NetworkMap, logf logger.Logf, flags netmap.WGConfigFlags,
 			cpeer.PersistentKeepalive = 25 // seconds
 		}
 
-		if !peer.DiscoKey.IsZero() {
+		cpeer.Endpoints = wgcfg.Endpoints{PublicKey: wgkey.Key(peer.Key), DiscoKey: peer.DiscoKey}
-			cpeer.Endpoints = fmt.Sprintf("%x.disco.tailscale:12345", peer.DiscoKey[:])
+		if peer.DiscoKey.IsZero() {
-		} else {
+			// Legacy connection. Add IP+port endpoints.
-			if err := appendEndpoint(cpeer, peer.DERP); err != nil {
+			var ipps []netaddr.IPPort
+			if err := appendEndpoint(cpeer, &ipps, peer.DERP); err != nil {
 				return nil, err
 			}
 			for _, ep := range peer.Endpoints {
-				if err := appendEndpoint(cpeer, ep); err != nil {
+				if err := appendEndpoint(cpeer, &ipps, ep); err != nil {
 					return nil, err
 				}
 			}
+			cpeer.Endpoints.IPPorts = wgcfg.NewIPPortSet(ipps...)
 		}
 		didExitNodeWarn := false
 		for _, allowedIP := range peer.AllowedIPs {
@@ -136,21 +136,14 @@ func WGCfg(nm *netmap.NetworkMap, logf logger.Logf, flags netmap.WGConfigFlags,
 	return cfg, nil
 }
 
-func appendEndpoint(peer *wgcfg.Peer, epStr string) error {
+func appendEndpoint(peer *wgcfg.Peer, ipps *[]netaddr.IPPort, epStr string) error {
 	if epStr == "" {
 		return nil
 	}
-	_, port, err := net.SplitHostPort(epStr)
+	ipp, err := netaddr.ParseIPPort(epStr)
 	if err != nil {
 		return fmt.Errorf("malformed endpoint %q for peer %v", epStr, peer.PublicKey.ShortString())
 	}
-	_, err = strconv.ParseUint(port, 10, 16)
+	*ipps = append(*ipps, ipp)
-	if err != nil {
-		return fmt.Errorf("invalid port in endpoint %q for peer %v", epStr, peer.PublicKey.ShortString())
-	}
-	if peer.Endpoints != "" {
-		peer.Endpoints += ","
-	}
-	peer.Endpoints += epStr
 	return nil
 }

wgengine/wgcfg/parser.go

@@ -7,6 +7,7 @@
 import (
 	"bufio"
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net"
@@ -26,21 +27,6 @@ func (e *ParseError) Error() string {
 	return fmt.Sprintf("%s: %q", e.why, e.offender)
 }
 
-func validateEndpoints(s string) error {
-	if s == "" {
-		// Otherwise strings.Split of the empty string produces [""].
-		return nil
-	}
-	vals := strings.Split(s, ",")
-	for _, val := range vals {
-		_, _, err := parseEndpoint(val)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 func parseEndpoint(s string) (host string, port uint16, err error) {
 	i := strings.LastIndexByte(s, ':')
 	if i < 0 {
@@ -103,6 +89,7 @@ func FromUAPI(r io.Reader) (*Config, error) {
 		}
 		key := parts[0]
 		value := parts[1]
+		valueBytes := scanner.Bytes()[len(key)+1:]
 
 		if key == "public_key" {
 			if deviceConfig {
@@ -121,7 +108,7 @@ func FromUAPI(r io.Reader) (*Config, error) {
 		if deviceConfig {
 			err = cfg.handleDeviceLine(key, value)
 		} else {
-			err = cfg.handlePeerLine(peer, key, value)
+			err = cfg.handlePeerLine(peer, key, value, valueBytes)
 		}
 		if err != nil {
 			return nil, err
@@ -165,14 +152,13 @@ func (cfg *Config) handlePublicKeyLine(value string) (*Peer, error) {
 	return peer, nil
 }
 
-func (cfg *Config) handlePeerLine(peer *Peer, key, value string) error {
+func (cfg *Config) handlePeerLine(peer *Peer, key, value string, valueBytes []byte) error {
 	switch key {
 	case "endpoint":
-		err := validateEndpoints(value)
+		err := json.Unmarshal(valueBytes, &peer.Endpoints)
 		if err != nil {
 			return err
 		}
-		peer.Endpoints = value
 	case "persistent_keepalive_interval":
 		n, err := strconv.ParseUint(value, 10, 16)
 		if err != nil {

wgengine/wgcfg/parser_test.go

@@ -53,21 +53,3 @@ func TestParseEndpoint(t *testing.T) {
 		t.Error("Error was expected")
 	}
 }
-
-func TestValidateEndpoints(t *testing.T) {
-	tests := []struct {
-		in   string
-		want error
-	}{
-		{"", nil},
-		{"1.2.3.4:5", nil},
-		{"1.2.3.4:5,6.7.8.9:10", nil},
-		{",", &ParseError{why: "Missing port from endpoint", offender: ""}},
-	}
-	for _, tt := range tests {
-		got := validateEndpoints(tt.in)
-		if !reflect.DeepEqual(got, tt.want) {
-			t.Errorf("%q = %#v (%s); want %#v (%s)", tt.in, got, got, tt.want, tt.want)
-		}
-	}
-}

wgengine/wgcfg/writer.go

@@ -5,11 +5,10 @@
 package wgcfg
 
 import (
+	"encoding/json"
 	"fmt"
 	"io"
-	"sort"
 	"strconv"
-	"strings"
 
 	"inet.af/netaddr"
 	"tailscale.com/types/wgkey"
@@ -53,8 +52,12 @@ func (cfg *Config) ToUAPI(w io.Writer, prev *Config) error {
 		setPeer(p)
 		set("protocol_version", "1")
 
-		if !endpointsEqual(oldPeer.Endpoints, p.Endpoints) {
+		if !oldPeer.Endpoints.Equal(p.Endpoints) {
-			set("endpoint", p.Endpoints)
+			buf, err := json.Marshal(p.Endpoints)
+			if err != nil {
+				return err
+			}
+			set("endpoint", string(buf))
 		}
 
 		// TODO: replace_allowed_ips is expensive.
@@ -90,24 +93,6 @@ func (cfg *Config) ToUAPI(w io.Writer, prev *Config) error {
 	return stickyErr
 }
 
-func endpointsEqual(x, y string) bool {
-	// Cheap comparisons.
-	if x == y {
-		return true
-	}
-	xs := strings.Split(x, ",")
-	ys := strings.Split(y, ",")
-	if len(xs) != len(ys) {
-		return false
-	}
-	// Otherwise, see if they're the same, but out of order.
-	sort.Strings(xs)
-	sort.Strings(ys)
-	x = strings.Join(xs, ",")
-	y = strings.Join(ys, ",")
-	return x == y
-}
-
 func cidrsEqual(x, y []netaddr.IPPrefix) bool {
 	// TODO: re-implement using netaddr.IPSet.Equal.
 	if len(x) != len(y) {