From 9d7e7288c68093f8e857ab83bc44203102a72ca0 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 18:47:15 -0500
Subject: [PATCH 001/130] start migrating the router to an actor

---
 go.mod                  |  1 +
 go.sum                  |  2 +
 src/yggdrasil/conn.go   |  2 +-
 src/yggdrasil/dht.go    |  2 -
 src/yggdrasil/peer.go   |  2 +-
 src/yggdrasil/router.go | 84 +++++++++++++++++++++--------------------
 src/yggdrasil/search.go |  2 +-
 src/yggdrasil/switch.go | 10 +----
 8 files changed, 51 insertions(+), 54 deletions(-)

diff --git a/go.mod b/go.mod
index 5569496c..3f5cae88 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,7 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
 require (
+	github.com/Arceliar/phony v0.0.0-20190821233739-c7f353f14438
 	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
 	github.com/hashicorp/go-syslog v1.0.0
 	github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222
diff --git a/go.sum b/go.sum
index 756ed5af..22276c2c 100644
--- a/go.sum
+++ b/go.sum
@@ -1,3 +1,5 @@
+github.com/Arceliar/phony v0.0.0-20190821233739-c7f353f14438 h1:t4tRgrItIq2ap4O31yOuWm17lUiyzf8gf/P+bEfgmrw=
+github.com/Arceliar/phony v0.0.0-20190821233739-c7f353f14438/go.mod h1:2Q9yJvg2PlMrnOEa3RTEy9hElWAICo/D8HTUDqAHUAo=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=
diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index 25330aae..c77e19d6 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -134,7 +134,7 @@ func (c *Conn) doSearch() {
 			sinfo.continueSearch()
 		}
 	}
-	go func() { c.core.router.admin <- routerWork }()
+	go c.core.router.doAdmin(routerWork)
 }
 
 func (c *Conn) getDeadlineCancellation(value *atomic.Value) (util.Cancellation, bool) {
diff --git a/src/yggdrasil/dht.go b/src/yggdrasil/dht.go
index b53e29c9..d35d3aaf 100644
--- a/src/yggdrasil/dht.go
+++ b/src/yggdrasil/dht.go
@@ -68,7 +68,6 @@ type dht struct {
 	core        *Core
 	reconfigure chan chan error
 	nodeID      crypto.NodeID
-	peers       chan *dhtInfo                    // other goroutines put incoming dht updates here
 	reqs        map[dhtReqKey]time.Time          // Keeps track of recent outstanding requests
 	callbacks   map[dhtReqKey][]dht_callbackInfo // Search and admin lookup callbacks
 	// These next two could be replaced by a single linked list or similar...
@@ -87,7 +86,6 @@ func (t *dht) init(c *Core) {
 		}
 	}()
 	t.nodeID = *t.core.NodeID()
-	t.peers = make(chan *dhtInfo, 1024)
 	t.callbacks = make(map[dhtReqKey][]dht_callbackInfo)
 	t.reset()
 }
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 379ca85b..fcd9364c 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -210,7 +210,7 @@ func (p *peer) linkLoop() {
 		case dinfo = <-p.dinfo:
 		case _ = <-tick.C:
 			if dinfo != nil {
-				p.core.dht.peers <- dinfo
+				p.core.router.insertPeer(&p.core.router, dinfo)
 			}
 		}
 	}
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index bdead848..4ce6b7ed 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -30,19 +30,19 @@ import (
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
+
+	"github.com/Arceliar/phony"
 )
 
 // The router struct has channels to/from the adapter device and a self peer (0), which is how messages are passed between this node and the peers/switch layer.
 // The router's mainLoop goroutine is responsible for managing all information related to the dht, searches, and crypto sessions.
 type router struct {
+	phony.Actor
 	core        *Core
 	reconfigure chan chan error
 	addr        address.Address
 	subnet      address.Subnet
-	in          <-chan [][]byte // packets we received from the network, link to peer's "out"
-	out         func([]byte)    // packets we're sending to the network, link to peer's "in"
-	reset       chan struct{}   // signal that coords changed (re-init sessions/dht)
-	admin       chan func()     // pass a lambda for the admin socket to query stuff
+	out         func([]byte) // packets we're sending to the network, link to peer's "in"
 	nodeinfo    nodeinfo
 }
 
@@ -52,7 +52,6 @@ func (r *router) init(core *Core) {
 	r.reconfigure = make(chan chan error, 1)
 	r.addr = *address.AddrForNodeID(&r.core.dht.nodeID)
 	r.subnet = *address.SubnetForNodeID(&r.core.dht.nodeID)
-	in := make(chan [][]byte, 1) // TODO something better than this...
 	self := linkInterface{
 		name: "(self)",
 		info: linkInfo{
@@ -62,8 +61,10 @@ func (r *router) init(core *Core) {
 		},
 	}
 	p := r.core.peers.newPeer(&r.core.boxPub, &r.core.sigPub, &crypto.BoxSharedKey{}, &self, nil)
-	p.out = func(packets [][]byte) { in <- packets }
-	r.in = in
+	p.out = func(packets [][]byte) {
+		// TODO make peers and/or the switch into actors, have them pass themselves as the from field
+		r.handlePackets(r, packets)
+	}
 	out := make(chan []byte, 32)
 	go func() {
 		for packet := range out {
@@ -90,8 +91,6 @@ func (r *router) init(core *Core) {
 		}
 	}()
 	r.out = func(packet []byte) { out2 <- packet }
-	r.reset = make(chan struct{}, 1)
-	r.admin = make(chan func(), 32)
 	r.nodeinfo.init(r.core)
 	r.core.config.Mutex.RLock()
 	r.nodeinfo.setNodeInfo(r.core.config.Current.NodeInfo, r.core.config.Current.NodeInfoPrivacy)
@@ -105,42 +104,55 @@ func (r *router) start() error {
 	return nil
 }
 
-// Takes traffic from the adapter and passes it to router.send, or from r.in and handles incoming traffic.
-// Also adds new peer info to the DHT.
-// Also resets the DHT and sesssions in the event of a coord change.
-// Also does periodic maintenance stuff.
+// In practice, the switch will call this with 1 packet
+func (r *router) handlePackets(from phony.IActor, packets [][]byte) {
+	r.EnqueueFrom(from, func() {
+		for _, packet := range packets {
+			r.handlePacket(packet)
+		}
+	})
+}
+
+// Insert a peer info into the dht, TODO? make the dht a separate actor
+func (r *router) insertPeer(from phony.IActor, info *dhtInfo) {
+	r.EnqueueFrom(from, func() {
+		r.core.dht.insertPeer(info)
+	})
+}
+
+// Reset sessions and DHT after the switch sees our coords change
+func (r *router) reset(from phony.IActor) {
+	r.EnqueueFrom(from, func() {
+		r.core.sessions.reset()
+		r.core.dht.reset()
+	})
+}
+
+// TODO remove reconfigure so this is just a ticker loop
+// and then find something better than a ticker loop to schedule things...
 func (r *router) mainLoop() {
 	ticker := time.NewTicker(time.Second)
 	defer ticker.Stop()
 	for {
 		select {
-		case ps := <-r.in:
-			for _, p := range ps {
-				r.handleIn(p)
-			}
-		case info := <-r.core.dht.peers:
-			r.core.dht.insertPeer(info)
-		case <-r.reset:
-			r.core.sessions.reset()
-			r.core.dht.reset()
 		case <-ticker.C:
-			{
+			r.SyncExec(func() {
 				// Any periodic maintenance stuff goes here
 				r.core.switchTable.doMaintenance()
 				r.core.dht.doMaintenance()
 				r.core.sessions.cleanup()
-			}
-		case f := <-r.admin:
-			f()
+			})
 		case e := <-r.reconfigure:
-			current := r.core.config.GetCurrent()
-			e <- r.nodeinfo.setNodeInfo(current.NodeInfo, current.NodeInfoPrivacy)
+			r.SyncExec(func() {
+				current := r.core.config.GetCurrent()
+				e <- r.nodeinfo.setNodeInfo(current.NodeInfo, current.NodeInfoPrivacy)
+			})
 		}
 	}
 }
 
 // Checks incoming traffic type and passes it to the appropriate handler.
-func (r *router) handleIn(packet []byte) {
+func (r *router) handlePacket(packet []byte) {
 	pType, pTypeLen := wire_decode_uint64(packet)
 	if pTypeLen == 0 {
 		return
@@ -263,17 +275,7 @@ func (r *router) handleNodeInfo(bs []byte, fromKey *crypto.BoxPubKey) {
 	r.nodeinfo.handleNodeInfo(&req)
 }
 
-// Passed a function to call.
-// This will send the function to r.admin and block until it finishes.
-// It's used by the admin socket to ask the router mainLoop goroutine about information in the session or dht structs, which cannot be read safely from outside that goroutine.
+// TODO remove this, have things either be actors that send message or else call SyncExec directly
 func (r *router) doAdmin(f func()) {
-	// Pass this a function that needs to be run by the router's main goroutine
-	// It will pass the function to the router and wait for the router to finish
-	done := make(chan struct{})
-	newF := func() {
-		f()
-		close(done)
-	}
-	r.admin <- newF
-	<-done
+	r.SyncExec(f)
 }
diff --git a/src/yggdrasil/search.go b/src/yggdrasil/search.go
index b970fe55..c035e72f 100644
--- a/src/yggdrasil/search.go
+++ b/src/yggdrasil/search.go
@@ -165,7 +165,7 @@ func (sinfo *searchInfo) continueSearch() {
 	}
 	go func() {
 		time.Sleep(search_RETRY_TIME)
-		sinfo.core.router.admin <- retryLater
+		sinfo.core.router.doAdmin(retryLater)
 	}()
 }
 
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index cc316d16..86ae102b 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -245,10 +245,7 @@ func (t *switchTable) cleanRoot() {
 		if t.data.locator.root != t.key {
 			t.data.seq++
 			t.updater.Store(&sync.Once{})
-			select {
-			case t.core.router.reset <- struct{}{}:
-			default:
-			}
+			t.core.router.reset(&t.core.router)
 		}
 		t.data.locator = switchLocator{root: t.key, tstamp: now.Unix()}
 		t.core.peers.sendSwitchMsgs()
@@ -511,10 +508,7 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort, rep
 		if !equiv(&sender.locator, &t.data.locator) {
 			doUpdate = true
 			t.data.seq++
-			select {
-			case t.core.router.reset <- struct{}{}:
-			default:
-			}
+			t.core.router.reset(&t.core.router)
 		}
 		if t.data.locator.tstamp != sender.locator.tstamp {
 			t.time = now

From 232e6d3cb3e6f63100e4af1aa9fa05f21c7cb78d Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 18:55:41 -0500
Subject: [PATCH 002/130] more router migration

---
 src/yggdrasil/router.go | 27 +--------------------------
 1 file changed, 1 insertion(+), 26 deletions(-)

diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index 4ce6b7ed..34dcb913 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -65,32 +65,7 @@ func (r *router) init(core *Core) {
 		// TODO make peers and/or the switch into actors, have them pass themselves as the from field
 		r.handlePackets(r, packets)
 	}
-	out := make(chan []byte, 32)
-	go func() {
-		for packet := range out {
-			p.handlePacket(packet)
-		}
-	}()
-	out2 := make(chan []byte, 32)
-	go func() {
-		// This worker makes sure r.out never blocks
-		// It will buffer traffic long enough for the switch worker to take it
-		// If (somehow) you can send faster than the switch can receive, then this would use unbounded memory
-		// But crypto slows sends enough that the switch should always be able to take the packets...
-		var buf [][]byte
-		for {
-			buf = append(buf, <-out2)
-			for len(buf) > 0 {
-				select {
-				case bs := <-out2:
-					buf = append(buf, bs)
-				case out <- buf[0]:
-					buf = buf[1:]
-				}
-			}
-		}
-	}()
-	r.out = func(packet []byte) { out2 <- packet }
+	r.out = p.handlePacket // TODO if the peer becomes its own actor, then send a message here
 	r.nodeinfo.init(r.core)
 	r.core.config.Mutex.RLock()
 	r.nodeinfo.setNodeInfo(r.core.config.Current.NodeInfo, r.core.config.Current.NodeInfoPrivacy)

From 8e89816099b842f73fa7c25cd441d0e54400231e Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 18:59:34 -0500
Subject: [PATCH 003/130] more router migration: rename functions that should
 only be called internally by the actor

---
 src/yggdrasil/router.go | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index 34dcb913..b24400da 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -75,7 +75,7 @@ func (r *router) init(core *Core) {
 // Starts the mainLoop goroutine.
 func (r *router) start() error {
 	r.core.log.Infoln("Starting router")
-	go r.mainLoop()
+	go r._mainLoop()
 	return nil
 }
 
@@ -83,7 +83,7 @@ func (r *router) start() error {
 func (r *router) handlePackets(from phony.IActor, packets [][]byte) {
 	r.EnqueueFrom(from, func() {
 		for _, packet := range packets {
-			r.handlePacket(packet)
+			r._handlePacket(packet)
 		}
 	})
 }
@@ -105,7 +105,7 @@ func (r *router) reset(from phony.IActor) {
 
 // TODO remove reconfigure so this is just a ticker loop
 // and then find something better than a ticker loop to schedule things...
-func (r *router) mainLoop() {
+func (r *router) _mainLoop() {
 	ticker := time.NewTicker(time.Second)
 	defer ticker.Stop()
 	for {
@@ -127,23 +127,23 @@ func (r *router) mainLoop() {
 }
 
 // Checks incoming traffic type and passes it to the appropriate handler.
-func (r *router) handlePacket(packet []byte) {
+func (r *router) _handlePacket(packet []byte) {
 	pType, pTypeLen := wire_decode_uint64(packet)
 	if pTypeLen == 0 {
 		return
 	}
 	switch pType {
 	case wire_Traffic:
-		r.handleTraffic(packet)
+		r._handleTraffic(packet)
 	case wire_ProtocolTraffic:
-		r.handleProto(packet)
+		r._handleProto(packet)
 	default:
 	}
 }
 
 // Handles incoming traffic, i.e. encapuslated ordinary IPv6 packets.
 // Passes them to the crypto session worker to be decrypted and sent to the adapter.
-func (r *router) handleTraffic(packet []byte) {
+func (r *router) _handleTraffic(packet []byte) {
 	defer util.PutBytes(packet)
 	p := wire_trafficPacket{}
 	if !p.decode(packet) {
@@ -162,7 +162,7 @@ func (r *router) handleTraffic(packet []byte) {
 }
 
 // Handles protocol traffic by decrypting it, checking its type, and passing it to the appropriate handler for that traffic type.
-func (r *router) handleProto(packet []byte) {
+func (r *router) _handleProto(packet []byte) {
 	// First parse the packet
 	p := wire_protoTrafficPacket{}
 	if !p.decode(packet) {
@@ -189,24 +189,24 @@ func (r *router) handleProto(packet []byte) {
 	}
 	switch bsType {
 	case wire_SessionPing:
-		r.handlePing(bs, &p.FromKey)
+		r._handlePing(bs, &p.FromKey)
 	case wire_SessionPong:
-		r.handlePong(bs, &p.FromKey)
+		r._handlePong(bs, &p.FromKey)
 	case wire_NodeInfoRequest:
 		fallthrough
 	case wire_NodeInfoResponse:
-		r.handleNodeInfo(bs, &p.FromKey)
+		r._handleNodeInfo(bs, &p.FromKey)
 	case wire_DHTLookupRequest:
-		r.handleDHTReq(bs, &p.FromKey)
+		r._handleDHTReq(bs, &p.FromKey)
 	case wire_DHTLookupResponse:
-		r.handleDHTRes(bs, &p.FromKey)
+		r._handleDHTRes(bs, &p.FromKey)
 	default:
 		util.PutBytes(packet)
 	}
 }
 
 // Decodes session pings from wire format and passes them to sessions.handlePing where they either create or update a session.
-func (r *router) handlePing(bs []byte, fromKey *crypto.BoxPubKey) {
+func (r *router) _handlePing(bs []byte, fromKey *crypto.BoxPubKey) {
 	ping := sessionPing{}
 	if !ping.decode(bs) {
 		return
@@ -216,12 +216,12 @@ func (r *router) handlePing(bs []byte, fromKey *crypto.BoxPubKey) {
 }
 
 // Handles session pongs (which are really pings with an extra flag to prevent acknowledgement).
-func (r *router) handlePong(bs []byte, fromKey *crypto.BoxPubKey) {
-	r.handlePing(bs, fromKey)
+func (r *router) _handlePong(bs []byte, fromKey *crypto.BoxPubKey) {
+	r._handlePing(bs, fromKey)
 }
 
 // Decodes dht requests and passes them to dht.handleReq to trigger a lookup/response.
-func (r *router) handleDHTReq(bs []byte, fromKey *crypto.BoxPubKey) {
+func (r *router) _handleDHTReq(bs []byte, fromKey *crypto.BoxPubKey) {
 	req := dhtReq{}
 	if !req.decode(bs) {
 		return
@@ -231,7 +231,7 @@ func (r *router) handleDHTReq(bs []byte, fromKey *crypto.BoxPubKey) {
 }
 
 // Decodes dht responses and passes them to dht.handleRes to update the DHT table and further pass them to the search code (if applicable).
-func (r *router) handleDHTRes(bs []byte, fromKey *crypto.BoxPubKey) {
+func (r *router) _handleDHTRes(bs []byte, fromKey *crypto.BoxPubKey) {
 	res := dhtRes{}
 	if !res.decode(bs) {
 		return
@@ -241,7 +241,7 @@ func (r *router) handleDHTRes(bs []byte, fromKey *crypto.BoxPubKey) {
 }
 
 // Decodes nodeinfo request
-func (r *router) handleNodeInfo(bs []byte, fromKey *crypto.BoxPubKey) {
+func (r *router) _handleNodeInfo(bs []byte, fromKey *crypto.BoxPubKey) {
 	req := nodeinfoReqRes{}
 	if !req.decode(bs) {
 		return

From bbcbbaf3b14d5b8ceb6f06ca33fc3f7969fc950b Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 20:05:18 -0500
Subject: [PATCH 004/130] start migrating sessionInfo to be an actor

---
 src/yggdrasil/api.go     |  2 +-
 src/yggdrasil/conn.go    |  8 ++---
 src/yggdrasil/router.go  |  8 ++---
 src/yggdrasil/search.go  |  2 +-
 src/yggdrasil/session.go | 71 ++++++++++++++++++++++------------------
 5 files changed, 49 insertions(+), 42 deletions(-)

diff --git a/src/yggdrasil/api.go b/src/yggdrasil/api.go
index 57ee5c65..785a5f72 100644
--- a/src/yggdrasil/api.go
+++ b/src/yggdrasil/api.go
@@ -213,7 +213,7 @@ func (c *Core) GetSessions() []Session {
 			workerFunc := func() {
 				session = Session{
 					Coords:      append([]uint64{}, wire_coordsBytestoUint64s(sinfo.coords)...),
-					MTU:         sinfo.getMTU(),
+					MTU:         sinfo._getMTU(),
 					BytesSent:   sinfo.bytesSent,
 					BytesRecvd:  sinfo.bytesRecvd,
 					Uptime:      time.Now().Sub(sinfo.timeOpened),
diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index c77e19d6..51dd9509 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -190,8 +190,8 @@ func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error {
 	var err error
 	sessionFunc := func() {
 		// Does the packet exceed the permitted size for the session?
-		if uint16(len(msg.Message)) > c.session.getMTU() {
-			err = ConnError{errors.New("packet too big"), true, false, false, int(c.session.getMTU())}
+		if uint16(len(msg.Message)) > c.session._getMTU() {
+			err = ConnError{errors.New("packet too big"), true, false, false, int(c.session._getMTU())}
 			return
 		}
 		// The rest of this work is session keep-alive traffic
@@ -201,10 +201,10 @@ func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error {
 				// TODO double check that the above condition is correct
 				c.doSearch()
 			} else {
-				c.core.sessions.ping(c.session)
+				c.session.ping(c.session) // TODO send from self if this becomes an actor
 			}
 		case c.session.reset && c.session.pingTime.Before(c.session.time):
-			c.core.sessions.ping(c.session)
+			c.session.ping(c.session) // TODO send from self if this becomes an actor
 		default: // Don't do anything, to keep traffic throttled
 		}
 	}
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index b24400da..95521841 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -98,7 +98,7 @@ func (r *router) insertPeer(from phony.IActor, info *dhtInfo) {
 // Reset sessions and DHT after the switch sees our coords change
 func (r *router) reset(from phony.IActor) {
 	r.EnqueueFrom(from, func() {
-		r.core.sessions.reset()
+		r.core.sessions.reset(r)
 		r.core.dht.reset()
 	})
 }
@@ -111,14 +111,14 @@ func (r *router) _mainLoop() {
 	for {
 		select {
 		case <-ticker.C:
-			r.SyncExec(func() {
+			<-r.SyncExec(func() {
 				// Any periodic maintenance stuff goes here
 				r.core.switchTable.doMaintenance()
 				r.core.dht.doMaintenance()
 				r.core.sessions.cleanup()
 			})
 		case e := <-r.reconfigure:
-			r.SyncExec(func() {
+			<-r.SyncExec(func() {
 				current := r.core.config.GetCurrent()
 				e <- r.nodeinfo.setNodeInfo(current.NodeInfo, current.NodeInfoPrivacy)
 			})
@@ -252,5 +252,5 @@ func (r *router) _handleNodeInfo(bs []byte, fromKey *crypto.BoxPubKey) {
 
 // TODO remove this, have things either be actors that send message or else call SyncExec directly
 func (r *router) doAdmin(f func()) {
-	r.SyncExec(f)
+	<-r.SyncExec(f)
 }
diff --git a/src/yggdrasil/search.go b/src/yggdrasil/search.go
index c035e72f..56adda80 100644
--- a/src/yggdrasil/search.go
+++ b/src/yggdrasil/search.go
@@ -213,7 +213,7 @@ func (sinfo *searchInfo) checkDHTRes(res *dhtRes) bool {
 	}
 	// FIXME (!) replay attacks could mess with coords? Give it a handle (tstamp)?
 	sess.coords = res.Coords
-	sinfo.core.sessions.ping(sess)
+	sess.ping(&sinfo.core.router)
 	sinfo.callback(sess, nil)
 	// Cleanup
 	delete(sinfo.core.searches.searches, res.Dest)
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 7cd92db9..f28e014a 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -14,6 +14,8 @@ import (
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
+
+	"github.com/Arceliar/phony"
 )
 
 // Duration that we keep track of old nonces per session, to allow some out-of-order packet delivery
@@ -37,7 +39,7 @@ func (h nonceHeap) peek() *crypto.BoxNonce { return &h[len(h)-1] }
 // All the information we know about an active session.
 // This includes coords, permanent and ephemeral keys, handles and nonces, various sorts of timing information for timeout and maintenance, and some metadata for the admin API.
 type sessionInfo struct {
-	mutex          sync.Mutex                    // Protects all of the below, use it any time you read/chance the contents of a session
+	phony.Actor                                  // Protects all of the below, use it any time you read/change the contents of a session
 	core           *Core                         //
 	reconfigure    chan chan error               //
 	theirAddr      address.Address               //
@@ -46,6 +48,7 @@ type sessionInfo struct {
 	theirSesPub    crypto.BoxPubKey              //
 	mySesPub       crypto.BoxPubKey              //
 	mySesPriv      crypto.BoxPrivKey             //
+	sharedPermKey  crypto.BoxSharedKey           // used for session pings
 	sharedSesKey   crypto.BoxSharedKey           // derived from session keys
 	theirHandle    crypto.Handle                 //
 	myHandle       crypto.Handle                 //
@@ -73,10 +76,9 @@ type sessionInfo struct {
 	send           chan FlowKeyMessage           // Packets with optional flow key go here, to be encrypted and sent
 }
 
+// TODO remove this, call SyncExec directly
 func (sinfo *sessionInfo) doFunc(f func()) {
-	sinfo.mutex.Lock()
-	defer sinfo.mutex.Unlock()
-	f()
+	<-sinfo.SyncExec(f)
 }
 
 // Represents a session ping/pong packet, andincludes information like public keys, a session handle, coords, a timestamp to prevent replays, and the tun/tap MTU.
@@ -92,7 +94,7 @@ type sessionPing struct {
 
 // Updates session info in response to a ping, after checking that the ping is OK.
 // Returns true if the session was updated, or false otherwise.
-func (s *sessionInfo) update(p *sessionPing) bool {
+func (s *sessionInfo) _update(p *sessionPing) bool {
 	if !(p.Tstamp > s.tstamp) {
 		// To protect against replay attacks
 		return false
@@ -214,6 +216,7 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 	sinfo.core = ss.core
 	sinfo.reconfigure = make(chan chan error, 1)
 	sinfo.theirPermPub = *theirPermKey
+	sinfo.sharedPermKey = *ss.getSharedKey(&ss.core.boxPriv, &sinfo.theirPermPub)
 	pub, priv := crypto.NewBoxKeys()
 	sinfo.mySesPub = *pub
 	sinfo.mySesPriv = *priv
@@ -257,7 +260,9 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 	go func() {
 		// Run cleanup when the session is canceled
 		<-sinfo.cancel.Finished()
-		sinfo.core.router.doAdmin(sinfo.close)
+		sinfo.core.router.doAdmin(func() {
+			sinfo.core.sessions.removeSession(&sinfo)
+		})
 	}()
 	go sinfo.startWorkers()
 	return &sinfo
@@ -292,7 +297,7 @@ func (ss *sessions) cleanup() {
 }
 
 // Closes a session, removing it from sessions maps.
-func (sinfo *sessionInfo) close() {
+func (ss *sessions) removeSession(sinfo *sessionInfo) {
 	if s := sinfo.core.sessions.sinfos[sinfo.myHandle]; s == sinfo {
 		delete(sinfo.core.sessions.sinfos, sinfo.myHandle)
 		delete(sinfo.core.sessions.byTheirPerm, sinfo.theirPermPub)
@@ -300,11 +305,11 @@ func (sinfo *sessionInfo) close() {
 }
 
 // Returns a session ping appropriate for the given session info.
-func (ss *sessions) getPing(sinfo *sessionInfo) sessionPing {
-	loc := ss.core.switchTable.getLocator()
+func (sinfo *sessionInfo) _getPing() sessionPing {
+	loc := sinfo.core.switchTable.getLocator()
 	coords := loc.getCoords()
-	ref := sessionPing{
-		SendPermPub: ss.core.boxPub,
+	ping := sessionPing{
+		SendPermPub: sinfo.core.boxPub,
 		Handle:      sinfo.myHandle,
 		SendSesPub:  sinfo.mySesPub,
 		Tstamp:      time.Now().Unix(),
@@ -312,7 +317,7 @@ func (ss *sessions) getPing(sinfo *sessionInfo) sessionPing {
 		MTU:         sinfo.myMTU,
 	}
 	sinfo.myNonce.Increment()
-	return ref
+	return ping
 }
 
 // Gets the shared key for a pair of box keys.
@@ -339,27 +344,29 @@ func (ss *sessions) getSharedKey(myPriv *crypto.BoxPrivKey,
 }
 
 // Sends a session ping by calling sendPingPong in ping mode.
-func (ss *sessions) ping(sinfo *sessionInfo) {
-	ss.sendPingPong(sinfo, false)
+func (sinfo *sessionInfo) ping(from phony.IActor) {
+	sinfo.EnqueueFrom(from, func() {
+		sinfo._sendPingPong(false)
+	})
 }
 
 // Calls getPing, sets the appropriate ping/pong flag, encodes to wire format, and send it.
 // Updates the time the last ping was sent in the session info.
-func (ss *sessions) sendPingPong(sinfo *sessionInfo, isPong bool) {
-	ping := ss.getPing(sinfo)
+func (sinfo *sessionInfo) _sendPingPong(isPong bool) {
+	ping := sinfo._getPing()
 	ping.IsPong = isPong
 	bs := ping.encode()
-	shared := ss.getSharedKey(&ss.core.boxPriv, &sinfo.theirPermPub)
-	payload, nonce := crypto.BoxSeal(shared, bs, nil)
+	payload, nonce := crypto.BoxSeal(&sinfo.sharedPermKey, bs, nil)
 	p := wire_protoTrafficPacket{
 		Coords:  sinfo.coords,
 		ToKey:   sinfo.theirPermPub,
-		FromKey: ss.core.boxPub,
+		FromKey: sinfo.core.boxPub,
 		Nonce:   *nonce,
 		Payload: payload,
 	}
 	packet := p.encode()
-	ss.core.router.out(packet)
+	// TODO rewrite the below if/when the peer struct becomes an actor, to not go through the router first
+	sinfo.core.router.EnqueueFrom(sinfo, func() { sinfo.core.router.out(packet) })
 	if sinfo.pingTime.Before(sinfo.time) {
 		sinfo.pingTime = time.Now()
 	}
@@ -371,9 +378,9 @@ func (ss *sessions) handlePing(ping *sessionPing) {
 	// Get the corresponding session (or create a new session)
 	sinfo, isIn := ss.getByTheirPerm(&ping.SendPermPub)
 	switch {
+	case ping.IsPong: // This is a response, not an initial ping, so ignore it.
 	case isIn: // Session already exists
 	case !ss.isSessionAllowed(&ping.SendPermPub, false): // Session is not allowed
-	case ping.IsPong: // This is a response, not an initial ping, so ignore it.
 	default:
 		ss.listenerMutex.Lock()
 		if ss.listener != nil {
@@ -393,13 +400,13 @@ func (ss *sessions) handlePing(ping *sessionPing) {
 		ss.listenerMutex.Unlock()
 	}
 	if sinfo != nil {
-		sinfo.doFunc(func() {
+		sinfo.EnqueueFrom(&ss.core.router, func() {
 			// Update the session
-			if !sinfo.update(ping) { /*panic("Should not happen in testing")*/
+			if !sinfo._update(ping) { /*panic("Should not happen in testing")*/
 				return
 			}
 			if !ping.IsPong {
-				ss.sendPingPong(sinfo, true)
+				sinfo._sendPingPong(true)
 			}
 		})
 	}
@@ -408,7 +415,7 @@ func (ss *sessions) handlePing(ping *sessionPing) {
 // Get the MTU of the session.
 // Will be equal to the smaller of this node's MTU or the remote node's MTU.
 // If sending over links with a maximum message size (this was a thing with the old UDP code), it could be further lowered, to a minimum of 1280.
-func (sinfo *sessionInfo) getMTU() uint16 {
+func (sinfo *sessionInfo) _getMTU() uint16 {
 	if sinfo.theirMTU == 0 || sinfo.myMTU == 0 {
 		return 0
 	}
@@ -419,7 +426,7 @@ func (sinfo *sessionInfo) getMTU() uint16 {
 }
 
 // Checks if a packet's nonce is recent enough to fall within the window of allowed packets, and not already received.
-func (sinfo *sessionInfo) nonceIsOK(theirNonce *crypto.BoxNonce) bool {
+func (sinfo *sessionInfo) _nonceIsOK(theirNonce *crypto.BoxNonce) bool {
 	// The bitmask is to allow for some non-duplicate out-of-order packets
 	if theirNonce.Minus(&sinfo.theirNonce) > 0 {
 		// This is newer than the newest nonce we've seen
@@ -437,7 +444,7 @@ func (sinfo *sessionInfo) nonceIsOK(theirNonce *crypto.BoxNonce) bool {
 }
 
 // Updates the nonce mask by (possibly) shifting the bitmask and setting the bit corresponding to this nonce to 1, and then updating the most recent nonce
-func (sinfo *sessionInfo) updateNonce(theirNonce *crypto.BoxNonce) {
+func (sinfo *sessionInfo) _updateNonce(theirNonce *crypto.BoxNonce) {
 	// Start with some cleanup
 	for len(sinfo.theirNonceHeap) > 64 {
 		if time.Since(sinfo.theirNonceMap[*sinfo.theirNonceHeap.peek()]) < nonceWindow {
@@ -459,9 +466,9 @@ func (sinfo *sessionInfo) updateNonce(theirNonce *crypto.BoxNonce) {
 
 // Resets all sessions to an uninitialized state.
 // Called after coord changes, so attemtps to use a session will trigger a new ping and notify the remote end of the coord change.
-func (ss *sessions) reset() {
+func (ss *sessions) reset(from phony.IActor) {
 	for _, sinfo := range ss.sinfos {
-		sinfo.doFunc(func() {
+		sinfo.EnqueueFrom(from, func() {
 			sinfo.reset = true
 		})
 	}
@@ -492,7 +499,7 @@ func (sinfo *sessionInfo) recvWorker() {
 		var err error
 		var k crypto.BoxSharedKey
 		sessionFunc := func() {
-			if !sinfo.nonceIsOK(&p.Nonce) {
+			if !sinfo._nonceIsOK(&p.Nonce) {
 				err = ConnError{errors.New("packet dropped due to invalid nonce"), false, true, false, 0}
 				return
 			}
@@ -514,12 +521,12 @@ func (sinfo *sessionInfo) recvWorker() {
 					return
 				}
 				sessionFunc = func() {
-					if k != sinfo.sharedSesKey || !sinfo.nonceIsOK(&p.Nonce) {
+					if k != sinfo.sharedSesKey || !sinfo._nonceIsOK(&p.Nonce) {
 						// The session updated in the mean time, so return an error
 						err = ConnError{errors.New("session updated during crypto operation"), false, true, false, 0}
 						return
 					}
-					sinfo.updateNonce(&p.Nonce)
+					sinfo._updateNonce(&p.Nonce)
 					sinfo.time = time.Now()
 					sinfo.bytesRecvd += uint64(len(bs))
 				}

From 9835c638180dc7a493c10ecb0a5e4772bacc145c Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 20:26:15 -0500
Subject: [PATCH 005/130] refactor things the router owns (dht, sessions,
 searches) into that struct, to make the ownership more explicit

---
 src/yggdrasil/api.go      | 26 +++++++++++++-------------
 src/yggdrasil/conn.go     |  8 ++++----
 src/yggdrasil/core.go     | 15 ++++++---------
 src/yggdrasil/dht.go      |  4 ++--
 src/yggdrasil/listener.go |  4 ++--
 src/yggdrasil/nodeinfo.go |  2 +-
 src/yggdrasil/router.go   | 27 +++++++++++++++------------
 src/yggdrasil/search.go   | 20 ++++++++++----------
 src/yggdrasil/session.go  | 12 ++++++------
 9 files changed, 59 insertions(+), 59 deletions(-)

diff --git a/src/yggdrasil/api.go b/src/yggdrasil/api.go
index 785a5f72..f50c8ceb 100644
--- a/src/yggdrasil/api.go
+++ b/src/yggdrasil/api.go
@@ -156,11 +156,11 @@ func (c *Core) GetDHT() []DHTEntry {
 	getDHT := func() {
 		now := time.Now()
 		var dhtentry []*dhtInfo
-		for _, v := range c.dht.table {
+		for _, v := range c.router.dht.table {
 			dhtentry = append(dhtentry, v)
 		}
 		sort.SliceStable(dhtentry, func(i, j int) bool {
-			return dht_ordered(&c.dht.nodeID, dhtentry[i].getNodeID(), dhtentry[j].getNodeID())
+			return dht_ordered(&c.router.dht.nodeID, dhtentry[i].getNodeID(), dhtentry[j].getNodeID())
 		})
 		for _, v := range dhtentry {
 			info := DHTEntry{
@@ -208,7 +208,7 @@ func (c *Core) GetSwitchQueues() SwitchQueues {
 func (c *Core) GetSessions() []Session {
 	var sessions []Session
 	getSessions := func() {
-		for _, sinfo := range c.sessions.sinfos {
+		for _, sinfo := range c.router.sessions.sinfos {
 			var session Session
 			workerFunc := func() {
 				session = Session{
@@ -243,17 +243,17 @@ func (c *Core) GetSessions() []Session {
 
 // ConnListen returns a listener for Yggdrasil session connections.
 func (c *Core) ConnListen() (*Listener, error) {
-	c.sessions.listenerMutex.Lock()
-	defer c.sessions.listenerMutex.Unlock()
-	if c.sessions.listener != nil {
+	c.router.sessions.listenerMutex.Lock()
+	defer c.router.sessions.listenerMutex.Unlock()
+	if c.router.sessions.listener != nil {
 		return nil, errors.New("a listener already exists")
 	}
-	c.sessions.listener = &Listener{
+	c.router.sessions.listener = &Listener{
 		core:  c,
 		conn:  make(chan *Conn),
 		close: make(chan interface{}),
 	}
-	return c.sessions.listener, nil
+	return c.router.sessions.listener, nil
 }
 
 // ConnDialer returns a dialer for Yggdrasil session connections.
@@ -356,10 +356,10 @@ func (c *Core) GetNodeInfo(key crypto.BoxPubKey, coords []uint64, nocache bool)
 // received an incoming session request. The function should return true to
 // allow the session or false to reject it.
 func (c *Core) SetSessionGatekeeper(f func(pubkey *crypto.BoxPubKey, initiator bool) bool) {
-	c.sessions.isAllowedMutex.Lock()
-	defer c.sessions.isAllowedMutex.Unlock()
+	c.router.sessions.isAllowedMutex.Lock()
+	defer c.router.sessions.isAllowedMutex.Unlock()
 
-	c.sessions.isAllowedHandler = f
+	c.router.sessions.isAllowedHandler = f
 }
 
 // SetLogger sets the output logger of the Yggdrasil node after startup. This
@@ -445,10 +445,10 @@ func (c *Core) DHTPing(key crypto.BoxPubKey, coords []uint64, target *crypto.Nod
 	}
 	rq := dhtReqKey{info.key, *target}
 	sendPing := func() {
-		c.dht.addCallback(&rq, func(res *dhtRes) {
+		c.router.dht.addCallback(&rq, func(res *dhtRes) {
 			resCh <- res
 		})
-		c.dht.ping(&info, &rq.dest)
+		c.router.dht.ping(&info, &rq.dest)
 	}
 	c.router.doAdmin(sendPing)
 	// TODO: do something better than the below...
diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index 51dd9509..b4ce71f9 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -84,7 +84,7 @@ func (c *Conn) String() string {
 func (c *Conn) search() error {
 	var sinfo *searchInfo
 	var isIn bool
-	c.core.router.doAdmin(func() { sinfo, isIn = c.core.searches.searches[*c.nodeID] })
+	c.core.router.doAdmin(func() { sinfo, isIn = c.core.router.searches.searches[*c.nodeID] })
 	if !isIn {
 		done := make(chan struct{}, 1)
 		var sess *sessionInfo
@@ -99,7 +99,7 @@ func (c *Conn) search() error {
 			}
 		}
 		c.core.router.doAdmin(func() {
-			sinfo = c.core.searches.newIterSearch(c.nodeID, c.nodeMask, searchCompleted)
+			sinfo = c.core.router.searches.newIterSearch(c.nodeID, c.nodeMask, searchCompleted)
 			sinfo.continueSearch()
 		})
 		<-done
@@ -124,11 +124,11 @@ func (c *Conn) search() error {
 func (c *Conn) doSearch() {
 	routerWork := func() {
 		// Check to see if there is a search already matching the destination
-		sinfo, isIn := c.core.searches.searches[*c.nodeID]
+		sinfo, isIn := c.core.router.searches.searches[*c.nodeID]
 		if !isIn {
 			// Nothing was found, so create a new search
 			searchCompleted := func(sinfo *sessionInfo, e error) {}
-			sinfo = c.core.searches.newIterSearch(c.nodeID, c.nodeMask, searchCompleted)
+			sinfo = c.core.router.searches.newIterSearch(c.nodeID, c.nodeMask, searchCompleted)
 			c.core.log.Debugf("%s DHT search started: %p", c.String(), sinfo)
 			// Start the search
 			sinfo.continueSearch()
diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index 0921ab9f..870597a9 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -26,10 +26,7 @@ type Core struct {
 	sigPriv     crypto.SigPrivKey
 	switchTable switchTable
 	peers       peers
-	sessions    sessions
 	router      router
-	dht         dht
-	searches    searches
 	link        link
 	log         *log.Logger
 }
@@ -76,9 +73,9 @@ func (c *Core) init() error {
 		c.log.Warnln("SigningPublicKey in config is incorrect, should be", sp)
 	}
 
-	c.searches.init(c)
-	c.dht.init(c)
-	c.sessions.init(c)
+	c.router.searches.init(c)
+	c.router.dht.init(c)
+	c.router.sessions.init(c)
 	c.peers.init(c)
 	c.router.init(c)
 	c.switchTable.init(c) // TODO move before peers? before router?
@@ -124,9 +121,9 @@ func (c *Core) UpdateConfig(config *config.NodeConfig) {
 	errors := 0
 
 	components := []chan chan error{
-		c.searches.reconfigure,
-		c.dht.reconfigure,
-		c.sessions.reconfigure,
+		c.router.searches.reconfigure,
+		c.router.dht.reconfigure,
+		c.router.sessions.reconfigure,
 		c.peers.reconfigure,
 		c.router.reconfigure,
 		c.switchTable.reconfigure,
diff --git a/src/yggdrasil/dht.go b/src/yggdrasil/dht.go
index d35d3aaf..d0d98558 100644
--- a/src/yggdrasil/dht.go
+++ b/src/yggdrasil/dht.go
@@ -221,7 +221,7 @@ func (t *dht) handleReq(req *dhtReq) {
 func (t *dht) sendRes(res *dhtRes, req *dhtReq) {
 	// Send a reply for a dhtReq
 	bs := res.encode()
-	shared := t.core.sessions.getSharedKey(&t.core.boxPriv, &req.Key)
+	shared := t.core.router.sessions.getSharedKey(&t.core.boxPriv, &req.Key)
 	payload, nonce := crypto.BoxSeal(shared, bs, nil)
 	p := wire_protoTrafficPacket{
 		Coords:  req.Coords,
@@ -285,7 +285,7 @@ func (t *dht) handleRes(res *dhtRes) {
 func (t *dht) sendReq(req *dhtReq, dest *dhtInfo) {
 	// Send a dhtReq to the node in dhtInfo
 	bs := req.encode()
-	shared := t.core.sessions.getSharedKey(&t.core.boxPriv, &dest.key)
+	shared := t.core.router.sessions.getSharedKey(&t.core.boxPriv, &dest.key)
 	payload, nonce := crypto.BoxSeal(shared, bs, nil)
 	p := wire_protoTrafficPacket{
 		Coords:  dest.coords,
diff --git a/src/yggdrasil/listener.go b/src/yggdrasil/listener.go
index 62225412..fec543f4 100644
--- a/src/yggdrasil/listener.go
+++ b/src/yggdrasil/listener.go
@@ -31,8 +31,8 @@ func (l *Listener) Close() (err error) {
 		recover()
 		err = errors.New("already closed")
 	}()
-	if l.core.sessions.listener == l {
-		l.core.sessions.listener = nil
+	if l.core.router.sessions.listener == l {
+		l.core.router.sessions.listener = nil
 	}
 	close(l.close)
 	close(l.conn)
diff --git a/src/yggdrasil/nodeinfo.go b/src/yggdrasil/nodeinfo.go
index 73d4e115..50f5bf9c 100644
--- a/src/yggdrasil/nodeinfo.go
+++ b/src/yggdrasil/nodeinfo.go
@@ -172,7 +172,7 @@ func (m *nodeinfo) sendNodeInfo(key crypto.BoxPubKey, coords []byte, isResponse
 		NodeInfo:   m.getNodeInfo(),
 	}
 	bs := nodeinfo.encode()
-	shared := m.core.sessions.getSharedKey(&m.core.boxPriv, &key)
+	shared := m.core.router.sessions.getSharedKey(&m.core.boxPriv, &key)
 	payload, nonce := crypto.BoxSeal(shared, bs, nil)
 	p := wire_protoTrafficPacket{
 		Coords:  coords,
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index 95521841..ed919983 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -43,15 +43,18 @@ type router struct {
 	addr        address.Address
 	subnet      address.Subnet
 	out         func([]byte) // packets we're sending to the network, link to peer's "in"
+	dht         dht
 	nodeinfo    nodeinfo
+	searches    searches
+	sessions    sessions
 }
 
 // Initializes the router struct, which includes setting up channels to/from the adapter.
 func (r *router) init(core *Core) {
 	r.core = core
 	r.reconfigure = make(chan chan error, 1)
-	r.addr = *address.AddrForNodeID(&r.core.dht.nodeID)
-	r.subnet = *address.SubnetForNodeID(&r.core.dht.nodeID)
+	r.addr = *address.AddrForNodeID(&r.dht.nodeID)
+	r.subnet = *address.SubnetForNodeID(&r.dht.nodeID)
 	self := linkInterface{
 		name: "(self)",
 		info: linkInfo{
@@ -91,15 +94,15 @@ func (r *router) handlePackets(from phony.IActor, packets [][]byte) {
 // Insert a peer info into the dht, TODO? make the dht a separate actor
 func (r *router) insertPeer(from phony.IActor, info *dhtInfo) {
 	r.EnqueueFrom(from, func() {
-		r.core.dht.insertPeer(info)
+		r.dht.insertPeer(info)
 	})
 }
 
 // Reset sessions and DHT after the switch sees our coords change
 func (r *router) reset(from phony.IActor) {
 	r.EnqueueFrom(from, func() {
-		r.core.sessions.reset(r)
-		r.core.dht.reset()
+		r.sessions.reset()
+		r.dht.reset()
 	})
 }
 
@@ -114,8 +117,8 @@ func (r *router) _mainLoop() {
 			<-r.SyncExec(func() {
 				// Any periodic maintenance stuff goes here
 				r.core.switchTable.doMaintenance()
-				r.core.dht.doMaintenance()
-				r.core.sessions.cleanup()
+				r.dht.doMaintenance()
+				r.sessions.cleanup()
 			})
 		case e := <-r.reconfigure:
 			<-r.SyncExec(func() {
@@ -149,7 +152,7 @@ func (r *router) _handleTraffic(packet []byte) {
 	if !p.decode(packet) {
 		return
 	}
-	sinfo, isIn := r.core.sessions.getSessionForHandle(&p.Handle)
+	sinfo, isIn := r.sessions.getSessionForHandle(&p.Handle)
 	if !isIn {
 		util.PutBytes(p.Payload)
 		return
@@ -172,7 +175,7 @@ func (r *router) _handleProto(packet []byte) {
 	var sharedKey *crypto.BoxSharedKey
 	if p.ToKey == r.core.boxPub {
 		// Try to open using our permanent key
-		sharedKey = r.core.sessions.getSharedKey(&r.core.boxPriv, &p.FromKey)
+		sharedKey = r.sessions.getSharedKey(&r.core.boxPriv, &p.FromKey)
 	} else {
 		return
 	}
@@ -212,7 +215,7 @@ func (r *router) _handlePing(bs []byte, fromKey *crypto.BoxPubKey) {
 		return
 	}
 	ping.SendPermPub = *fromKey
-	r.core.sessions.handlePing(&ping)
+	r.sessions.handlePing(&ping)
 }
 
 // Handles session pongs (which are really pings with an extra flag to prevent acknowledgement).
@@ -227,7 +230,7 @@ func (r *router) _handleDHTReq(bs []byte, fromKey *crypto.BoxPubKey) {
 		return
 	}
 	req.Key = *fromKey
-	r.core.dht.handleReq(&req)
+	r.dht.handleReq(&req)
 }
 
 // Decodes dht responses and passes them to dht.handleRes to update the DHT table and further pass them to the search code (if applicable).
@@ -237,7 +240,7 @@ func (r *router) _handleDHTRes(bs []byte, fromKey *crypto.BoxPubKey) {
 		return
 	}
 	res.Key = *fromKey
-	r.core.dht.handleRes(&res)
+	r.dht.handleRes(&res)
 }
 
 // Decodes nodeinfo request
diff --git a/src/yggdrasil/search.go b/src/yggdrasil/search.go
index 56adda80..5751043f 100644
--- a/src/yggdrasil/search.go
+++ b/src/yggdrasil/search.go
@@ -100,7 +100,7 @@ func (sinfo *searchInfo) addToSearch(res *dhtRes) {
 	from := dhtInfo{key: res.Key, coords: res.Coords}
 	sinfo.visited[*from.getNodeID()] = true
 	for _, info := range res.Infos {
-		if *info.getNodeID() == sinfo.core.dht.nodeID || sinfo.visited[*info.getNodeID()] {
+		if *info.getNodeID() == sinfo.core.router.dht.nodeID || sinfo.visited[*info.getNodeID()] {
 			continue
 		}
 		if dht_ordered(&sinfo.dest, info.getNodeID(), from.getNodeID()) {
@@ -134,7 +134,7 @@ func (sinfo *searchInfo) doSearchStep() {
 	if len(sinfo.toVisit) == 0 {
 		if time.Since(sinfo.time) > search_RETRY_TIME {
 			// Dead end and no response in too long, do cleanup
-			delete(sinfo.core.searches.searches, sinfo.dest)
+			delete(sinfo.core.router.searches.searches, sinfo.dest)
 			sinfo.callback(nil, errors.New("search reached dead end"))
 		}
 		return
@@ -143,8 +143,8 @@ func (sinfo *searchInfo) doSearchStep() {
 	var next *dhtInfo
 	next, sinfo.toVisit = sinfo.toVisit[0], sinfo.toVisit[1:]
 	rq := dhtReqKey{next.key, sinfo.dest}
-	sinfo.core.dht.addCallback(&rq, sinfo.handleDHTRes)
-	sinfo.core.dht.ping(next, &sinfo.dest)
+	sinfo.core.router.dht.addCallback(&rq, sinfo.handleDHTRes)
+	sinfo.core.router.dht.ping(next, &sinfo.dest)
 	sinfo.time = time.Now()
 }
 
@@ -157,7 +157,7 @@ func (sinfo *searchInfo) continueSearch() {
 	// Any that die aren't restarted, but a new one will start later
 	retryLater := func() {
 		// FIXME this keeps the search alive forever if not for the searches map, fix that
-		newSearchInfo := sinfo.core.searches.searches[sinfo.dest]
+		newSearchInfo := sinfo.core.router.searches.searches[sinfo.dest]
 		if newSearchInfo != sinfo {
 			return
 		}
@@ -196,17 +196,17 @@ func (sinfo *searchInfo) checkDHTRes(res *dhtRes) bool {
 		return false
 	}
 	// They match, so create a session and send a sessionRequest
-	sess, isIn := sinfo.core.sessions.getByTheirPerm(&res.Key)
+	sess, isIn := sinfo.core.router.sessions.getByTheirPerm(&res.Key)
 	if !isIn {
-		sess = sinfo.core.sessions.createSession(&res.Key)
+		sess = sinfo.core.router.sessions.createSession(&res.Key)
 		if sess == nil {
 			// nil if the DHT search finished but the session wasn't allowed
 			sinfo.callback(nil, errors.New("session not allowed"))
 			// Cleanup
-			delete(sinfo.core.searches.searches, res.Dest)
+			delete(sinfo.core.router.searches.searches, res.Dest)
 			return true
 		}
-		_, isIn := sinfo.core.sessions.getByTheirPerm(&res.Key)
+		_, isIn := sinfo.core.router.sessions.getByTheirPerm(&res.Key)
 		if !isIn {
 			panic("This should never happen")
 		}
@@ -216,6 +216,6 @@ func (sinfo *searchInfo) checkDHTRes(res *dhtRes) bool {
 	sess.ping(&sinfo.core.router)
 	sinfo.callback(sess, nil)
 	// Cleanup
-	delete(sinfo.core.searches.searches, res.Dest)
+	delete(sinfo.core.router.searches.searches, res.Dest)
 	return true
 }
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index f28e014a..02d4dc82 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -261,7 +261,7 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 		// Run cleanup when the session is canceled
 		<-sinfo.cancel.Finished()
 		sinfo.core.router.doAdmin(func() {
-			sinfo.core.sessions.removeSession(&sinfo)
+			sinfo.core.router.sessions.removeSession(&sinfo)
 		})
 	}()
 	go sinfo.startWorkers()
@@ -298,9 +298,9 @@ func (ss *sessions) cleanup() {
 
 // Closes a session, removing it from sessions maps.
 func (ss *sessions) removeSession(sinfo *sessionInfo) {
-	if s := sinfo.core.sessions.sinfos[sinfo.myHandle]; s == sinfo {
-		delete(sinfo.core.sessions.sinfos, sinfo.myHandle)
-		delete(sinfo.core.sessions.byTheirPerm, sinfo.theirPermPub)
+	if s := sinfo.core.router.sessions.sinfos[sinfo.myHandle]; s == sinfo {
+		delete(sinfo.core.router.sessions.sinfos, sinfo.myHandle)
+		delete(sinfo.core.router.sessions.byTheirPerm, sinfo.theirPermPub)
 	}
 }
 
@@ -466,9 +466,9 @@ func (sinfo *sessionInfo) _updateNonce(theirNonce *crypto.BoxNonce) {
 
 // Resets all sessions to an uninitialized state.
 // Called after coord changes, so attemtps to use a session will trigger a new ping and notify the remote end of the coord change.
-func (ss *sessions) reset(from phony.IActor) {
+func (ss *sessions) reset() {
 	for _, sinfo := range ss.sinfos {
-		sinfo.EnqueueFrom(from, func() {
+		sinfo.EnqueueFrom(&ss.core.router, func() {
 			sinfo.reset = true
 		})
 	}

From ebd806f27a9396d5184345425e4cf7d409f66582 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 20:29:16 -0500
Subject: [PATCH 006/130] move router member initialization into router.init

---
 src/yggdrasil/core.go   | 3 ---
 src/yggdrasil/router.go | 3 +++
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index 870597a9..fb2142c0 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -73,9 +73,6 @@ func (c *Core) init() error {
 		c.log.Warnln("SigningPublicKey in config is incorrect, should be", sp)
 	}
 
-	c.router.searches.init(c)
-	c.router.dht.init(c)
-	c.router.sessions.init(c)
 	c.peers.init(c)
 	c.router.init(c)
 	c.switchTable.init(c) // TODO move before peers? before router?
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index ed919983..29bdf1d4 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -73,6 +73,9 @@ func (r *router) init(core *Core) {
 	r.core.config.Mutex.RLock()
 	r.nodeinfo.setNodeInfo(r.core.config.Current.NodeInfo, r.core.config.Current.NodeInfoPrivacy)
 	r.core.config.Mutex.RUnlock()
+	r.dht.init(r.core)
+	r.searches.init(r.core)
+	r.sessions.init(r.core)
 }
 
 // Starts the mainLoop goroutine.

From e7024a00e712c7097ed8c194a937546566d80f10 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 20:35:54 -0500
Subject: [PATCH 007/130] have dht store a pointer to router instead of core

---
 src/yggdrasil/dht.go    | 34 +++++++++++++++++-----------------
 src/yggdrasil/router.go |  2 +-
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/src/yggdrasil/dht.go b/src/yggdrasil/dht.go
index d0d98558..adfc40e7 100644
--- a/src/yggdrasil/dht.go
+++ b/src/yggdrasil/dht.go
@@ -65,7 +65,7 @@ type dhtReqKey struct {
 
 // The main DHT struct.
 type dht struct {
-	core        *Core
+	router      *router
 	reconfigure chan chan error
 	nodeID      crypto.NodeID
 	reqs        map[dhtReqKey]time.Time          // Keeps track of recent outstanding requests
@@ -76,8 +76,8 @@ type dht struct {
 }
 
 // Initializes the DHT.
-func (t *dht) init(c *Core) {
-	t.core = c
+func (t *dht) init(r *router) {
+	t.router = r
 	t.reconfigure = make(chan chan error, 1)
 	go func() {
 		for {
@@ -85,7 +85,7 @@ func (t *dht) init(c *Core) {
 			e <- nil
 		}
 	}()
-	t.nodeID = *t.core.NodeID()
+	t.nodeID = *t.router.core.NodeID()
 	t.callbacks = make(map[dhtReqKey][]dht_callbackInfo)
 	t.reset()
 }
@@ -190,10 +190,10 @@ func dht_ordered(first, second, third *crypto.NodeID) bool {
 // Update info about the node that sent the request.
 func (t *dht) handleReq(req *dhtReq) {
 	// Send them what they asked for
-	loc := t.core.switchTable.getLocator()
+	loc := t.router.core.switchTable.getLocator()
 	coords := loc.getCoords()
 	res := dhtRes{
-		Key:    t.core.boxPub,
+		Key:    t.router.core.boxPub,
 		Coords: coords,
 		Dest:   req.Dest,
 		Infos:  t.lookup(&req.Dest, false),
@@ -221,17 +221,17 @@ func (t *dht) handleReq(req *dhtReq) {
 func (t *dht) sendRes(res *dhtRes, req *dhtReq) {
 	// Send a reply for a dhtReq
 	bs := res.encode()
-	shared := t.core.router.sessions.getSharedKey(&t.core.boxPriv, &req.Key)
+	shared := t.router.sessions.getSharedKey(&t.router.core.boxPriv, &req.Key)
 	payload, nonce := crypto.BoxSeal(shared, bs, nil)
 	p := wire_protoTrafficPacket{
 		Coords:  req.Coords,
 		ToKey:   req.Key,
-		FromKey: t.core.boxPub,
+		FromKey: t.router.core.boxPub,
 		Nonce:   *nonce,
 		Payload: payload,
 	}
 	packet := p.encode()
-	t.core.router.out(packet)
+	t.router.out(packet)
 }
 
 type dht_callbackInfo struct {
@@ -285,17 +285,17 @@ func (t *dht) handleRes(res *dhtRes) {
 func (t *dht) sendReq(req *dhtReq, dest *dhtInfo) {
 	// Send a dhtReq to the node in dhtInfo
 	bs := req.encode()
-	shared := t.core.router.sessions.getSharedKey(&t.core.boxPriv, &dest.key)
+	shared := t.router.sessions.getSharedKey(&t.router.core.boxPriv, &dest.key)
 	payload, nonce := crypto.BoxSeal(shared, bs, nil)
 	p := wire_protoTrafficPacket{
 		Coords:  dest.coords,
 		ToKey:   dest.key,
-		FromKey: t.core.boxPub,
+		FromKey: t.router.core.boxPub,
 		Nonce:   *nonce,
 		Payload: payload,
 	}
 	packet := p.encode()
-	t.core.router.out(packet)
+	t.router.out(packet)
 	rq := dhtReqKey{dest.key, req.Dest}
 	t.reqs[rq] = time.Now()
 }
@@ -306,10 +306,10 @@ func (t *dht) ping(info *dhtInfo, target *crypto.NodeID) {
 	if target == nil {
 		target = &t.nodeID
 	}
-	loc := t.core.switchTable.getLocator()
+	loc := t.router.core.switchTable.getLocator()
 	coords := loc.getCoords()
 	req := dhtReq{
-		Key:    t.core.boxPub,
+		Key:    t.router.core.boxPub,
 		Coords: coords,
 		Dest:   *target,
 	}
@@ -384,7 +384,7 @@ func (t *dht) getImportant() []*dhtInfo {
 		})
 		// Keep the ones that are no further than the closest seen so far
 		minDist := ^uint64(0)
-		loc := t.core.switchTable.getLocator()
+		loc := t.router.core.switchTable.getLocator()
 		important := infos[:0]
 		for _, info := range infos {
 			dist := uint64(loc.dist(info.coords))
@@ -413,12 +413,12 @@ func (t *dht) getImportant() []*dhtInfo {
 
 // Returns true if this is a node we need to keep track of for the DHT to work.
 func (t *dht) isImportant(ninfo *dhtInfo) bool {
-	if ninfo.key == t.core.boxPub {
+	if ninfo.key == t.router.core.boxPub {
 		return false
 	}
 	important := t.getImportant()
 	// Check if ninfo is of equal or greater importance to what we already know
-	loc := t.core.switchTable.getLocator()
+	loc := t.router.core.switchTable.getLocator()
 	ndist := uint64(loc.dist(ninfo.coords))
 	minDist := ^uint64(0)
 	for _, info := range important {
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index 29bdf1d4..bd23768b 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -73,7 +73,7 @@ func (r *router) init(core *Core) {
 	r.core.config.Mutex.RLock()
 	r.nodeinfo.setNodeInfo(r.core.config.Current.NodeInfo, r.core.config.Current.NodeInfoPrivacy)
 	r.core.config.Mutex.RUnlock()
-	r.dht.init(r.core)
+	r.dht.init(r)
 	r.searches.init(r.core)
 	r.sessions.init(r.core)
 }

From 5bb85cf07b7bc7d75f6a1afd4648e3d88b10d676 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 20:42:38 -0500
Subject: [PATCH 008/130] refactor searches to store a pointer to router
 instead of core

---
 src/yggdrasil/router.go |  2 +-
 src/yggdrasil/search.go | 38 +++++++++++++++++++-------------------
 2 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index bd23768b..ad9bc15a 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -74,7 +74,7 @@ func (r *router) init(core *Core) {
 	r.nodeinfo.setNodeInfo(r.core.config.Current.NodeInfo, r.core.config.Current.NodeInfoPrivacy)
 	r.core.config.Mutex.RUnlock()
 	r.dht.init(r)
-	r.searches.init(r.core)
+	r.searches.init(r)
 	r.sessions.init(r.core)
 }
 
diff --git a/src/yggdrasil/search.go b/src/yggdrasil/search.go
index 5751043f..397c28a9 100644
--- a/src/yggdrasil/search.go
+++ b/src/yggdrasil/search.go
@@ -33,7 +33,7 @@ const search_RETRY_TIME = time.Second
 // Information about an ongoing search.
 // Includes the target NodeID, the bitmask to match it to an IP, and the list of nodes to visit / already visited.
 type searchInfo struct {
-	core     *Core
+	searches *searches
 	dest     crypto.NodeID
 	mask     crypto.NodeID
 	time     time.Time
@@ -45,14 +45,14 @@ type searchInfo struct {
 
 // This stores a map of active searches.
 type searches struct {
-	core        *Core
+	router      *router
 	reconfigure chan chan error
 	searches    map[crypto.NodeID]*searchInfo
 }
 
 // Initializes the searches struct.
-func (s *searches) init(core *Core) {
-	s.core = core
+func (s *searches) init(r *router) {
+	s.router = r
 	s.reconfigure = make(chan chan error, 1)
 	go func() {
 		for {
@@ -66,7 +66,7 @@ func (s *searches) init(core *Core) {
 // Creates a new search info, adds it to the searches struct, and returns a pointer to the info.
 func (s *searches) createSearch(dest *crypto.NodeID, mask *crypto.NodeID, callback func(*sessionInfo, error)) *searchInfo {
 	info := searchInfo{
-		core:     s.core,
+		searches: s,
 		dest:     *dest,
 		mask:     *mask,
 		time:     time.Now(),
@@ -100,7 +100,7 @@ func (sinfo *searchInfo) addToSearch(res *dhtRes) {
 	from := dhtInfo{key: res.Key, coords: res.Coords}
 	sinfo.visited[*from.getNodeID()] = true
 	for _, info := range res.Infos {
-		if *info.getNodeID() == sinfo.core.router.dht.nodeID || sinfo.visited[*info.getNodeID()] {
+		if *info.getNodeID() == sinfo.searches.router.dht.nodeID || sinfo.visited[*info.getNodeID()] {
 			continue
 		}
 		if dht_ordered(&sinfo.dest, info.getNodeID(), from.getNodeID()) {
@@ -134,7 +134,7 @@ func (sinfo *searchInfo) doSearchStep() {
 	if len(sinfo.toVisit) == 0 {
 		if time.Since(sinfo.time) > search_RETRY_TIME {
 			// Dead end and no response in too long, do cleanup
-			delete(sinfo.core.router.searches.searches, sinfo.dest)
+			delete(sinfo.searches.searches, sinfo.dest)
 			sinfo.callback(nil, errors.New("search reached dead end"))
 		}
 		return
@@ -143,8 +143,8 @@ func (sinfo *searchInfo) doSearchStep() {
 	var next *dhtInfo
 	next, sinfo.toVisit = sinfo.toVisit[0], sinfo.toVisit[1:]
 	rq := dhtReqKey{next.key, sinfo.dest}
-	sinfo.core.router.dht.addCallback(&rq, sinfo.handleDHTRes)
-	sinfo.core.router.dht.ping(next, &sinfo.dest)
+	sinfo.searches.router.dht.addCallback(&rq, sinfo.handleDHTRes)
+	sinfo.searches.router.dht.ping(next, &sinfo.dest)
 	sinfo.time = time.Now()
 }
 
@@ -157,7 +157,7 @@ func (sinfo *searchInfo) continueSearch() {
 	// Any that die aren't restarted, but a new one will start later
 	retryLater := func() {
 		// FIXME this keeps the search alive forever if not for the searches map, fix that
-		newSearchInfo := sinfo.core.router.searches.searches[sinfo.dest]
+		newSearchInfo := sinfo.searches.searches[sinfo.dest]
 		if newSearchInfo != sinfo {
 			return
 		}
@@ -165,7 +165,7 @@ func (sinfo *searchInfo) continueSearch() {
 	}
 	go func() {
 		time.Sleep(search_RETRY_TIME)
-		sinfo.core.router.doAdmin(retryLater)
+		sinfo.searches.router.doAdmin(retryLater)
 	}()
 }
 
@@ -173,9 +173,9 @@ func (sinfo *searchInfo) continueSearch() {
 func (s *searches) newIterSearch(dest *crypto.NodeID, mask *crypto.NodeID, callback func(*sessionInfo, error)) *searchInfo {
 	sinfo := s.createSearch(dest, mask, callback)
 	sinfo.visited = make(map[crypto.NodeID]bool)
-	loc := s.core.switchTable.getLocator()
+	loc := s.router.core.switchTable.getLocator()
 	sinfo.toVisit = append(sinfo.toVisit, &dhtInfo{
-		key:    s.core.boxPub,
+		key:    s.router.core.boxPub,
 		coords: loc.getCoords(),
 	}) // Start the search by asking ourself, useful if we're the destination
 	return sinfo
@@ -196,26 +196,26 @@ func (sinfo *searchInfo) checkDHTRes(res *dhtRes) bool {
 		return false
 	}
 	// They match, so create a session and send a sessionRequest
-	sess, isIn := sinfo.core.router.sessions.getByTheirPerm(&res.Key)
+	sess, isIn := sinfo.searches.router.sessions.getByTheirPerm(&res.Key)
 	if !isIn {
-		sess = sinfo.core.router.sessions.createSession(&res.Key)
+		sess = sinfo.searches.router.sessions.createSession(&res.Key)
 		if sess == nil {
 			// nil if the DHT search finished but the session wasn't allowed
 			sinfo.callback(nil, errors.New("session not allowed"))
 			// Cleanup
-			delete(sinfo.core.router.searches.searches, res.Dest)
+			delete(sinfo.searches.searches, res.Dest)
 			return true
 		}
-		_, isIn := sinfo.core.router.sessions.getByTheirPerm(&res.Key)
+		_, isIn := sinfo.searches.router.sessions.getByTheirPerm(&res.Key)
 		if !isIn {
 			panic("This should never happen")
 		}
 	}
 	// FIXME (!) replay attacks could mess with coords? Give it a handle (tstamp)?
 	sess.coords = res.Coords
-	sess.ping(&sinfo.core.router)
+	sess.ping(sinfo.searches.router)
 	sinfo.callback(sess, nil)
 	// Cleanup
-	delete(sinfo.core.router.searches.searches, res.Dest)
+	delete(sinfo.searches.searches, res.Dest)
 	return true
 }

From 436c84ca3311cf9c57eb8eec6901a0edd408645f Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 20:53:00 -0500
Subject: [PATCH 009/130] refactor sessions to store a pointer to router
 instead of core

---
 src/yggdrasil/conn.go    |  2 +-
 src/yggdrasil/router.go  |  2 +-
 src/yggdrasil/session.go | 53 +++++++++++++++++++++-------------------
 3 files changed, 30 insertions(+), 27 deletions(-)

diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index b4ce71f9..4ba0b2aa 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -252,7 +252,7 @@ func (c *Conn) Close() (err error) {
 }
 
 func (c *Conn) LocalAddr() crypto.NodeID {
-	return *crypto.GetNodeID(&c.session.core.boxPub)
+	return *crypto.GetNodeID(&c.core.boxPub)
 }
 
 func (c *Conn) RemoteAddr() crypto.NodeID {
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index ad9bc15a..25f8e800 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -75,7 +75,7 @@ func (r *router) init(core *Core) {
 	r.core.config.Mutex.RUnlock()
 	r.dht.init(r)
 	r.searches.init(r)
-	r.sessions.init(r.core)
+	r.sessions.init(r)
 }
 
 // Starts the mainLoop goroutine.
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 02d4dc82..13f64424 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -40,7 +40,7 @@ func (h nonceHeap) peek() *crypto.BoxNonce { return &h[len(h)-1] }
 // This includes coords, permanent and ephemeral keys, handles and nonces, various sorts of timing information for timeout and maintenance, and some metadata for the admin API.
 type sessionInfo struct {
 	phony.Actor                                  // Protects all of the below, use it any time you read/change the contents of a session
-	core           *Core                         //
+	sessions       *sessions                     //
 	reconfigure    chan chan error               //
 	theirAddr      address.Address               //
 	theirSubnet    address.Subnet                //
@@ -136,7 +136,7 @@ func (s *sessionInfo) _update(p *sessionPing) bool {
 // Sessions are indexed by handle.
 // Additionally, stores maps of address/subnet onto keys, and keys onto handles.
 type sessions struct {
-	core             *Core
+	router           *router
 	listener         *Listener
 	listenerMutex    sync.Mutex
 	reconfigure      chan chan error
@@ -149,8 +149,8 @@ type sessions struct {
 }
 
 // Initializes the session struct.
-func (ss *sessions) init(core *Core) {
-	ss.core = core
+func (ss *sessions) init(r *router) {
+	ss.router = r
 	ss.reconfigure = make(chan chan error, 1)
 	go func() {
 		for {
@@ -213,18 +213,18 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 		return nil
 	}
 	sinfo := sessionInfo{}
-	sinfo.core = ss.core
+	sinfo.sessions = ss
 	sinfo.reconfigure = make(chan chan error, 1)
 	sinfo.theirPermPub = *theirPermKey
-	sinfo.sharedPermKey = *ss.getSharedKey(&ss.core.boxPriv, &sinfo.theirPermPub)
+	sinfo.sharedPermKey = *ss.getSharedKey(&ss.router.core.boxPriv, &sinfo.theirPermPub)
 	pub, priv := crypto.NewBoxKeys()
 	sinfo.mySesPub = *pub
 	sinfo.mySesPriv = *priv
 	sinfo.myNonce = *crypto.NewBoxNonce()
 	sinfo.theirMTU = 1280
-	ss.core.config.Mutex.RLock()
-	sinfo.myMTU = uint16(ss.core.config.Current.IfMTU)
-	ss.core.config.Mutex.RUnlock()
+	ss.router.core.config.Mutex.RLock()
+	sinfo.myMTU = uint16(ss.router.core.config.Current.IfMTU)
+	ss.router.core.config.Mutex.RUnlock()
 	now := time.Now()
 	sinfo.timeOpened = now
 	sinfo.time = now
@@ -234,11 +234,11 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 	sinfo.init = make(chan struct{})
 	sinfo.cancel = util.NewCancellation()
 	higher := false
-	for idx := range ss.core.boxPub {
-		if ss.core.boxPub[idx] > sinfo.theirPermPub[idx] {
+	for idx := range ss.router.core.boxPub {
+		if ss.router.core.boxPub[idx] > sinfo.theirPermPub[idx] {
 			higher = true
 			break
-		} else if ss.core.boxPub[idx] < sinfo.theirPermPub[idx] {
+		} else if ss.router.core.boxPub[idx] < sinfo.theirPermPub[idx] {
 			break
 		}
 	}
@@ -260,8 +260,8 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 	go func() {
 		// Run cleanup when the session is canceled
 		<-sinfo.cancel.Finished()
-		sinfo.core.router.doAdmin(func() {
-			sinfo.core.router.sessions.removeSession(&sinfo)
+		sinfo.sessions.router.doAdmin(func() {
+			sinfo.sessions.removeSession(&sinfo)
 		})
 	}()
 	go sinfo.startWorkers()
@@ -298,18 +298,18 @@ func (ss *sessions) cleanup() {
 
 // Closes a session, removing it from sessions maps.
 func (ss *sessions) removeSession(sinfo *sessionInfo) {
-	if s := sinfo.core.router.sessions.sinfos[sinfo.myHandle]; s == sinfo {
-		delete(sinfo.core.router.sessions.sinfos, sinfo.myHandle)
-		delete(sinfo.core.router.sessions.byTheirPerm, sinfo.theirPermPub)
+	if s := sinfo.sessions.sinfos[sinfo.myHandle]; s == sinfo {
+		delete(sinfo.sessions.sinfos, sinfo.myHandle)
+		delete(sinfo.sessions.byTheirPerm, sinfo.theirPermPub)
 	}
 }
 
 // Returns a session ping appropriate for the given session info.
 func (sinfo *sessionInfo) _getPing() sessionPing {
-	loc := sinfo.core.switchTable.getLocator()
+	loc := sinfo.sessions.router.core.switchTable.getLocator()
 	coords := loc.getCoords()
 	ping := sessionPing{
-		SendPermPub: sinfo.core.boxPub,
+		SendPermPub: sinfo.sessions.router.core.boxPub,
 		Handle:      sinfo.myHandle,
 		SendSesPub:  sinfo.mySesPub,
 		Tstamp:      time.Now().Unix(),
@@ -360,13 +360,13 @@ func (sinfo *sessionInfo) _sendPingPong(isPong bool) {
 	p := wire_protoTrafficPacket{
 		Coords:  sinfo.coords,
 		ToKey:   sinfo.theirPermPub,
-		FromKey: sinfo.core.boxPub,
+		FromKey: sinfo.sessions.router.core.boxPub,
 		Nonce:   *nonce,
 		Payload: payload,
 	}
 	packet := p.encode()
 	// TODO rewrite the below if/when the peer struct becomes an actor, to not go through the router first
-	sinfo.core.router.EnqueueFrom(sinfo, func() { sinfo.core.router.out(packet) })
+	sinfo.sessions.router.EnqueueFrom(sinfo, func() { sinfo.sessions.router.out(packet) })
 	if sinfo.pingTime.Before(sinfo.time) {
 		sinfo.pingTime = time.Now()
 	}
@@ -390,7 +390,7 @@ func (ss *sessions) handlePing(ping *sessionPing) {
 			if s, _ := ss.getByTheirPerm(&ping.SendPermPub); s != sinfo {
 				panic("This should not happen")
 			}
-			conn := newConn(ss.core, crypto.GetNodeID(&sinfo.theirPermPub), &crypto.NodeID{}, sinfo)
+			conn := newConn(ss.router.core, crypto.GetNodeID(&sinfo.theirPermPub), &crypto.NodeID{}, sinfo)
 			for i := range conn.nodeMask {
 				conn.nodeMask[i] = 0xFF
 			}
@@ -400,7 +400,7 @@ func (ss *sessions) handlePing(ping *sessionPing) {
 		ss.listenerMutex.Unlock()
 	}
 	if sinfo != nil {
-		sinfo.EnqueueFrom(&ss.core.router, func() {
+		sinfo.EnqueueFrom(ss.router, func() {
 			// Update the session
 			if !sinfo._update(ping) { /*panic("Should not happen in testing")*/
 				return
@@ -468,7 +468,7 @@ func (sinfo *sessionInfo) _updateNonce(theirNonce *crypto.BoxNonce) {
 // Called after coord changes, so attemtps to use a session will trigger a new ping and notify the remote end of the coord change.
 func (ss *sessions) reset() {
 	for _, sinfo := range ss.sinfos {
-		sinfo.EnqueueFrom(&ss.core.router, func() {
+		sinfo.EnqueueFrom(ss.router, func() {
 			sinfo.reset = true
 		})
 	}
@@ -639,7 +639,10 @@ func (sinfo *sessionInfo) sendWorker() {
 				util.PutBytes(msg.Message)
 				util.PutBytes(p.Payload)
 				// Send the packet
-				sinfo.core.router.out(packet)
+				// TODO replace this with a send to the peer struct if that becomes an actor
+				sinfo.sessions.router.EnqueueFrom(sinfo, func() {
+					sinfo.sessions.router.out(packet)
+				})
 			}
 			ch <- callback
 		}

From 533da351f9b0f0b1ffa6a2d459aeece7c05d2b2c Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 22:23:01 -0500
Subject: [PATCH 010/130] fix actor EnqueueFrom stack overflow (use nil now to
 send from self) and replace session send/recv workers with actor functions

---
 go.mod                   |   2 +-
 go.sum                   |   4 +-
 src/yggdrasil/conn.go    |   4 +-
 src/yggdrasil/peer.go    |   2 +-
 src/yggdrasil/router.go  |   4 +-
 src/yggdrasil/session.go | 129 ++++++++++++++++++++++++++++++++++++---
 src/yggdrasil/switch.go  |   4 +-
 7 files changed, 132 insertions(+), 17 deletions(-)

diff --git a/go.mod b/go.mod
index 3f5cae88..fff4ae6f 100644
--- a/go.mod
+++ b/go.mod
@@ -1,7 +1,7 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
 require (
-	github.com/Arceliar/phony v0.0.0-20190821233739-c7f353f14438
+	github.com/Arceliar/phony v0.0.0-20190824031448-b53e115f69b5
 	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
 	github.com/hashicorp/go-syslog v1.0.0
 	github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222
diff --git a/go.sum b/go.sum
index 22276c2c..29854cbf 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-github.com/Arceliar/phony v0.0.0-20190821233739-c7f353f14438 h1:t4tRgrItIq2ap4O31yOuWm17lUiyzf8gf/P+bEfgmrw=
-github.com/Arceliar/phony v0.0.0-20190821233739-c7f353f14438/go.mod h1:2Q9yJvg2PlMrnOEa3RTEy9hElWAICo/D8HTUDqAHUAo=
+github.com/Arceliar/phony v0.0.0-20190824031448-b53e115f69b5 h1:D2Djqo/q7mftrtHLCpW4Rpplm8jj+Edc9jNz8Ll6E0A=
+github.com/Arceliar/phony v0.0.0-20190824031448-b53e115f69b5/go.mod h1:2Q9yJvg2PlMrnOEa3RTEy9hElWAICo/D8HTUDqAHUAo=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=
diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index 4ba0b2aa..1828b556 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -162,7 +162,7 @@ func (c *Conn) ReadNoCopy() ([]byte, error) {
 		} else {
 			return nil, ConnError{errors.New("session closed"), false, false, true, 0}
 		}
-	case bs := <-c.session.recv:
+	case bs := <-c.session.toConn:
 		return bs, nil
 	}
 }
@@ -221,7 +221,7 @@ func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error {
 			} else {
 				err = ConnError{errors.New("session closed"), false, false, true, 0}
 			}
-		case c.session.send <- msg:
+		case <-c.session.SyncExec(func() { c.session._send(msg) }):
 		}
 	}
 	return err
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index fcd9364c..8869bd2a 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -210,7 +210,7 @@ func (p *peer) linkLoop() {
 		case dinfo = <-p.dinfo:
 		case _ = <-tick.C:
 			if dinfo != nil {
-				p.core.router.insertPeer(&p.core.router, dinfo)
+				p.core.router.insertPeer(nil, dinfo)
 			}
 		}
 	}
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index 25f8e800..b5df107e 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -66,7 +66,7 @@ func (r *router) init(core *Core) {
 	p := r.core.peers.newPeer(&r.core.boxPub, &r.core.sigPub, &crypto.BoxSharedKey{}, &self, nil)
 	p.out = func(packets [][]byte) {
 		// TODO make peers and/or the switch into actors, have them pass themselves as the from field
-		r.handlePackets(r, packets)
+		r.handlePackets(nil, packets)
 	}
 	r.out = p.handlePacket // TODO if the peer becomes its own actor, then send a message here
 	r.nodeinfo.init(r.core)
@@ -160,6 +160,8 @@ func (r *router) _handleTraffic(packet []byte) {
 		util.PutBytes(p.Payload)
 		return
 	}
+	sinfo.recv(r, &p)
+	return
 	select {
 	case sinfo.fromRouter <- p:
 	case <-sinfo.cancel.Finished():
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 13f64424..84ea92af 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -72,8 +72,9 @@ type sessionInfo struct {
 	init           chan struct{}                 // Closed when the first session pong arrives, used to signal that the session is ready for initial use
 	cancel         util.Cancellation             // Used to terminate workers
 	fromRouter     chan wire_trafficPacket       // Received packets go here, to be decrypted by the session
-	recv           chan []byte                   // Decrypted packets go here, picked up by the associated Conn
-	send           chan FlowKeyMessage           // Packets with optional flow key go here, to be encrypted and sent
+	toConn         chan []byte                   // Decrypted packets go here, picked up by the associated Conn
+	fromConn       chan FlowKeyMessage           // Packets with optional flow key go here, to be encrypted and sent
+	callbacks      []chan func()                 // Finished work from crypto workers
 }
 
 // TODO remove this, call SyncExec directly
@@ -253,8 +254,8 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 	sinfo.theirAddr = *address.AddrForNodeID(crypto.GetNodeID(&sinfo.theirPermPub))
 	sinfo.theirSubnet = *address.SubnetForNodeID(crypto.GetNodeID(&sinfo.theirPermPub))
 	sinfo.fromRouter = make(chan wire_trafficPacket, 1)
-	sinfo.recv = make(chan []byte, 32)
-	sinfo.send = make(chan FlowKeyMessage, 32)
+	sinfo.toConn = make(chan []byte, 32)
+	sinfo.fromConn = make(chan FlowKeyMessage, 32)
 	ss.sinfos[sinfo.myHandle] = &sinfo
 	ss.byTheirPerm[sinfo.theirPermPub] = &sinfo.myHandle
 	go func() {
@@ -264,7 +265,7 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 			sinfo.sessions.removeSession(&sinfo)
 		})
 	}()
-	go sinfo.startWorkers()
+	//go sinfo.startWorkers()
 	return &sinfo
 }
 
@@ -539,7 +540,7 @@ func (sinfo *sessionInfo) recvWorker() {
 					select {
 					case <-sinfo.cancel.Finished():
 						util.PutBytes(bs)
-					case sinfo.recv <- bs:
+					case sinfo.toConn <- bs:
 					}
 				}
 			}
@@ -664,15 +665,127 @@ func (sinfo *sessionInfo) sendWorker() {
 				f()
 			case <-sinfo.cancel.Finished():
 				return
-			case msg := <-sinfo.send:
+			case msg := <-sinfo.fromConn:
 				doSend(msg)
 			}
 		}
 		select {
 		case <-sinfo.cancel.Finished():
 			return
-		case bs := <-sinfo.send:
+		case bs := <-sinfo.fromConn:
 			doSend(bs)
 		}
 	}
 }
+
+////////////////////////////////////////////////////////////////////////////////
+
+func (sinfo *sessionInfo) recv(from phony.IActor, packet *wire_trafficPacket) {
+	sinfo.EnqueueFrom(from, func() {
+		sinfo._recvPacket(packet)
+	})
+}
+
+func (sinfo *sessionInfo) _recvPacket(p *wire_trafficPacket) {
+	select {
+	case <-sinfo.init:
+	default:
+		// TODO find a better way to drop things until initialized
+		util.PutBytes(p.Payload)
+		return
+	}
+	switch {
+	case sinfo._nonceIsOK(&p.Nonce):
+	case len(sinfo.toConn) < cap(sinfo.toConn):
+	default:
+		// We're either full or don't like this nonce
+		util.PutBytes(p.Payload)
+		return
+	}
+
+	k := sinfo.sharedSesKey
+	var isOK bool
+	var bs []byte
+	ch := make(chan func(), 1)
+	poolFunc := func() {
+		bs, isOK = crypto.BoxOpen(&k, p.Payload, &p.Nonce)
+		callback := func() {
+			util.PutBytes(p.Payload)
+			if !isOK || k != sinfo.sharedSesKey || !sinfo._nonceIsOK(&p.Nonce) {
+				// Either we failed to decrypt, or the session was updated, or we received this packet in the mean time
+				util.PutBytes(bs)
+				return
+			}
+			sinfo._updateNonce(&p.Nonce)
+			sinfo.time = time.Now()
+			sinfo.bytesRecvd += uint64(len(bs))
+			select {
+			case sinfo.toConn <- bs:
+			default:
+				// We seem to have filled up the buffer in the mean time, so drop it
+				util.PutBytes(bs)
+			}
+		}
+		ch <- callback
+		sinfo.checkCallbacks()
+	}
+	sinfo.callbacks = append(sinfo.callbacks, ch)
+	util.WorkerGo(poolFunc)
+}
+
+func (sinfo *sessionInfo) _send(msg FlowKeyMessage) {
+	select {
+	case <-sinfo.init:
+	default:
+		// TODO find a better way to drop things until initialized
+		util.PutBytes(msg.Message)
+		return
+	}
+	sinfo.bytesSent += uint64(len(msg.Message))
+	coords := append([]byte(nil), sinfo.coords...)
+	if msg.FlowKey != 0 {
+		coords = append(coords, 0)
+		coords = append(coords, wire_encode_uint64(msg.FlowKey)...)
+	}
+	p := wire_trafficPacket{
+		Coords: coords,
+		Handle: sinfo.theirHandle,
+		Nonce:  sinfo.myNonce,
+	}
+	sinfo.myNonce.Increment()
+	k := sinfo.sharedSesKey
+	ch := make(chan func(), 1)
+	poolFunc := func() {
+		p.Payload, _ = crypto.BoxSeal(&k, msg.Message, &p.Nonce)
+		callback := func() {
+			// Encoding may block on a util.GetBytes(), so kept out of the worker pool
+			packet := p.encode()
+			// Cleanup
+			util.PutBytes(msg.Message)
+			util.PutBytes(p.Payload)
+			// Send the packet
+			// TODO replace this with a send to the peer struct if that becomes an actor
+			sinfo.sessions.router.EnqueueFrom(sinfo, func() {
+				sinfo.sessions.router.out(packet)
+			})
+		}
+		ch <- callback
+		sinfo.checkCallbacks()
+	}
+	sinfo.callbacks = append(sinfo.callbacks, ch)
+	util.WorkerGo(poolFunc)
+}
+
+func (sinfo *sessionInfo) checkCallbacks() {
+	sinfo.EnqueueFrom(nil, func() {
+		if len(sinfo.callbacks) > 0 {
+			select {
+			case callback := <-sinfo.callbacks[0]:
+				sinfo.callbacks = sinfo.callbacks[1:]
+				callback()
+				sinfo.checkCallbacks()
+			default:
+			}
+		}
+	})
+}
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 86ae102b..1fa75a6c 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -245,7 +245,7 @@ func (t *switchTable) cleanRoot() {
 		if t.data.locator.root != t.key {
 			t.data.seq++
 			t.updater.Store(&sync.Once{})
-			t.core.router.reset(&t.core.router)
+			t.core.router.reset(nil)
 		}
 		t.data.locator = switchLocator{root: t.key, tstamp: now.Unix()}
 		t.core.peers.sendSwitchMsgs()
@@ -508,7 +508,7 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort, rep
 		if !equiv(&sender.locator, &t.data.locator) {
 			doUpdate = true
 			t.data.seq++
-			t.core.router.reset(&t.core.router)
+			t.core.router.reset(nil)
 		}
 		if t.data.locator.tstamp != sender.locator.tstamp {
 			t.time = now

From e3603c0462fba858e330d7bc69d1c41d1016a98d Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 22:25:40 -0500
Subject: [PATCH 011/130] clean up unused session code

---
 src/yggdrasil/router.go  |   6 --
 src/yggdrasil/session.go | 201 ---------------------------------------
 2 files changed, 207 deletions(-)

diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index b5df107e..464a4778 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -161,12 +161,6 @@ func (r *router) _handleTraffic(packet []byte) {
 		return
 	}
 	sinfo.recv(r, &p)
-	return
-	select {
-	case sinfo.fromRouter <- p:
-	case <-sinfo.cancel.Finished():
-		util.PutBytes(p.Payload)
-	}
 }
 
 // Handles protocol traffic by decrypting it, checking its type, and passing it to the appropriate handler for that traffic type.
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 84ea92af..d17b119f 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -7,7 +7,6 @@ package yggdrasil
 import (
 	"bytes"
 	"container/heap"
-	"errors"
 	"sync"
 	"time"
 
@@ -71,9 +70,7 @@ type sessionInfo struct {
 	bytesRecvd     uint64                        // Bytes of real traffic received in this session
 	init           chan struct{}                 // Closed when the first session pong arrives, used to signal that the session is ready for initial use
 	cancel         util.Cancellation             // Used to terminate workers
-	fromRouter     chan wire_trafficPacket       // Received packets go here, to be decrypted by the session
 	toConn         chan []byte                   // Decrypted packets go here, picked up by the associated Conn
-	fromConn       chan FlowKeyMessage           // Packets with optional flow key go here, to be encrypted and sent
 	callbacks      []chan func()                 // Finished work from crypto workers
 }
 
@@ -253,9 +250,7 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 	sinfo.myHandle = *crypto.NewHandle()
 	sinfo.theirAddr = *address.AddrForNodeID(crypto.GetNodeID(&sinfo.theirPermPub))
 	sinfo.theirSubnet = *address.SubnetForNodeID(crypto.GetNodeID(&sinfo.theirPermPub))
-	sinfo.fromRouter = make(chan wire_trafficPacket, 1)
 	sinfo.toConn = make(chan []byte, 32)
-	sinfo.fromConn = make(chan FlowKeyMessage, 32)
 	ss.sinfos[sinfo.myHandle] = &sinfo
 	ss.byTheirPerm[sinfo.theirPermPub] = &sinfo.myHandle
 	go func() {
@@ -479,207 +474,11 @@ func (ss *sessions) reset() {
 //////////////////////////// Worker Functions Below ////////////////////////////
 ////////////////////////////////////////////////////////////////////////////////
 
-func (sinfo *sessionInfo) startWorkers() {
-	go sinfo.recvWorker()
-	go sinfo.sendWorker()
-}
-
 type FlowKeyMessage struct {
 	FlowKey uint64
 	Message []byte
 }
 
-func (sinfo *sessionInfo) recvWorker() {
-	// TODO move theirNonce etc into a struct that gets stored here, passed in over a channel
-	//  Since there's no reason for anywhere else in the session code to need to *read* it...
-	//  Only needs to be updated from the outside if a ping resets it...
-	//  That would get rid of the need to take a mutex for the sessionFunc
-	var callbacks []chan func()
-	doRecv := func(p wire_trafficPacket) {
-		var bs []byte
-		var err error
-		var k crypto.BoxSharedKey
-		sessionFunc := func() {
-			if !sinfo._nonceIsOK(&p.Nonce) {
-				err = ConnError{errors.New("packet dropped due to invalid nonce"), false, true, false, 0}
-				return
-			}
-			k = sinfo.sharedSesKey
-		}
-		sinfo.doFunc(sessionFunc)
-		if err != nil {
-			util.PutBytes(p.Payload)
-			return
-		}
-		var isOK bool
-		ch := make(chan func(), 1)
-		poolFunc := func() {
-			bs, isOK = crypto.BoxOpen(&k, p.Payload, &p.Nonce)
-			callback := func() {
-				util.PutBytes(p.Payload)
-				if !isOK {
-					util.PutBytes(bs)
-					return
-				}
-				sessionFunc = func() {
-					if k != sinfo.sharedSesKey || !sinfo._nonceIsOK(&p.Nonce) {
-						// The session updated in the mean time, so return an error
-						err = ConnError{errors.New("session updated during crypto operation"), false, true, false, 0}
-						return
-					}
-					sinfo._updateNonce(&p.Nonce)
-					sinfo.time = time.Now()
-					sinfo.bytesRecvd += uint64(len(bs))
-				}
-				sinfo.doFunc(sessionFunc)
-				if err != nil {
-					// Not sure what else to do with this packet, I guess just drop it
-					util.PutBytes(bs)
-				} else {
-					// Pass the packet to the buffer for Conn.Read
-					select {
-					case <-sinfo.cancel.Finished():
-						util.PutBytes(bs)
-					case sinfo.toConn <- bs:
-					}
-				}
-			}
-			ch <- callback
-		}
-		// Send to the worker and wait for it to finish
-		util.WorkerGo(poolFunc)
-		callbacks = append(callbacks, ch)
-	}
-	fromHelper := make(chan wire_trafficPacket, 1)
-	go func() {
-		var buf []wire_trafficPacket
-		for {
-			for len(buf) > 0 {
-				select {
-				case <-sinfo.cancel.Finished():
-					return
-				case p := <-sinfo.fromRouter:
-					buf = append(buf, p)
-					for len(buf) > 64 { // Based on nonce window size
-						util.PutBytes(buf[0].Payload)
-						buf = buf[1:]
-					}
-				case fromHelper <- buf[0]:
-					buf = buf[1:]
-				}
-			}
-			select {
-			case <-sinfo.cancel.Finished():
-				return
-			case p := <-sinfo.fromRouter:
-				buf = append(buf, p)
-			}
-		}
-	}()
-	select {
-	case <-sinfo.cancel.Finished():
-		return
-	case <-sinfo.init:
-		// Wait until the session has finished initializing before processing any packets
-	}
-	for {
-		for len(callbacks) > 0 {
-			select {
-			case f := <-callbacks[0]:
-				callbacks = callbacks[1:]
-				f()
-			case <-sinfo.cancel.Finished():
-				return
-			case p := <-fromHelper:
-				doRecv(p)
-			}
-		}
-		select {
-		case <-sinfo.cancel.Finished():
-			return
-		case p := <-fromHelper:
-			doRecv(p)
-		}
-	}
-}
-
-func (sinfo *sessionInfo) sendWorker() {
-	// TODO move info that this worker needs here, send updates via a channel
-	//  Otherwise we need to take a mutex to avoid races with update()
-	var callbacks []chan func()
-	doSend := func(msg FlowKeyMessage) {
-		var p wire_trafficPacket
-		var k crypto.BoxSharedKey
-		sessionFunc := func() {
-			sinfo.bytesSent += uint64(len(msg.Message))
-			p = wire_trafficPacket{
-				Coords: append([]byte(nil), sinfo.coords...),
-				Handle: sinfo.theirHandle,
-				Nonce:  sinfo.myNonce,
-			}
-			if msg.FlowKey != 0 {
-				// Helps ensure that traffic from this flow ends up in a separate queue from other flows
-				// The zero padding relies on the fact that the self-peer is always on port 0
-				p.Coords = append(p.Coords, 0)
-				p.Coords = wire_put_uint64(msg.FlowKey, p.Coords)
-			}
-			sinfo.myNonce.Increment()
-			k = sinfo.sharedSesKey
-		}
-		// Get the mutex-protected info needed to encrypt the packet
-		sinfo.doFunc(sessionFunc)
-		ch := make(chan func(), 1)
-		poolFunc := func() {
-			// Encrypt the packet
-			p.Payload, _ = crypto.BoxSeal(&k, msg.Message, &p.Nonce)
-			// The callback will send the packet
-			callback := func() {
-				// Encoding may block on a util.GetBytes(), so kept out of the worker pool
-				packet := p.encode()
-				// Cleanup
-				util.PutBytes(msg.Message)
-				util.PutBytes(p.Payload)
-				// Send the packet
-				// TODO replace this with a send to the peer struct if that becomes an actor
-				sinfo.sessions.router.EnqueueFrom(sinfo, func() {
-					sinfo.sessions.router.out(packet)
-				})
-			}
-			ch <- callback
-		}
-		// Send to the worker and wait for it to finish
-		util.WorkerGo(poolFunc)
-		callbacks = append(callbacks, ch)
-	}
-	select {
-	case <-sinfo.cancel.Finished():
-		return
-	case <-sinfo.init:
-		// Wait until the session has finished initializing before processing any packets
-	}
-	for {
-		for len(callbacks) > 0 {
-			select {
-			case f := <-callbacks[0]:
-				callbacks = callbacks[1:]
-				f()
-			case <-sinfo.cancel.Finished():
-				return
-			case msg := <-sinfo.fromConn:
-				doSend(msg)
-			}
-		}
-		select {
-		case <-sinfo.cancel.Finished():
-			return
-		case bs := <-sinfo.fromConn:
-			doSend(bs)
-		}
-	}
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
 func (sinfo *sessionInfo) recv(from phony.IActor, packet *wire_trafficPacket) {
 	sinfo.EnqueueFrom(from, func() {
 		sinfo._recvPacket(packet)

From cf9880464bdf100ea6c88aebd58dc02afef35e07 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 22:36:59 -0500
Subject: [PATCH 012/130] explicitly consider the session finished case, and
 make a note that we could fix the packet drop situation by making the Conn
 into an actor too

---
 src/yggdrasil/session.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index d17b119f..3f5e913e 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -520,8 +520,12 @@ func (sinfo *sessionInfo) _recvPacket(p *wire_trafficPacket) {
 			sinfo.bytesRecvd += uint64(len(bs))
 			select {
 			case sinfo.toConn <- bs:
+			case <-sinfo.cancel.Finished():
+				util.PutBytes(bs)
 			default:
-				// We seem to have filled up the buffer in the mean time, so drop it
+				// We seem to have filled up the buffer in the mean time
+				// Since we need to not block, but the conn isn't an actor, we need to drop this packet
+				// TODO find some nicer way to interact with the Conn...
 				util.PutBytes(bs)
 			}
 		}

From cac3444d9ae0a2e45aabe142430461843f91948b Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 22:40:13 -0500
Subject: [PATCH 013/130] fix debug builds

---
 src/yggdrasil/debug.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/yggdrasil/debug.go b/src/yggdrasil/debug.go
index 9f7707e2..d90bf1f4 100644
--- a/src/yggdrasil/debug.go
+++ b/src/yggdrasil/debug.go
@@ -258,7 +258,7 @@ func DEBUG_wire_encode_coords(coords []byte) []byte {
 func (c *Core) DEBUG_getDHTSize() int {
 	var total int
 	c.router.doAdmin(func() {
-		total = len(c.dht.table)
+		total = len(c.router.dht.table)
 	})
 	return total
 }
@@ -320,7 +320,7 @@ func (c *Core) DEBUG_startLoopbackUDPInterface() {
 ////////////////////////////////////////////////////////////////////////////////
 
 func (c *Core) DEBUG_getAddr() *address.Address {
-	return address.AddrForNodeID(&c.dht.nodeID)
+	return address.AddrForNodeID(&c.router.dht.nodeID)
 }
 
 /*

From 6ecbc439f0c0755597ab56fe1fe1b364fcea9be4 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 23:36:00 -0500
Subject: [PATCH 014/130] start migrating Conn to be an actor

---
 src/yggdrasil/conn.go | 54 +++++++++++++++++++++++--------------------
 1 file changed, 29 insertions(+), 25 deletions(-)

diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index 1828b556..fa05fff0 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -3,12 +3,12 @@ package yggdrasil
 import (
 	"errors"
 	"fmt"
-	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
+
+	"github.com/Arceliar/phony"
 )
 
 // ConnError implements the net.Error interface
@@ -54,10 +54,10 @@ func (e *ConnError) Closed() bool {
 }
 
 type Conn struct {
+	phony.Actor
 	core          *Core
-	readDeadline  atomic.Value // time.Time // TODO timer
-	writeDeadline atomic.Value // time.Time // TODO timer
-	mutex         sync.RWMutex // protects the below
+	readDeadline  *time.Time
+	writeDeadline *time.Time
 	nodeID        *crypto.NodeID
 	nodeMask      *crypto.NodeID
 	session       *sessionInfo
@@ -75,9 +75,9 @@ func newConn(core *Core, nodeID *crypto.NodeID, nodeMask *crypto.NodeID, session
 }
 
 func (c *Conn) String() string {
-	c.mutex.RLock()
-	defer c.mutex.RUnlock()
-	return fmt.Sprintf("conn=%p", c)
+	var s string
+	<-c.SyncExec(func() { s = fmt.Sprintf("conn=%p", c) })
+	return s
 }
 
 // This should never be called from the router goroutine, used in the dial functions
@@ -137,10 +137,10 @@ func (c *Conn) doSearch() {
 	go c.core.router.doAdmin(routerWork)
 }
 
-func (c *Conn) getDeadlineCancellation(value *atomic.Value) (util.Cancellation, bool) {
-	if deadline, ok := value.Load().(time.Time); ok {
+func (c *Conn) _getDeadlineCancellation(t *time.Time) (util.Cancellation, bool) {
+	if t != nil {
 		// A deadline is set, so return a Cancellation that uses it
-		c := util.CancellationWithDeadline(c.session.cancel, deadline)
+		c := util.CancellationWithDeadline(c.session.cancel, *t)
 		return c, true
 	} else {
 		// No deadline was set, so just return the existinc cancellation and a dummy value
@@ -150,7 +150,9 @@ func (c *Conn) getDeadlineCancellation(value *atomic.Value) (util.Cancellation,
 
 // Used internally by Read, the caller is responsible for util.PutBytes when they're done.
 func (c *Conn) ReadNoCopy() ([]byte, error) {
-	cancel, doCancel := c.getDeadlineCancellation(&c.readDeadline)
+	var cancel util.Cancellation
+	var doCancel bool
+	<-c.SyncExec(func() { cancel, doCancel = c._getDeadlineCancellation(c.readDeadline) })
 	if doCancel {
 		defer cancel.Cancel(nil)
 	}
@@ -210,7 +212,9 @@ func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error {
 	}
 	c.session.doFunc(sessionFunc)
 	if err == nil {
-		cancel, doCancel := c.getDeadlineCancellation(&c.writeDeadline)
+		var cancel util.Cancellation
+		var doCancel bool
+		<-c.SyncExec(func() { cancel, doCancel = c._getDeadlineCancellation(c.writeDeadline) })
 		if doCancel {
 			defer cancel.Cancel(nil)
 		}
@@ -240,14 +244,14 @@ func (c *Conn) Write(b []byte) (int, error) {
 }
 
 func (c *Conn) Close() (err error) {
-	c.mutex.Lock()
-	defer c.mutex.Unlock()
-	if c.session != nil {
-		// Close the session, if it hasn't been closed already
-		if e := c.session.cancel.Cancel(errors.New("connection closed")); e != nil {
-			err = ConnError{errors.New("close failed, session already closed"), false, false, true, 0}
+	<-c.SyncExec(func() {
+		if c.session != nil {
+			// Close the session, if it hasn't been closed already
+			if e := c.session.cancel.Cancel(errors.New("connection closed")); e != nil {
+				err = ConnError{errors.New("close failed, session already closed"), false, false, true, 0}
+			}
 		}
-	}
+	})
 	return
 }
 
@@ -256,9 +260,9 @@ func (c *Conn) LocalAddr() crypto.NodeID {
 }
 
 func (c *Conn) RemoteAddr() crypto.NodeID {
-	c.mutex.RLock()
-	defer c.mutex.RUnlock()
-	return *c.nodeID
+	var n crypto.NodeID
+	<-c.SyncExec(func() { n = *c.nodeID })
+	return n
 }
 
 func (c *Conn) SetDeadline(t time.Time) error {
@@ -268,11 +272,11 @@ func (c *Conn) SetDeadline(t time.Time) error {
 }
 
 func (c *Conn) SetReadDeadline(t time.Time) error {
-	c.readDeadline.Store(t)
+	<-c.SyncExec(func() { c.readDeadline = &t })
 	return nil
 }
 
 func (c *Conn) SetWriteDeadline(t time.Time) error {
-	c.writeDeadline.Store(t)
+	<-c.SyncExec(func() { c.writeDeadline = &t })
 	return nil
 }

From da9f7151e3da50547b7320a6bbe9e9bbede23694 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 00:17:37 -0500
Subject: [PATCH 015/130] more conn migration

---
 src/yggdrasil/conn.go    | 63 ++++++++++++++++++++++------------------
 src/yggdrasil/dialer.go  |  1 +
 src/yggdrasil/session.go | 12 ++++++++
 3 files changed, 47 insertions(+), 29 deletions(-)

diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index fa05fff0..7121fc15 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -61,6 +61,7 @@ type Conn struct {
 	nodeID        *crypto.NodeID
 	nodeMask      *crypto.NodeID
 	session       *sessionInfo
+	mtu           uint16
 }
 
 // TODO func NewConn() that initializes additional fields as needed
@@ -80,6 +81,10 @@ func (c *Conn) String() string {
 	return s
 }
 
+func (c *Conn) setMTU(from phony.IActor, mtu uint16) {
+	c.EnqueueFrom(from, func() { c.mtu = mtu })
+}
+
 // This should never be called from the router goroutine, used in the dial functions
 func (c *Conn) search() error {
 	var sinfo *searchInfo
@@ -112,6 +117,7 @@ func (c *Conn) search() error {
 			for i := range c.nodeMask {
 				c.nodeMask[i] = 0xFF
 			}
+			c.session.conn = c
 		}
 		return err
 	} else {
@@ -120,7 +126,7 @@ func (c *Conn) search() error {
 	return nil
 }
 
-// Used in session keep-alive traffic in Conn.Write
+// Used in session keep-alive traffic
 func (c *Conn) doSearch() {
 	routerWork := func() {
 		// Check to see if there is a search already matching the destination
@@ -134,7 +140,7 @@ func (c *Conn) doSearch() {
 			sinfo.continueSearch()
 		}
 	}
-	go c.core.router.doAdmin(routerWork)
+	c.core.router.EnqueueFrom(c.session, routerWork)
 }
 
 func (c *Conn) _getDeadlineCancellation(t *time.Time) (util.Cancellation, bool) {
@@ -187,16 +193,14 @@ func (c *Conn) Read(b []byte) (int, error) {
 	return n, err
 }
 
-// Used internally by Write, the caller must not reuse the argument bytes when no error occurs
-func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error {
-	var err error
-	sessionFunc := func() {
-		// Does the packet exceed the permitted size for the session?
-		if uint16(len(msg.Message)) > c.session._getMTU() {
-			err = ConnError{errors.New("packet too big"), true, false, false, int(c.session._getMTU())}
-			return
-		}
-		// The rest of this work is session keep-alive traffic
+func (c *Conn) _write(msg FlowKeyMessage) error {
+	if len(msg.Message) > int(c.mtu) {
+		return ConnError{errors.New("packet too big"), true, false, false, int(c.mtu)}
+	}
+	c.session.EnqueueFrom(c, func() {
+		// Send the packet
+		c.session._send(msg)
+		// Session keep-alive, while we wait for the crypto workers from send
 		switch {
 		case time.Since(c.session.time) > 6*time.Second:
 			if c.session.time.Before(c.session.pingTime) && time.Since(c.session.pingTime) > 6*time.Second {
@@ -209,24 +213,25 @@ func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error {
 			c.session.ping(c.session) // TODO send from self if this becomes an actor
 		default: // Don't do anything, to keep traffic throttled
 		}
-	}
-	c.session.doFunc(sessionFunc)
-	if err == nil {
-		var cancel util.Cancellation
-		var doCancel bool
-		<-c.SyncExec(func() { cancel, doCancel = c._getDeadlineCancellation(c.writeDeadline) })
-		if doCancel {
-			defer cancel.Cancel(nil)
-		}
-		select {
-		case <-cancel.Finished():
-			if cancel.Error() == util.CancellationTimeoutError {
-				err = ConnError{errors.New("write timeout"), true, false, false, 0}
-			} else {
-				err = ConnError{errors.New("session closed"), false, false, true, 0}
-			}
-		case <-c.session.SyncExec(func() { c.session._send(msg) }):
+	})
+	return nil
+}
+
+// Used internally by Write, the caller must not reuse the argument bytes when no error occurs
+func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error {
+	var cancel util.Cancellation
+	var doCancel bool
+	<-c.SyncExec(func() { cancel, doCancel = c._getDeadlineCancellation(c.writeDeadline) })
+	var err error
+	select {
+	case <-cancel.Finished():
+		if cancel.Error() == util.CancellationTimeoutError {
+			err = ConnError{errors.New("write timeout"), true, false, false, 0}
+		} else {
+			err = ConnError{errors.New("session closed"), false, false, true, 0}
 		}
+	default:
+		<-c.SyncExec(func() { err = c._write(msg) })
 	}
 	return err
 }
diff --git a/src/yggdrasil/dialer.go b/src/yggdrasil/dialer.go
index 6b24cfb4..04410855 100644
--- a/src/yggdrasil/dialer.go
+++ b/src/yggdrasil/dialer.go
@@ -65,6 +65,7 @@ func (d *Dialer) DialByNodeIDandMask(nodeID, nodeMask *crypto.NodeID) (*Conn, er
 		conn.Close()
 		return nil, err
 	}
+	conn.session.setConn(nil, conn)
 	t := time.NewTimer(6 * time.Second) // TODO use a context instead
 	defer t.Stop()
 	select {
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 3f5e913e..854eb24d 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -71,6 +71,7 @@ type sessionInfo struct {
 	init           chan struct{}                 // Closed when the first session pong arrives, used to signal that the session is ready for initial use
 	cancel         util.Cancellation             // Used to terminate workers
 	toConn         chan []byte                   // Decrypted packets go here, picked up by the associated Conn
+	conn           *Conn                         // The associated Conn object
 	callbacks      []chan func()                 // Finished work from crypto workers
 }
 
@@ -112,6 +113,9 @@ func (s *sessionInfo) _update(p *sessionPing) bool {
 	}
 	if p.MTU >= 1280 || p.MTU == 0 {
 		s.theirMTU = p.MTU
+		if s.conn != nil {
+			s.conn.setMTU(s, s._getMTU())
+		}
 	}
 	if !bytes.Equal(s.coords, p.Coords) {
 		// allocate enough space for additional coords
@@ -368,6 +372,13 @@ func (sinfo *sessionInfo) _sendPingPong(isPong bool) {
 	}
 }
 
+func (sinfo *sessionInfo) setConn(from phony.IActor, conn *Conn) {
+	sinfo.EnqueueFrom(from, func() {
+		sinfo.conn = conn
+		sinfo.conn.setMTU(sinfo, sinfo._getMTU())
+	})
+}
+
 // Handles a session ping, creating a session if needed and calling update, then possibly responding with a pong if the ping was in ping mode and the update was successful.
 // If the session has a packet cached (common when first setting up a session), it will be sent.
 func (ss *sessions) handlePing(ping *sessionPing) {
@@ -390,6 +401,7 @@ func (ss *sessions) handlePing(ping *sessionPing) {
 			for i := range conn.nodeMask {
 				conn.nodeMask[i] = 0xFF
 			}
+			sinfo.setConn(ss.router, conn)
 			c := ss.listener.conn
 			go func() { c <- conn }()
 		}

From 9948e3d6591e905dd6095b10eae5f1232d37bd0f Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 00:44:02 -0500
Subject: [PATCH 016/130] add Conn.WriteFrom to allow actor-based sending

---
 src/yggdrasil/conn.go | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index 7121fc15..9e237436 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -217,7 +217,17 @@ func (c *Conn) _write(msg FlowKeyMessage) error {
 	return nil
 }
 
-// Used internally by Write, the caller must not reuse the argument bytes when no error occurs
+// WriteFrom should be called by a phony.IActor, and tells the Conn to send a message.
+// This is used internaly by WriteNoCopy and Write.
+// If the callback is called with a non-nil value, then it is safe to reuse the argument FlowKeyMessage.
+func (c *Conn) WriteFrom(from phony.IActor, msg FlowKeyMessage, callback func(error)) {
+	c.EnqueueFrom(from, func() {
+		callback(c._write(msg))
+	})
+}
+
+// WriteNoCopy is used internally by Write and makes use of WriteFrom under the hood.
+// The caller must not reuse the argument FlowKeyMessage when a nil error is returned.
 func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error {
 	var cancel util.Cancellation
 	var doCancel bool
@@ -231,12 +241,15 @@ func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error {
 			err = ConnError{errors.New("session closed"), false, false, true, 0}
 		}
 	default:
-		<-c.SyncExec(func() { err = c._write(msg) })
+		done := make(chan struct{})
+		callback := func(e error) { err = e; close(done) }
+		c.WriteFrom(nil, msg, callback)
+		<-done
 	}
 	return err
 }
 
-// Implements net.Conn.Write
+// Write implement the Write function of a net.Conn, and makes use of WriteNoCopy under the hood.
 func (c *Conn) Write(b []byte) (int, error) {
 	written := len(b)
 	msg := FlowKeyMessage{Message: append(util.GetBytes(), b...)}

From 1e346aaad052a23ada257f07e34355b622dafc90 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 01:52:21 -0500
Subject: [PATCH 017/130] have the conn actor receive messages from the session
 actor and either pass them to a callback or buffer them in a channel for Read
 to use if no callback was set

---
 src/yggdrasil/conn.go    | 55 ++++++++++++++++++++++++++++++++++++----
 src/yggdrasil/session.go | 20 ++-------------
 2 files changed, 52 insertions(+), 23 deletions(-)

diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index 9e237436..18c0b880 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -62,15 +62,18 @@ type Conn struct {
 	nodeMask      *crypto.NodeID
 	session       *sessionInfo
 	mtu           uint16
+	readCallback  func([]byte)
+	readBuffer    chan []byte
 }
 
 // TODO func NewConn() that initializes additional fields as needed
 func newConn(core *Core, nodeID *crypto.NodeID, nodeMask *crypto.NodeID, session *sessionInfo) *Conn {
 	conn := Conn{
-		core:     core,
-		nodeID:   nodeID,
-		nodeMask: nodeMask,
-		session:  session,
+		core:       core,
+		nodeID:     nodeID,
+		nodeMask:   nodeMask,
+		session:    session,
+		readBuffer: make(chan []byte, 1024),
 	}
 	return &conn
 }
@@ -154,6 +157,45 @@ func (c *Conn) _getDeadlineCancellation(t *time.Time) (util.Cancellation, bool)
 	}
 }
 
+// SetReadCallback sets a callback which will be called whenever a packet is received.
+// Note that calls to Read will fail if the callback has been set to a non-nil value.
+func (c *Conn) SetReadCallback(callback func([]byte)) {
+	c.EnqueueFrom(nil, func() {
+		c._setReadCallback(callback)
+	})
+}
+
+func (c *Conn) _setReadCallback(callback func([]byte)) {
+	c.readCallback = callback
+	c._drainReadBuffer()
+}
+
+func (c *Conn) _drainReadBuffer() {
+	if c.readCallback == nil {
+		return
+	}
+	select {
+	case bs := <-c.readBuffer:
+		c.readCallback(bs)
+		c.EnqueueFrom(nil, c._drainReadBuffer) // In case there's more
+	default:
+	}
+}
+
+// Called by the session to pass a new message to the Conn
+func (c *Conn) recvMsg(from phony.IActor, msg []byte) {
+	c.EnqueueFrom(from, func() {
+		if c.readCallback != nil {
+			c.readCallback(msg)
+		} else {
+			select {
+			case c.readBuffer <- msg:
+			default:
+			}
+		}
+	})
+}
+
 // Used internally by Read, the caller is responsible for util.PutBytes when they're done.
 func (c *Conn) ReadNoCopy() ([]byte, error) {
 	var cancel util.Cancellation
@@ -170,7 +212,7 @@ func (c *Conn) ReadNoCopy() ([]byte, error) {
 		} else {
 			return nil, ConnError{errors.New("session closed"), false, false, true, 0}
 		}
-	case bs := <-c.session.toConn:
+	case bs := <-c.readBuffer:
 		return bs, nil
 	}
 }
@@ -278,6 +320,7 @@ func (c *Conn) LocalAddr() crypto.NodeID {
 }
 
 func (c *Conn) RemoteAddr() crypto.NodeID {
+	// TODO warn that this can block while waiting for the Conn actor to run, so don't call it from other actors...
 	var n crypto.NodeID
 	<-c.SyncExec(func() { n = *c.nodeID })
 	return n
@@ -290,11 +333,13 @@ func (c *Conn) SetDeadline(t time.Time) error {
 }
 
 func (c *Conn) SetReadDeadline(t time.Time) error {
+	// TODO warn that this can block while waiting for the Conn actor to run, so don't call it from other actors...
 	<-c.SyncExec(func() { c.readDeadline = &t })
 	return nil
 }
 
 func (c *Conn) SetWriteDeadline(t time.Time) error {
+	// TODO warn that this can block while waiting for the Conn actor to run, so don't call it from other actors...
 	<-c.SyncExec(func() { c.writeDeadline = &t })
 	return nil
 }
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 854eb24d..a483a059 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -70,7 +70,6 @@ type sessionInfo struct {
 	bytesRecvd     uint64                        // Bytes of real traffic received in this session
 	init           chan struct{}                 // Closed when the first session pong arrives, used to signal that the session is ready for initial use
 	cancel         util.Cancellation             // Used to terminate workers
-	toConn         chan []byte                   // Decrypted packets go here, picked up by the associated Conn
 	conn           *Conn                         // The associated Conn object
 	callbacks      []chan func()                 // Finished work from crypto workers
 }
@@ -254,7 +253,6 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 	sinfo.myHandle = *crypto.NewHandle()
 	sinfo.theirAddr = *address.AddrForNodeID(crypto.GetNodeID(&sinfo.theirPermPub))
 	sinfo.theirSubnet = *address.SubnetForNodeID(crypto.GetNodeID(&sinfo.theirPermPub))
-	sinfo.toConn = make(chan []byte, 32)
 	ss.sinfos[sinfo.myHandle] = &sinfo
 	ss.byTheirPerm[sinfo.theirPermPub] = &sinfo.myHandle
 	go func() {
@@ -505,15 +503,10 @@ func (sinfo *sessionInfo) _recvPacket(p *wire_trafficPacket) {
 		util.PutBytes(p.Payload)
 		return
 	}
-	switch {
-	case sinfo._nonceIsOK(&p.Nonce):
-	case len(sinfo.toConn) < cap(sinfo.toConn):
-	default:
-		// We're either full or don't like this nonce
+	if !sinfo._nonceIsOK(&p.Nonce) {
 		util.PutBytes(p.Payload)
 		return
 	}
-
 	k := sinfo.sharedSesKey
 	var isOK bool
 	var bs []byte
@@ -530,16 +523,7 @@ func (sinfo *sessionInfo) _recvPacket(p *wire_trafficPacket) {
 			sinfo._updateNonce(&p.Nonce)
 			sinfo.time = time.Now()
 			sinfo.bytesRecvd += uint64(len(bs))
-			select {
-			case sinfo.toConn <- bs:
-			case <-sinfo.cancel.Finished():
-				util.PutBytes(bs)
-			default:
-				// We seem to have filled up the buffer in the mean time
-				// Since we need to not block, but the conn isn't an actor, we need to drop this packet
-				// TODO find some nicer way to interact with the Conn...
-				util.PutBytes(bs)
-			}
+			sinfo.conn.recvMsg(sinfo, bs)
 		}
 		ch <- callback
 		sinfo.checkCallbacks()

From b582c444f876cf50866fefdd70cf88357fd87f3a Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 01:57:08 -0500
Subject: [PATCH 018/130] minor cleanup

---
 src/yggdrasil/conn.go | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index 18c0b880..2f69139e 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -158,18 +158,13 @@ func (c *Conn) _getDeadlineCancellation(t *time.Time) (util.Cancellation, bool)
 }
 
 // SetReadCallback sets a callback which will be called whenever a packet is received.
-// Note that calls to Read will fail if the callback has been set to a non-nil value.
 func (c *Conn) SetReadCallback(callback func([]byte)) {
 	c.EnqueueFrom(nil, func() {
-		c._setReadCallback(callback)
+		c.readCallback = callback
+		c._drainReadBuffer()
 	})
 }
 
-func (c *Conn) _setReadCallback(callback func([]byte)) {
-	c.readCallback = callback
-	c._drainReadBuffer()
-}
-
 func (c *Conn) _drainReadBuffer() {
 	if c.readCallback == nil {
 		return

From 4893a07696c25aaed69f71cb4304f7f3312764b6 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 11:38:47 -0500
Subject: [PATCH 019/130] start migrating tunConn to the actor model

---
 src/tuntap/conn.go  | 183 ++++++++++++++++++++++++++++++++++++++++++++
 src/tuntap/iface.go |   4 +-
 src/tuntap/tun.go   |   9 ++-
 3 files changed, 192 insertions(+), 4 deletions(-)

diff --git a/src/tuntap/conn.go b/src/tuntap/conn.go
index 3fb7a544..32998e99 100644
--- a/src/tuntap/conn.go
+++ b/src/tuntap/conn.go
@@ -5,6 +5,7 @@ import (
 	"errors"
 	"time"
 
+	"github.com/Arceliar/phony"
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
@@ -16,6 +17,7 @@ import (
 const tunConnTimeout = 2 * time.Minute
 
 type tunConn struct {
+	phony.Actor
 	tun   *TunAdapter
 	conn  *yggdrasil.Conn
 	addr  address.Address
@@ -45,6 +47,83 @@ func (s *tunConn) _close_nomutex() {
 	}()
 }
 
+func (s *tunConn) _read(bs []byte) (err error) {
+	select {
+	case <-s.stop:
+		err = errors.New("session was already closed")
+		util.PutBytes(bs)
+		return
+	default:
+	}
+	if len(bs) == 0 {
+		err = errors.New("read packet with 0 size")
+		util.PutBytes(bs)
+		return
+	}
+	ipv4 := len(bs) > 20 && bs[0]&0xf0 == 0x40
+	ipv6 := len(bs) > 40 && bs[0]&0xf0 == 0x60
+	isCGA := true
+	// Check source addresses
+	switch {
+	case ipv6 && bs[8] == 0x02 && bytes.Equal(s.addr[:16], bs[8:24]): // source
+	case ipv6 && bs[8] == 0x03 && bytes.Equal(s.snet[:8], bs[8:16]): // source
+	default:
+		isCGA = false
+	}
+	// Check destiantion addresses
+	switch {
+	case ipv6 && bs[24] == 0x02 && bytes.Equal(s.tun.addr[:16], bs[24:40]): // destination
+	case ipv6 && bs[24] == 0x03 && bytes.Equal(s.tun.subnet[:8], bs[24:32]): // destination
+	default:
+		isCGA = false
+	}
+	// Decide how to handle the packet
+	var skip bool
+	switch {
+	case isCGA: // Allowed
+	case s.tun.ckr.isEnabled() && (ipv4 || ipv6):
+		var srcAddr address.Address
+		var dstAddr address.Address
+		var addrlen int
+		if ipv4 {
+			copy(srcAddr[:], bs[12:16])
+			copy(dstAddr[:], bs[16:20])
+			addrlen = 4
+		}
+		if ipv6 {
+			copy(srcAddr[:], bs[8:24])
+			copy(dstAddr[:], bs[24:40])
+			addrlen = 16
+		}
+		if !s.tun.ckr.isValidLocalAddress(dstAddr, addrlen) {
+			// The destination address isn't in our CKR allowed range
+			skip = true
+		} else if key, err := s.tun.ckr.getPublicKeyForAddress(srcAddr, addrlen); err == nil {
+			srcNodeID := crypto.GetNodeID(&key)
+			if s.conn.RemoteAddr() == *srcNodeID {
+				// This is the one allowed CKR case, where source and destination addresses are both good
+			} else {
+				// The CKR key associated with this address doesn't match the sender's NodeID
+				skip = true
+			}
+		} else {
+			// We have no CKR route for this source address
+			skip = true
+		}
+	default:
+		skip = true
+	}
+	if skip {
+		err = errors.New("address not allowed")
+		util.PutBytes(bs)
+		return
+	}
+	// FIXME this send can block if the tuntap isn't running, which isn't really safe...
+	s.tun.send <- bs
+	s.stillAlive()
+	return
+}
+
 func (s *tunConn) reader() (err error) {
 	select {
 	case _, ok := <-s.stop:
@@ -137,6 +216,110 @@ func (s *tunConn) reader() (err error) {
 	}
 }
 
+func (s *tunConn) _write(bs []byte) (err error) {
+	select {
+	case <-s.stop:
+		err = errors.New("session was already closed")
+		util.PutBytes(bs)
+		return
+	default:
+	}
+	v4 := len(bs) > 20 && bs[0]&0xf0 == 0x40
+	v6 := len(bs) > 40 && bs[0]&0xf0 == 0x60
+	isCGA := true
+	// Check source addresses
+	switch {
+	case v6 && bs[8] == 0x02 && bytes.Equal(s.tun.addr[:16], bs[8:24]): // source
+	case v6 && bs[8] == 0x03 && bytes.Equal(s.tun.subnet[:8], bs[8:16]): // source
+	default:
+		isCGA = false
+	}
+	// Check destiantion addresses
+	switch {
+	case v6 && bs[24] == 0x02 && bytes.Equal(s.addr[:16], bs[24:40]): // destination
+	case v6 && bs[24] == 0x03 && bytes.Equal(s.snet[:8], bs[24:32]): // destination
+	default:
+		isCGA = false
+	}
+	// Decide how to handle the packet
+	var skip bool
+	switch {
+	case isCGA: // Allowed
+	case s.tun.ckr.isEnabled() && (v4 || v6):
+		var srcAddr address.Address
+		var dstAddr address.Address
+		var addrlen int
+		if v4 {
+			copy(srcAddr[:], bs[12:16])
+			copy(dstAddr[:], bs[16:20])
+			addrlen = 4
+		}
+		if v6 {
+			copy(srcAddr[:], bs[8:24])
+			copy(dstAddr[:], bs[24:40])
+			addrlen = 16
+		}
+		if !s.tun.ckr.isValidLocalAddress(srcAddr, addrlen) {
+			// The source address isn't in our CKR allowed range
+			skip = true
+		} else if key, err := s.tun.ckr.getPublicKeyForAddress(dstAddr, addrlen); err == nil {
+			dstNodeID := crypto.GetNodeID(&key)
+			if s.conn.RemoteAddr() == *dstNodeID {
+				// This is the one allowed CKR case, where source and destination addresses are both good
+			} else {
+				// The CKR key associated with this address doesn't match the sender's NodeID
+				skip = true
+			}
+		} else {
+			// We have no CKR route for this destination address... why do we have the packet in the first place?
+			skip = true
+		}
+	default:
+		skip = true
+	}
+	if skip {
+		err = errors.New("address not allowed")
+		util.PutBytes(bs)
+		return
+	}
+	msg := yggdrasil.FlowKeyMessage{
+		FlowKey: util.GetFlowKey(bs),
+		Message: bs,
+	}
+	s.conn.WriteFrom(s, msg, func(err error) {
+		if err == nil {
+			// No point in wasting resources to send back an error if there was none
+			return
+		}
+		s.EnqueueFrom(s.conn, func() {
+			if e, eok := err.(yggdrasil.ConnError); !eok {
+				if e.Closed() {
+					s.tun.log.Debugln(s.conn.String(), "TUN/TAP generic write debug:", err)
+				} else {
+					s.tun.log.Errorln(s.conn.String(), "TUN/TAP generic write error:", err)
+				}
+			} else if e.PacketTooBig() {
+				// TODO: This currently isn't aware of IPv4 for CKR
+				ptb := &icmp.PacketTooBig{
+					MTU:  int(e.PacketMaximumSize()),
+					Data: bs[:900],
+				}
+				if packet, err := CreateICMPv6(bs[8:24], bs[24:40], ipv6.ICMPTypePacketTooBig, 0, ptb); err == nil {
+					s.tun.send <- packet
+				}
+			} else {
+				if e.Closed() {
+					s.tun.log.Debugln(s.conn.String(), "TUN/TAP conn write debug:", err)
+				} else {
+					s.tun.log.Errorln(s.conn.String(), "TUN/TAP conn write error:", err)
+				}
+			}
+		})
+	})
+	s.stillAlive()
+	return
+}
+
 func (s *tunConn) writer() error {
 	select {
 	case _, ok := <-s.stop:
diff --git a/src/tuntap/iface.go b/src/tuntap/iface.go
index 35657a9f..e0693e56 100644
--- a/src/tuntap/iface.go
+++ b/src/tuntap/iface.go
@@ -232,7 +232,7 @@ func (tun *TunAdapter) readerPacketHandler(ch chan []byte) {
 				if tc != nil {
 					for _, packet := range packets {
 						p := packet // Possibly required because of how range
-						tc.send <- p
+						<-tc.SyncExec(func() { tc._write(p) })
 					}
 				}
 			}()
@@ -242,7 +242,7 @@ func (tun *TunAdapter) readerPacketHandler(ch chan []byte) {
 		}
 		// If we have a connection now, try writing to it
 		if isIn && session != nil {
-			session.send <- bs
+			<-session.SyncExec(func() { session._write(bs) })
 		}
 	}
 }
diff --git a/src/tuntap/tun.go b/src/tuntap/tun.go
index eef05b87..dfc9ac3a 100644
--- a/src/tuntap/tun.go
+++ b/src/tuntap/tun.go
@@ -257,8 +257,13 @@ func (tun *TunAdapter) wrap(conn *yggdrasil.Conn) (c *tunConn, err error) {
 	tun.addrToConn[s.addr] = &s
 	tun.subnetToConn[s.snet] = &s
 	// Start the connection goroutines
-	go s.reader()
-	go s.writer()
+	conn.SetReadCallback(func(bs []byte) {
+		s.EnqueueFrom(conn, func() {
+			s._read(bs)
+		})
+	})
+	//go s.reader()
+	//go s.writer()
 	go s.checkForTimeouts()
 	// Return
 	return c, err

From ef15a6bd794145e49abdab53ca7247004046af3b Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 11:44:21 -0500
Subject: [PATCH 020/130] tunConn cleanup

---
 src/tuntap/conn.go | 202 ---------------------------------------------
 src/tuntap/tun.go  |   5 +-
 2 files changed, 1 insertion(+), 206 deletions(-)

diff --git a/src/tuntap/conn.go b/src/tuntap/conn.go
index 32998e99..744ca9fc 100644
--- a/src/tuntap/conn.go
+++ b/src/tuntap/conn.go
@@ -22,7 +22,6 @@ type tunConn struct {
 	conn  *yggdrasil.Conn
 	addr  address.Address
 	snet  address.Subnet
-	send  chan []byte
 	stop  chan struct{}
 	alive chan struct{}
 }
@@ -124,98 +123,6 @@ func (s *tunConn) _read(bs []byte) (err error) {
 	return
 }
 
-func (s *tunConn) reader() (err error) {
-	select {
-	case _, ok := <-s.stop:
-		if !ok {
-			return errors.New("session was already closed")
-		}
-	default:
-	}
-	s.tun.log.Debugln("Starting conn reader for", s.conn.String())
-	defer s.tun.log.Debugln("Stopping conn reader for", s.conn.String())
-	for {
-		select {
-		case <-s.stop:
-			return nil
-		default:
-		}
-		var bs []byte
-		if bs, err = s.conn.ReadNoCopy(); err != nil {
-			if e, eok := err.(yggdrasil.ConnError); eok && !e.Temporary() {
-				if e.Closed() {
-					s.tun.log.Debugln(s.conn.String(), "TUN/TAP conn read debug:", err)
-				} else {
-					s.tun.log.Errorln(s.conn.String(), "TUN/TAP conn read error:", err)
-				}
-				return e
-			}
-		} else if len(bs) > 0 {
-			ipv4 := len(bs) > 20 && bs[0]&0xf0 == 0x40
-			ipv6 := len(bs) > 40 && bs[0]&0xf0 == 0x60
-			isCGA := true
-			// Check source addresses
-			switch {
-			case ipv6 && bs[8] == 0x02 && bytes.Equal(s.addr[:16], bs[8:24]): // source
-			case ipv6 && bs[8] == 0x03 && bytes.Equal(s.snet[:8], bs[8:16]): // source
-			default:
-				isCGA = false
-			}
-			// Check destiantion addresses
-			switch {
-			case ipv6 && bs[24] == 0x02 && bytes.Equal(s.tun.addr[:16], bs[24:40]): // destination
-			case ipv6 && bs[24] == 0x03 && bytes.Equal(s.tun.subnet[:8], bs[24:32]): // destination
-			default:
-				isCGA = false
-			}
-			// Decide how to handle the packet
-			var skip bool
-			switch {
-			case isCGA: // Allowed
-			case s.tun.ckr.isEnabled() && (ipv4 || ipv6):
-				var srcAddr address.Address
-				var dstAddr address.Address
-				var addrlen int
-				if ipv4 {
-					copy(srcAddr[:], bs[12:16])
-					copy(dstAddr[:], bs[16:20])
-					addrlen = 4
-				}
-				if ipv6 {
-					copy(srcAddr[:], bs[8:24])
-					copy(dstAddr[:], bs[24:40])
-					addrlen = 16
-				}
-				if !s.tun.ckr.isValidLocalAddress(dstAddr, addrlen) {
-					// The destination address isn't in our CKR allowed range
-					skip = true
-				} else if key, err := s.tun.ckr.getPublicKeyForAddress(srcAddr, addrlen); err == nil {
-					srcNodeID := crypto.GetNodeID(&key)
-					if s.conn.RemoteAddr() == *srcNodeID {
-						// This is the one allowed CKR case, where source and destination addresses are both good
-					} else {
-						// The CKR key associated with this address doesn't match the sender's NodeID
-						skip = true
-					}
-				} else {
-					// We have no CKR route for this source address
-					skip = true
-				}
-			default:
-				skip = true
-			}
-			if skip {
-				util.PutBytes(bs)
-				continue
-			}
-			s.tun.send <- bs
-			s.stillAlive()
-		} else {
-			util.PutBytes(bs)
-		}
-	}
-}
-
 func (s *tunConn) _write(bs []byte) (err error) {
 	select {
 	case <-s.stop:
@@ -320,115 +227,6 @@ func (s *tunConn) _write(bs []byte) (err error) {
 	return
 }
 
-func (s *tunConn) writer() error {
-	select {
-	case _, ok := <-s.stop:
-		if !ok {
-			return errors.New("session was already closed")
-		}
-	default:
-	}
-	s.tun.log.Debugln("Starting conn writer for", s.conn.String())
-	defer s.tun.log.Debugln("Stopping conn writer for", s.conn.String())
-	for {
-		select {
-		case <-s.stop:
-			return nil
-		case bs, ok := <-s.send:
-			if !ok {
-				return errors.New("send closed")
-			}
-			v4 := len(bs) > 20 && bs[0]&0xf0 == 0x40
-			v6 := len(bs) > 40 && bs[0]&0xf0 == 0x60
-			isCGA := true
-			// Check source addresses
-			switch {
-			case v6 && bs[8] == 0x02 && bytes.Equal(s.tun.addr[:16], bs[8:24]): // source
-			case v6 && bs[8] == 0x03 && bytes.Equal(s.tun.subnet[:8], bs[8:16]): // source
-			default:
-				isCGA = false
-			}
-			// Check destiantion addresses
-			switch {
-			case v6 && bs[24] == 0x02 && bytes.Equal(s.addr[:16], bs[24:40]): // destination
-			case v6 && bs[24] == 0x03 && bytes.Equal(s.snet[:8], bs[24:32]): // destination
-			default:
-				isCGA = false
-			}
-			// Decide how to handle the packet
-			var skip bool
-			switch {
-			case isCGA: // Allowed
-			case s.tun.ckr.isEnabled() && (v4 || v6):
-				var srcAddr address.Address
-				var dstAddr address.Address
-				var addrlen int
-				if v4 {
-					copy(srcAddr[:], bs[12:16])
-					copy(dstAddr[:], bs[16:20])
-					addrlen = 4
-				}
-				if v6 {
-					copy(srcAddr[:], bs[8:24])
-					copy(dstAddr[:], bs[24:40])
-					addrlen = 16
-				}
-				if !s.tun.ckr.isValidLocalAddress(srcAddr, addrlen) {
-					// The source address isn't in our CKR allowed range
-					skip = true
-				} else if key, err := s.tun.ckr.getPublicKeyForAddress(dstAddr, addrlen); err == nil {
-					dstNodeID := crypto.GetNodeID(&key)
-					if s.conn.RemoteAddr() == *dstNodeID {
-						// This is the one allowed CKR case, where source and destination addresses are both good
-					} else {
-						// The CKR key associated with this address doesn't match the sender's NodeID
-						skip = true
-					}
-				} else {
-					// We have no CKR route for this destination address... why do we have the packet in the first place?
-					skip = true
-				}
-			default:
-				skip = true
-			}
-			if skip {
-				util.PutBytes(bs)
-				continue
-			}
-			msg := yggdrasil.FlowKeyMessage{
-				FlowKey: util.GetFlowKey(bs),
-				Message: bs,
-			}
-			if err := s.conn.WriteNoCopy(msg); err != nil {
-				if e, eok := err.(yggdrasil.ConnError); !eok {
-					if e.Closed() {
-						s.tun.log.Debugln(s.conn.String(), "TUN/TAP generic write debug:", err)
-					} else {
-						s.tun.log.Errorln(s.conn.String(), "TUN/TAP generic write error:", err)
-					}
-				} else if e.PacketTooBig() {
-					// TODO: This currently isn't aware of IPv4 for CKR
-					ptb := &icmp.PacketTooBig{
-						MTU:  int(e.PacketMaximumSize()),
-						Data: bs[:900],
-					}
-					if packet, err := CreateICMPv6(bs[8:24], bs[24:40], ipv6.ICMPTypePacketTooBig, 0, ptb); err == nil {
-						s.tun.send <- packet
-					}
-				} else {
-					if e.Closed() {
-						s.tun.log.Debugln(s.conn.String(), "TUN/TAP conn write debug:", err)
-					} else {
-						s.tun.log.Errorln(s.conn.String(), "TUN/TAP conn write error:", err)
-					}
-				}
-			} else {
-				s.stillAlive()
-			}
-		}
-	}
-}
-
 func (s *tunConn) stillAlive() {
 	defer func() { recover() }()
 	select {
diff --git a/src/tuntap/tun.go b/src/tuntap/tun.go
index dfc9ac3a..895f893b 100644
--- a/src/tuntap/tun.go
+++ b/src/tuntap/tun.go
@@ -229,7 +229,6 @@ func (tun *TunAdapter) wrap(conn *yggdrasil.Conn) (c *tunConn, err error) {
 	s := tunConn{
 		tun:   tun,
 		conn:  conn,
-		send:  make(chan []byte, 32), // TODO: is this a sensible value?
 		stop:  make(chan struct{}),
 		alive: make(chan struct{}, 1),
 	}
@@ -256,14 +255,12 @@ func (tun *TunAdapter) wrap(conn *yggdrasil.Conn) (c *tunConn, err error) {
 	// we receive a packet through the interface for this address
 	tun.addrToConn[s.addr] = &s
 	tun.subnetToConn[s.snet] = &s
-	// Start the connection goroutines
+	// Set the read callback and start the timeout goroutine
 	conn.SetReadCallback(func(bs []byte) {
 		s.EnqueueFrom(conn, func() {
 			s._read(bs)
 		})
 	})
-	//go s.reader()
-	//go s.writer()
 	go s.checkForTimeouts()
 	// Return
 	return c, err

From 775fb535dce6ad3d1639dff10fa50d518a9f041b Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 12:46:24 -0500
Subject: [PATCH 021/130] start converting the peer struct into an actor

---
 src/yggdrasil/peer.go   | 34 +++++++++++++++++++++-------------
 src/yggdrasil/switch.go |  8 +++++---
 2 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 8869bd2a..851a376a 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -12,6 +12,8 @@ import (
 
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
+
+	"github.com/Arceliar/phony"
 )
 
 // The peers struct represents peers with an active connection.
@@ -97,6 +99,7 @@ type peer struct {
 	bytesSent  uint64 // To track bandwidth usage for getPeers
 	bytesRecvd uint64 // To track bandwidth usage for getPeers
 	// BUG: sync/atomic, 32 bit platforms need the above to be the first element
+	phony.Actor
 	core       *Core
 	intf       *linkInterface
 	port       switchPort
@@ -206,7 +209,7 @@ func (p *peer) linkLoop() {
 			if !ok {
 				return
 			}
-			p.sendSwitchMsg()
+			<-p.SyncExec(p._sendSwitchMsg)
 		case dinfo = <-p.dinfo:
 		case _ = <-tick.C:
 			if dinfo != nil {
@@ -227,20 +230,19 @@ func (p *peer) handlePacket(packet []byte) {
 	}
 	switch pType {
 	case wire_Traffic:
-		p.handleTraffic(packet, pTypeLen)
+		p._handleTraffic(packet, pTypeLen)
 	case wire_ProtocolTraffic:
-		p.handleTraffic(packet, pTypeLen)
+		p._handleTraffic(packet, pTypeLen)
 	case wire_LinkProtocolTraffic:
-		p.handleLinkTraffic(packet)
+		p._handleLinkTraffic(packet)
 	default:
 		util.PutBytes(packet)
 	}
-	return
 }
 
 // Called to handle traffic or protocolTraffic packets.
 // In either case, this reads from the coords of the packet header, does a switch lookup, and forwards to the next node.
-func (p *peer) handleTraffic(packet []byte, pTypeLen int) {
+func (p *peer) _handleTraffic(packet []byte, pTypeLen int) {
 	table := p.core.switchTable.getTable()
 	if _, isIn := table.elems[p.port]; !isIn && p.port != 0 {
 		// Drop traffic if the peer isn't in the switch
@@ -249,8 +251,14 @@ func (p *peer) handleTraffic(packet []byte, pTypeLen int) {
 	p.core.switchTable.packetIn <- packet
 }
 
+func (p *peer) sendPacketsFrom(from phony.IActor, packets [][]byte) {
+	p.EnqueueFrom(from, func() {
+		p._sendPackets(packets)
+	})
+}
+
 // This just calls p.out(packet) for now.
-func (p *peer) sendPackets(packets [][]byte) {
+func (p *peer) _sendPackets(packets [][]byte) {
 	// Is there ever a case where something more complicated is needed?
 	// What if p.out blocks?
 	var size int
@@ -263,7 +271,7 @@ func (p *peer) sendPackets(packets [][]byte) {
 
 // This wraps the packet in the inner (ephemeral) and outer (permanent) crypto layers.
 // It sends it to p.linkOut, which bypasses the usual packet queues.
-func (p *peer) sendLinkPacket(packet []byte) {
+func (p *peer) _sendLinkPacket(packet []byte) {
 	innerPayload, innerNonce := crypto.BoxSeal(&p.linkShared, packet, nil)
 	innerLinkPacket := wire_linkProtoTrafficPacket{
 		Nonce:   *innerNonce,
@@ -281,7 +289,7 @@ func (p *peer) sendLinkPacket(packet []byte) {
 
 // Decrypts the outer (permanent) and inner (ephemeral) crypto layers on link traffic.
 // Identifies the link traffic type and calls the appropriate handler.
-func (p *peer) handleLinkTraffic(bs []byte) {
+func (p *peer) _handleLinkTraffic(bs []byte) {
 	packet := wire_linkProtoTrafficPacket{}
 	if !packet.decode(bs) {
 		return
@@ -304,14 +312,14 @@ func (p *peer) handleLinkTraffic(bs []byte) {
 	}
 	switch pType {
 	case wire_SwitchMsg:
-		p.handleSwitchMsg(payload)
+		p._handleSwitchMsg(payload)
 	default:
 		util.PutBytes(bs)
 	}
 }
 
 // Gets a switchMsg from the switch, adds signed next-hop info for this peer, and sends it to them.
-func (p *peer) sendSwitchMsg() {
+func (p *peer) _sendSwitchMsg() {
 	msg := p.core.switchTable.getMsg()
 	if msg == nil {
 		return
@@ -323,12 +331,12 @@ func (p *peer) sendSwitchMsg() {
 		Sig:  *crypto.Sign(&p.core.sigPriv, bs),
 	})
 	packet := msg.encode()
-	p.sendLinkPacket(packet)
+	p._sendLinkPacket(packet)
 }
 
 // Handles a switchMsg from the peer, checking signatures and passing good messages to the switch.
 // Also creates a dhtInfo struct and arranges for it to be added to the dht (this is how dht bootstrapping begins).
-func (p *peer) handleSwitchMsg(packet []byte) {
+func (p *peer) _handleSwitchMsg(packet []byte) {
 	var msg switchMsg
 	if !msg.decode(packet) {
 		return
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 1fa75a6c..d87b7880 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -703,7 +703,7 @@ func (t *switchTable) handleIn(packet []byte, idle map[switchPort]time.Time) boo
 	if best != nil {
 		// Send to the best idle next hop
 		delete(idle, best.port)
-		best.sendPackets([][]byte{packet})
+		best.sendPacketsFrom(nil, [][]byte{packet})
 		return true
 	}
 	// Didn't find anyone idle to send it to
@@ -817,7 +817,8 @@ func (t *switchTable) handleIdle(port switchPort) bool {
 		}
 	}
 	if len(packets) > 0 {
-		to.sendPackets(packets)
+		// TODO rewrite if/when the switch becomes an actor
+		to.sendPacketsFrom(nil, packets)
 		return true
 	}
 	return false
@@ -830,7 +831,8 @@ func (t *switchTable) doWorker() {
 		// Keep sending packets to the router
 		self := t.core.peers.getPorts()[0]
 		for bs := range sendingToRouter {
-			self.sendPackets([][]byte{bs})
+			// TODO remove this ugly mess of goroutines if/when the switch becomes an actor
+			<-self.SyncExec(func() { self._sendPackets([][]byte{bs}) })
 		}
 	}()
 	go func() {

From 88161009e94740511fb25b73dbb5b19c68f00af1 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 12:55:49 -0500
Subject: [PATCH 022/130] more peer migration

---
 src/yggdrasil/link.go   | 3 ++-
 src/yggdrasil/peer.go   | 8 +++++++-
 src/yggdrasil/router.go | 7 ++-----
 3 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index 4ce374b4..824afd34 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -387,7 +387,8 @@ func (intf *linkInterface) handler() error {
 	for {
 		msg, err := intf.msgIO.readMsg()
 		if len(msg) > 0 {
-			intf.peer.handlePacket(msg)
+			// TODO rewrite this if the link becomes an actor
+			<-intf.peer.SyncExec(func() { intf.peer._handlePacket(msg) })
 		}
 		if err != nil {
 			if err != io.EOF {
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 851a376a..22db88d1 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -219,9 +219,15 @@ func (p *peer) linkLoop() {
 	}
 }
 
+func (p *peer) handlePacketFrom(from phony.IActor, packet []byte) {
+	p.EnqueueFrom(from, func() {
+		p._handlePacket(packet)
+	})
+}
+
 // Called to handle incoming packets.
 // Passes the packet to a handler for that packet type.
-func (p *peer) handlePacket(packet []byte) {
+func (p *peer) _handlePacket(packet []byte) {
 	// FIXME this is off by stream padding and msg length overhead, should be done in tcp.go
 	atomic.AddUint64(&p.bytesRecvd, uint64(len(packet)))
 	pType, pTypeLen := wire_decode_uint64(packet)
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index 464a4778..adf1b1d4 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -64,11 +64,8 @@ func (r *router) init(core *Core) {
 		},
 	}
 	p := r.core.peers.newPeer(&r.core.boxPub, &r.core.sigPub, &crypto.BoxSharedKey{}, &self, nil)
-	p.out = func(packets [][]byte) {
-		// TODO make peers and/or the switch into actors, have them pass themselves as the from field
-		r.handlePackets(nil, packets)
-	}
-	r.out = p.handlePacket // TODO if the peer becomes its own actor, then send a message here
+	p.out = func(packets [][]byte) { r.handlePackets(p, packets) }
+	r.out = func(bs []byte) { p.handlePacketFrom(r, bs) }
 	r.nodeinfo.init(r.core)
 	r.core.config.Mutex.RLock()
 	r.nodeinfo.setNodeInfo(r.core.config.Current.NodeInfo, r.core.config.Current.NodeInfoPrivacy)

From ecd23ce9fc1e46c016697c4ac945162b9fb159e5 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 12:59:20 -0500
Subject: [PATCH 023/130] safer linkloop

---
 src/yggdrasil/peer.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 22db88d1..8c38c06a 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -213,7 +213,7 @@ func (p *peer) linkLoop() {
 		case dinfo = <-p.dinfo:
 		case _ = <-tick.C:
 			if dinfo != nil {
-				p.core.router.insertPeer(nil, dinfo)
+				<-p.SyncExec(func() { p.core.router.insertPeer(p, dinfo) })
 			}
 		}
 	}

From 034fece33f4fbabbf2bdf8c4fb643b91791438a7 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 13:15:29 -0500
Subject: [PATCH 024/130] more peer migration

---
 src/yggdrasil/peer.go   | 50 ++++++++++++++++-------------------------
 src/yggdrasil/switch.go |  4 ++--
 2 files changed, 21 insertions(+), 33 deletions(-)

diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 8c38c06a..c94cf266 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -110,9 +110,9 @@ type peer struct {
 	endpoint   string
 	firstSeen  time.Time       // To track uptime for getPeers
 	linkOut    (chan []byte)   // used for protocol traffic (to bypass queues)
-	doSend     (chan struct{}) // tell the linkLoop to send a switchMsg
-	dinfo      (chan *dhtInfo) // used to keep the DHT working
+	dinfo      *dhtInfo        // used to keep the DHT working
 	out        func([][]byte)  // Set up by whatever created the peers struct, used to send packets to other nodes
+	done       (chan struct{}) // closed to exit the linkLoop
 	close      func()          // Called when a peer is removed, to close the underlying connection, or via admin api
 }
 
@@ -124,8 +124,7 @@ func (ps *peers) newPeer(box *crypto.BoxPubKey, sig *crypto.SigPubKey, linkShare
 		shared:     *crypto.GetSharedKey(&ps.core.boxPriv, box),
 		linkShared: *linkShared,
 		firstSeen:  now,
-		doSend:     make(chan struct{}, 1),
-		dinfo:      make(chan *dhtInfo, 1),
+		done:       make(chan struct{}),
 		close:      closer,
 		core:       ps.core,
 		intf:       intf,
@@ -170,29 +169,19 @@ func (ps *peers) removePeer(port switchPort) {
 		if p.close != nil {
 			p.close()
 		}
-		close(p.doSend)
+		close(p.done)
 	}
 }
 
 // If called, sends a notification to each peer that they should send a new switch message.
 // Mainly called by the switch after an update.
-func (ps *peers) sendSwitchMsgs() {
+func (ps *peers) sendSwitchMsgs(from phony.IActor) {
 	ports := ps.getPorts()
 	for _, p := range ports {
 		if p.port == 0 {
 			continue
 		}
-		p.doSendSwitchMsgs()
-	}
-}
-
-// If called, sends a notification to the peer's linkLoop to trigger a switchMsg send.
-// Mainly called by sendSwitchMsgs or during linkLoop startup.
-func (p *peer) doSendSwitchMsgs() {
-	defer func() { recover() }() // In case there's a race with close(p.doSend)
-	select {
-	case p.doSend <- struct{}{}:
-	default:
+		p.EnqueueFrom(from, p._sendSwitchMsg)
 	}
 }
 
@@ -201,24 +190,23 @@ func (p *peer) doSendSwitchMsgs() {
 func (p *peer) linkLoop() {
 	tick := time.NewTicker(time.Second)
 	defer tick.Stop()
-	p.doSendSwitchMsgs()
-	var dinfo *dhtInfo
+	<-p.SyncExec(p._sendSwitchMsg) // Startup message
 	for {
 		select {
-		case _, ok := <-p.doSend:
-			if !ok {
-				return
-			}
-			<-p.SyncExec(p._sendSwitchMsg)
-		case dinfo = <-p.dinfo:
+		case <-p.done:
+			return
 		case _ = <-tick.C:
-			if dinfo != nil {
-				<-p.SyncExec(func() { p.core.router.insertPeer(p, dinfo) })
-			}
+			<-p.SyncExec(p._updateDHT)
 		}
 	}
 }
 
+func (p *peer) _updateDHT() {
+	if p.dinfo != nil {
+		p.core.router.insertPeer(p, p.dinfo)
+	}
+}
+
 func (p *peer) handlePacketFrom(from phony.IActor, packet []byte) {
 	p.EnqueueFrom(from, func() {
 		p._handlePacket(packet)
@@ -366,16 +354,16 @@ func (p *peer) _handleSwitchMsg(packet []byte) {
 	p.core.switchTable.handleMsg(&msg, p.port)
 	if !p.core.switchTable.checkRoot(&msg) {
 		// Bad switch message
-		p.dinfo <- nil
+		p.dinfo = nil
 		return
 	}
 	// Pass a mesage to the dht informing it that this peer (still) exists
 	loc.coords = loc.coords[:len(loc.coords)-1]
-	dinfo := dhtInfo{
+	p.dinfo = &dhtInfo{
 		key:    p.box,
 		coords: loc.getCoords(),
 	}
-	p.dinfo <- &dinfo
+	p._updateDHT()
 }
 
 // This generates the bytes that we sign or check the signature of for a switchMsg.
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index d87b7880..0be01240 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -248,7 +248,7 @@ func (t *switchTable) cleanRoot() {
 			t.core.router.reset(nil)
 		}
 		t.data.locator = switchLocator{root: t.key, tstamp: now.Unix()}
-		t.core.peers.sendSwitchMsgs()
+		t.core.peers.sendSwitchMsgs(nil) // TODO update if/when the switch becomes an actor
 	}
 }
 
@@ -515,7 +515,7 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort, rep
 		}
 		t.data.locator = sender.locator
 		t.parent = sender.port
-		t.core.peers.sendSwitchMsgs()
+		t.core.peers.sendSwitchMsgs(nil) // TODO update if/when the switch becomes an actor
 	}
 	if doUpdate {
 		t.updater.Store(&sync.Once{})

From 0539dee900342a9fad784e83bea51f32459cba9f Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 13:25:38 -0500
Subject: [PATCH 025/130] warning about possible deadlock in legacy channel
 send, need to migrate the link code to fix it

---
 src/yggdrasil/peer.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index c94cf266..f3af140f 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -278,6 +278,8 @@ func (p *peer) _sendLinkPacket(packet []byte) {
 		Payload: bs,
 	}
 	packet = linkPacket.encode()
+	// TODO replace this with a message send if/when the link becomes an actor
+	// FIXME not 100% sure the channel send version is deadlock-free...
 	p.linkOut <- packet
 }
 

From b337228aa45436736640492b54cd64059f21a199 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 14:24:42 -0500
Subject: [PATCH 026/130] minor fixes to peer stuff

---
 src/yggdrasil/api.go   | 43 +++++++++++++++++++++++-------------------
 src/yggdrasil/debug.go |  2 +-
 src/yggdrasil/peer.go  | 20 +++++++++++++-------
 3 files changed, 38 insertions(+), 27 deletions(-)

diff --git a/src/yggdrasil/api.go b/src/yggdrasil/api.go
index f50c8ceb..cbf232aa 100644
--- a/src/yggdrasil/api.go
+++ b/src/yggdrasil/api.go
@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"net"
 	"sort"
-	"sync/atomic"
 	"time"
 
 	"github.com/gologme/log"
@@ -106,15 +105,18 @@ func (c *Core) GetPeers() []Peer {
 	sort.Slice(ps, func(i, j int) bool { return ps[i] < ps[j] })
 	for _, port := range ps {
 		p := ports[port]
-		info := Peer{
-			Endpoint:   p.intf.name,
-			BytesSent:  atomic.LoadUint64(&p.bytesSent),
-			BytesRecvd: atomic.LoadUint64(&p.bytesRecvd),
-			Protocol:   p.intf.info.linkType,
-			Port:       uint64(port),
-			Uptime:     time.Since(p.firstSeen),
-		}
-		copy(info.PublicKey[:], p.box[:])
+		var info Peer
+		<-p.SyncExec(func() {
+			info = Peer{
+				Endpoint:   p.intf.name,
+				BytesSent:  p.bytesSent,
+				BytesRecvd: p.bytesRecvd,
+				Protocol:   p.intf.info.linkType,
+				Port:       uint64(port),
+				Uptime:     time.Since(p.firstSeen),
+			}
+			copy(info.PublicKey[:], p.box[:])
+		})
 		peers = append(peers, info)
 	}
 	return peers
@@ -135,15 +137,18 @@ func (c *Core) GetSwitchPeers() []SwitchPeer {
 			continue
 		}
 		coords := elem.locator.getCoords()
-		info := SwitchPeer{
-			Coords:     append([]uint64{}, wire_coordsBytestoUint64s(coords)...),
-			BytesSent:  atomic.LoadUint64(&peer.bytesSent),
-			BytesRecvd: atomic.LoadUint64(&peer.bytesRecvd),
-			Port:       uint64(elem.port),
-			Protocol:   peer.intf.info.linkType,
-			Endpoint:   peer.intf.info.remote,
-		}
-		copy(info.PublicKey[:], peer.box[:])
+		var info SwitchPeer
+		<-peer.SyncExec(func() {
+			info = SwitchPeer{
+				Coords:     append([]uint64{}, wire_coordsBytestoUint64s(coords)...),
+				BytesSent:  peer.bytesSent,
+				BytesRecvd: peer.bytesRecvd,
+				Port:       uint64(elem.port),
+				Protocol:   peer.intf.info.linkType,
+				Endpoint:   peer.intf.info.remote,
+			}
+			copy(info.PublicKey[:], peer.box[:])
+		})
 		switchpeers = append(switchpeers, info)
 	}
 	return switchpeers
diff --git a/src/yggdrasil/debug.go b/src/yggdrasil/debug.go
index d90bf1f4..30eb874d 100644
--- a/src/yggdrasil/debug.go
+++ b/src/yggdrasil/debug.go
@@ -576,7 +576,7 @@ func DEBUG_simLinkPeers(p, q *peer) {
 				default:
 				}
 				if len(packets) > 0 {
-					dest.handlePacket(packets[0])
+					<-dest.SyncExec(func() { dest._handlePacket(packets[0]) })
 					packets = packets[1:]
 					continue
 				}
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index f3af140f..4bebc17e 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -96,9 +96,6 @@ func (ps *peers) putPorts(ports map[switchPort]*peer) {
 
 // Information known about a peer, including thier box/sig keys, precomputed shared keys (static and ephemeral) and a handler for their outgoing traffic
 type peer struct {
-	bytesSent  uint64 // To track bandwidth usage for getPeers
-	bytesRecvd uint64 // To track bandwidth usage for getPeers
-	// BUG: sync/atomic, 32 bit platforms need the above to be the first element
 	phony.Actor
 	core       *Core
 	intf       *linkInterface
@@ -114,6 +111,9 @@ type peer struct {
 	out        func([][]byte)  // Set up by whatever created the peers struct, used to send packets to other nodes
 	done       (chan struct{}) // closed to exit the linkLoop
 	close      func()          // Called when a peer is removed, to close the underlying connection, or via admin api
+	// The below aren't actually useful internally, they're just gathered for getPeers statistics
+	bytesSent  uint64
+	bytesRecvd uint64
 }
 
 // Creates a new peer with the specified box, sig, and linkShared keys, using the lowest unoccupied port number.
@@ -217,7 +217,7 @@ func (p *peer) handlePacketFrom(from phony.IActor, packet []byte) {
 // Passes the packet to a handler for that packet type.
 func (p *peer) _handlePacket(packet []byte) {
 	// FIXME this is off by stream padding and msg length overhead, should be done in tcp.go
-	atomic.AddUint64(&p.bytesRecvd, uint64(len(packet)))
+	p.bytesRecvd += uint64(len(packet))
 	pType, pTypeLen := wire_decode_uint64(packet)
 	if pTypeLen == 0 {
 		return
@@ -259,10 +259,12 @@ func (p *peer) _sendPackets(packets [][]byte) {
 	for _, packet := range packets {
 		size += len(packet)
 	}
-	atomic.AddUint64(&p.bytesSent, uint64(size))
+	p.bytesSent += uint64(size)
 	p.out(packets)
 }
 
+var peerLinkOutHelper phony.Actor
+
 // This wraps the packet in the inner (ephemeral) and outer (permanent) crypto layers.
 // It sends it to p.linkOut, which bypasses the usual packet queues.
 func (p *peer) _sendLinkPacket(packet []byte) {
@@ -279,8 +281,12 @@ func (p *peer) _sendLinkPacket(packet []byte) {
 	}
 	packet = linkPacket.encode()
 	// TODO replace this with a message send if/when the link becomes an actor
-	// FIXME not 100% sure the channel send version is deadlock-free...
-	p.linkOut <- packet
+	peerLinkOutHelper.EnqueueFrom(nil, func() {
+		select {
+		case p.linkOut <- packet:
+		case <-p.done:
+		}
+	})
 }
 
 // Decrypts the outer (permanent) and inner (ephemeral) crypto layers on link traffic.

From 498bc395e2ed8adc55813f8aeda24f2827c3d4d3 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 14:56:33 -0500
Subject: [PATCH 027/130] start migrating switch to the actor model

---
 src/yggdrasil/switch.go | 117 ++++++++++++++++------------------------
 1 file changed, 47 insertions(+), 70 deletions(-)

diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 0be01240..7f3c7a78 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -19,6 +19,8 @@ import (
 
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
+
+	"github.com/Arceliar/phony"
 )
 
 const (
@@ -172,12 +174,13 @@ type switchTable struct {
 	data              switchData                 //
 	updater           atomic.Value               // *sync.Once
 	table             atomic.Value               // lookupTable
+	phony.Actor                                  // Owns the below
 	packetIn          chan []byte                // Incoming packets for the worker to handle
 	idleIn            chan switchPort            // Incoming idle notifications from peer links
 	admin             chan func()                // Pass a lambda for the admin socket to query stuff
 	queues            switch_buffers             // Queues - not atomic so ONLY use through admin chan
 	queueTotalMaxSize uint64                     // Maximum combined size of queues
-	toRouter          chan []byte                // Packets to be sent to the router
+	idle              map[switchPort]time.Time   // idle peers
 }
 
 // Minimum allowed total size of switch queues.
@@ -199,7 +202,7 @@ func (t *switchTable) init(core *Core) {
 	t.idleIn = make(chan switchPort, 1024)
 	t.admin = make(chan func())
 	t.queueTotalMaxSize = SwitchQueueTotalMinSize
-	t.toRouter = make(chan []byte, 1)
+	t.idle = make(map[switchPort]time.Time)
 }
 
 // Safely gets a copy of this node's locator.
@@ -653,12 +656,13 @@ func (t *switchTable) bestPortForCoords(coords []byte) switchPort {
 // Handle an incoming packet
 // Either send it to ourself, or to the first idle peer that's free
 // Returns true if the packet has been handled somehow, false if it should be queued
-func (t *switchTable) handleIn(packet []byte, idle map[switchPort]time.Time) bool {
+func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]time.Time) bool {
 	coords := switch_getPacketCoords(packet)
 	closer := t.getCloser(coords)
 	if len(closer) == 0 {
 		// TODO? call the router directly, and remove the whole concept of a self peer?
-		t.toRouter <- packet
+		self := t.core.peers.getPorts()[0]
+		self.sendPacketsFrom(t, [][]byte{packet})
 		return true
 	}
 	var best *peer
@@ -731,7 +735,7 @@ type switch_buffers struct {
 	closer      []closerInfo // Scratch space
 }
 
-func (b *switch_buffers) cleanup(t *switchTable) {
+func (b *switch_buffers) _cleanup(t *switchTable) {
 	for streamID, buf := range b.bufs {
 		// Remove queues for which we have no next hop
 		packet := buf.packets[0]
@@ -773,14 +777,14 @@ func (b *switch_buffers) cleanup(t *switchTable) {
 // Handles incoming idle notifications
 // Loops over packets and sends the newest one that's OK for this peer to send
 // Returns true if the peer is no longer idle, false if it should be added to the idle list
-func (t *switchTable) handleIdle(port switchPort) bool {
+func (t *switchTable) _handleIdle(port switchPort) bool {
 	to := t.core.peers.getPorts()[port]
 	if to == nil {
 		return true
 	}
 	var packets [][]byte
 	var psize int
-	t.queues.cleanup(t)
+	t.queues._cleanup(t)
 	now := time.Now()
 	for psize < 65535 {
 		var best string
@@ -824,79 +828,52 @@ func (t *switchTable) handleIdle(port switchPort) bool {
 	return false
 }
 
-// The switch worker does routing lookups and sends packets to where they need to be
-func (t *switchTable) doWorker() {
-	sendingToRouter := make(chan []byte, 1)
-	go func() {
-		// Keep sending packets to the router
-		self := t.core.peers.getPorts()[0]
-		for bs := range sendingToRouter {
-			// TODO remove this ugly mess of goroutines if/when the switch becomes an actor
-			<-self.SyncExec(func() { self._sendPackets([][]byte{bs}) })
+func (t *switchTable) _packetIn(bytes []byte) {
+	// Try to send it somewhere (or drop it if it's corrupt or at a dead end)
+	if !t._handleIn(bytes, t.idle) {
+		// There's nobody free to take it right now, so queue it for later
+		packet := switch_packetInfo{bytes, time.Now()}
+		streamID := switch_getPacketStreamID(packet.bytes)
+		buf, bufExists := t.queues.bufs[streamID]
+		buf.packets = append(buf.packets, packet)
+		buf.size += uint64(len(packet.bytes))
+		t.queues.size += uint64(len(packet.bytes))
+		// Keep a track of the max total queue size
+		if t.queues.size > t.queues.maxsize {
+			t.queues.maxsize = t.queues.size
 		}
-	}()
-	go func() {
-		// Keep taking packets from the idle worker and sending them to the above whenever it's idle, keeping anything extra in a (fifo, head-drop) buffer
-		var buf [][]byte
-		var size int
-		for {
-			bs := <-t.toRouter
-			size += len(bs)
-			buf = append(buf, bs)
-			for len(buf) > 0 {
-				select {
-				case bs := <-t.toRouter:
-					size += len(bs)
-					buf = append(buf, bs)
-					for size > int(t.queueTotalMaxSize) {
-						size -= len(buf[0])
-						util.PutBytes(buf[0])
-						buf = buf[1:]
-					}
-				case sendingToRouter <- buf[0]:
-					size -= len(buf[0])
-					buf = buf[1:]
-				}
+		t.queues.bufs[streamID] = buf
+		if !bufExists {
+			// Keep a track of the max total queue count. Only recalculate this
+			// when the queue is new because otherwise repeating len(dict) might
+			// cause unnecessary processing overhead
+			if len(t.queues.bufs) > t.queues.maxbufs {
+				t.queues.maxbufs = len(t.queues.bufs)
 			}
 		}
-	}()
+		t.queues._cleanup(t)
+	}
+}
+
+func (t *switchTable) _idleIn(port switchPort) {
+	// Try to find something to send to this peer
+	if !t._handleIdle(port) {
+		// Didn't find anything ready to send yet, so stay idle
+		t.idle[port] = time.Now()
+	}
+}
+
+// The switch worker does routing lookups and sends packets to where they need to be
+func (t *switchTable) doWorker() {
 	t.queues.switchTable = t
 	t.queues.bufs = make(map[string]switch_buffer) // Packets per PacketStreamID (string)
-	idle := make(map[switchPort]time.Time)         // this is to deduplicate things
 	for {
 		//t.core.log.Debugf("Switch state: idle = %d, buffers = %d", len(idle), len(t.queues.bufs))
 		select {
 		case bytes := <-t.packetIn:
-			// Try to send it somewhere (or drop it if it's corrupt or at a dead end)
-			if !t.handleIn(bytes, idle) {
-				// There's nobody free to take it right now, so queue it for later
-				packet := switch_packetInfo{bytes, time.Now()}
-				streamID := switch_getPacketStreamID(packet.bytes)
-				buf, bufExists := t.queues.bufs[streamID]
-				buf.packets = append(buf.packets, packet)
-				buf.size += uint64(len(packet.bytes))
-				t.queues.size += uint64(len(packet.bytes))
-				// Keep a track of the max total queue size
-				if t.queues.size > t.queues.maxsize {
-					t.queues.maxsize = t.queues.size
-				}
-				t.queues.bufs[streamID] = buf
-				if !bufExists {
-					// Keep a track of the max total queue count. Only recalculate this
-					// when the queue is new because otherwise repeating len(dict) might
-					// cause unnecessary processing overhead
-					if len(t.queues.bufs) > t.queues.maxbufs {
-						t.queues.maxbufs = len(t.queues.bufs)
-					}
-				}
-				t.queues.cleanup(t)
-			}
+			<-t.SyncExec(func() { t._packetIn(bytes) })
 		case port := <-t.idleIn:
-			// Try to find something to send to this peer
-			if !t.handleIdle(port) {
-				// Didn't find anything ready to send yet, so stay idle
-				idle[port] = time.Now()
-			}
+			<-t.SyncExec(func() { t._idleIn(port) })
 		case f := <-t.admin:
 			f()
 		case e := <-t.reconfigure:

From 555b4c18d4938dd2ad70941e8ffaf95490057059 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 15:05:18 -0500
Subject: [PATCH 028/130] a little switch cleanup

---
 src/yggdrasil/switch.go | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)

diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 7f3c7a78..3f4c41fe 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -177,7 +177,6 @@ type switchTable struct {
 	phony.Actor                                  // Owns the below
 	packetIn          chan []byte                // Incoming packets for the worker to handle
 	idleIn            chan switchPort            // Incoming idle notifications from peer links
-	admin             chan func()                // Pass a lambda for the admin socket to query stuff
 	queues            switch_buffers             // Queues - not atomic so ONLY use through admin chan
 	queueTotalMaxSize uint64                     // Maximum combined size of queues
 	idle              map[switchPort]time.Time   // idle peers
@@ -200,9 +199,10 @@ func (t *switchTable) init(core *Core) {
 	t.drop = make(map[crypto.SigPubKey]int64)
 	t.packetIn = make(chan []byte, 1024)
 	t.idleIn = make(chan switchPort, 1024)
-	t.admin = make(chan func())
 	t.queueTotalMaxSize = SwitchQueueTotalMinSize
 	t.idle = make(map[switchPort]time.Time)
+	t.queues.switchTable = t
+	t.queues.bufs = make(map[string]switch_buffer) // Packets per PacketStreamID (string)
 }
 
 // Safely gets a copy of this node's locator.
@@ -865,8 +865,6 @@ func (t *switchTable) _idleIn(port switchPort) {
 
 // The switch worker does routing lookups and sends packets to where they need to be
 func (t *switchTable) doWorker() {
-	t.queues.switchTable = t
-	t.queues.bufs = make(map[string]switch_buffer) // Packets per PacketStreamID (string)
 	for {
 		//t.core.log.Debugf("Switch state: idle = %d, buffers = %d", len(idle), len(t.queues.bufs))
 		select {
@@ -874,8 +872,6 @@ func (t *switchTable) doWorker() {
 			<-t.SyncExec(func() { t._packetIn(bytes) })
 		case port := <-t.idleIn:
 			<-t.SyncExec(func() { t._idleIn(port) })
-		case f := <-t.admin:
-			f()
 		case e := <-t.reconfigure:
 			e <- nil
 		}
@@ -885,13 +881,5 @@ func (t *switchTable) doWorker() {
 // Passed a function to call.
 // This will send the function to t.admin and block until it finishes.
 func (t *switchTable) doAdmin(f func()) {
-	// Pass this a function that needs to be run by the router's main goroutine
-	// It will pass the function to the router and wait for the router to finish
-	done := make(chan struct{})
-	newF := func() {
-		f()
-		close(done)
-	}
-	t.admin <- newF
-	<-done
+	<-t.SyncExec(f)
 }

From 998c76fd8cd3c97131a4ba31a24b53e0475b8f10 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 15:22:46 -0500
Subject: [PATCH 029/130] more switch migration

---
 src/yggdrasil/api.go    |  2 +-
 src/yggdrasil/core.go   |  4 ++-
 src/yggdrasil/link.go   |  8 ++++--
 src/yggdrasil/peer.go   |  2 +-
 src/yggdrasil/switch.go | 64 ++++++++++++++++++++---------------------
 5 files changed, 42 insertions(+), 38 deletions(-)

diff --git a/src/yggdrasil/api.go b/src/yggdrasil/api.go
index cbf232aa..2bc5c816 100644
--- a/src/yggdrasil/api.go
+++ b/src/yggdrasil/api.go
@@ -191,7 +191,7 @@ func (c *Core) GetSwitchQueues() SwitchQueues {
 			Size:         switchTable.queues.size,
 			HighestCount: uint64(switchTable.queues.maxbufs),
 			HighestSize:  switchTable.queues.maxsize,
-			MaximumSize:  switchTable.queueTotalMaxSize,
+			MaximumSize:  switchTable.queues.totalMaxSize,
 		}
 		for k, v := range switchTable.queues.bufs {
 			nexthop := switchTable.bestPortForCoords([]byte(k))
diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index fb2142c0..40982cdd 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -174,7 +174,9 @@ func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState,
 
 	c.config.Mutex.RLock()
 	if c.config.Current.SwitchOptions.MaxTotalQueueSize >= SwitchQueueTotalMinSize {
-		c.switchTable.queueTotalMaxSize = c.config.Current.SwitchOptions.MaxTotalQueueSize
+		c.switchTable.doAdmin(func() {
+			c.switchTable.queues.totalMaxSize = c.config.Current.SwitchOptions.MaxTotalQueueSize
+		})
 	}
 	c.config.Mutex.RUnlock()
 
diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index 824afd34..1389f41e 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -318,7 +318,9 @@ func (intf *linkInterface) handler() error {
 				isAlive = true
 				if !isReady {
 					// (Re-)enable in the switch
-					intf.link.core.switchTable.idleIn <- intf.peer.port
+					intf.link.core.switchTable.EnqueueFrom(nil, func() {
+						intf.link.core.switchTable._idleIn(intf.peer.port)
+					})
 					isReady = true
 				}
 				if gotMsg && !sendTimerRunning {
@@ -355,7 +357,9 @@ func (intf *linkInterface) handler() error {
 					isReady = false
 				} else {
 					// Keep enabled in the switch
-					intf.link.core.switchTable.idleIn <- intf.peer.port
+					intf.link.core.switchTable.EnqueueFrom(nil, func() {
+						intf.link.core.switchTable._idleIn(intf.peer.port)
+					})
 					isReady = true
 				}
 			case <-sendBlocked.C:
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 4bebc17e..9a6f75f1 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -242,7 +242,7 @@ func (p *peer) _handleTraffic(packet []byte, pTypeLen int) {
 		// Drop traffic if the peer isn't in the switch
 		return
 	}
-	p.core.switchTable.packetIn <- packet
+	p.core.switchTable.packetInFrom(p, packet)
 }
 
 func (p *peer) sendPacketsFrom(from phony.IActor, packets [][]byte) {
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 3f4c41fe..20552ae0 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -164,22 +164,19 @@ type switchData struct {
 
 // All the information stored by the switch.
 type switchTable struct {
-	core              *Core
-	reconfigure       chan chan error
-	key               crypto.SigPubKey           // Our own key
-	time              time.Time                  // Time when locator.tstamp was last updated
-	drop              map[crypto.SigPubKey]int64 // Tstamp associated with a dropped root
-	mutex             sync.RWMutex               // Lock for reads/writes of switchData
-	parent            switchPort                 // Port of whatever peer is our parent, or self if we're root
-	data              switchData                 //
-	updater           atomic.Value               // *sync.Once
-	table             atomic.Value               // lookupTable
-	phony.Actor                                  // Owns the below
-	packetIn          chan []byte                // Incoming packets for the worker to handle
-	idleIn            chan switchPort            // Incoming idle notifications from peer links
-	queues            switch_buffers             // Queues - not atomic so ONLY use through admin chan
-	queueTotalMaxSize uint64                     // Maximum combined size of queues
-	idle              map[switchPort]time.Time   // idle peers
+	core        *Core
+	reconfigure chan chan error
+	key         crypto.SigPubKey           // Our own key
+	time        time.Time                  // Time when locator.tstamp was last updated
+	drop        map[crypto.SigPubKey]int64 // Tstamp associated with a dropped root
+	mutex       sync.RWMutex               // Lock for reads/writes of switchData
+	parent      switchPort                 // Port of whatever peer is our parent, or self if we're root
+	data        switchData                 //
+	updater     atomic.Value               // *sync.Once
+	table       atomic.Value               // lookupTable
+	phony.Actor                            // Owns the below
+	queues      switch_buffers             // Queues - not atomic so ONLY use through the actor
+	idle        map[switchPort]time.Time   // idle peers - not atomic so ONLY use through the actor
 }
 
 // Minimum allowed total size of switch queues.
@@ -197,12 +194,11 @@ func (t *switchTable) init(core *Core) {
 	t.updater.Store(&sync.Once{})
 	t.table.Store(lookupTable{})
 	t.drop = make(map[crypto.SigPubKey]int64)
-	t.packetIn = make(chan []byte, 1024)
-	t.idleIn = make(chan switchPort, 1024)
-	t.queueTotalMaxSize = SwitchQueueTotalMinSize
-	t.idle = make(map[switchPort]time.Time)
-	t.queues.switchTable = t
-	t.queues.bufs = make(map[string]switch_buffer) // Packets per PacketStreamID (string)
+	<-t.SyncExec(func() {
+		t.queues.totalMaxSize = SwitchQueueTotalMinSize
+		t.queues.bufs = make(map[string]switch_buffer)
+		t.idle = make(map[switchPort]time.Time)
+	})
 }
 
 // Safely gets a copy of this node's locator.
@@ -727,12 +723,12 @@ type switch_buffer struct {
 }
 
 type switch_buffers struct {
-	switchTable *switchTable
-	bufs        map[string]switch_buffer // Buffers indexed by StreamID
-	size        uint64                   // Total size of all buffers, in bytes
-	maxbufs     int
-	maxsize     uint64
-	closer      []closerInfo // Scratch space
+	totalMaxSize uint64
+	bufs         map[string]switch_buffer // Buffers indexed by StreamID
+	size         uint64                   // Total size of all buffers, in bytes
+	maxbufs      int
+	maxsize      uint64
+	closer       []closerInfo // Scratch space
 }
 
 func (b *switch_buffers) _cleanup(t *switchTable) {
@@ -749,7 +745,7 @@ func (b *switch_buffers) _cleanup(t *switchTable) {
 		}
 	}
 
-	for b.size > b.switchTable.queueTotalMaxSize {
+	for b.size > b.totalMaxSize {
 		// Drop a random queue
 		target := rand.Uint64() % b.size
 		var size uint64 // running total
@@ -828,6 +824,12 @@ func (t *switchTable) _handleIdle(port switchPort) bool {
 	return false
 }
 
+func (t *switchTable) packetInFrom(from phony.IActor, bytes []byte) {
+	t.EnqueueFrom(from, func() {
+		t._packetIn(bytes)
+	})
+}
+
 func (t *switchTable) _packetIn(bytes []byte) {
 	// Try to send it somewhere (or drop it if it's corrupt or at a dead end)
 	if !t._handleIn(bytes, t.idle) {
@@ -868,10 +870,6 @@ func (t *switchTable) doWorker() {
 	for {
 		//t.core.log.Debugf("Switch state: idle = %d, buffers = %d", len(idle), len(t.queues.bufs))
 		select {
-		case bytes := <-t.packetIn:
-			<-t.SyncExec(func() { t._packetIn(bytes) })
-		case port := <-t.idleIn:
-			<-t.SyncExec(func() { t._idleIn(port) })
 		case e := <-t.reconfigure:
 			e <- nil
 		}

From c573170886a85e9a88fe3d8e770949ccfd78c4ea Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 15:27:56 -0500
Subject: [PATCH 030/130] remove switch doworker loop, start a dummy loop to
 respond to (unused) reconfiguration instead

---
 src/yggdrasil/switch.go | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 20552ae0..905f2a79 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -566,7 +566,12 @@ func (t *switchTable) getTable() lookupTable {
 // Starts the switch worker
 func (t *switchTable) start() error {
 	t.core.log.Infoln("Starting switch")
-	go t.doWorker()
+	go func() {
+		// TODO find a better way to handle reconfiguration... and have the switch do something with the new configuration
+		for ch := range t.reconfigure {
+			ch <- nil
+		}
+	}()
 	return nil
 }
 
@@ -865,17 +870,6 @@ func (t *switchTable) _idleIn(port switchPort) {
 	}
 }
 
-// The switch worker does routing lookups and sends packets to where they need to be
-func (t *switchTable) doWorker() {
-	for {
-		//t.core.log.Debugf("Switch state: idle = %d, buffers = %d", len(idle), len(t.queues.bufs))
-		select {
-		case e := <-t.reconfigure:
-			e <- nil
-		}
-	}
-}
-
 // Passed a function to call.
 // This will send the function to t.admin and block until it finishes.
 func (t *switchTable) doAdmin(f func()) {

From 8c7e9ec7c0266fce7ca074a45857233e5b583d4a Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 15:32:19 -0500
Subject: [PATCH 031/130] fix debug builds

---
 src/yggdrasil/debug.go | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/yggdrasil/debug.go b/src/yggdrasil/debug.go
index 30eb874d..f6ec8716 100644
--- a/src/yggdrasil/debug.go
+++ b/src/yggdrasil/debug.go
@@ -571,7 +571,9 @@ func DEBUG_simLinkPeers(p, q *peer) {
 					continue
 				case packet := <-send:
 					packets = append(packets, packet)
-					source.core.switchTable.idleIn <- source.port
+					<-source.core.switchTable.SyncExec(func() {
+						source.core.switchTable._idleIn(source.port)
+					})
 					continue
 				default:
 				}
@@ -585,15 +587,21 @@ func DEBUG_simLinkPeers(p, q *peer) {
 					packets = append(packets, packet)
 				case packet := <-send:
 					packets = append(packets, packet)
-					source.core.switchTable.idleIn <- source.port
+					<-source.core.switchTable.SyncExec(func() {
+						source.core.switchTable._idleIn(source.port)
+					})
 				}
 			}
 		}()
 	}
 	goWorkers(p, q)
 	goWorkers(q, p)
-	p.core.switchTable.idleIn <- p.port
-	q.core.switchTable.idleIn <- q.port
+	<-p.core.switchTable.SyncExec(func() {
+		p.core.switchTable._idleIn(p.port)
+	})
+	<-q.core.switchTable.SyncExec(func() {
+		q.core.switchTable._idleIn(q.port)
+	})
 }
 
 /*

From 209d2ffea55a6ce3f1f3d22a2f06bda4ec25aae3 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 16:04:05 -0500
Subject: [PATCH 032/130] correctly call peer.sendPacketsFrom in the switch

---
 src/yggdrasil/peer.go   | 6 +++---
 src/yggdrasil/switch.go | 5 ++---
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 9a6f75f1..7fe33ecd 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -224,9 +224,9 @@ func (p *peer) _handlePacket(packet []byte) {
 	}
 	switch pType {
 	case wire_Traffic:
-		p._handleTraffic(packet, pTypeLen)
+		p._handleTraffic(packet)
 	case wire_ProtocolTraffic:
-		p._handleTraffic(packet, pTypeLen)
+		p._handleTraffic(packet)
 	case wire_LinkProtocolTraffic:
 		p._handleLinkTraffic(packet)
 	default:
@@ -236,7 +236,7 @@ func (p *peer) _handlePacket(packet []byte) {
 
 // Called to handle traffic or protocolTraffic packets.
 // In either case, this reads from the coords of the packet header, does a switch lookup, and forwards to the next node.
-func (p *peer) _handleTraffic(packet []byte, pTypeLen int) {
+func (p *peer) _handleTraffic(packet []byte) {
 	table := p.core.switchTable.getTable()
 	if _, isIn := table.elems[p.port]; !isIn && p.port != 0 {
 		// Drop traffic if the peer isn't in the switch
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 905f2a79..b6fc26f8 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -708,7 +708,7 @@ func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]time.Time) bo
 	if best != nil {
 		// Send to the best idle next hop
 		delete(idle, best.port)
-		best.sendPacketsFrom(nil, [][]byte{packet})
+		best.sendPacketsFrom(t, [][]byte{packet})
 		return true
 	}
 	// Didn't find anyone idle to send it to
@@ -822,8 +822,7 @@ func (t *switchTable) _handleIdle(port switchPort) bool {
 		}
 	}
 	if len(packets) > 0 {
-		// TODO rewrite if/when the switch becomes an actor
-		to.sendPacketsFrom(nil, packets)
+		to.sendPacketsFrom(t, packets)
 		return true
 	}
 	return false

From 99be6b037d5cccccf5f7103eaf582bec002d4eee Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 16:13:34 -0500
Subject: [PATCH 033/130] stop synchronizing message reads for now, not 100%
 safe but I don't have any better ideas

---
 src/yggdrasil/link.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index 1389f41e..4b7f766d 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -392,7 +392,10 @@ func (intf *linkInterface) handler() error {
 		msg, err := intf.msgIO.readMsg()
 		if len(msg) > 0 {
 			// TODO rewrite this if the link becomes an actor
-			<-intf.peer.SyncExec(func() { intf.peer._handlePacket(msg) })
+			// FIXME this could theoretically send traffic faster than the peer can handle
+			//  The alternative is to SyncExec, but that causes traffic to block while *other* links work
+			//  Need to figure out why and find a workaround
+			intf.peer.handlePacketFrom(nil, msg)
 		}
 		if err != nil {
 			if err != io.EOF {

From 48bbdac9b3596c513e2530a5bc20dad6bfc4faea Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 16:27:12 -0500
Subject: [PATCH 034/130] add a helper actor to the link reader to make it play
 nicer with backpressure

---
 src/yggdrasil/link.go | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index 4b7f766d..98f44bc7 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -16,6 +16,8 @@ import (
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
+
+	"github.com/Arceliar/phony"
 )
 
 type link struct {
@@ -388,14 +390,17 @@ func (intf *linkInterface) handler() error {
 		}
 	}()
 	// Run reader loop
-	for {
+	var helper phony.Actor
+	done := make(chan struct{})
+	var helperFunc func()
+	helperFunc = func() {
+		// The helper reads in a loop and sends to the peer
+		// It loops by sending itself a message, which can be delayed by backpressure
+		// So if the peer is busy, backpressure will pause reading until the peer catches up
 		msg, err := intf.msgIO.readMsg()
 		if len(msg) > 0 {
 			// TODO rewrite this if the link becomes an actor
-			// FIXME this could theoretically send traffic faster than the peer can handle
-			//  The alternative is to SyncExec, but that causes traffic to block while *other* links work
-			//  Need to figure out why and find a workaround
-			intf.peer.handlePacketFrom(nil, msg)
+			intf.peer.handlePacketFrom(&helper, msg)
 		}
 		if err != nil {
 			if err != io.EOF {
@@ -404,13 +409,19 @@ func (intf *linkInterface) handler() error {
 				default:
 				}
 			}
-			break
+			close(done)
+			return
 		}
 		select {
 		case signalAlive <- len(msg) > 0:
 		default:
 		}
+		// Now try to read again
+		helper.EnqueueFrom(nil, helperFunc)
 	}
+	// Start the read loop
+	helper.EnqueueFrom(nil, helperFunc)
+	<-done // Wait for the helper to exit
 	////////////////////////////////////////////////////////////////////////////////
 	// Remember to set `err` to something useful before returning
 	select {

From 68c380ff452c4bc5fcbe70786f39b8db05b041a6 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 17:03:19 -0500
Subject: [PATCH 035/130] update phony dependency

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index fff4ae6f..db307d33 100644
--- a/go.mod
+++ b/go.mod
@@ -1,7 +1,7 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
 require (
-	github.com/Arceliar/phony v0.0.0-20190824031448-b53e115f69b5
+	github.com/Arceliar/phony v0.0.0-20190824220211-2c0698579651
 	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
 	github.com/hashicorp/go-syslog v1.0.0
 	github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222
diff --git a/go.sum b/go.sum
index 29854cbf..3839b51a 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-github.com/Arceliar/phony v0.0.0-20190824031448-b53e115f69b5 h1:D2Djqo/q7mftrtHLCpW4Rpplm8jj+Edc9jNz8Ll6E0A=
-github.com/Arceliar/phony v0.0.0-20190824031448-b53e115f69b5/go.mod h1:2Q9yJvg2PlMrnOEa3RTEy9hElWAICo/D8HTUDqAHUAo=
+github.com/Arceliar/phony v0.0.0-20190824220211-2c0698579651 h1:FDO8ctwDOOs8mI7EVE/hR3lpGjWoY2y21tAlblIYvs8=
+github.com/Arceliar/phony v0.0.0-20190824220211-2c0698579651/go.mod h1:+/sVcxsqK1Sjm3Vd+yCfMAohJOfTRyNh24apkxhqU3Q=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=

From f62bc842ae0a01b89b13779ce6320286fef2e10d Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 18:23:54 -0500
Subject: [PATCH 036/130] fix memory leak in session nonce map

---
 src/yggdrasil/session.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 7cd92db9..6f6a96a8 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -32,7 +32,7 @@ func (h *nonceHeap) Pop() interface{} {
 	n, *h = (*h)[l-1], (*h)[:l-1]
 	return n
 }
-func (h nonceHeap) peek() *crypto.BoxNonce { return &h[len(h)-1] }
+func (h nonceHeap) peek() *crypto.BoxNonce { return &h[0] }
 
 // All the information we know about an active session.
 // This includes coords, permanent and ephemeral keys, handles and nonces, various sorts of timing information for timeout and maintenance, and some metadata for the admin API.

From cff1366146456e6421ab6eb4199879794938b784 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 22:28:20 -0500
Subject: [PATCH 037/130] update phony dependency

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index db307d33..a71eb4bc 100644
--- a/go.mod
+++ b/go.mod
@@ -1,7 +1,7 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
 require (
-	github.com/Arceliar/phony v0.0.0-20190824220211-2c0698579651
+	github.com/Arceliar/phony v0.0.0-20190825032731-f8ba56f9093e
 	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
 	github.com/hashicorp/go-syslog v1.0.0
 	github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222
diff --git a/go.sum b/go.sum
index 3839b51a..0aba11ef 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-github.com/Arceliar/phony v0.0.0-20190824220211-2c0698579651 h1:FDO8ctwDOOs8mI7EVE/hR3lpGjWoY2y21tAlblIYvs8=
-github.com/Arceliar/phony v0.0.0-20190824220211-2c0698579651/go.mod h1:+/sVcxsqK1Sjm3Vd+yCfMAohJOfTRyNh24apkxhqU3Q=
+github.com/Arceliar/phony v0.0.0-20190825032731-f8ba56f9093e h1:qsNZzfxSvlSE4JZ3OpmDmAeqCRpOO3RI9eJ7U6z23Gk=
+github.com/Arceliar/phony v0.0.0-20190825032731-f8ba56f9093e/go.mod h1:+/sVcxsqK1Sjm3Vd+yCfMAohJOfTRyNh24apkxhqU3Q=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=

From aa30c6cc98071cb4c953049baf3ff8e19e213044 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 25 Aug 2019 10:36:09 -0500
Subject: [PATCH 038/130] upgrade phony dependency and switch to its new
 interface

---
 go.mod                   |  2 +-
 go.sum                   |  4 ++--
 src/tuntap/conn.go       |  4 ++--
 src/tuntap/tun.go        |  2 +-
 src/yggdrasil/conn.go    | 24 ++++++++++++------------
 src/yggdrasil/link.go    | 10 +++++-----
 src/yggdrasil/peer.go    | 18 +++++++++---------
 src/yggdrasil/router.go  | 14 +++++++-------
 src/yggdrasil/session.go | 24 ++++++++++++------------
 src/yggdrasil/switch.go  |  6 +++---
 10 files changed, 54 insertions(+), 54 deletions(-)

diff --git a/go.mod b/go.mod
index a71eb4bc..f3d8417f 100644
--- a/go.mod
+++ b/go.mod
@@ -1,7 +1,7 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
 require (
-	github.com/Arceliar/phony v0.0.0-20190825032731-f8ba56f9093e
+	github.com/Arceliar/phony v0.0.0-20190825152505-180ac75690fe
 	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
 	github.com/hashicorp/go-syslog v1.0.0
 	github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222
diff --git a/go.sum b/go.sum
index 0aba11ef..29e0dfec 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-github.com/Arceliar/phony v0.0.0-20190825032731-f8ba56f9093e h1:qsNZzfxSvlSE4JZ3OpmDmAeqCRpOO3RI9eJ7U6z23Gk=
-github.com/Arceliar/phony v0.0.0-20190825032731-f8ba56f9093e/go.mod h1:+/sVcxsqK1Sjm3Vd+yCfMAohJOfTRyNh24apkxhqU3Q=
+github.com/Arceliar/phony v0.0.0-20190825152505-180ac75690fe h1:U5bediuXjZ1y6bByIXXraoE319yFp9kx1z8K6el7Ftc=
+github.com/Arceliar/phony v0.0.0-20190825152505-180ac75690fe/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=
diff --git a/src/tuntap/conn.go b/src/tuntap/conn.go
index 744ca9fc..0e0dd461 100644
--- a/src/tuntap/conn.go
+++ b/src/tuntap/conn.go
@@ -17,7 +17,7 @@ import (
 const tunConnTimeout = 2 * time.Minute
 
 type tunConn struct {
-	phony.Actor
+	phony.Inbox
 	tun   *TunAdapter
 	conn  *yggdrasil.Conn
 	addr  address.Address
@@ -198,7 +198,7 @@ func (s *tunConn) _write(bs []byte) (err error) {
 			// No point in wasting resources to send back an error if there was none
 			return
 		}
-		s.EnqueueFrom(s.conn, func() {
+		s.RecvFrom(s.conn, func() {
 			if e, eok := err.(yggdrasil.ConnError); !eok {
 				if e.Closed() {
 					s.tun.log.Debugln(s.conn.String(), "TUN/TAP generic write debug:", err)
diff --git a/src/tuntap/tun.go b/src/tuntap/tun.go
index 895f893b..dfc343df 100644
--- a/src/tuntap/tun.go
+++ b/src/tuntap/tun.go
@@ -257,7 +257,7 @@ func (tun *TunAdapter) wrap(conn *yggdrasil.Conn) (c *tunConn, err error) {
 	tun.subnetToConn[s.snet] = &s
 	// Set the read callback and start the timeout goroutine
 	conn.SetReadCallback(func(bs []byte) {
-		s.EnqueueFrom(conn, func() {
+		s.RecvFrom(conn, func() {
 			s._read(bs)
 		})
 	})
diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index 2f69139e..b0c26867 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -54,7 +54,7 @@ func (e *ConnError) Closed() bool {
 }
 
 type Conn struct {
-	phony.Actor
+	phony.Inbox
 	core          *Core
 	readDeadline  *time.Time
 	writeDeadline *time.Time
@@ -84,8 +84,8 @@ func (c *Conn) String() string {
 	return s
 }
 
-func (c *Conn) setMTU(from phony.IActor, mtu uint16) {
-	c.EnqueueFrom(from, func() { c.mtu = mtu })
+func (c *Conn) setMTU(from phony.Actor, mtu uint16) {
+	c.RecvFrom(from, func() { c.mtu = mtu })
 }
 
 // This should never be called from the router goroutine, used in the dial functions
@@ -143,7 +143,7 @@ func (c *Conn) doSearch() {
 			sinfo.continueSearch()
 		}
 	}
-	c.core.router.EnqueueFrom(c.session, routerWork)
+	c.core.router.RecvFrom(c.session, routerWork)
 }
 
 func (c *Conn) _getDeadlineCancellation(t *time.Time) (util.Cancellation, bool) {
@@ -159,7 +159,7 @@ func (c *Conn) _getDeadlineCancellation(t *time.Time) (util.Cancellation, bool)
 
 // SetReadCallback sets a callback which will be called whenever a packet is received.
 func (c *Conn) SetReadCallback(callback func([]byte)) {
-	c.EnqueueFrom(nil, func() {
+	c.RecvFrom(nil, func() {
 		c.readCallback = callback
 		c._drainReadBuffer()
 	})
@@ -172,14 +172,14 @@ func (c *Conn) _drainReadBuffer() {
 	select {
 	case bs := <-c.readBuffer:
 		c.readCallback(bs)
-		c.EnqueueFrom(nil, c._drainReadBuffer) // In case there's more
+		c.RecvFrom(nil, c._drainReadBuffer) // In case there's more
 	default:
 	}
 }
 
 // Called by the session to pass a new message to the Conn
-func (c *Conn) recvMsg(from phony.IActor, msg []byte) {
-	c.EnqueueFrom(from, func() {
+func (c *Conn) recvMsg(from phony.Actor, msg []byte) {
+	c.RecvFrom(from, func() {
 		if c.readCallback != nil {
 			c.readCallback(msg)
 		} else {
@@ -234,7 +234,7 @@ func (c *Conn) _write(msg FlowKeyMessage) error {
 	if len(msg.Message) > int(c.mtu) {
 		return ConnError{errors.New("packet too big"), true, false, false, int(c.mtu)}
 	}
-	c.session.EnqueueFrom(c, func() {
+	c.session.RecvFrom(c, func() {
 		// Send the packet
 		c.session._send(msg)
 		// Session keep-alive, while we wait for the crypto workers from send
@@ -254,11 +254,11 @@ func (c *Conn) _write(msg FlowKeyMessage) error {
 	return nil
 }
 
-// WriteFrom should be called by a phony.IActor, and tells the Conn to send a message.
+// WriteFrom should be called by a phony.Actor, and tells the Conn to send a message.
 // This is used internaly by WriteNoCopy and Write.
 // If the callback is called with a non-nil value, then it is safe to reuse the argument FlowKeyMessage.
-func (c *Conn) WriteFrom(from phony.IActor, msg FlowKeyMessage, callback func(error)) {
-	c.EnqueueFrom(from, func() {
+func (c *Conn) WriteFrom(from phony.Actor, msg FlowKeyMessage, callback func(error)) {
+	c.RecvFrom(from, func() {
 		callback(c._write(msg))
 	})
 }
diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index 98f44bc7..d4779ead 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -320,7 +320,7 @@ func (intf *linkInterface) handler() error {
 				isAlive = true
 				if !isReady {
 					// (Re-)enable in the switch
-					intf.link.core.switchTable.EnqueueFrom(nil, func() {
+					intf.link.core.switchTable.RecvFrom(nil, func() {
 						intf.link.core.switchTable._idleIn(intf.peer.port)
 					})
 					isReady = true
@@ -359,7 +359,7 @@ func (intf *linkInterface) handler() error {
 					isReady = false
 				} else {
 					// Keep enabled in the switch
-					intf.link.core.switchTable.EnqueueFrom(nil, func() {
+					intf.link.core.switchTable.RecvFrom(nil, func() {
 						intf.link.core.switchTable._idleIn(intf.peer.port)
 					})
 					isReady = true
@@ -390,7 +390,7 @@ func (intf *linkInterface) handler() error {
 		}
 	}()
 	// Run reader loop
-	var helper phony.Actor
+	var helper phony.Inbox
 	done := make(chan struct{})
 	var helperFunc func()
 	helperFunc = func() {
@@ -417,10 +417,10 @@ func (intf *linkInterface) handler() error {
 		default:
 		}
 		// Now try to read again
-		helper.EnqueueFrom(nil, helperFunc)
+		helper.RecvFrom(nil, helperFunc)
 	}
 	// Start the read loop
-	helper.EnqueueFrom(nil, helperFunc)
+	helper.RecvFrom(nil, helperFunc)
 	<-done // Wait for the helper to exit
 	////////////////////////////////////////////////////////////////////////////////
 	// Remember to set `err` to something useful before returning
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 7fe33ecd..aa31bb1b 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -96,7 +96,7 @@ func (ps *peers) putPorts(ports map[switchPort]*peer) {
 
 // Information known about a peer, including thier box/sig keys, precomputed shared keys (static and ephemeral) and a handler for their outgoing traffic
 type peer struct {
-	phony.Actor
+	phony.Inbox
 	core       *Core
 	intf       *linkInterface
 	port       switchPort
@@ -175,13 +175,13 @@ func (ps *peers) removePeer(port switchPort) {
 
 // If called, sends a notification to each peer that they should send a new switch message.
 // Mainly called by the switch after an update.
-func (ps *peers) sendSwitchMsgs(from phony.IActor) {
+func (ps *peers) sendSwitchMsgs(from phony.Actor) {
 	ports := ps.getPorts()
 	for _, p := range ports {
 		if p.port == 0 {
 			continue
 		}
-		p.EnqueueFrom(from, p._sendSwitchMsg)
+		p.RecvFrom(from, p._sendSwitchMsg)
 	}
 }
 
@@ -207,8 +207,8 @@ func (p *peer) _updateDHT() {
 	}
 }
 
-func (p *peer) handlePacketFrom(from phony.IActor, packet []byte) {
-	p.EnqueueFrom(from, func() {
+func (p *peer) handlePacketFrom(from phony.Actor, packet []byte) {
+	p.RecvFrom(from, func() {
 		p._handlePacket(packet)
 	})
 }
@@ -245,8 +245,8 @@ func (p *peer) _handleTraffic(packet []byte) {
 	p.core.switchTable.packetInFrom(p, packet)
 }
 
-func (p *peer) sendPacketsFrom(from phony.IActor, packets [][]byte) {
-	p.EnqueueFrom(from, func() {
+func (p *peer) sendPacketsFrom(from phony.Actor, packets [][]byte) {
+	p.RecvFrom(from, func() {
 		p._sendPackets(packets)
 	})
 }
@@ -263,7 +263,7 @@ func (p *peer) _sendPackets(packets [][]byte) {
 	p.out(packets)
 }
 
-var peerLinkOutHelper phony.Actor
+var peerLinkOutHelper phony.Inbox
 
 // This wraps the packet in the inner (ephemeral) and outer (permanent) crypto layers.
 // It sends it to p.linkOut, which bypasses the usual packet queues.
@@ -281,7 +281,7 @@ func (p *peer) _sendLinkPacket(packet []byte) {
 	}
 	packet = linkPacket.encode()
 	// TODO replace this with a message send if/when the link becomes an actor
-	peerLinkOutHelper.EnqueueFrom(nil, func() {
+	peerLinkOutHelper.RecvFrom(nil, func() {
 		select {
 		case p.linkOut <- packet:
 		case <-p.done:
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index adf1b1d4..7b6a9b80 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -37,7 +37,7 @@ import (
 // The router struct has channels to/from the adapter device and a self peer (0), which is how messages are passed between this node and the peers/switch layer.
 // The router's mainLoop goroutine is responsible for managing all information related to the dht, searches, and crypto sessions.
 type router struct {
-	phony.Actor
+	phony.Inbox
 	core        *Core
 	reconfigure chan chan error
 	addr        address.Address
@@ -83,8 +83,8 @@ func (r *router) start() error {
 }
 
 // In practice, the switch will call this with 1 packet
-func (r *router) handlePackets(from phony.IActor, packets [][]byte) {
-	r.EnqueueFrom(from, func() {
+func (r *router) handlePackets(from phony.Actor, packets [][]byte) {
+	r.RecvFrom(from, func() {
 		for _, packet := range packets {
 			r._handlePacket(packet)
 		}
@@ -92,15 +92,15 @@ func (r *router) handlePackets(from phony.IActor, packets [][]byte) {
 }
 
 // Insert a peer info into the dht, TODO? make the dht a separate actor
-func (r *router) insertPeer(from phony.IActor, info *dhtInfo) {
-	r.EnqueueFrom(from, func() {
+func (r *router) insertPeer(from phony.Actor, info *dhtInfo) {
+	r.RecvFrom(from, func() {
 		r.dht.insertPeer(info)
 	})
 }
 
 // Reset sessions and DHT after the switch sees our coords change
-func (r *router) reset(from phony.IActor) {
-	r.EnqueueFrom(from, func() {
+func (r *router) reset(from phony.Actor) {
+	r.RecvFrom(from, func() {
 		r.sessions.reset()
 		r.dht.reset()
 	})
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 5243fb0f..f1263379 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -38,7 +38,7 @@ func (h nonceHeap) peek() *crypto.BoxNonce { return &h[0] }
 // All the information we know about an active session.
 // This includes coords, permanent and ephemeral keys, handles and nonces, various sorts of timing information for timeout and maintenance, and some metadata for the admin API.
 type sessionInfo struct {
-	phony.Actor                                  // Protects all of the below, use it any time you read/change the contents of a session
+	phony.Inbox                                  // Protects all of the below, use it any time you read/change the contents of a session
 	sessions       *sessions                     //
 	reconfigure    chan chan error               //
 	theirAddr      address.Address               //
@@ -342,8 +342,8 @@ func (ss *sessions) getSharedKey(myPriv *crypto.BoxPrivKey,
 }
 
 // Sends a session ping by calling sendPingPong in ping mode.
-func (sinfo *sessionInfo) ping(from phony.IActor) {
-	sinfo.EnqueueFrom(from, func() {
+func (sinfo *sessionInfo) ping(from phony.Actor) {
+	sinfo.RecvFrom(from, func() {
 		sinfo._sendPingPong(false)
 	})
 }
@@ -364,14 +364,14 @@ func (sinfo *sessionInfo) _sendPingPong(isPong bool) {
 	}
 	packet := p.encode()
 	// TODO rewrite the below if/when the peer struct becomes an actor, to not go through the router first
-	sinfo.sessions.router.EnqueueFrom(sinfo, func() { sinfo.sessions.router.out(packet) })
+	sinfo.sessions.router.RecvFrom(sinfo, func() { sinfo.sessions.router.out(packet) })
 	if sinfo.pingTime.Before(sinfo.time) {
 		sinfo.pingTime = time.Now()
 	}
 }
 
-func (sinfo *sessionInfo) setConn(from phony.IActor, conn *Conn) {
-	sinfo.EnqueueFrom(from, func() {
+func (sinfo *sessionInfo) setConn(from phony.Actor, conn *Conn) {
+	sinfo.RecvFrom(from, func() {
 		sinfo.conn = conn
 		sinfo.conn.setMTU(sinfo, sinfo._getMTU())
 	})
@@ -406,7 +406,7 @@ func (ss *sessions) handlePing(ping *sessionPing) {
 		ss.listenerMutex.Unlock()
 	}
 	if sinfo != nil {
-		sinfo.EnqueueFrom(ss.router, func() {
+		sinfo.RecvFrom(ss.router, func() {
 			// Update the session
 			if !sinfo._update(ping) { /*panic("Should not happen in testing")*/
 				return
@@ -474,7 +474,7 @@ func (sinfo *sessionInfo) _updateNonce(theirNonce *crypto.BoxNonce) {
 // Called after coord changes, so attemtps to use a session will trigger a new ping and notify the remote end of the coord change.
 func (ss *sessions) reset() {
 	for _, sinfo := range ss.sinfos {
-		sinfo.EnqueueFrom(ss.router, func() {
+		sinfo.RecvFrom(ss.router, func() {
 			sinfo.reset = true
 		})
 	}
@@ -489,8 +489,8 @@ type FlowKeyMessage struct {
 	Message []byte
 }
 
-func (sinfo *sessionInfo) recv(from phony.IActor, packet *wire_trafficPacket) {
-	sinfo.EnqueueFrom(from, func() {
+func (sinfo *sessionInfo) recv(from phony.Actor, packet *wire_trafficPacket) {
+	sinfo.RecvFrom(from, func() {
 		sinfo._recvPacket(packet)
 	})
 }
@@ -564,7 +564,7 @@ func (sinfo *sessionInfo) _send(msg FlowKeyMessage) {
 			util.PutBytes(p.Payload)
 			// Send the packet
 			// TODO replace this with a send to the peer struct if that becomes an actor
-			sinfo.sessions.router.EnqueueFrom(sinfo, func() {
+			sinfo.sessions.router.RecvFrom(sinfo, func() {
 				sinfo.sessions.router.out(packet)
 			})
 		}
@@ -576,7 +576,7 @@ func (sinfo *sessionInfo) _send(msg FlowKeyMessage) {
 }
 
 func (sinfo *sessionInfo) checkCallbacks() {
-	sinfo.EnqueueFrom(nil, func() {
+	sinfo.RecvFrom(nil, func() {
 		if len(sinfo.callbacks) > 0 {
 			select {
 			case callback := <-sinfo.callbacks[0]:
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index b6fc26f8..db63a845 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -174,7 +174,7 @@ type switchTable struct {
 	data        switchData                 //
 	updater     atomic.Value               // *sync.Once
 	table       atomic.Value               // lookupTable
-	phony.Actor                            // Owns the below
+	phony.Inbox                            // Owns the below
 	queues      switch_buffers             // Queues - not atomic so ONLY use through the actor
 	idle        map[switchPort]time.Time   // idle peers - not atomic so ONLY use through the actor
 }
@@ -828,8 +828,8 @@ func (t *switchTable) _handleIdle(port switchPort) bool {
 	return false
 }
 
-func (t *switchTable) packetInFrom(from phony.IActor, bytes []byte) {
-	t.EnqueueFrom(from, func() {
+func (t *switchTable) packetInFrom(from phony.Actor, bytes []byte) {
+	t.RecvFrom(from, func() {
 		t._packetIn(bytes)
 	})
 }

From a3d4d8125b4acb438779d4d5263ea0faa48d0125 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 25 Aug 2019 12:10:59 -0500
Subject: [PATCH 039/130] make the main library reconfiguration more
 actor-friendly

---
 src/yggdrasil/core.go    | 15 ++++----
 src/yggdrasil/dht.go     | 21 +++++------
 src/yggdrasil/link.go    | 32 +++++++----------
 src/yggdrasil/peer.go    | 19 +++++-----
 src/yggdrasil/router.go  | 63 +++++++++++++++++---------------
 src/yggdrasil/search.go  | 17 ++++-----
 src/yggdrasil/session.go | 42 +++++++++++-----------
 src/yggdrasil/switch.go  | 16 ++++-----
 src/yggdrasil/tcp.go     | 78 ++++++++++++++++++----------------------
 9 files changed, 144 insertions(+), 159 deletions(-)

diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index 40982cdd..ec530745 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -117,20 +117,21 @@ func (c *Core) UpdateConfig(config *config.NodeConfig) {
 
 	errors := 0
 
-	components := []chan chan error{
-		c.router.searches.reconfigure,
-		c.router.dht.reconfigure,
-		c.router.sessions.reconfigure,
+	// Each reconfigure function should pass any errors to the channel, then close it
+	components := []func(chan error){
+		c.link.reconfigure,
 		c.peers.reconfigure,
 		c.router.reconfigure,
+		c.router.dht.reconfigure,
+		c.router.searches.reconfigure,
+		c.router.sessions.reconfigure,
 		c.switchTable.reconfigure,
-		c.link.reconfigure,
 	}
 
 	for _, component := range components {
 		response := make(chan error)
-		component <- response
-		if err := <-response; err != nil {
+		go component(response)
+		for err := range response {
 			c.log.Errorln(err)
 			errors++
 		}
diff --git a/src/yggdrasil/dht.go b/src/yggdrasil/dht.go
index adfc40e7..4f380363 100644
--- a/src/yggdrasil/dht.go
+++ b/src/yggdrasil/dht.go
@@ -65,11 +65,10 @@ type dhtReqKey struct {
 
 // The main DHT struct.
 type dht struct {
-	router      *router
-	reconfigure chan chan error
-	nodeID      crypto.NodeID
-	reqs        map[dhtReqKey]time.Time          // Keeps track of recent outstanding requests
-	callbacks   map[dhtReqKey][]dht_callbackInfo // Search and admin lookup callbacks
+	router    *router
+	nodeID    crypto.NodeID
+	reqs      map[dhtReqKey]time.Time          // Keeps track of recent outstanding requests
+	callbacks map[dhtReqKey][]dht_callbackInfo // Search and admin lookup callbacks
 	// These next two could be replaced by a single linked list or similar...
 	table map[crypto.NodeID]*dhtInfo
 	imp   []*dhtInfo
@@ -78,18 +77,16 @@ type dht struct {
 // Initializes the DHT.
 func (t *dht) init(r *router) {
 	t.router = r
-	t.reconfigure = make(chan chan error, 1)
-	go func() {
-		for {
-			e := <-t.reconfigure
-			e <- nil
-		}
-	}()
 	t.nodeID = *t.router.core.NodeID()
 	t.callbacks = make(map[dhtReqKey][]dht_callbackInfo)
 	t.reset()
 }
 
+func (t *dht) reconfigure(e chan error) {
+	defer close(e)
+	// This is where reconfiguration would go, if we had anything to do
+}
+
 // Resets the DHT in response to coord changes.
 // This empties all info from the DHT and drops outstanding requests.
 func (t *dht) reset() {
diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index d4779ead..bfbcc99b 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -21,11 +21,10 @@ import (
 )
 
 type link struct {
-	core        *Core
-	reconfigure chan chan error
-	mutex       sync.RWMutex // protects interfaces below
-	interfaces  map[linkInfo]*linkInterface
-	tcp         tcp // TCP interface support
+	core       *Core
+	mutex      sync.RWMutex // protects interfaces below
+	interfaces map[linkInfo]*linkInterface
+	tcp        tcp // TCP interface support
 	// TODO timeout (to remove from switch), read from config.ReadTimeout
 }
 
@@ -61,7 +60,6 @@ func (l *link) init(c *Core) error {
 	l.core = c
 	l.mutex.Lock()
 	l.interfaces = make(map[linkInfo]*linkInterface)
-	l.reconfigure = make(chan chan error)
 	l.mutex.Unlock()
 
 	if err := l.tcp.init(l); err != nil {
@@ -69,22 +67,18 @@ func (l *link) init(c *Core) error {
 		return err
 	}
 
-	go func() {
-		for {
-			e := <-l.reconfigure
-			tcpresponse := make(chan error)
-			l.tcp.reconfigure <- tcpresponse
-			if err := <-tcpresponse; err != nil {
-				e <- err
-				continue
-			}
-			e <- nil
-		}
-	}()
-
 	return nil
 }
 
+func (l *link) reconfigure(e chan error) {
+	defer close(e)
+	tcpResponse := make(chan error)
+	go l.tcp.reconfigure(tcpResponse)
+	for err := range tcpResponse {
+		e <- err
+	}
+}
+
 func (l *link) call(uri string, sintf string) error {
 	u, err := url.Parse(uri)
 	if err != nil {
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index aa31bb1b..989d9ee1 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -21,10 +21,9 @@ import (
 // In most cases, this involves passing the packet to the handler for outgoing traffic to another peer.
 // In other cases, it's link protocol traffic used to build the spanning tree, in which case this checks signatures and passes the message along to the switch.
 type peers struct {
-	core        *Core
-	reconfigure chan chan error
-	mutex       sync.Mutex   // Synchronize writes to atomic
-	ports       atomic.Value //map[switchPort]*peer, use CoW semantics
+	core  *Core
+	mutex sync.Mutex   // Synchronize writes to atomic
+	ports atomic.Value //map[switchPort]*peer, use CoW semantics
 }
 
 // Initializes the peers struct.
@@ -33,13 +32,11 @@ func (ps *peers) init(c *Core) {
 	defer ps.mutex.Unlock()
 	ps.putPorts(make(map[switchPort]*peer))
 	ps.core = c
-	ps.reconfigure = make(chan chan error, 1)
-	go func() {
-		for {
-			e := <-ps.reconfigure
-			e <- nil
-		}
-	}()
+}
+
+func (ps *peers) reconfigure(e chan error) {
+	defer close(e)
+	// This is where reconfiguration would go, if we had anything to do
 }
 
 // Returns true if an incoming peer connection to a key is allowed, either
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index 7b6a9b80..002905bc 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -35,24 +35,22 @@ import (
 )
 
 // The router struct has channels to/from the adapter device and a self peer (0), which is how messages are passed between this node and the peers/switch layer.
-// The router's mainLoop goroutine is responsible for managing all information related to the dht, searches, and crypto sessions.
+// The router's phony.Inbox goroutine is responsible for managing all information related to the dht, searches, and crypto sessions.
 type router struct {
 	phony.Inbox
-	core        *Core
-	reconfigure chan chan error
-	addr        address.Address
-	subnet      address.Subnet
-	out         func([]byte) // packets we're sending to the network, link to peer's "in"
-	dht         dht
-	nodeinfo    nodeinfo
-	searches    searches
-	sessions    sessions
+	core     *Core
+	addr     address.Address
+	subnet   address.Subnet
+	out      func([]byte) // packets we're sending to the network, link to peer's "in"
+	dht      dht
+	nodeinfo nodeinfo
+	searches searches
+	sessions sessions
 }
 
 // Initializes the router struct, which includes setting up channels to/from the adapter.
 func (r *router) init(core *Core) {
 	r.core = core
-	r.reconfigure = make(chan chan error, 1)
 	r.addr = *address.AddrForNodeID(&r.dht.nodeID)
 	r.subnet = *address.SubnetForNodeID(&r.dht.nodeID)
 	self := linkInterface{
@@ -75,10 +73,26 @@ func (r *router) init(core *Core) {
 	r.sessions.init(r)
 }
 
-// Starts the mainLoop goroutine.
+func (r *router) reconfigure(e chan error) {
+	defer close(e)
+	var errs []error
+	// Reconfigure the router
+	<-r.SyncExec(func() {
+		current := r.core.config.GetCurrent()
+		err := r.nodeinfo.setNodeInfo(current.NodeInfo, current.NodeInfoPrivacy)
+		if err != nil {
+			errs = append(errs, err)
+		}
+	})
+	for _, err := range errs {
+		e <- err
+	}
+}
+
+// Starts the tickerLoop goroutine.
 func (r *router) start() error {
 	r.core.log.Infoln("Starting router")
-	go r._mainLoop()
+	go r.tickerLoop()
 	return nil
 }
 
@@ -108,24 +122,17 @@ func (r *router) reset(from phony.Actor) {
 
 // TODO remove reconfigure so this is just a ticker loop
 // and then find something better than a ticker loop to schedule things...
-func (r *router) _mainLoop() {
+func (r *router) tickerLoop() {
 	ticker := time.NewTicker(time.Second)
 	defer ticker.Stop()
 	for {
-		select {
-		case <-ticker.C:
-			<-r.SyncExec(func() {
-				// Any periodic maintenance stuff goes here
-				r.core.switchTable.doMaintenance()
-				r.dht.doMaintenance()
-				r.sessions.cleanup()
-			})
-		case e := <-r.reconfigure:
-			<-r.SyncExec(func() {
-				current := r.core.config.GetCurrent()
-				e <- r.nodeinfo.setNodeInfo(current.NodeInfo, current.NodeInfoPrivacy)
-			})
-		}
+		<-ticker.C
+		<-r.SyncExec(func() {
+			// Any periodic maintenance stuff goes here
+			r.core.switchTable.doMaintenance()
+			r.dht.doMaintenance()
+			r.sessions.cleanup()
+		})
 	}
 }
 
diff --git a/src/yggdrasil/search.go b/src/yggdrasil/search.go
index 397c28a9..5fb36584 100644
--- a/src/yggdrasil/search.go
+++ b/src/yggdrasil/search.go
@@ -45,24 +45,21 @@ type searchInfo struct {
 
 // This stores a map of active searches.
 type searches struct {
-	router      *router
-	reconfigure chan chan error
-	searches    map[crypto.NodeID]*searchInfo
+	router   *router
+	searches map[crypto.NodeID]*searchInfo
 }
 
 // Initializes the searches struct.
 func (s *searches) init(r *router) {
 	s.router = r
-	s.reconfigure = make(chan chan error, 1)
-	go func() {
-		for {
-			e := <-s.reconfigure
-			e <- nil
-		}
-	}()
 	s.searches = make(map[crypto.NodeID]*searchInfo)
 }
 
+func (s *searches) reconfigure(e chan error) {
+	defer close(e)
+	// This is where reconfiguration would go, if we had anything to do
+}
+
 // Creates a new search info, adds it to the searches struct, and returns a pointer to the info.
 func (s *searches) createSearch(dest *crypto.NodeID, mask *crypto.NodeID, callback func(*sessionInfo, error)) *searchInfo {
 	info := searchInfo{
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index f1263379..e448cf25 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -40,7 +40,6 @@ func (h nonceHeap) peek() *crypto.BoxNonce { return &h[0] }
 type sessionInfo struct {
 	phony.Inbox                                  // Protects all of the below, use it any time you read/change the contents of a session
 	sessions       *sessions                     //
-	reconfigure    chan chan error               //
 	theirAddr      address.Address               //
 	theirSubnet    address.Subnet                //
 	theirPermPub   crypto.BoxPubKey              //
@@ -74,6 +73,11 @@ type sessionInfo struct {
 	callbacks      []chan func()                 // Finished work from crypto workers
 }
 
+func (sinfo *sessionInfo) reconfigure(e chan error) {
+	defer close(e)
+	// This is where reconfiguration would go, if we had anything to do
+}
+
 // TODO remove this, call SyncExec directly
 func (sinfo *sessionInfo) doFunc(f func()) {
 	<-sinfo.SyncExec(f)
@@ -140,7 +144,6 @@ type sessions struct {
 	router           *router
 	listener         *Listener
 	listenerMutex    sync.Mutex
-	reconfigure      chan chan error
 	lastCleanup      time.Time
 	isAllowedHandler func(pubkey *crypto.BoxPubKey, initiator bool) bool // Returns true or false if session setup is allowed
 	isAllowedMutex   sync.RWMutex                                        // Protects the above
@@ -152,30 +155,28 @@ type sessions struct {
 // Initializes the session struct.
 func (ss *sessions) init(r *router) {
 	ss.router = r
-	ss.reconfigure = make(chan chan error, 1)
-	go func() {
-		for {
-			e := <-ss.reconfigure
-			responses := make(map[crypto.Handle]chan error)
-			for index, session := range ss.sinfos {
-				responses[index] = make(chan error)
-				session.reconfigure <- responses[index]
-			}
-			for _, response := range responses {
-				if err := <-response; err != nil {
-					e <- err
-					continue
-				}
-			}
-			e <- nil
-		}
-	}()
 	ss.permShared = make(map[crypto.BoxPubKey]*crypto.BoxSharedKey)
 	ss.sinfos = make(map[crypto.Handle]*sessionInfo)
 	ss.byTheirPerm = make(map[crypto.BoxPubKey]*crypto.Handle)
 	ss.lastCleanup = time.Now()
 }
 
+func (ss *sessions) reconfigure(e chan error) {
+	defer close(e)
+	responses := make(map[crypto.Handle]chan error)
+	<-ss.router.SyncExec(func() {
+		for index, session := range ss.sinfos {
+			responses[index] = make(chan error)
+			go session.reconfigure(responses[index])
+		}
+	})
+	for _, response := range responses {
+		for err := range response {
+			e <- err
+		}
+	}
+}
+
 // Determines whether the session with a given publickey is allowed based on
 // session firewall rules.
 func (ss *sessions) isSessionAllowed(pubkey *crypto.BoxPubKey, initiator bool) bool {
@@ -215,7 +216,6 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 	}
 	sinfo := sessionInfo{}
 	sinfo.sessions = ss
-	sinfo.reconfigure = make(chan chan error, 1)
 	sinfo.theirPermPub = *theirPermKey
 	sinfo.sharedPermKey = *ss.getSharedKey(&ss.router.core.boxPriv, &sinfo.theirPermPub)
 	pub, priv := crypto.NewBoxKeys()
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index db63a845..cb5cf1eb 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -165,7 +165,6 @@ type switchData struct {
 // All the information stored by the switch.
 type switchTable struct {
 	core        *Core
-	reconfigure chan chan error
 	key         crypto.SigPubKey           // Our own key
 	time        time.Time                  // Time when locator.tstamp was last updated
 	drop        map[crypto.SigPubKey]int64 // Tstamp associated with a dropped root
@@ -186,7 +185,6 @@ const SwitchQueueTotalMinSize = 4 * 1024 * 1024
 func (t *switchTable) init(core *Core) {
 	now := time.Now()
 	t.core = core
-	t.reconfigure = make(chan chan error, 1)
 	t.key = t.core.sigPub
 	locator := switchLocator{root: t.key, tstamp: now.Unix()}
 	peers := make(map[switchPort]peerInfo)
@@ -201,6 +199,13 @@ func (t *switchTable) init(core *Core) {
 	})
 }
 
+func (t *switchTable) reconfigure(e chan error) {
+	go func() {
+		defer close(e)
+		// This is where reconfiguration would go, if we had anything useful to do.
+	}()
+}
+
 // Safely gets a copy of this node's locator.
 func (t *switchTable) getLocator() switchLocator {
 	t.mutex.RLock()
@@ -566,12 +571,7 @@ func (t *switchTable) getTable() lookupTable {
 // Starts the switch worker
 func (t *switchTable) start() error {
 	t.core.log.Infoln("Starting switch")
-	go func() {
-		// TODO find a better way to handle reconfiguration... and have the switch do something with the new configuration
-		for ch := range t.reconfigure {
-			ch <- nil
-		}
-	}()
+	// There's actually nothing to do to start it...
 	return nil
 }
 
diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go
index dfb41510..ccb488f9 100644
--- a/src/yggdrasil/tcp.go
+++ b/src/yggdrasil/tcp.go
@@ -33,12 +33,11 @@ const tcp_ping_interval = (default_timeout * 2 / 3)
 
 // The TCP listener and information about active TCP connections, to avoid duplication.
 type tcp struct {
-	link        *link
-	reconfigure chan chan error
-	mutex       sync.Mutex // Protecting the below
-	listeners   map[string]*TcpListener
-	calls       map[string]struct{}
-	conns       map[linkInfo](chan struct{})
+	link      *link
+	mutex     sync.Mutex // Protecting the below
+	listeners map[string]*TcpListener
+	calls     map[string]struct{}
+	conns     map[linkInfo](chan struct{})
 }
 
 // TcpListener is a stoppable TCP listener interface. These are typically
@@ -76,49 +75,12 @@ func (t *tcp) getAddr() *net.TCPAddr {
 // Initializes the struct.
 func (t *tcp) init(l *link) error {
 	t.link = l
-	t.reconfigure = make(chan chan error, 1)
 	t.mutex.Lock()
 	t.calls = make(map[string]struct{})
 	t.conns = make(map[linkInfo](chan struct{}))
 	t.listeners = make(map[string]*TcpListener)
 	t.mutex.Unlock()
 
-	go func() {
-		for {
-			e := <-t.reconfigure
-			t.link.core.config.Mutex.RLock()
-			added := util.Difference(t.link.core.config.Current.Listen, t.link.core.config.Previous.Listen)
-			deleted := util.Difference(t.link.core.config.Previous.Listen, t.link.core.config.Current.Listen)
-			t.link.core.config.Mutex.RUnlock()
-			if len(added) > 0 || len(deleted) > 0 {
-				for _, a := range added {
-					if a[:6] != "tcp://" {
-						continue
-					}
-					if _, err := t.listen(a[6:]); err != nil {
-						e <- err
-						continue
-					}
-				}
-				for _, d := range deleted {
-					if d[:6] != "tcp://" {
-						continue
-					}
-					t.mutex.Lock()
-					if listener, ok := t.listeners[d[6:]]; ok {
-						t.mutex.Unlock()
-						listener.Stop <- true
-					} else {
-						t.mutex.Unlock()
-					}
-				}
-				e <- nil
-			} else {
-				e <- nil
-			}
-		}
-	}()
-
 	t.link.core.config.Mutex.RLock()
 	defer t.link.core.config.Mutex.RUnlock()
 	for _, listenaddr := range t.link.core.config.Current.Listen {
@@ -133,6 +95,36 @@ func (t *tcp) init(l *link) error {
 	return nil
 }
 
+func (t *tcp) reconfigure(e chan error) {
+	defer close(e)
+	t.link.core.config.Mutex.RLock()
+	added := util.Difference(t.link.core.config.Current.Listen, t.link.core.config.Previous.Listen)
+	deleted := util.Difference(t.link.core.config.Previous.Listen, t.link.core.config.Current.Listen)
+	t.link.core.config.Mutex.RUnlock()
+	if len(added) > 0 || len(deleted) > 0 {
+		for _, a := range added {
+			if a[:6] != "tcp://" {
+				continue
+			}
+			if _, err := t.listen(a[6:]); err != nil {
+				e <- err
+			}
+		}
+		for _, d := range deleted {
+			if d[:6] != "tcp://" {
+				continue
+			}
+			t.mutex.Lock()
+			if listener, ok := t.listeners[d[6:]]; ok {
+				t.mutex.Unlock()
+				listener.Stop <- true
+			} else {
+				t.mutex.Unlock()
+			}
+		}
+	}
+}
+
 func (t *tcp) listen(listenaddr string) (*TcpListener, error) {
 	var err error
 

From 502f2937a9419fabc7985da9a55d5913a40499db Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 25 Aug 2019 17:00:02 -0500
Subject: [PATCH 040/130] a couple race fixes and use timer.AfterFunc instead
 of sleeping goroutines or ticker in a few places

---
 src/tuntap/conn.go        | 31 ++++---------------------------
 src/tuntap/tun.go         | 11 +++++------
 src/yggdrasil/api.go      |  6 ++----
 src/yggdrasil/debug.go    |  4 ++--
 src/yggdrasil/link.go     |  2 +-
 src/yggdrasil/nodeinfo.go | 34 +++++++++++++++++-----------------
 src/yggdrasil/peer.go     | 25 ++++++++++++++-----------
 src/yggdrasil/router.go   | 22 +++++++++-------------
 src/yggdrasil/search.go   | 22 ++++++++++------------
 src/yggdrasil/session.go  |  1 -
 src/yggdrasil/switch.go   |  6 ++----
 11 files changed, 66 insertions(+), 98 deletions(-)

diff --git a/src/tuntap/conn.go b/src/tuntap/conn.go
index 0e0dd461..31490916 100644
--- a/src/tuntap/conn.go
+++ b/src/tuntap/conn.go
@@ -23,7 +23,7 @@ type tunConn struct {
 	addr  address.Address
 	snet  address.Subnet
 	stop  chan struct{}
-	alive chan struct{}
+	alive *time.Timer // From calling time.AfterFunc
 }
 
 func (s *tunConn) close() {
@@ -40,10 +40,6 @@ func (s *tunConn) _close_nomutex() {
 		defer func() { recover() }()
 		close(s.stop) // Closes reader/writer goroutines
 	}()
-	func() {
-		defer func() { recover() }()
-		close(s.alive) // Closes timeout goroutine
-	}()
 }
 
 func (s *tunConn) _read(bs []byte) (err error) {
@@ -228,27 +224,8 @@ func (s *tunConn) _write(bs []byte) (err error) {
 }
 
 func (s *tunConn) stillAlive() {
-	defer func() { recover() }()
-	select {
-	case s.alive <- struct{}{}:
-	default:
-	}
-}
-
-func (s *tunConn) checkForTimeouts() error {
-	timer := time.NewTimer(tunConnTimeout)
-	defer util.TimerStop(timer)
-	defer s.close()
-	for {
-		select {
-		case _, ok := <-s.alive:
-			if !ok {
-				return errors.New("connection closed")
-			}
-			util.TimerStop(timer)
-			timer.Reset(tunConnTimeout)
-		case <-timer.C:
-			return errors.New("timed out")
-		}
+	if s.alive != nil {
+		s.alive.Stop()
 	}
+	s.alive = time.AfterFunc(tunConnTimeout, s.close)
 }
diff --git a/src/tuntap/tun.go b/src/tuntap/tun.go
index dfc343df..d73384d4 100644
--- a/src/tuntap/tun.go
+++ b/src/tuntap/tun.go
@@ -227,10 +227,9 @@ func (tun *TunAdapter) handler() error {
 func (tun *TunAdapter) wrap(conn *yggdrasil.Conn) (c *tunConn, err error) {
 	// Prepare a session wrapper for the given connection
 	s := tunConn{
-		tun:   tun,
-		conn:  conn,
-		stop:  make(chan struct{}),
-		alive: make(chan struct{}, 1),
+		tun:  tun,
+		conn: conn,
+		stop: make(chan struct{}),
 	}
 	c = &s
 	// Get the remote address and subnet of the other side
@@ -255,13 +254,13 @@ func (tun *TunAdapter) wrap(conn *yggdrasil.Conn) (c *tunConn, err error) {
 	// we receive a packet through the interface for this address
 	tun.addrToConn[s.addr] = &s
 	tun.subnetToConn[s.snet] = &s
-	// Set the read callback and start the timeout goroutine
+	// Set the read callback and start the timeout
 	conn.SetReadCallback(func(bs []byte) {
 		s.RecvFrom(conn, func() {
 			s._read(bs)
 		})
 	})
-	go s.checkForTimeouts()
+	s.RecvFrom(nil, s.stillAlive)
 	// Return
 	return c, err
 }
diff --git a/src/yggdrasil/api.go b/src/yggdrasil/api.go
index 2bc5c816..c6c15e24 100644
--- a/src/yggdrasil/api.go
+++ b/src/yggdrasil/api.go
@@ -344,10 +344,8 @@ func (c *Core) GetNodeInfo(key crypto.BoxPubKey, coords []uint64, nocache bool)
 		c.router.nodeinfo.sendNodeInfo(key, wire_coordsUint64stoBytes(coords), false)
 	}
 	c.router.doAdmin(sendNodeInfoRequest)
-	go func() {
-		time.Sleep(6 * time.Second)
-		close(response)
-	}()
+	timer := time.AfterFunc(6*time.Second, func() { close(response) })
+	defer timer.Stop()
 	for res := range response {
 		return *res, nil
 	}
diff --git a/src/yggdrasil/debug.go b/src/yggdrasil/debug.go
index f6ec8716..c6f98ca0 100644
--- a/src/yggdrasil/debug.go
+++ b/src/yggdrasil/debug.go
@@ -552,7 +552,7 @@ func (c *Core) DEBUG_addAllowedEncryptionPublicKey(boxStr string) {
 ////////////////////////////////////////////////////////////////////////////////
 
 func DEBUG_simLinkPeers(p, q *peer) {
-	// Sets q.out() to point to p and starts p.linkLoop()
+	// Sets q.out() to point to p and starts p.start()
 	goWorkers := func(source, dest *peer) {
 		source.linkOut = make(chan []byte, 1)
 		send := make(chan []byte, 1)
@@ -561,7 +561,7 @@ func DEBUG_simLinkPeers(p, q *peer) {
 				send <- bs
 			}
 		}
-		go source.linkLoop()
+		go source.start()
 		go func() {
 			var packets [][]byte
 			for {
diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index bfbcc99b..1b48f391 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -216,7 +216,7 @@ func (intf *linkInterface) handler() error {
 	intf.link.core.log.Infof("Connected %s: %s, source %s",
 		strings.ToUpper(intf.info.linkType), themString, intf.info.local)
 	// Start the link loop
-	go intf.peer.linkLoop()
+	go intf.peer.start()
 	// Start the writer
 	signalReady := make(chan struct{}, 1)
 	signalSent := make(chan bool, 1)
diff --git a/src/yggdrasil/nodeinfo.go b/src/yggdrasil/nodeinfo.go
index 50f5bf9c..8a5d7872 100644
--- a/src/yggdrasil/nodeinfo.go
+++ b/src/yggdrasil/nodeinfo.go
@@ -47,25 +47,25 @@ func (m *nodeinfo) init(core *Core) {
 	m.callbacks = make(map[crypto.BoxPubKey]nodeinfoCallback)
 	m.cache = make(map[crypto.BoxPubKey]nodeinfoCached)
 
-	go func() {
-		for {
-			m.callbacksMutex.Lock()
-			for boxPubKey, callback := range m.callbacks {
-				if time.Since(callback.created) > time.Minute {
-					delete(m.callbacks, boxPubKey)
-				}
+	var f func()
+	f = func() {
+		m.callbacksMutex.Lock()
+		for boxPubKey, callback := range m.callbacks {
+			if time.Since(callback.created) > time.Minute {
+				delete(m.callbacks, boxPubKey)
 			}
-			m.callbacksMutex.Unlock()
-			m.cacheMutex.Lock()
-			for boxPubKey, cache := range m.cache {
-				if time.Since(cache.created) > time.Hour {
-					delete(m.cache, boxPubKey)
-				}
-			}
-			m.cacheMutex.Unlock()
-			time.Sleep(time.Second * 30)
 		}
-	}()
+		m.callbacksMutex.Unlock()
+		m.cacheMutex.Lock()
+		for boxPubKey, cache := range m.cache {
+			if time.Since(cache.created) > time.Hour {
+				delete(m.cache, boxPubKey)
+			}
+		}
+		m.cacheMutex.Unlock()
+		time.AfterFunc(time.Second*30, f)
+	}
+	go f()
 }
 
 // Add a callback for a nodeinfo lookup
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 989d9ee1..50fb03f8 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -184,18 +184,21 @@ func (ps *peers) sendSwitchMsgs(from phony.Actor) {
 
 // This must be launched in a separate goroutine by whatever sets up the peer struct.
 // It handles link protocol traffic.
-func (p *peer) linkLoop() {
-	tick := time.NewTicker(time.Second)
-	defer tick.Stop()
-	<-p.SyncExec(p._sendSwitchMsg) // Startup message
-	for {
-		select {
-		case <-p.done:
-			return
-		case _ = <-tick.C:
-			<-p.SyncExec(p._updateDHT)
-		}
+func (p *peer) start() {
+	var updateDHT func()
+	updateDHT = func() {
+		<-p.SyncExec(func() {
+			select {
+			case <-p.done:
+			default:
+				p._updateDHT()
+				time.AfterFunc(time.Second, updateDHT)
+			}
+		})
 	}
+	updateDHT()
+	// Just for good measure, immediately send a switch message to this peer when we start
+	<-p.SyncExec(p._sendSwitchMsg)
 }
 
 func (p *peer) _updateDHT() {
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index 002905bc..fec55b7c 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -92,7 +92,7 @@ func (r *router) reconfigure(e chan error) {
 // Starts the tickerLoop goroutine.
 func (r *router) start() error {
 	r.core.log.Infoln("Starting router")
-	go r.tickerLoop()
+	go r.doMaintenance()
 	return nil
 }
 
@@ -122,18 +122,14 @@ func (r *router) reset(from phony.Actor) {
 
 // TODO remove reconfigure so this is just a ticker loop
 // and then find something better than a ticker loop to schedule things...
-func (r *router) tickerLoop() {
-	ticker := time.NewTicker(time.Second)
-	defer ticker.Stop()
-	for {
-		<-ticker.C
-		<-r.SyncExec(func() {
-			// Any periodic maintenance stuff goes here
-			r.core.switchTable.doMaintenance()
-			r.dht.doMaintenance()
-			r.sessions.cleanup()
-		})
-	}
+func (r *router) doMaintenance() {
+	<-r.SyncExec(func() {
+		// Any periodic maintenance stuff goes here
+		r.core.switchTable.doMaintenance()
+		r.dht.doMaintenance()
+		r.sessions.cleanup()
+	})
+	time.AfterFunc(time.Second, r.doMaintenance)
 }
 
 // Checks incoming traffic type and passes it to the appropriate handler.
diff --git a/src/yggdrasil/search.go b/src/yggdrasil/search.go
index 5fb36584..9cf5f06e 100644
--- a/src/yggdrasil/search.go
+++ b/src/yggdrasil/search.go
@@ -152,18 +152,16 @@ func (sinfo *searchInfo) continueSearch() {
 	// In case the search dies, try to spawn another thread later
 	// Note that this will spawn multiple parallel searches as time passes
 	// Any that die aren't restarted, but a new one will start later
-	retryLater := func() {
-		// FIXME this keeps the search alive forever if not for the searches map, fix that
-		newSearchInfo := sinfo.searches.searches[sinfo.dest]
-		if newSearchInfo != sinfo {
-			return
-		}
-		sinfo.continueSearch()
-	}
-	go func() {
-		time.Sleep(search_RETRY_TIME)
-		sinfo.searches.router.doAdmin(retryLater)
-	}()
+	time.AfterFunc(search_RETRY_TIME, func() {
+		sinfo.searches.router.RecvFrom(nil, func() {
+			// FIXME this keeps the search alive forever if not for the searches map, fix that
+			newSearchInfo := sinfo.searches.searches[sinfo.dest]
+			if newSearchInfo != sinfo {
+				return
+			}
+			sinfo.continueSearch()
+		})
+	})
 }
 
 // Calls create search, and initializes the iterative search parts of the struct before returning it.
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index e448cf25..94ee41e6 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -262,7 +262,6 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 			sinfo.sessions.removeSession(&sinfo)
 		})
 	}()
-	//go sinfo.startWorkers()
 	return &sinfo
 }
 
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index cb5cf1eb..6d882252 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -200,10 +200,8 @@ func (t *switchTable) init(core *Core) {
 }
 
 func (t *switchTable) reconfigure(e chan error) {
-	go func() {
-		defer close(e)
-		// This is where reconfiguration would go, if we had anything useful to do.
-	}()
+	defer close(e)
+	// This is where reconfiguration would go, if we had anything useful to do.
 }
 
 // Safely gets a copy of this node's locator.

From aaf34c63049ed8314fd222367cfbc1fa51813ca7 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 25 Aug 2019 18:08:43 -0500
Subject: [PATCH 041/130] start migrating the TunAdapter to the actor model

---
 src/tuntap/conn.go  |   5 +-
 src/tuntap/iface.go | 501 +++++++++++++++++++++++---------------------
 src/tuntap/tun.go   |  11 +-
 3 files changed, 267 insertions(+), 250 deletions(-)

diff --git a/src/tuntap/conn.go b/src/tuntap/conn.go
index 31490916..cdb8c32e 100644
--- a/src/tuntap/conn.go
+++ b/src/tuntap/conn.go
@@ -113,8 +113,7 @@ func (s *tunConn) _read(bs []byte) (err error) {
 		util.PutBytes(bs)
 		return
 	}
-	// FIXME this send can block if the tuntap isn't running, which isn't really safe...
-	s.tun.send <- bs
+	s.tun.writer.writeFrom(s, bs)
 	s.stillAlive()
 	return
 }
@@ -208,7 +207,7 @@ func (s *tunConn) _write(bs []byte) (err error) {
 					Data: bs[:900],
 				}
 				if packet, err := CreateICMPv6(bs[8:24], bs[24:40], ipv6.ICMPTypePacketTooBig, 0, ptb); err == nil {
-					s.tun.send <- packet
+					s.tun.writer.writeFrom(s, packet)
 				}
 			} else {
 				if e.Closed() {
diff --git a/src/tuntap/iface.go b/src/tuntap/iface.go
index e0693e56..3f923513 100644
--- a/src/tuntap/iface.go
+++ b/src/tuntap/iface.go
@@ -9,264 +9,279 @@ import (
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
+
+	"github.com/Arceliar/phony"
 )
 
-func (tun *TunAdapter) writer() error {
-	var w int
+type tunWriter struct {
+	phony.Inbox
+	tun *TunAdapter
+}
+
+func (w *tunWriter) writeFrom(from phony.Actor, b []byte) {
+	w.RecvFrom(from, func() {
+		w._write(b)
+	})
+}
+
+// write is pretty loose with the memory safety rules, e.g. it assumes it can read w.tun.iface.IsTap() safely
+func (w *tunWriter) _write(b []byte) {
+	var written int
 	var err error
-	for {
-		b := <-tun.send
-		n := len(b)
-		if n == 0 {
-			continue
-		}
-		if tun.iface.IsTAP() {
-			sendndp := func(dstAddr address.Address) {
-				neigh, known := tun.icmpv6.getNeighbor(dstAddr)
-				known = known && (time.Since(neigh.lastsolicitation).Seconds() < 30)
-				if !known {
-					tun.icmpv6.Solicit(dstAddr)
-				}
-			}
-			peermac := net.HardwareAddr{0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
-			var dstAddr address.Address
-			var peerknown bool
-			if b[0]&0xf0 == 0x40 {
-				dstAddr = tun.addr
-			} else if b[0]&0xf0 == 0x60 {
-				if !bytes.Equal(tun.addr[:16], dstAddr[:16]) && !bytes.Equal(tun.subnet[:8], dstAddr[:8]) {
-					dstAddr = tun.addr
-				}
-			}
-			if neighbor, ok := tun.icmpv6.getNeighbor(dstAddr); ok && neighbor.learned {
-				// If we've learned the MAC of a 300::/7 address, for example, or a CKR
-				// address, use the MAC address of that
-				peermac = neighbor.mac
-				peerknown = true
-			} else if neighbor, ok := tun.icmpv6.getNeighbor(tun.addr); ok && neighbor.learned {
-				// Otherwise send directly to the MAC address of the host if that's
-				// known instead
-				peermac = neighbor.mac
-				peerknown = true
-			} else {
-				// Nothing has been discovered, try to discover the destination
-				sendndp(tun.addr)
-			}
-			if peerknown {
-				var proto ethernet.Ethertype
-				switch {
-				case b[0]&0xf0 == 0x60:
-					proto = ethernet.IPv6
-				case b[0]&0xf0 == 0x40:
-					proto = ethernet.IPv4
-				}
-				var frame ethernet.Frame
-				frame.Prepare(
-					peermac[:6],          // Destination MAC address
-					tun.icmpv6.mymac[:6], // Source MAC address
-					ethernet.NotTagged,   // VLAN tagging
-					proto,                // Ethertype
-					len(b))               // Payload length
-				copy(frame[tun_ETHER_HEADER_LENGTH:], b[:n])
-				n += tun_ETHER_HEADER_LENGTH
-				w, err = tun.iface.Write(frame[:n])
-			} else {
-				tun.log.Errorln("TUN/TAP iface write error: no peer MAC known for", net.IP(dstAddr[:]).String(), "- dropping packet")
-			}
-		} else {
-			w, err = tun.iface.Write(b[:n])
-			util.PutBytes(b)
-		}
-		if err != nil {
-			if !tun.isOpen {
-				return err
-			}
-			tun.log.Errorln("TUN/TAP iface write error:", err)
-			continue
-		}
-		if w != n {
-			tun.log.Errorln("TUN/TAP iface write mismatch:", w, "bytes written vs", n, "bytes given")
-			continue
-		}
+	n := len(b)
+	if n == 0 {
+		return
 	}
-}
-
-// Run in a separate goroutine by the reader
-// Does all of the per-packet ICMP checks, passes packets to the right Conn worker
-func (tun *TunAdapter) readerPacketHandler(ch chan []byte) {
-	for recvd := range ch {
-		// If it's a TAP adapter, update the buffer slice so that we no longer
-		// include the ethernet headers
-		offset := 0
-		if tun.iface.IsTAP() {
-			// Set our offset to beyond the ethernet headers
-			offset = tun_ETHER_HEADER_LENGTH
-			// Check first of all that we can go beyond the ethernet headers
-			if len(recvd) <= offset {
-				continue
+	if w.tun.iface.IsTAP() {
+		sendndp := func(dstAddr address.Address) {
+			neigh, known := w.tun.icmpv6.getNeighbor(dstAddr)
+			known = known && (time.Since(neigh.lastsolicitation).Seconds() < 30)
+			if !known {
+				w.tun.icmpv6.Solicit(dstAddr)
 			}
 		}
-		// Offset the buffer from now on so that we can ignore ethernet frames if
-		// they are present
-		bs := recvd[offset:]
-		// If we detect an ICMP packet then hand it to the ICMPv6 module
-		if bs[6] == 58 {
-			// Found an ICMPv6 packet - we need to make sure to give ICMPv6 the full
-			// Ethernet frame rather than just the IPv6 packet as this is needed for
-			// NDP to work correctly
-			if err := tun.icmpv6.ParsePacket(recvd); err == nil {
-				// We acted on the packet in the ICMPv6 module so don't forward or do
-				// anything else with it
-				continue
-			}
-		}
-		if offset != 0 {
-			// Shift forward to avoid leaking bytes off the front of the slice when we eventually store it
-			bs = append(recvd[:0], bs...)
-		}
-		// From the IP header, work out what our source and destination addresses
-		// and node IDs are. We will need these in order to work out where to send
-		// the packet
+		peermac := net.HardwareAddr{0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
 		var dstAddr address.Address
-		var dstSnet address.Subnet
-		var addrlen int
-		n := len(bs)
-		// Check the IP protocol - if it doesn't match then we drop the packet and
-		// do nothing with it
-		if bs[0]&0xf0 == 0x60 {
-			// Check if we have a fully-sized IPv6 header
-			if len(bs) < 40 {
-				continue
+		var peerknown bool
+		if b[0]&0xf0 == 0x40 {
+			dstAddr = w.tun.addr
+		} else if b[0]&0xf0 == 0x60 {
+			if !bytes.Equal(w.tun.addr[:16], dstAddr[:16]) && !bytes.Equal(w.tun.subnet[:8], dstAddr[:8]) {
+				dstAddr = w.tun.addr
 			}
-			// Check the packet size
-			if n-tun_IPv6_HEADER_LENGTH != 256*int(bs[4])+int(bs[5]) {
-				continue
-			}
-			// IPv6 address
-			addrlen = 16
-			copy(dstAddr[:addrlen], bs[24:])
-			copy(dstSnet[:addrlen/2], bs[24:])
-		} else if bs[0]&0xf0 == 0x40 {
-			// Check if we have a fully-sized IPv4 header
-			if len(bs) < 20 {
-				continue
-			}
-			// Check the packet size
-			if n != 256*int(bs[2])+int(bs[3]) {
-				continue
-			}
-			// IPv4 address
-			addrlen = 4
-			copy(dstAddr[:addrlen], bs[16:])
+		}
+		if neighbor, ok := w.tun.icmpv6.getNeighbor(dstAddr); ok && neighbor.learned {
+			// If we've learned the MAC of a 300::/7 address, for example, or a CKR
+			// address, use the MAC address of that
+			peermac = neighbor.mac
+			peerknown = true
+		} else if neighbor, ok := w.tun.icmpv6.getNeighbor(w.tun.addr); ok && neighbor.learned {
+			// Otherwise send directly to the MAC address of the host if that's
+			// known instead
+			peermac = neighbor.mac
+			peerknown = true
 		} else {
-			// Unknown address length or protocol, so drop the packet and ignore it
-			tun.log.Traceln("Unknown packet type, dropping")
-			continue
+			// Nothing has been discovered, try to discover the destination
+			sendndp(w.tun.addr)
 		}
-		if tun.ckr.isEnabled() {
-			if addrlen != 16 || (!dstAddr.IsValid() && !dstSnet.IsValid()) {
-				if key, err := tun.ckr.getPublicKeyForAddress(dstAddr, addrlen); err == nil {
-					// A public key was found, get the node ID for the search
-					dstNodeID := crypto.GetNodeID(&key)
-					dstAddr = *address.AddrForNodeID(dstNodeID)
-					dstSnet = *address.SubnetForNodeID(dstNodeID)
-					addrlen = 16
-				}
+		if peerknown {
+			var proto ethernet.Ethertype
+			switch {
+			case b[0]&0xf0 == 0x60:
+				proto = ethernet.IPv6
+			case b[0]&0xf0 == 0x40:
+				proto = ethernet.IPv4
 			}
+			var frame ethernet.Frame
+			frame.Prepare(
+				peermac[:6],            // Destination MAC address
+				w.tun.icmpv6.mymac[:6], // Source MAC address
+				ethernet.NotTagged,     // VLAN tagging
+				proto,                  // Ethertype
+				len(b))                 // Payload length
+			copy(frame[tun_ETHER_HEADER_LENGTH:], b[:n])
+			n += tun_ETHER_HEADER_LENGTH
+			written, err = w.tun.iface.Write(frame[:n])
+		} else {
+			w.tun.log.Errorln("TUN/TAP iface write error: no peer MAC known for", net.IP(dstAddr[:]).String(), "- dropping packet")
 		}
-		if addrlen != 16 || (!dstAddr.IsValid() && !dstSnet.IsValid()) {
-			// Couldn't find this node's ygg IP
-			continue
-		}
-		// Do we have an active connection for this node address?
-		var dstNodeID, dstNodeIDMask *crypto.NodeID
-		tun.mutex.RLock()
-		session, isIn := tun.addrToConn[dstAddr]
-		if !isIn || session == nil {
-			session, isIn = tun.subnetToConn[dstSnet]
-			if !isIn || session == nil {
-				// Neither an address nor a subnet mapping matched, therefore populate
-				// the node ID and mask to commence a search
-				if dstAddr.IsValid() {
-					dstNodeID, dstNodeIDMask = dstAddr.GetNodeIDandMask()
-				} else {
-					dstNodeID, dstNodeIDMask = dstSnet.GetNodeIDandMask()
-				}
-			}
-		}
-		tun.mutex.RUnlock()
-		// If we don't have a connection then we should open one
-		if !isIn || session == nil {
-			// Check we haven't been given empty node ID, really this shouldn't ever
-			// happen but just to be sure...
-			if dstNodeID == nil || dstNodeIDMask == nil {
-				panic("Given empty dstNodeID and dstNodeIDMask - this shouldn't happen")
-			}
-			// Dial to the remote node
-			go func() {
-				// FIXME just spitting out a goroutine to do this is kind of ugly and means we drop packets until the dial finishes
-				tun.mutex.Lock()
-				_, known := tun.dials[*dstNodeID]
-				tun.dials[*dstNodeID] = append(tun.dials[*dstNodeID], bs)
-				for len(tun.dials[*dstNodeID]) > 32 {
-					util.PutBytes(tun.dials[*dstNodeID][0])
-					tun.dials[*dstNodeID] = tun.dials[*dstNodeID][1:]
-				}
-				tun.mutex.Unlock()
-				if known {
-					return
-				}
-				var tc *tunConn
-				if conn, err := tun.dialer.DialByNodeIDandMask(dstNodeID, dstNodeIDMask); err == nil {
-					// We've been given a connection so prepare the session wrapper
-					if tc, err = tun.wrap(conn); err != nil {
-						// Something went wrong when storing the connection, typically that
-						// something already exists for this address or subnet
-						tun.log.Debugln("TUN/TAP iface wrap:", err)
-					}
-				}
-				tun.mutex.Lock()
-				packets := tun.dials[*dstNodeID]
-				delete(tun.dials, *dstNodeID)
-				tun.mutex.Unlock()
-				if tc != nil {
-					for _, packet := range packets {
-						p := packet // Possibly required because of how range
-						<-tc.SyncExec(func() { tc._write(p) })
-					}
-				}
-			}()
-			// While the dial is going on we can't do much else
-			// continuing this iteration - skip to the next one
-			continue
-		}
-		// If we have a connection now, try writing to it
-		if isIn && session != nil {
-			<-session.SyncExec(func() { session._write(bs) })
+	} else {
+		written, err = w.tun.iface.Write(b[:n])
+		util.PutBytes(b)
+	}
+	if err != nil {
+		w.tun.mutex.Lock()
+		open := w.tun.isOpen
+		w.tun.mutex.Unlock()
+		if !open {
+			return
 		}
+		w.tun.log.Errorln("TUN/TAP iface write error:", err)
+	}
+	if written != n {
+		w.tun.log.Errorln("TUN/TAP iface write mismatch:", written, "bytes written vs", n, "bytes given")
 	}
 }
 
-func (tun *TunAdapter) reader() error {
-	toWorker := make(chan []byte, 32)
-	defer close(toWorker)
-	go tun.readerPacketHandler(toWorker)
-	for {
-		// Get a slice to store the packet in
-		recvd := util.ResizeBytes(util.GetBytes(), 65535+tun_ETHER_HEADER_LENGTH)
-		// Wait for a packet to be delivered to us through the TUN/TAP adapter
-		n, err := tun.iface.Read(recvd)
-		if err != nil {
-			if !tun.isOpen {
-				return err
-			}
-			panic(err)
-		}
-		if n == 0 {
-			util.PutBytes(recvd)
-			continue
-		}
-		// Send the packet to the worker
-		toWorker <- recvd[:n]
+type tunReader struct {
+	phony.Inbox
+	tun *TunAdapter
+}
+
+func (r *tunReader) _read() {
+	// Get a slice to store the packet in
+	recvd := util.ResizeBytes(util.GetBytes(), 65535+tun_ETHER_HEADER_LENGTH)
+	// Wait for a packet to be delivered to us through the TUN/TAP adapter
+	n, err := r.tun.iface.Read(recvd)
+	if n == 0 {
+		util.PutBytes(recvd)
+	} else {
+		r.tun.handlePacketFrom(r, recvd[:n], err)
+	}
+	if err == nil {
+		// Now read again
+		r.RecvFrom(nil, r._read)
+	}
+}
+
+func (tun *TunAdapter) handlePacketFrom(from phony.Actor, packet []byte, err error) {
+	tun.RecvFrom(from, func() {
+		tun._handlePacket(packet, err)
+	})
+}
+
+// does the work of reading a packet and sending it to the correct tunConn
+func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
+	if err != nil {
+		tun.log.Errorln("TUN/TAP iface read error:", err)
+		return
+	}
+	// If it's a TAP adapter, update the buffer slice so that we no longer
+	// include the ethernet headers
+	offset := 0
+	if tun.iface.IsTAP() {
+		// Set our offset to beyond the ethernet headers
+		offset = tun_ETHER_HEADER_LENGTH
+		// Check first of all that we can go beyond the ethernet headers
+		if len(recvd) <= offset {
+			return
+		}
+	}
+	// Offset the buffer from now on so that we can ignore ethernet frames if
+	// they are present
+	bs := recvd[offset:]
+	// If we detect an ICMP packet then hand it to the ICMPv6 module
+	if bs[6] == 58 {
+		// Found an ICMPv6 packet - we need to make sure to give ICMPv6 the full
+		// Ethernet frame rather than just the IPv6 packet as this is needed for
+		// NDP to work correctly
+		if err := tun.icmpv6.ParsePacket(recvd); err == nil {
+			// We acted on the packet in the ICMPv6 module so don't forward or do
+			// anything else with it
+			return
+		}
+	}
+	if offset != 0 {
+		// Shift forward to avoid leaking bytes off the front of the slice when we eventually store it
+		bs = append(recvd[:0], bs...)
+	}
+	// From the IP header, work out what our source and destination addresses
+	// and node IDs are. We will need these in order to work out where to send
+	// the packet
+	var dstAddr address.Address
+	var dstSnet address.Subnet
+	var addrlen int
+	n := len(bs)
+	// Check the IP protocol - if it doesn't match then we drop the packet and
+	// do nothing with it
+	if bs[0]&0xf0 == 0x60 {
+		// Check if we have a fully-sized IPv6 header
+		if len(bs) < 40 {
+			return
+		}
+		// Check the packet size
+		if n-tun_IPv6_HEADER_LENGTH != 256*int(bs[4])+int(bs[5]) {
+			return
+		}
+		// IPv6 address
+		addrlen = 16
+		copy(dstAddr[:addrlen], bs[24:])
+		copy(dstSnet[:addrlen/2], bs[24:])
+	} else if bs[0]&0xf0 == 0x40 {
+		// Check if we have a fully-sized IPv4 header
+		if len(bs) < 20 {
+			return
+		}
+		// Check the packet size
+		if n != 256*int(bs[2])+int(bs[3]) {
+			return
+		}
+		// IPv4 address
+		addrlen = 4
+		copy(dstAddr[:addrlen], bs[16:])
+	} else {
+		// Unknown address length or protocol, so drop the packet and ignore it
+		tun.log.Traceln("Unknown packet type, dropping")
+		return
+	}
+	if tun.ckr.isEnabled() {
+		if addrlen != 16 || (!dstAddr.IsValid() && !dstSnet.IsValid()) {
+			if key, err := tun.ckr.getPublicKeyForAddress(dstAddr, addrlen); err == nil {
+				// A public key was found, get the node ID for the search
+				dstNodeID := crypto.GetNodeID(&key)
+				dstAddr = *address.AddrForNodeID(dstNodeID)
+				dstSnet = *address.SubnetForNodeID(dstNodeID)
+				addrlen = 16
+			}
+		}
+	}
+	if addrlen != 16 || (!dstAddr.IsValid() && !dstSnet.IsValid()) {
+		// Couldn't find this node's ygg IP
+		return
+	}
+	// Do we have an active connection for this node address?
+	var dstNodeID, dstNodeIDMask *crypto.NodeID
+	tun.mutex.RLock()
+	session, isIn := tun.addrToConn[dstAddr]
+	if !isIn || session == nil {
+		session, isIn = tun.subnetToConn[dstSnet]
+		if !isIn || session == nil {
+			// Neither an address nor a subnet mapping matched, therefore populate
+			// the node ID and mask to commence a search
+			if dstAddr.IsValid() {
+				dstNodeID, dstNodeIDMask = dstAddr.GetNodeIDandMask()
+			} else {
+				dstNodeID, dstNodeIDMask = dstSnet.GetNodeIDandMask()
+			}
+		}
+	}
+	tun.mutex.RUnlock()
+	// If we don't have a connection then we should open one
+	if !isIn || session == nil {
+		// Check we haven't been given empty node ID, really this shouldn't ever
+		// happen but just to be sure...
+		if dstNodeID == nil || dstNodeIDMask == nil {
+			panic("Given empty dstNodeID and dstNodeIDMask - this shouldn't happen")
+		}
+		// Dial to the remote node
+		go func() {
+			// FIXME just spitting out a goroutine to do this is kind of ugly and means we drop packets until the dial finishes
+			tun.mutex.Lock()
+			_, known := tun.dials[*dstNodeID]
+			tun.dials[*dstNodeID] = append(tun.dials[*dstNodeID], bs)
+			for len(tun.dials[*dstNodeID]) > 32 {
+				util.PutBytes(tun.dials[*dstNodeID][0])
+				tun.dials[*dstNodeID] = tun.dials[*dstNodeID][1:]
+			}
+			tun.mutex.Unlock()
+			if known {
+				return
+			}
+			var tc *tunConn
+			if conn, err := tun.dialer.DialByNodeIDandMask(dstNodeID, dstNodeIDMask); err == nil {
+				// We've been given a connection so prepare the session wrapper
+				if tc, err = tun.wrap(conn); err != nil {
+					// Something went wrong when storing the connection, typically that
+					// something already exists for this address or subnet
+					tun.log.Debugln("TUN/TAP iface wrap:", err)
+				}
+			}
+			tun.mutex.Lock()
+			packets := tun.dials[*dstNodeID]
+			delete(tun.dials, *dstNodeID)
+			tun.mutex.Unlock()
+			if tc != nil {
+				for _, packet := range packets {
+					p := packet // Possibly required because of how range
+					<-tc.SyncExec(func() { tc._write(p) })
+				}
+			}
+		}()
+		// While the dial is going on we can't do much else
+		return
+	}
+	// If we have a connection now, try writing to it
+	if isIn && session != nil {
+		session.RecvFrom(tun, func() { session._write(bs) })
 	}
 }
diff --git a/src/tuntap/tun.go b/src/tuntap/tun.go
index d73384d4..7bb3a59f 100644
--- a/src/tuntap/tun.go
+++ b/src/tuntap/tun.go
@@ -15,6 +15,7 @@ import (
 	"net"
 	"sync"
 
+	"github.com/Arceliar/phony"
 	"github.com/gologme/log"
 	"github.com/yggdrasil-network/water"
 
@@ -33,6 +34,8 @@ const tun_ETHER_HEADER_LENGTH = 14
 // you should pass this object to the yggdrasil.SetRouterAdapter() function
 // before calling yggdrasil.Start().
 type TunAdapter struct {
+	writer       tunWriter
+	reader       tunReader
 	config       *config.NodeState
 	log          *log.Logger
 	reconfigure  chan chan error
@@ -44,7 +47,7 @@ type TunAdapter struct {
 	icmpv6       ICMPv6
 	mtu          int
 	iface        *water.Interface
-	send         chan []byte
+	phony.Inbox               // Currently only used for _handlePacket from the reader, TODO: all the stuff that currently needs a mutex below
 	mutex        sync.RWMutex // Protects the below
 	addrToConn   map[address.Address]*tunConn
 	subnetToConn map[address.Subnet]*tunConn
@@ -114,6 +117,8 @@ func (tun *TunAdapter) Init(config *config.NodeState, log *log.Logger, listener
 	tun.addrToConn = make(map[address.Address]*tunConn)
 	tun.subnetToConn = make(map[address.Subnet]*tunConn)
 	tun.dials = make(map[crypto.NodeID][][]byte)
+	tun.writer.tun = tun
+	tun.reader.tun = tun
 }
 
 // Start the setup process for the TUN/TAP adapter. If successful, starts the
@@ -147,7 +152,6 @@ func (tun *TunAdapter) Start() error {
 	}
 	tun.mutex.Lock()
 	tun.isOpen = true
-	tun.send = make(chan []byte, 32) // TODO: is this a sensible value?
 	tun.reconfigure = make(chan chan error)
 	tun.mutex.Unlock()
 	go func() {
@@ -157,8 +161,7 @@ func (tun *TunAdapter) Start() error {
 		}
 	}()
 	go tun.handler()
-	go tun.reader()
-	go tun.writer()
+	tun.reader.RecvFrom(nil, tun.reader._read) // Start the reader
 	tun.icmpv6.Init(tun)
 	if iftapmode {
 		go tun.icmpv6.Solicit(tun.addr)

From b2a2e251ad173686c752463f38efdf77e6814487 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 25 Aug 2019 18:53:11 -0500
Subject: [PATCH 042/130] more TunAdapter migration

---
 src/tuntap/conn.go  | 12 ++++---
 src/tuntap/iface.go | 78 +++++++++++++++++++--------------------------
 src/tuntap/tun.go   | 72 ++++++++++++++++++++++++-----------------
 3 files changed, 84 insertions(+), 78 deletions(-)

diff --git a/src/tuntap/conn.go b/src/tuntap/conn.go
index cdb8c32e..afb0f09e 100644
--- a/src/tuntap/conn.go
+++ b/src/tuntap/conn.go
@@ -27,12 +27,10 @@ type tunConn struct {
 }
 
 func (s *tunConn) close() {
-	s.tun.mutex.Lock()
-	defer s.tun.mutex.Unlock()
-	s._close_nomutex()
+	s.tun.RecvFrom(s, s._close_from_tun)
 }
 
-func (s *tunConn) _close_nomutex() {
+func (s *tunConn) _close_from_tun() {
 	s.conn.Close()
 	delete(s.tun.addrToConn, s.addr)
 	delete(s.tun.subnetToConn, s.snet)
@@ -118,6 +116,12 @@ func (s *tunConn) _read(bs []byte) (err error) {
 	return
 }
 
+func (s *tunConn) writeFrom(from phony.Actor, bs []byte) {
+	s.RecvFrom(from, func() {
+		s._write(bs)
+	})
+}
+
 func (s *tunConn) _write(bs []byte) (err error) {
 	select {
 	case <-s.stop:
diff --git a/src/tuntap/iface.go b/src/tuntap/iface.go
index 3f923513..16e8d25d 100644
--- a/src/tuntap/iface.go
+++ b/src/tuntap/iface.go
@@ -90,13 +90,11 @@ func (w *tunWriter) _write(b []byte) {
 		util.PutBytes(b)
 	}
 	if err != nil {
-		w.tun.mutex.Lock()
-		open := w.tun.isOpen
-		w.tun.mutex.Unlock()
-		if !open {
-			return
-		}
-		w.tun.log.Errorln("TUN/TAP iface write error:", err)
+		w.tun.RecvFrom(w, func() {
+			if !w.tun.isOpen {
+				w.tun.log.Errorln("TUN/TAP iface write error:", err)
+			}
+		})
 	}
 	if written != n {
 		w.tun.log.Errorln("TUN/TAP iface write mismatch:", written, "bytes written vs", n, "bytes given")
@@ -221,7 +219,6 @@ func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
 	}
 	// Do we have an active connection for this node address?
 	var dstNodeID, dstNodeIDMask *crypto.NodeID
-	tun.mutex.RLock()
 	session, isIn := tun.addrToConn[dstAddr]
 	if !isIn || session == nil {
 		session, isIn = tun.subnetToConn[dstSnet]
@@ -235,7 +232,6 @@ func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
 			}
 		}
 	}
-	tun.mutex.RUnlock()
 	// If we don't have a connection then we should open one
 	if !isIn || session == nil {
 		// Check we haven't been given empty node ID, really this shouldn't ever
@@ -243,45 +239,37 @@ func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
 		if dstNodeID == nil || dstNodeIDMask == nil {
 			panic("Given empty dstNodeID and dstNodeIDMask - this shouldn't happen")
 		}
-		// Dial to the remote node
-		go func() {
-			// FIXME just spitting out a goroutine to do this is kind of ugly and means we drop packets until the dial finishes
-			tun.mutex.Lock()
-			_, known := tun.dials[*dstNodeID]
-			tun.dials[*dstNodeID] = append(tun.dials[*dstNodeID], bs)
-			for len(tun.dials[*dstNodeID]) > 32 {
-				util.PutBytes(tun.dials[*dstNodeID][0])
-				tun.dials[*dstNodeID] = tun.dials[*dstNodeID][1:]
-			}
-			tun.mutex.Unlock()
-			if known {
-				return
-			}
-			var tc *tunConn
-			if conn, err := tun.dialer.DialByNodeIDandMask(dstNodeID, dstNodeIDMask); err == nil {
-				// We've been given a connection so prepare the session wrapper
-				if tc, err = tun.wrap(conn); err != nil {
-					// Something went wrong when storing the connection, typically that
-					// something already exists for this address or subnet
-					tun.log.Debugln("TUN/TAP iface wrap:", err)
+		_, known := tun.dials[*dstNodeID]
+		tun.dials[*dstNodeID] = append(tun.dials[*dstNodeID], bs)
+		for len(tun.dials[*dstNodeID]) > 32 {
+			util.PutBytes(tun.dials[*dstNodeID][0])
+			tun.dials[*dstNodeID] = tun.dials[*dstNodeID][1:]
+		}
+		if !known {
+			go func() {
+				if conn, err := tun.dialer.DialByNodeIDandMask(dstNodeID, dstNodeIDMask); err == nil {
+					tun.RecvFrom(nil, func() {
+						// We've been given a connection so prepare the session wrapper
+						packets := tun.dials[*dstNodeID]
+						delete(tun.dials, *dstNodeID)
+						var tc *tunConn
+						var err error
+						if tc, err = tun._wrap(conn); err != nil {
+							// Something went wrong when storing the connection, typically that
+							// something already exists for this address or subnet
+							tun.log.Debugln("TUN/TAP iface wrap:", err)
+							return
+						}
+						for _, packet := range packets {
+							tc.writeFrom(nil, packet)
+						}
+					})
 				}
-			}
-			tun.mutex.Lock()
-			packets := tun.dials[*dstNodeID]
-			delete(tun.dials, *dstNodeID)
-			tun.mutex.Unlock()
-			if tc != nil {
-				for _, packet := range packets {
-					p := packet // Possibly required because of how range
-					<-tc.SyncExec(func() { tc._write(p) })
-				}
-			}
-		}()
-		// While the dial is going on we can't do much else
-		return
+			}()
+		}
 	}
 	// If we have a connection now, try writing to it
 	if isIn && session != nil {
-		session.RecvFrom(tun, func() { session._write(bs) })
+		session.writeFrom(tun, bs)
 	}
 }
diff --git a/src/tuntap/tun.go b/src/tuntap/tun.go
index 7bb3a59f..2b163e3b 100644
--- a/src/tuntap/tun.go
+++ b/src/tuntap/tun.go
@@ -13,7 +13,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
-	"sync"
+	//"sync"
 
 	"github.com/Arceliar/phony"
 	"github.com/gologme/log"
@@ -34,21 +34,21 @@ const tun_ETHER_HEADER_LENGTH = 14
 // you should pass this object to the yggdrasil.SetRouterAdapter() function
 // before calling yggdrasil.Start().
 type TunAdapter struct {
-	writer       tunWriter
-	reader       tunReader
-	config       *config.NodeState
-	log          *log.Logger
-	reconfigure  chan chan error
-	listener     *yggdrasil.Listener
-	dialer       *yggdrasil.Dialer
-	addr         address.Address
-	subnet       address.Subnet
-	ckr          cryptokey
-	icmpv6       ICMPv6
-	mtu          int
-	iface        *water.Interface
-	phony.Inbox               // Currently only used for _handlePacket from the reader, TODO: all the stuff that currently needs a mutex below
-	mutex        sync.RWMutex // Protects the below
+	writer      tunWriter
+	reader      tunReader
+	config      *config.NodeState
+	log         *log.Logger
+	reconfigure chan chan error
+	listener    *yggdrasil.Listener
+	dialer      *yggdrasil.Dialer
+	addr        address.Address
+	subnet      address.Subnet
+	ckr         cryptokey
+	icmpv6      ICMPv6
+	mtu         int
+	iface       *water.Interface
+	phony.Inbox // Currently only used for _handlePacket from the reader, TODO: all the stuff that currently needs a mutex below
+	//mutex        sync.RWMutex // Protects the below
 	addrToConn   map[address.Address]*tunConn
 	subnetToConn map[address.Subnet]*tunConn
 	dials        map[crypto.NodeID][][]byte // Buffer of packets to send after dialing finishes
@@ -122,8 +122,16 @@ func (tun *TunAdapter) Init(config *config.NodeState, log *log.Logger, listener
 }
 
 // Start the setup process for the TUN/TAP adapter. If successful, starts the
-// read/write goroutines to handle packets on that interface.
+// reader actor to handle packets on that interface.
 func (tun *TunAdapter) Start() error {
+	var err error
+	<-tun.SyncExec(func() {
+		err = tun._start()
+	})
+	return err
+}
+
+func (tun *TunAdapter) _start() error {
 	current := tun.config.GetCurrent()
 	if tun.config == nil || tun.listener == nil || tun.dialer == nil {
 		return errors.New("No configuration available to TUN/TAP")
@@ -150,10 +158,8 @@ func (tun *TunAdapter) Start() error {
 		tun.log.Debugln("Not starting TUN/TAP as ifname is none or dummy")
 		return nil
 	}
-	tun.mutex.Lock()
 	tun.isOpen = true
 	tun.reconfigure = make(chan chan error)
-	tun.mutex.Unlock()
 	go func() {
 		for {
 			e := <-tun.reconfigure
@@ -173,6 +179,14 @@ func (tun *TunAdapter) Start() error {
 // Start the setup process for the TUN/TAP adapter. If successful, starts the
 // read/write goroutines to handle packets on that interface.
 func (tun *TunAdapter) Stop() error {
+	var err error
+	<-tun.SyncExec(func() {
+		err = tun._stop()
+	})
+	return err
+}
+
+func (tun *TunAdapter) _stop() error {
 	tun.isOpen = false
 	// TODO: we have nothing that cleanly stops all the various goroutines opened
 	// by TUN/TAP, e.g. readers/writers, sessions
@@ -219,15 +233,17 @@ func (tun *TunAdapter) handler() error {
 			tun.log.Errorln("TUN/TAP connection accept error:", err)
 			return err
 		}
-		if _, err := tun.wrap(conn); err != nil {
-			// Something went wrong when storing the connection, typically that
-			// something already exists for this address or subnet
-			tun.log.Debugln("TUN/TAP handler wrap:", err)
-		}
+		<-tun.SyncExec(func() {
+			if _, err := tun._wrap(conn); err != nil {
+				// Something went wrong when storing the connection, typically that
+				// something already exists for this address or subnet
+				tun.log.Debugln("TUN/TAP handler wrap:", err)
+			}
+		})
 	}
 }
 
-func (tun *TunAdapter) wrap(conn *yggdrasil.Conn) (c *tunConn, err error) {
+func (tun *TunAdapter) _wrap(conn *yggdrasil.Conn) (c *tunConn, err error) {
 	// Prepare a session wrapper for the given connection
 	s := tunConn{
 		tun:  tun,
@@ -240,17 +256,15 @@ func (tun *TunAdapter) wrap(conn *yggdrasil.Conn) (c *tunConn, err error) {
 	s.addr = *address.AddrForNodeID(&remoteNodeID)
 	s.snet = *address.SubnetForNodeID(&remoteNodeID)
 	// Work out if this is already a destination we already know about
-	tun.mutex.Lock()
-	defer tun.mutex.Unlock()
 	atc, aok := tun.addrToConn[s.addr]
 	stc, sok := tun.subnetToConn[s.snet]
 	// If we know about a connection for this destination already then assume it
 	// is no longer valid and close it
 	if aok {
-		atc._close_nomutex()
+		atc._close_from_tun()
 		err = errors.New("replaced connection for address")
 	} else if sok {
-		stc._close_nomutex()
+		stc._close_from_tun()
 		err = errors.New("replaced connection for subnet")
 	}
 	// Save the session wrapper so that we can look it up quickly next time

From dffd70119da10ef92665a14edc224d9db690a036 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 25 Aug 2019 19:13:47 -0500
Subject: [PATCH 043/130] remove session shutdown goroutine, just send a
 message instead

---
 src/yggdrasil/conn.go    |  2 ++
 src/yggdrasil/session.go | 13 ++++++-------
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index b0c26867..efc0c81e 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -304,6 +304,8 @@ func (c *Conn) Close() (err error) {
 			// Close the session, if it hasn't been closed already
 			if e := c.session.cancel.Cancel(errors.New("connection closed")); e != nil {
 				err = ConnError{errors.New("close failed, session already closed"), false, false, true, 0}
+			} else {
+				c.session.doRemove()
 			}
 		}
 	})
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 94ee41e6..0fc7ec80 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -255,13 +255,6 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 	sinfo.theirSubnet = *address.SubnetForNodeID(crypto.GetNodeID(&sinfo.theirPermPub))
 	ss.sinfos[sinfo.myHandle] = &sinfo
 	ss.byTheirPerm[sinfo.theirPermPub] = &sinfo.myHandle
-	go func() {
-		// Run cleanup when the session is canceled
-		<-sinfo.cancel.Finished()
-		sinfo.sessions.router.doAdmin(func() {
-			sinfo.sessions.removeSession(&sinfo)
-		})
-	}()
 	return &sinfo
 }
 
@@ -293,6 +286,12 @@ func (ss *sessions) cleanup() {
 	ss.lastCleanup = time.Now()
 }
 
+func (sinfo *sessionInfo) doRemove() {
+	sinfo.sessions.router.RecvFrom(nil, func() {
+		sinfo.sessions.removeSession(sinfo)
+	})
+}
+
 // Closes a session, removing it from sessions maps.
 func (ss *sessions) removeSession(sinfo *sessionInfo) {
 	if s := sinfo.sessions.sinfos[sinfo.myHandle]; s == sinfo {

From b5b179904b89a668c03e735a430e9a534f5140b2 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 25 Aug 2019 22:19:20 -0500
Subject: [PATCH 044/130] ugly work-in-progress to migrate link to the actor
 model

---
 src/yggdrasil/link.go | 211 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 210 insertions(+), 1 deletion(-)

diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index 1b48f391..5489c015 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -54,6 +54,15 @@ type linkInterface struct {
 	incoming bool
 	force    bool
 	closed   chan struct{}
+  reader linkReader // Reads packets, notifies this linkInterface, passes packets to switch
+  writer linkWriter // Writes packets, notifies this linkInterface
+  phony.Inbox // Protects the below
+  sendTimer *time.Timer // Fires to signal that sending is blocked
+  stallTimer *time.Timer // Fires to signal that no incoming traffic (including keep-alive) has been seen
+  recvTimer *time.Timer // Fires to send keep-alive traffic
+  closeTimer *time.Timer // Fires when the link has been idle so long we need to close it
+  inSwitch bool // True if the switch is tracking this link
+  stalled bool // True if we haven't been receiving any response traffic
 }
 
 func (l *link) init(c *Core) error {
@@ -124,6 +133,9 @@ func (l *link) create(msgIO linkInterfaceMsgIO, name, linkType, local, remote st
 		incoming: incoming,
 		force:    force,
 	}
+  intf.writer.intf = &intf
+  intf.reader.intf = &intf
+  intf.reader.err = make(chan error)
 	return &intf, nil
 }
 
@@ -187,7 +199,7 @@ func (intf *linkInterface) handler() error {
 			intf.link.mutex.Lock()
 			delete(intf.link.interfaces, intf.info)
 			intf.link.mutex.Unlock()
-			close(intf.closed)
+			//close(intf.closed)
 		}()
 		intf.link.core.log.Debugln("DEBUG: registered interface for", intf.name)
 	}
@@ -203,13 +215,24 @@ func (intf *linkInterface) handler() error {
 		intf.link.core.peers.removePeer(intf.peer.port)
 	}()
 	// Finish setting up the peer struct
+  /*
 	out := make(chan [][]byte, 1)
 	defer close(out)
 	intf.peer.out = func(msgs [][]byte) {
 		defer func() { recover() }()
 		out <- msgs
 	}
+  */
+  intf.peer.out = func(msgs [][]byte) {
+    intf.writer.sendFrom(intf.peer, msgs, false)
+  }
 	intf.peer.linkOut = make(chan []byte, 1)
+  go func() {
+    // TODO fix this
+    for bs := range intf.peer.linkOut {
+      intf.writer.sendFrom(intf.peer, [][]byte{bs}, true)
+    }
+  }()
 	themAddr := address.AddrForNodeID(crypto.GetNodeID(&intf.info.box))
 	themAddrString := net.IP(themAddr[:]).String()
 	themString := fmt.Sprintf("%s@%s", themAddrString, intf.info.remote)
@@ -218,6 +241,7 @@ func (intf *linkInterface) handler() error {
 	// Start the link loop
 	go intf.peer.start()
 	// Start the writer
+  /*
 	signalReady := make(chan struct{}, 1)
 	signalSent := make(chan bool, 1)
 	sendAck := make(chan struct{}, 1)
@@ -413,6 +437,7 @@ func (intf *linkInterface) handler() error {
 		// Now try to read again
 		helper.RecvFrom(nil, helperFunc)
 	}
+  
 	// Start the read loop
 	helper.RecvFrom(nil, helperFunc)
 	<-done // Wait for the helper to exit
@@ -427,5 +452,189 @@ func (intf *linkInterface) handler() error {
 		intf.link.core.log.Infof("Disconnected %s: %s, source %s",
 			strings.ToUpper(intf.info.linkType), themString, intf.info.local)
 	}
+  */
+  // Start the reader
+  intf.reader.RecvFrom(nil, intf.reader._read)
+  // Wait for the reader to finish
+  err = <- intf.reader.err
+  if err != nil {
+		intf.link.core.log.Infof("Disconnected %s: %s, source %s; error: %s",
+			strings.ToUpper(intf.info.linkType), themString, intf.info.local, err)
+  } else {
+		intf.link.core.log.Infof("Disconnected %s: %s, source %s",
+			strings.ToUpper(intf.info.linkType), themString, intf.info.local)
+  }
 	return err
 }
+
+////////////////////////////////////////////////////////////////////////////////
+
+
+/*
+  phony.Inbox // Protects the below
+  sendTimer *time.Timer // Fires to signal that sending is blocked
+  stallTimer *time.Time // Fires to signal that no incoming traffic (including keep-alive) has been seen
+  recvTimer *time.Timer // Fires to send keep-alive traffic
+  closeTimer *time.Timer // Fires when the link has been idle so long we need to close it
+  inSwitch bool // True if the switch is tracking this link
+  stalled bool // True if we haven't been receiving any response traffic
+*/
+
+const (
+  sendBlockedTime = time.Second // How long to wait before deciding a send is blocked
+  keepAliveTime = 2*time.Second // How long to wait before sending a keep-alive response if we have no real traffic to send
+  stallTime = 6*time.Second // How long to wait for response traffic before deciding the connection has stalled
+  closeTime = 2*switch_timeout // How long to wait before closing the link
+)
+
+// notify the intf that we're currently sending
+func (intf *linkInterface) notifySending(size int, isLinkTraffic bool) {
+  intf.RecvFrom(nil, func() {
+    if !isLinkTraffic && size > 0 {
+      intf.inSwitch = false
+    }
+    intf.sendTimer = time.AfterFunc(sendBlockedTime, intf.notifyBlockedSend)
+    intf._cancelRecvTimer()
+  })
+}
+
+// we just sent something, so cancel any pending timer to send keep-alive traffic
+func (intf *linkInterface) _cancelRecvTimer() {
+  intf.RecvFrom(nil, func() {
+    if intf.recvTimer != nil {
+      intf.recvTimer.Stop()
+      intf.recvTimer = nil
+    }
+  })
+}
+
+// called by an AfterFunc if we appear to have timed out
+func (intf *linkInterface) notifyBlockedSend() {
+  intf.RecvFrom(nil, func() {
+    if intf.sendTimer != nil {
+      //As far as we know, we're still trying to send, and the timer fired.
+			intf.link.core.switchTable.blockPeer(intf.peer.port)
+    }
+  })
+}
+
+// notify the intf that we've finished sending, returning the peer to the switch
+func (intf *linkInterface) notifySent(size int, isLinkTraffic bool) {
+  intf.RecvFrom(nil, func() {
+    intf.sendTimer.Stop()
+    intf.sendTimer = nil
+    if !isLinkTraffic {
+      intf._notifySwitch()
+    }
+    if size > 0 && intf.stallTimer == nil {
+      intf.stallTimer = time.AfterFunc(stallTime, intf.notifyStalled)
+    }
+  })
+}
+
+// Notify the switch that we're ready for more traffic, assuming we're not in a stalled state
+func (intf *linkInterface) _notifySwitch() {
+  if !intf.inSwitch && !intf.stalled {
+    intf.inSwitch = true
+	  intf.link.core.switchTable.RecvFrom(intf, func() {
+		  intf.link.core.switchTable._idleIn(intf.peer.port)
+	  })
+  }
+}
+
+// Set the peer as stalled, to prevent them from returning to the switch until a read succeeds
+func (intf *linkInterface) notifyStalled() {
+  intf.RecvFrom(nil, func() {
+    if intf.stallTimer != nil {
+      intf.stallTimer = nil
+      intf.stalled = true
+			intf.link.core.switchTable.blockPeer(intf.peer.port)
+    }
+  })
+}
+
+
+// reset the close timer
+func (intf *linkInterface) notifyReading(from phony.Actor) {
+  intf.RecvFrom(from, func() {
+    if intf.closeTimer != nil {
+      intf.closeTimer.Stop()
+    }
+    intf.closeTimer = time.AfterFunc(closeTime, func() { intf.msgIO.close() })
+  })
+}
+
+// wake up the link if it was stalled, and (if size > 0) prepare to send keep-alive traffic
+func (intf *linkInterface) notifyReadFrom(from phony.Actor, size int) {
+  intf.RecvFrom(from, func() {
+    intf.link.core.log.Printf("DEBUG notifyReadFrom: inSwitch %v, stalled %v\n", intf.inSwitch, intf.stalled)
+    if intf.stallTimer != nil {
+      intf.stallTimer.Stop()
+      intf.stallTimer = nil
+    }
+    intf.stalled = false
+    intf._notifySwitch()
+    if size > 0 && intf.recvTimer == nil {
+      intf.recvTimer = time.AfterFunc(keepAliveTime, intf.notifyDoKeepAlive)
+    }
+  })
+}
+
+// We need to send keep-alive traffic now
+func (intf *linkInterface) notifyDoKeepAlive() {
+  intf.RecvFrom(nil, func() {
+    if intf.recvTimer != nil {
+      intf.recvTimer.Stop()
+      intf.recvTimer = nil
+      intf.writer.sendFrom(nil, [][]byte{nil}, true) // Empty keep-alive traffic
+    }
+  })
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+type linkWriter struct {
+  phony.Inbox
+  intf *linkInterface
+}
+
+func (w *linkWriter) sendFrom(from phony.Actor, bss [][]byte, isLinkTraffic bool) {
+  w.RecvFrom(from, func() {
+    var size int
+    for _, bs := range bss {
+      size += len(bs)
+    }
+    w.intf.notifySending(size, isLinkTraffic)
+	  w.intf.msgIO.writeMsgs(bss)
+    w.intf.notifySent(size, isLinkTraffic)
+    w.intf.link.core.log.Println("DEBUG: wrote something, size:", size, "isLinkTraffic:", isLinkTraffic)
+  })
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+type linkReader struct {
+  phony.Inbox
+  intf *linkInterface
+  err chan error
+}
+
+func (r *linkReader) _read() {
+    r.intf.notifyReading(r)
+		msg, err := r.intf.msgIO.readMsg()
+    r.intf.link.core.log.Println("DEBUG read something")
+    r.intf.notifyReadFrom(r, len(msg))
+		if len(msg) > 0 {
+			r.intf.peer.handlePacketFrom(r, msg)
+		}
+		if err != nil {
+			if err != io.EOF {
+        r.err<-err
+			}
+      close(r.err)
+			return
+		}
+		// Now try to read again
+		r.RecvFrom(nil, r._read)
+}
+

From bd3eaefb72675473ca46a0c0b943ffc9ac2ffdee Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 25 Aug 2019 22:55:17 -0500
Subject: [PATCH 045/130] more link migration

---
 src/yggdrasil/debug.go  |   7 +-
 src/yggdrasil/link.go   | 497 ++++++++++------------------------------
 src/yggdrasil/peer.go   |  12 +-
 src/yggdrasil/switch.go |   4 +-
 4 files changed, 135 insertions(+), 385 deletions(-)

diff --git a/src/yggdrasil/debug.go b/src/yggdrasil/debug.go
index c6f98ca0..1cc7eae4 100644
--- a/src/yggdrasil/debug.go
+++ b/src/yggdrasil/debug.go
@@ -554,7 +554,8 @@ func (c *Core) DEBUG_addAllowedEncryptionPublicKey(boxStr string) {
 func DEBUG_simLinkPeers(p, q *peer) {
 	// Sets q.out() to point to p and starts p.start()
 	goWorkers := func(source, dest *peer) {
-		source.linkOut = make(chan []byte, 1)
+		linkOut := make(chan []byte, 1)
+		source.linkOut = func(bs []byte) { linkOut <- bs }
 		send := make(chan []byte, 1)
 		source.out = func(bss [][]byte) {
 			for _, bs := range bss {
@@ -566,7 +567,7 @@ func DEBUG_simLinkPeers(p, q *peer) {
 			var packets [][]byte
 			for {
 				select {
-				case packet := <-source.linkOut:
+				case packet := <-linkOut:
 					packets = append(packets, packet)
 					continue
 				case packet := <-send:
@@ -583,7 +584,7 @@ func DEBUG_simLinkPeers(p, q *peer) {
 					continue
 				}
 				select {
-				case packet := <-source.linkOut:
+				case packet := <-linkOut:
 					packets = append(packets, packet)
 				case packet := <-send:
 					packets = append(packets, packet)
diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index 5489c015..b4ca38c4 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -46,23 +46,23 @@ type linkInterfaceMsgIO interface {
 }
 
 type linkInterface struct {
-	name     string
-	link     *link
-	peer     *peer
-	msgIO    linkInterfaceMsgIO
-	info     linkInfo
-	incoming bool
-	force    bool
-	closed   chan struct{}
-  reader linkReader // Reads packets, notifies this linkInterface, passes packets to switch
-  writer linkWriter // Writes packets, notifies this linkInterface
-  phony.Inbox // Protects the below
-  sendTimer *time.Timer // Fires to signal that sending is blocked
-  stallTimer *time.Timer // Fires to signal that no incoming traffic (including keep-alive) has been seen
-  recvTimer *time.Timer // Fires to send keep-alive traffic
-  closeTimer *time.Timer // Fires when the link has been idle so long we need to close it
-  inSwitch bool // True if the switch is tracking this link
-  stalled bool // True if we haven't been receiving any response traffic
+	name        string
+	link        *link
+	peer        *peer
+	msgIO       linkInterfaceMsgIO
+	info        linkInfo
+	incoming    bool
+	force       bool
+	closed      chan struct{}
+	reader      linkReader  // Reads packets, notifies this linkInterface, passes packets to switch
+	writer      linkWriter  // Writes packets, notifies this linkInterface
+	phony.Inbox             // Protects the below
+	sendTimer   *time.Timer // Fires to signal that sending is blocked
+	stallTimer  *time.Timer // Fires to signal that no incoming traffic (including keep-alive) has been seen
+	recvTimer   *time.Timer // Fires to send keep-alive traffic
+	closeTimer  *time.Timer // Fires when the link has been idle so long we need to close it
+	inSwitch    bool        // True if the switch is tracking this link
+	stalled     bool        // True if we haven't been receiving any response traffic
 }
 
 func (l *link) init(c *Core) error {
@@ -133,9 +133,9 @@ func (l *link) create(msgIO linkInterfaceMsgIO, name, linkType, local, remote st
 		incoming: incoming,
 		force:    force,
 	}
-  intf.writer.intf = &intf
-  intf.reader.intf = &intf
-  intf.reader.err = make(chan error)
+	intf.writer.intf = &intf
+	intf.reader.intf = &intf
+	intf.reader.err = make(chan error)
 	return &intf, nil
 }
 
@@ -199,7 +199,7 @@ func (intf *linkInterface) handler() error {
 			intf.link.mutex.Lock()
 			delete(intf.link.interfaces, intf.info)
 			intf.link.mutex.Unlock()
-			//close(intf.closed)
+			close(intf.closed)
 		}()
 		intf.link.core.log.Debugln("DEBUG: registered interface for", intf.name)
 	}
@@ -214,427 +214,184 @@ func (intf *linkInterface) handler() error {
 		// More cleanup can go here
 		intf.link.core.peers.removePeer(intf.peer.port)
 	}()
-	// Finish setting up the peer struct
-  /*
-	out := make(chan [][]byte, 1)
-	defer close(out)
 	intf.peer.out = func(msgs [][]byte) {
-		defer func() { recover() }()
-		out <- msgs
+		intf.writer.sendFrom(intf.peer, msgs, false)
+	}
+	intf.peer.linkOut = func(bs []byte) {
+		intf.writer.sendFrom(intf.peer, [][]byte{bs}, true)
 	}
-  */
-  intf.peer.out = func(msgs [][]byte) {
-    intf.writer.sendFrom(intf.peer, msgs, false)
-  }
-	intf.peer.linkOut = make(chan []byte, 1)
-  go func() {
-    // TODO fix this
-    for bs := range intf.peer.linkOut {
-      intf.writer.sendFrom(intf.peer, [][]byte{bs}, true)
-    }
-  }()
 	themAddr := address.AddrForNodeID(crypto.GetNodeID(&intf.info.box))
 	themAddrString := net.IP(themAddr[:]).String()
 	themString := fmt.Sprintf("%s@%s", themAddrString, intf.info.remote)
 	intf.link.core.log.Infof("Connected %s: %s, source %s",
 		strings.ToUpper(intf.info.linkType), themString, intf.info.local)
-	// Start the link loop
+	// Start things
 	go intf.peer.start()
-	// Start the writer
-  /*
-	signalReady := make(chan struct{}, 1)
-	signalSent := make(chan bool, 1)
-	sendAck := make(chan struct{}, 1)
-	sendBlocked := time.NewTimer(time.Second)
-	defer util.TimerStop(sendBlocked)
-	util.TimerStop(sendBlocked)
-	go func() {
-		defer close(signalReady)
-		defer close(signalSent)
-		interval := 4 * time.Second
-		tcpTimer := time.NewTimer(interval) // used for backwards compat with old tcp
-		defer util.TimerStop(tcpTimer)
-		send := func(bss [][]byte) {
-			sendBlocked.Reset(time.Second)
-			size, _ := intf.msgIO.writeMsgs(bss)
-			util.TimerStop(sendBlocked)
-			select {
-			case signalSent <- size > 0:
-			default:
-			}
-		}
-		for {
-			// First try to send any link protocol traffic
-			select {
-			case msg := <-intf.peer.linkOut:
-				send([][]byte{msg})
-				continue
-			default:
-			}
-			// No protocol traffic to send, so reset the timer
-			util.TimerStop(tcpTimer)
-			tcpTimer.Reset(interval)
-			// Now block until something is ready or the timer triggers keepalive traffic
-			select {
-			case <-tcpTimer.C:
-				intf.link.core.log.Tracef("Sending (legacy) keep-alive to %s: %s, source %s",
-					strings.ToUpper(intf.info.linkType), themString, intf.info.local)
-				send([][]byte{nil})
-			case <-sendAck:
-				intf.link.core.log.Tracef("Sending ack to %s: %s, source %s",
-					strings.ToUpper(intf.info.linkType), themString, intf.info.local)
-				send([][]byte{nil})
-			case msg := <-intf.peer.linkOut:
-				send([][]byte{msg})
-			case msgs, ok := <-out:
-				if !ok {
-					return
-				}
-				send(msgs)
-				for _, msg := range msgs {
-					util.PutBytes(msg)
-				}
-				select {
-				case signalReady <- struct{}{}:
-				default:
-				}
-				//intf.link.core.log.Tracef("Sending packet to %s: %s, source %s",
-				//	strings.ToUpper(intf.info.linkType), themString, intf.info.local)
-			}
-		}
-	}()
-	//intf.link.core.switchTable.idleIn <- intf.peer.port // notify switch that we're idle
-	// Used to enable/disable activity in the switch
-	signalAlive := make(chan bool, 1) // True = real packet, false = keep-alive
-	defer close(signalAlive)
-	ret := make(chan error, 1) // How we signal the return value when multiple goroutines are involved
-	go func() {
-		var isAlive bool
-		var isReady bool
-		var sendTimerRunning bool
-		var recvTimerRunning bool
-		recvTime := 6 * time.Second     // TODO set to ReadTimeout from the config, reset if it gets changed
-		closeTime := 2 * switch_timeout // TODO or maybe this makes more sense for ReadTimeout?...
-		sendTime := time.Second
-		sendTimer := time.NewTimer(sendTime)
-		defer util.TimerStop(sendTimer)
-		recvTimer := time.NewTimer(recvTime)
-		defer util.TimerStop(recvTimer)
-		closeTimer := time.NewTimer(closeTime)
-		defer util.TimerStop(closeTimer)
-		for {
-			//intf.link.core.log.Debugf("State of %s: %s, source %s :: isAlive %t isReady %t sendTimerRunning %t recvTimerRunning %t",
-			//	strings.ToUpper(intf.info.linkType), themString, intf.info.local,
-			//	isAlive, isReady, sendTimerRunning, recvTimerRunning)
-			select {
-			case gotMsg, ok := <-signalAlive:
-				if !ok {
-					return
-				}
-				util.TimerStop(closeTimer)
-				closeTimer.Reset(closeTime)
-				util.TimerStop(recvTimer)
-				recvTimerRunning = false
-				isAlive = true
-				if !isReady {
-					// (Re-)enable in the switch
-					intf.link.core.switchTable.RecvFrom(nil, func() {
-						intf.link.core.switchTable._idleIn(intf.peer.port)
-					})
-					isReady = true
-				}
-				if gotMsg && !sendTimerRunning {
-					// We got a message
-					// Start a timer, if it expires then send a 0-sized ack to let them know we're alive
-					util.TimerStop(sendTimer)
-					sendTimer.Reset(sendTime)
-					sendTimerRunning = true
-				}
-				if !gotMsg {
-					intf.link.core.log.Tracef("Received ack from %s: %s, source %s",
-						strings.ToUpper(intf.info.linkType), themString, intf.info.local)
-				}
-			case sentMsg, ok := <-signalSent:
-				// Stop any running ack timer
-				if !ok {
-					return
-				}
-				util.TimerStop(sendTimer)
-				sendTimerRunning = false
-				if sentMsg && !recvTimerRunning {
-					// We sent a message
-					// Start a timer, if it expires and we haven't gotten any return traffic (including a 0-sized ack), then assume there's a problem
-					util.TimerStop(recvTimer)
-					recvTimer.Reset(recvTime)
-					recvTimerRunning = true
-				}
-			case _, ok := <-signalReady:
-				if !ok {
-					return
-				}
-				if !isAlive {
-					// Disable in the switch
-					isReady = false
-				} else {
-					// Keep enabled in the switch
-					intf.link.core.switchTable.RecvFrom(nil, func() {
-						intf.link.core.switchTable._idleIn(intf.peer.port)
-					})
-					isReady = true
-				}
-			case <-sendBlocked.C:
-				// We blocked while trying to send something
-				isReady = false
-				intf.link.core.switchTable.blockPeer(intf.peer.port)
-			case <-sendTimer.C:
-				// We haven't sent anything, so signal a send of a 0 packet to let them know we're alive
-				select {
-				case sendAck <- struct{}{}:
-				default:
-				}
-			case <-recvTimer.C:
-				// We haven't received anything, so assume there's a problem and don't return this node to the switch until they start responding
-				isAlive = false
-				intf.link.core.switchTable.blockPeer(intf.peer.port)
-			case <-closeTimer.C:
-				// We haven't received anything in a really long time, so things have died at the switch level and then some...
-				// Just close the connection at this point...
-				select {
-				case ret <- errors.New("timeout"):
-				default:
-				}
-				intf.msgIO.close()
-			}
-		}
-	}()
-	// Run reader loop
-	var helper phony.Inbox
-	done := make(chan struct{})
-	var helperFunc func()
-	helperFunc = func() {
-		// The helper reads in a loop and sends to the peer
-		// It loops by sending itself a message, which can be delayed by backpressure
-		// So if the peer is busy, backpressure will pause reading until the peer catches up
-		msg, err := intf.msgIO.readMsg()
-		if len(msg) > 0 {
-			// TODO rewrite this if the link becomes an actor
-			intf.peer.handlePacketFrom(&helper, msg)
-		}
-		if err != nil {
-			if err != io.EOF {
-				select {
-				case ret <- err:
-				default:
-				}
-			}
-			close(done)
-			return
-		}
-		select {
-		case signalAlive <- len(msg) > 0:
-		default:
-		}
-		// Now try to read again
-		helper.RecvFrom(nil, helperFunc)
-	}
-  
-	// Start the read loop
-	helper.RecvFrom(nil, helperFunc)
-	<-done // Wait for the helper to exit
-	////////////////////////////////////////////////////////////////////////////////
-	// Remember to set `err` to something useful before returning
-	select {
-	case err = <-ret:
+	intf.reader.RecvFrom(nil, intf.reader._read)
+	// Wait for the reader to finish
+	err = <-intf.reader.err
+	if err != nil {
 		intf.link.core.log.Infof("Disconnected %s: %s, source %s; error: %s",
 			strings.ToUpper(intf.info.linkType), themString, intf.info.local, err)
-	default:
-		err = nil
+	} else {
 		intf.link.core.log.Infof("Disconnected %s: %s, source %s",
 			strings.ToUpper(intf.info.linkType), themString, intf.info.local)
 	}
-  */
-  // Start the reader
-  intf.reader.RecvFrom(nil, intf.reader._read)
-  // Wait for the reader to finish
-  err = <- intf.reader.err
-  if err != nil {
-		intf.link.core.log.Infof("Disconnected %s: %s, source %s; error: %s",
-			strings.ToUpper(intf.info.linkType), themString, intf.info.local, err)
-  } else {
-		intf.link.core.log.Infof("Disconnected %s: %s, source %s",
-			strings.ToUpper(intf.info.linkType), themString, intf.info.local)
-  }
 	return err
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 
-
-/*
-  phony.Inbox // Protects the below
-  sendTimer *time.Timer // Fires to signal that sending is blocked
-  stallTimer *time.Time // Fires to signal that no incoming traffic (including keep-alive) has been seen
-  recvTimer *time.Timer // Fires to send keep-alive traffic
-  closeTimer *time.Timer // Fires when the link has been idle so long we need to close it
-  inSwitch bool // True if the switch is tracking this link
-  stalled bool // True if we haven't been receiving any response traffic
-*/
-
 const (
-  sendBlockedTime = time.Second // How long to wait before deciding a send is blocked
-  keepAliveTime = 2*time.Second // How long to wait before sending a keep-alive response if we have no real traffic to send
-  stallTime = 6*time.Second // How long to wait for response traffic before deciding the connection has stalled
-  closeTime = 2*switch_timeout // How long to wait before closing the link
+	sendBlockedTime = time.Second        // How long to wait before deciding a send is blocked
+	keepAliveTime   = 2 * time.Second    // How long to wait before sending a keep-alive response if we have no real traffic to send
+	stallTime       = 6 * time.Second    // How long to wait for response traffic before deciding the connection has stalled
+	closeTime       = 2 * switch_timeout // How long to wait before closing the link
 )
 
 // notify the intf that we're currently sending
 func (intf *linkInterface) notifySending(size int, isLinkTraffic bool) {
-  intf.RecvFrom(nil, func() {
-    if !isLinkTraffic && size > 0 {
-      intf.inSwitch = false
-    }
-    intf.sendTimer = time.AfterFunc(sendBlockedTime, intf.notifyBlockedSend)
-    intf._cancelRecvTimer()
-  })
+	intf.RecvFrom(nil, func() {
+		if !isLinkTraffic && size > 0 {
+			intf.inSwitch = false
+		}
+		intf.sendTimer = time.AfterFunc(sendBlockedTime, intf.notifyBlockedSend)
+		intf._cancelRecvTimer()
+	})
 }
 
 // we just sent something, so cancel any pending timer to send keep-alive traffic
 func (intf *linkInterface) _cancelRecvTimer() {
-  intf.RecvFrom(nil, func() {
-    if intf.recvTimer != nil {
-      intf.recvTimer.Stop()
-      intf.recvTimer = nil
-    }
-  })
+	intf.RecvFrom(nil, func() {
+		if intf.recvTimer != nil {
+			intf.recvTimer.Stop()
+			intf.recvTimer = nil
+		}
+	})
 }
 
 // called by an AfterFunc if we appear to have timed out
 func (intf *linkInterface) notifyBlockedSend() {
-  intf.RecvFrom(nil, func() {
-    if intf.sendTimer != nil {
-      //As far as we know, we're still trying to send, and the timer fired.
+	intf.RecvFrom(nil, func() {
+		if intf.sendTimer != nil {
+			//As far as we know, we're still trying to send, and the timer fired.
 			intf.link.core.switchTable.blockPeer(intf.peer.port)
-    }
-  })
+		}
+	})
 }
 
 // notify the intf that we've finished sending, returning the peer to the switch
 func (intf *linkInterface) notifySent(size int, isLinkTraffic bool) {
-  intf.RecvFrom(nil, func() {
-    intf.sendTimer.Stop()
-    intf.sendTimer = nil
-    if !isLinkTraffic {
-      intf._notifySwitch()
-    }
-    if size > 0 && intf.stallTimer == nil {
-      intf.stallTimer = time.AfterFunc(stallTime, intf.notifyStalled)
-    }
-  })
+	intf.RecvFrom(nil, func() {
+		intf.sendTimer.Stop()
+		intf.sendTimer = nil
+		if !isLinkTraffic {
+			intf._notifySwitch()
+		}
+		if size > 0 && intf.stallTimer == nil {
+			intf.stallTimer = time.AfterFunc(stallTime, intf.notifyStalled)
+		}
+	})
 }
 
 // Notify the switch that we're ready for more traffic, assuming we're not in a stalled state
 func (intf *linkInterface) _notifySwitch() {
-  if !intf.inSwitch && !intf.stalled {
-    intf.inSwitch = true
-	  intf.link.core.switchTable.RecvFrom(intf, func() {
-		  intf.link.core.switchTable._idleIn(intf.peer.port)
-	  })
-  }
+	if !intf.inSwitch && !intf.stalled {
+		intf.inSwitch = true
+		intf.link.core.switchTable.RecvFrom(intf, func() {
+			intf.link.core.switchTable._idleIn(intf.peer.port)
+		})
+	}
 }
 
 // Set the peer as stalled, to prevent them from returning to the switch until a read succeeds
 func (intf *linkInterface) notifyStalled() {
-  intf.RecvFrom(nil, func() {
-    if intf.stallTimer != nil {
-      intf.stallTimer = nil
-      intf.stalled = true
+	intf.RecvFrom(nil, func() {
+		if intf.stallTimer != nil {
+			intf.stallTimer = nil
+			intf.stalled = true
 			intf.link.core.switchTable.blockPeer(intf.peer.port)
-    }
-  })
+		}
+	})
 }
 
-
 // reset the close timer
 func (intf *linkInterface) notifyReading(from phony.Actor) {
-  intf.RecvFrom(from, func() {
-    if intf.closeTimer != nil {
-      intf.closeTimer.Stop()
-    }
-    intf.closeTimer = time.AfterFunc(closeTime, func() { intf.msgIO.close() })
-  })
+	intf.RecvFrom(from, func() {
+		if intf.closeTimer != nil {
+			intf.closeTimer.Stop()
+		}
+		intf.closeTimer = time.AfterFunc(closeTime, func() { intf.msgIO.close() })
+	})
 }
 
 // wake up the link if it was stalled, and (if size > 0) prepare to send keep-alive traffic
 func (intf *linkInterface) notifyReadFrom(from phony.Actor, size int) {
-  intf.RecvFrom(from, func() {
-    intf.link.core.log.Printf("DEBUG notifyReadFrom: inSwitch %v, stalled %v\n", intf.inSwitch, intf.stalled)
-    if intf.stallTimer != nil {
-      intf.stallTimer.Stop()
-      intf.stallTimer = nil
-    }
-    intf.stalled = false
-    intf._notifySwitch()
-    if size > 0 && intf.recvTimer == nil {
-      intf.recvTimer = time.AfterFunc(keepAliveTime, intf.notifyDoKeepAlive)
-    }
-  })
+	intf.RecvFrom(from, func() {
+		if intf.stallTimer != nil {
+			intf.stallTimer.Stop()
+			intf.stallTimer = nil
+		}
+		intf.stalled = false
+		intf._notifySwitch()
+		if size > 0 && intf.recvTimer == nil {
+			intf.recvTimer = time.AfterFunc(keepAliveTime, intf.notifyDoKeepAlive)
+		}
+	})
 }
 
 // We need to send keep-alive traffic now
 func (intf *linkInterface) notifyDoKeepAlive() {
-  intf.RecvFrom(nil, func() {
-    if intf.recvTimer != nil {
-      intf.recvTimer.Stop()
-      intf.recvTimer = nil
-      intf.writer.sendFrom(nil, [][]byte{nil}, true) // Empty keep-alive traffic
-    }
-  })
+	intf.RecvFrom(nil, func() {
+		if intf.recvTimer != nil {
+			intf.recvTimer.Stop()
+			intf.recvTimer = nil
+			intf.writer.sendFrom(nil, [][]byte{nil}, true) // Empty keep-alive traffic
+		}
+	})
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 
 type linkWriter struct {
-  phony.Inbox
-  intf *linkInterface
+	phony.Inbox
+	intf *linkInterface
 }
 
 func (w *linkWriter) sendFrom(from phony.Actor, bss [][]byte, isLinkTraffic bool) {
-  w.RecvFrom(from, func() {
-    var size int
-    for _, bs := range bss {
-      size += len(bs)
-    }
-    w.intf.notifySending(size, isLinkTraffic)
-	  w.intf.msgIO.writeMsgs(bss)
-    w.intf.notifySent(size, isLinkTraffic)
-    w.intf.link.core.log.Println("DEBUG: wrote something, size:", size, "isLinkTraffic:", isLinkTraffic)
-  })
+	w.RecvFrom(from, func() {
+		var size int
+		for _, bs := range bss {
+			size += len(bs)
+		}
+		w.intf.notifySending(size, isLinkTraffic)
+		w.intf.msgIO.writeMsgs(bss)
+		w.intf.notifySent(size, isLinkTraffic)
+	})
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 
 type linkReader struct {
-  phony.Inbox
-  intf *linkInterface
-  err chan error
+	phony.Inbox
+	intf *linkInterface
+	err  chan error
 }
 
 func (r *linkReader) _read() {
-    r.intf.notifyReading(r)
-		msg, err := r.intf.msgIO.readMsg()
-    r.intf.link.core.log.Println("DEBUG read something")
-    r.intf.notifyReadFrom(r, len(msg))
-		if len(msg) > 0 {
-			r.intf.peer.handlePacketFrom(r, msg)
+	r.intf.notifyReading(r)
+	msg, err := r.intf.msgIO.readMsg()
+	r.intf.notifyReadFrom(r, len(msg))
+	if len(msg) > 0 {
+		r.intf.peer.handlePacketFrom(r, msg)
+	}
+	if err != nil {
+		if err != io.EOF {
+			r.err <- err
 		}
-		if err != nil {
-			if err != io.EOF {
-        r.err<-err
-			}
-      close(r.err)
-			return
-		}
-		// Now try to read again
-		r.RecvFrom(nil, r._read)
+		close(r.err)
+		return
+	}
+	// Now try to read again
+	r.RecvFrom(nil, r._read)
 }
-
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 50fb03f8..4cf0068b 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -103,7 +103,7 @@ type peer struct {
 	linkShared crypto.BoxSharedKey
 	endpoint   string
 	firstSeen  time.Time       // To track uptime for getPeers
-	linkOut    (chan []byte)   // used for protocol traffic (to bypass queues)
+	linkOut    func([]byte)    // used for protocol traffic (bypasses the switch)
 	dinfo      *dhtInfo        // used to keep the DHT working
 	out        func([][]byte)  // Set up by whatever created the peers struct, used to send packets to other nodes
 	done       (chan struct{}) // closed to exit the linkLoop
@@ -263,8 +263,6 @@ func (p *peer) _sendPackets(packets [][]byte) {
 	p.out(packets)
 }
 
-var peerLinkOutHelper phony.Inbox
-
 // This wraps the packet in the inner (ephemeral) and outer (permanent) crypto layers.
 // It sends it to p.linkOut, which bypasses the usual packet queues.
 func (p *peer) _sendLinkPacket(packet []byte) {
@@ -280,13 +278,7 @@ func (p *peer) _sendLinkPacket(packet []byte) {
 		Payload: bs,
 	}
 	packet = linkPacket.encode()
-	// TODO replace this with a message send if/when the link becomes an actor
-	peerLinkOutHelper.RecvFrom(nil, func() {
-		select {
-		case p.linkOut <- packet:
-		case <-p.done:
-		}
-	})
+	p.linkOut(packet)
 }
 
 // Decrypts the outer (permanent) and inner (ephemeral) crypto layers on link traffic.
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 6d882252..b30e59d5 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -250,7 +250,7 @@ func (t *switchTable) cleanRoot() {
 			t.core.router.reset(nil)
 		}
 		t.data.locator = switchLocator{root: t.key, tstamp: now.Unix()}
-		t.core.peers.sendSwitchMsgs(nil) // TODO update if/when the switch becomes an actor
+		t.core.peers.sendSwitchMsgs(t)
 	}
 }
 
@@ -517,7 +517,7 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort, rep
 		}
 		t.data.locator = sender.locator
 		t.parent = sender.port
-		t.core.peers.sendSwitchMsgs(nil) // TODO update if/when the switch becomes an actor
+		t.core.peers.sendSwitchMsgs(t)
 	}
 	if doUpdate {
 		t.updater.Store(&sync.Once{})

From e5b88c0da3998cd7582d07b2385054f6005f402f Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 25 Aug 2019 23:07:56 -0500
Subject: [PATCH 046/130] update switch

---
 src/yggdrasil/switch.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 6d882252..b30e59d5 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -250,7 +250,7 @@ func (t *switchTable) cleanRoot() {
 			t.core.router.reset(nil)
 		}
 		t.data.locator = switchLocator{root: t.key, tstamp: now.Unix()}
-		t.core.peers.sendSwitchMsgs(nil) // TODO update if/when the switch becomes an actor
+		t.core.peers.sendSwitchMsgs(t)
 	}
 }
 
@@ -517,7 +517,7 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort, rep
 		}
 		t.data.locator = sender.locator
 		t.parent = sender.port
-		t.core.peers.sendSwitchMsgs(nil) // TODO update if/when the switch becomes an actor
+		t.core.peers.sendSwitchMsgs(t)
 	}
 	if doUpdate {
 		t.updater.Store(&sync.Once{})

From ab591295578e66aae15452b843194562c56ab490 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 25 Aug 2019 23:24:18 -0500
Subject: [PATCH 047/130] have the writer clean things up. note that their
 still seem to be bugs in the main linkInterface actor's state machine--links
 sometimes just die, presumably because they're dropped from the switch and
 never replaced

---
 src/yggdrasil/link.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index b4ca38c4..548f5341 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -367,6 +367,10 @@ func (w *linkWriter) sendFrom(from phony.Actor, bss [][]byte, isLinkTraffic bool
 		w.intf.notifySending(size, isLinkTraffic)
 		w.intf.msgIO.writeMsgs(bss)
 		w.intf.notifySent(size, isLinkTraffic)
+		// Cleanup
+		for _, bs := range bss {
+			util.PutBytes(bs)
+		}
 	})
 }
 

From c97dd4ad28c167b09e0b3ea9d26d41128085273e Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Mon, 26 Aug 2019 00:38:14 -0500
Subject: [PATCH 048/130] fix dial bug

---
 src/tuntap/iface.go   | 38 ++++++++++++++++++++------------------
 src/yggdrasil/link.go |  2 +-
 2 files changed, 21 insertions(+), 19 deletions(-)

diff --git a/src/tuntap/iface.go b/src/tuntap/iface.go
index 16e8d25d..da419369 100644
--- a/src/tuntap/iface.go
+++ b/src/tuntap/iface.go
@@ -247,24 +247,26 @@ func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
 		}
 		if !known {
 			go func() {
-				if conn, err := tun.dialer.DialByNodeIDandMask(dstNodeID, dstNodeIDMask); err == nil {
-					tun.RecvFrom(nil, func() {
-						// We've been given a connection so prepare the session wrapper
-						packets := tun.dials[*dstNodeID]
-						delete(tun.dials, *dstNodeID)
-						var tc *tunConn
-						var err error
-						if tc, err = tun._wrap(conn); err != nil {
-							// Something went wrong when storing the connection, typically that
-							// something already exists for this address or subnet
-							tun.log.Debugln("TUN/TAP iface wrap:", err)
-							return
-						}
-						for _, packet := range packets {
-							tc.writeFrom(nil, packet)
-						}
-					})
-				}
+				conn, err := tun.dialer.DialByNodeIDandMask(dstNodeID, dstNodeIDMask)
+				tun.RecvFrom(nil, func() {
+					packets := tun.dials[*dstNodeID]
+					delete(tun.dials, *dstNodeID)
+					if err != nil {
+						return
+					}
+					// We've been given a connection so prepare the session wrapper
+					var tc *tunConn
+					if tc, err = tun._wrap(conn); err != nil {
+						// Something went wrong when storing the connection, typically that
+						// something already exists for this address or subnet
+						tun.log.Debugln("TUN/TAP iface wrap:", err)
+						return
+					}
+					for _, packet := range packets {
+						tc.writeFrom(nil, packet)
+					}
+				})
+				return
 			}()
 		}
 	}
diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index 548f5341..8fbd0317 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -252,7 +252,7 @@ const (
 // notify the intf that we're currently sending
 func (intf *linkInterface) notifySending(size int, isLinkTraffic bool) {
 	intf.RecvFrom(nil, func() {
-		if !isLinkTraffic && size > 0 {
+		if !isLinkTraffic {
 			intf.inSwitch = false
 		}
 		intf.sendTimer = time.AfterFunc(sendBlockedTime, intf.notifyBlockedSend)

From 4d9c6342a7383a4481851be11c9357af770b6f18 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Mon, 26 Aug 2019 18:37:38 -0500
Subject: [PATCH 049/130] more link updates

---
 src/yggdrasil/link.go | 91 +++++++++++++++++++++----------------------
 1 file changed, 45 insertions(+), 46 deletions(-)

diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index 8fbd0317..6b7f446f 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -46,23 +46,23 @@ type linkInterfaceMsgIO interface {
 }
 
 type linkInterface struct {
-	name        string
-	link        *link
-	peer        *peer
-	msgIO       linkInterfaceMsgIO
-	info        linkInfo
-	incoming    bool
-	force       bool
-	closed      chan struct{}
-	reader      linkReader  // Reads packets, notifies this linkInterface, passes packets to switch
-	writer      linkWriter  // Writes packets, notifies this linkInterface
-	phony.Inbox             // Protects the below
-	sendTimer   *time.Timer // Fires to signal that sending is blocked
-	stallTimer  *time.Timer // Fires to signal that no incoming traffic (including keep-alive) has been seen
-	recvTimer   *time.Timer // Fires to send keep-alive traffic
-	closeTimer  *time.Timer // Fires when the link has been idle so long we need to close it
-	inSwitch    bool        // True if the switch is tracking this link
-	stalled     bool        // True if we haven't been receiving any response traffic
+	name           string
+	link           *link
+	peer           *peer
+	msgIO          linkInterfaceMsgIO
+	info           linkInfo
+	incoming       bool
+	force          bool
+	closed         chan struct{}
+	reader         linkReader  // Reads packets, notifies this linkInterface, passes packets to switch
+	writer         linkWriter  // Writes packets, notifies this linkInterface
+	phony.Inbox                // Protects the below
+	sendTimer      *time.Timer // Fires to signal that sending is blocked
+	keepAliveTimer *time.Timer // Fires to send keep-alive traffic
+	stallTimer     *time.Timer // Fires to signal that no incoming traffic (including keep-alive) has been seen
+	closeTimer     *time.Timer // Fires when the link has been idle so long we need to close it
+	inSwitch       bool        // True if the switch is tracking this link
+	stalled        bool        // True if we haven't been receiving any response traffic
 }
 
 func (l *link) init(c *Core) error {
@@ -243,36 +243,34 @@ func (intf *linkInterface) handler() error {
 ////////////////////////////////////////////////////////////////////////////////
 
 const (
-	sendBlockedTime = time.Second        // How long to wait before deciding a send is blocked
-	keepAliveTime   = 2 * time.Second    // How long to wait before sending a keep-alive response if we have no real traffic to send
-	stallTime       = 6 * time.Second    // How long to wait for response traffic before deciding the connection has stalled
-	closeTime       = 2 * switch_timeout // How long to wait before closing the link
+	sendTime      = 1 * time.Second    // How long to wait before deciding a send is blocked
+	keepAliveTime = 2 * time.Second    // How long to wait before sending a keep-alive response if we have no real traffic to send
+	stallTime     = 6 * time.Second    // How long to wait for response traffic before deciding the connection has stalled
+	closeTime     = 2 * switch_timeout // How long to wait before closing the link
 )
 
 // notify the intf that we're currently sending
 func (intf *linkInterface) notifySending(size int, isLinkTraffic bool) {
-	intf.RecvFrom(nil, func() {
+	intf.RecvFrom(&intf.writer, func() {
 		if !isLinkTraffic {
 			intf.inSwitch = false
 		}
-		intf.sendTimer = time.AfterFunc(sendBlockedTime, intf.notifyBlockedSend)
-		intf._cancelRecvTimer()
+		intf.sendTimer = time.AfterFunc(sendTime, intf.notifyBlockedSend)
+		intf._cancelStallTimer()
 	})
 }
 
 // we just sent something, so cancel any pending timer to send keep-alive traffic
-func (intf *linkInterface) _cancelRecvTimer() {
-	intf.RecvFrom(nil, func() {
-		if intf.recvTimer != nil {
-			intf.recvTimer.Stop()
-			intf.recvTimer = nil
-		}
-	})
+func (intf *linkInterface) _cancelStallTimer() {
+	if intf.stallTimer != nil {
+		intf.stallTimer.Stop()
+		intf.stallTimer = nil
+	}
 }
 
 // called by an AfterFunc if we appear to have timed out
 func (intf *linkInterface) notifyBlockedSend() {
-	intf.RecvFrom(nil, func() {
+	intf.RecvFrom(nil, func() { // Sent from a time.AfterFunc
 		if intf.sendTimer != nil {
 			//As far as we know, we're still trying to send, and the timer fired.
 			intf.link.core.switchTable.blockPeer(intf.peer.port)
@@ -282,7 +280,7 @@ func (intf *linkInterface) notifyBlockedSend() {
 
 // notify the intf that we've finished sending, returning the peer to the switch
 func (intf *linkInterface) notifySent(size int, isLinkTraffic bool) {
-	intf.RecvFrom(nil, func() {
+	intf.RecvFrom(&intf.writer, func() {
 		intf.sendTimer.Stop()
 		intf.sendTimer = nil
 		if !isLinkTraffic {
@@ -306,8 +304,9 @@ func (intf *linkInterface) _notifySwitch() {
 
 // Set the peer as stalled, to prevent them from returning to the switch until a read succeeds
 func (intf *linkInterface) notifyStalled() {
-	intf.RecvFrom(nil, func() {
+	intf.RecvFrom(nil, func() { // Sent from a time.AfterFunc
 		if intf.stallTimer != nil {
+			intf.stallTimer.Stop()
 			intf.stallTimer = nil
 			intf.stalled = true
 			intf.link.core.switchTable.blockPeer(intf.peer.port)
@@ -316,8 +315,8 @@ func (intf *linkInterface) notifyStalled() {
 }
 
 // reset the close timer
-func (intf *linkInterface) notifyReading(from phony.Actor) {
-	intf.RecvFrom(from, func() {
+func (intf *linkInterface) notifyReading() {
+	intf.RecvFrom(&intf.reader, func() {
 		if intf.closeTimer != nil {
 			intf.closeTimer.Stop()
 		}
@@ -326,26 +325,26 @@ func (intf *linkInterface) notifyReading(from phony.Actor) {
 }
 
 // wake up the link if it was stalled, and (if size > 0) prepare to send keep-alive traffic
-func (intf *linkInterface) notifyReadFrom(from phony.Actor, size int) {
-	intf.RecvFrom(from, func() {
+func (intf *linkInterface) notifyRead(size int) {
+	intf.RecvFrom(&intf.reader, func() {
 		if intf.stallTimer != nil {
 			intf.stallTimer.Stop()
 			intf.stallTimer = nil
 		}
 		intf.stalled = false
 		intf._notifySwitch()
-		if size > 0 && intf.recvTimer == nil {
-			intf.recvTimer = time.AfterFunc(keepAliveTime, intf.notifyDoKeepAlive)
+		if size > 0 && intf.stallTimer == nil {
+			intf.stallTimer = time.AfterFunc(keepAliveTime, intf.notifyDoKeepAlive)
 		}
 	})
 }
 
 // We need to send keep-alive traffic now
 func (intf *linkInterface) notifyDoKeepAlive() {
-	intf.RecvFrom(nil, func() {
-		if intf.recvTimer != nil {
-			intf.recvTimer.Stop()
-			intf.recvTimer = nil
+	intf.RecvFrom(nil, func() { // Sent from a time.AfterFunc
+		if intf.stallTimer != nil {
+			intf.stallTimer.Stop()
+			intf.stallTimer = nil
 			intf.writer.sendFrom(nil, [][]byte{nil}, true) // Empty keep-alive traffic
 		}
 	})
@@ -383,9 +382,9 @@ type linkReader struct {
 }
 
 func (r *linkReader) _read() {
-	r.intf.notifyReading(r)
+	r.intf.notifyReading()
 	msg, err := r.intf.msgIO.readMsg()
-	r.intf.notifyReadFrom(r, len(msg))
+	r.intf.notifyRead(len(msg))
 	if len(msg) > 0 {
 		r.intf.peer.handlePacketFrom(r, msg)
 	}

From 3845f81357dae0f29062e7caa60b1a723efddf05 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Tue, 27 Aug 2019 19:43:54 -0500
Subject: [PATCH 050/130] update to latest phony, adjust interface use
 accordingly

---
 go.mod                   |  2 +-
 go.sum                   |  4 ++--
 src/tuntap/conn.go       |  6 +++---
 src/tuntap/iface.go      | 10 +++++-----
 src/tuntap/tun.go        | 12 ++++++------
 src/yggdrasil/api.go     |  6 ++++--
 src/yggdrasil/conn.go    | 28 ++++++++++++++--------------
 src/yggdrasil/debug.go   | 12 +++++++-----
 src/yggdrasil/link.go    | 22 +++++++++++-----------
 src/yggdrasil/peer.go    | 10 +++++-----
 src/yggdrasil/router.go  | 14 +++++++-------
 src/yggdrasil/search.go  |  2 +-
 src/yggdrasil/session.go | 22 +++++++++++-----------
 src/yggdrasil/switch.go  |  6 +++---
 14 files changed, 80 insertions(+), 76 deletions(-)

diff --git a/go.mod b/go.mod
index f3d8417f..9ff6031a 100644
--- a/go.mod
+++ b/go.mod
@@ -1,7 +1,7 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
 require (
-	github.com/Arceliar/phony v0.0.0-20190825152505-180ac75690fe
+	github.com/Arceliar/phony v0.0.0-20190828002416-0337564e2c44
 	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
 	github.com/hashicorp/go-syslog v1.0.0
 	github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222
diff --git a/go.sum b/go.sum
index 29e0dfec..60388964 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-github.com/Arceliar/phony v0.0.0-20190825152505-180ac75690fe h1:U5bediuXjZ1y6bByIXXraoE319yFp9kx1z8K6el7Ftc=
-github.com/Arceliar/phony v0.0.0-20190825152505-180ac75690fe/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
+github.com/Arceliar/phony v0.0.0-20190828002416-0337564e2c44 h1:8EiuIp65v8aLkLc4LWxtn4NTH+P2LwDmrKKWdBzn9cI=
+github.com/Arceliar/phony v0.0.0-20190828002416-0337564e2c44/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=
diff --git a/src/tuntap/conn.go b/src/tuntap/conn.go
index afb0f09e..31875d94 100644
--- a/src/tuntap/conn.go
+++ b/src/tuntap/conn.go
@@ -27,7 +27,7 @@ type tunConn struct {
 }
 
 func (s *tunConn) close() {
-	s.tun.RecvFrom(s, s._close_from_tun)
+	s.tun.Act(s, s._close_from_tun)
 }
 
 func (s *tunConn) _close_from_tun() {
@@ -117,7 +117,7 @@ func (s *tunConn) _read(bs []byte) (err error) {
 }
 
 func (s *tunConn) writeFrom(from phony.Actor, bs []byte) {
-	s.RecvFrom(from, func() {
+	s.Act(from, func() {
 		s._write(bs)
 	})
 }
@@ -197,7 +197,7 @@ func (s *tunConn) _write(bs []byte) (err error) {
 			// No point in wasting resources to send back an error if there was none
 			return
 		}
-		s.RecvFrom(s.conn, func() {
+		s.Act(s.conn, func() {
 			if e, eok := err.(yggdrasil.ConnError); !eok {
 				if e.Closed() {
 					s.tun.log.Debugln(s.conn.String(), "TUN/TAP generic write debug:", err)
diff --git a/src/tuntap/iface.go b/src/tuntap/iface.go
index da419369..92ba36ab 100644
--- a/src/tuntap/iface.go
+++ b/src/tuntap/iface.go
@@ -19,7 +19,7 @@ type tunWriter struct {
 }
 
 func (w *tunWriter) writeFrom(from phony.Actor, b []byte) {
-	w.RecvFrom(from, func() {
+	w.Act(from, func() {
 		w._write(b)
 	})
 }
@@ -90,7 +90,7 @@ func (w *tunWriter) _write(b []byte) {
 		util.PutBytes(b)
 	}
 	if err != nil {
-		w.tun.RecvFrom(w, func() {
+		w.tun.Act(w, func() {
 			if !w.tun.isOpen {
 				w.tun.log.Errorln("TUN/TAP iface write error:", err)
 			}
@@ -118,12 +118,12 @@ func (r *tunReader) _read() {
 	}
 	if err == nil {
 		// Now read again
-		r.RecvFrom(nil, r._read)
+		r.Act(nil, r._read)
 	}
 }
 
 func (tun *TunAdapter) handlePacketFrom(from phony.Actor, packet []byte, err error) {
-	tun.RecvFrom(from, func() {
+	tun.Act(from, func() {
 		tun._handlePacket(packet, err)
 	})
 }
@@ -248,7 +248,7 @@ func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
 		if !known {
 			go func() {
 				conn, err := tun.dialer.DialByNodeIDandMask(dstNodeID, dstNodeIDMask)
-				tun.RecvFrom(nil, func() {
+				tun.Act(nil, func() {
 					packets := tun.dials[*dstNodeID]
 					delete(tun.dials, *dstNodeID)
 					if err != nil {
diff --git a/src/tuntap/tun.go b/src/tuntap/tun.go
index 2b163e3b..6317459c 100644
--- a/src/tuntap/tun.go
+++ b/src/tuntap/tun.go
@@ -125,7 +125,7 @@ func (tun *TunAdapter) Init(config *config.NodeState, log *log.Logger, listener
 // reader actor to handle packets on that interface.
 func (tun *TunAdapter) Start() error {
 	var err error
-	<-tun.SyncExec(func() {
+	phony.Block(tun, func() {
 		err = tun._start()
 	})
 	return err
@@ -167,7 +167,7 @@ func (tun *TunAdapter) _start() error {
 		}
 	}()
 	go tun.handler()
-	tun.reader.RecvFrom(nil, tun.reader._read) // Start the reader
+	tun.reader.Act(nil, tun.reader._read) // Start the reader
 	tun.icmpv6.Init(tun)
 	if iftapmode {
 		go tun.icmpv6.Solicit(tun.addr)
@@ -180,7 +180,7 @@ func (tun *TunAdapter) _start() error {
 // read/write goroutines to handle packets on that interface.
 func (tun *TunAdapter) Stop() error {
 	var err error
-	<-tun.SyncExec(func() {
+	phony.Block(tun, func() {
 		err = tun._stop()
 	})
 	return err
@@ -233,7 +233,7 @@ func (tun *TunAdapter) handler() error {
 			tun.log.Errorln("TUN/TAP connection accept error:", err)
 			return err
 		}
-		<-tun.SyncExec(func() {
+		phony.Block(tun, func() {
 			if _, err := tun._wrap(conn); err != nil {
 				// Something went wrong when storing the connection, typically that
 				// something already exists for this address or subnet
@@ -273,11 +273,11 @@ func (tun *TunAdapter) _wrap(conn *yggdrasil.Conn) (c *tunConn, err error) {
 	tun.subnetToConn[s.snet] = &s
 	// Set the read callback and start the timeout
 	conn.SetReadCallback(func(bs []byte) {
-		s.RecvFrom(conn, func() {
+		s.Act(conn, func() {
 			s._read(bs)
 		})
 	})
-	s.RecvFrom(nil, s.stillAlive)
+	s.Act(nil, s.stillAlive)
 	// Return
 	return c, err
 }
diff --git a/src/yggdrasil/api.go b/src/yggdrasil/api.go
index c6c15e24..44cbad1d 100644
--- a/src/yggdrasil/api.go
+++ b/src/yggdrasil/api.go
@@ -11,6 +11,8 @@ import (
 	"github.com/gologme/log"
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
+
+	"github.com/Arceliar/phony"
 )
 
 // Peer represents a single peer object. This contains information from the
@@ -106,7 +108,7 @@ func (c *Core) GetPeers() []Peer {
 	for _, port := range ps {
 		p := ports[port]
 		var info Peer
-		<-p.SyncExec(func() {
+		phony.Block(p, func() {
 			info = Peer{
 				Endpoint:   p.intf.name,
 				BytesSent:  p.bytesSent,
@@ -138,7 +140,7 @@ func (c *Core) GetSwitchPeers() []SwitchPeer {
 		}
 		coords := elem.locator.getCoords()
 		var info SwitchPeer
-		<-peer.SyncExec(func() {
+		phony.Block(peer, func() {
 			info = SwitchPeer{
 				Coords:     append([]uint64{}, wire_coordsBytestoUint64s(coords)...),
 				BytesSent:  peer.bytesSent,
diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index efc0c81e..c56c2a90 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -80,12 +80,12 @@ func newConn(core *Core, nodeID *crypto.NodeID, nodeMask *crypto.NodeID, session
 
 func (c *Conn) String() string {
 	var s string
-	<-c.SyncExec(func() { s = fmt.Sprintf("conn=%p", c) })
+	phony.Block(c, func() { s = fmt.Sprintf("conn=%p", c) })
 	return s
 }
 
 func (c *Conn) setMTU(from phony.Actor, mtu uint16) {
-	c.RecvFrom(from, func() { c.mtu = mtu })
+	c.Act(from, func() { c.mtu = mtu })
 }
 
 // This should never be called from the router goroutine, used in the dial functions
@@ -143,7 +143,7 @@ func (c *Conn) doSearch() {
 			sinfo.continueSearch()
 		}
 	}
-	c.core.router.RecvFrom(c.session, routerWork)
+	c.core.router.Act(c.session, routerWork)
 }
 
 func (c *Conn) _getDeadlineCancellation(t *time.Time) (util.Cancellation, bool) {
@@ -159,7 +159,7 @@ func (c *Conn) _getDeadlineCancellation(t *time.Time) (util.Cancellation, bool)
 
 // SetReadCallback sets a callback which will be called whenever a packet is received.
 func (c *Conn) SetReadCallback(callback func([]byte)) {
-	c.RecvFrom(nil, func() {
+	c.Act(nil, func() {
 		c.readCallback = callback
 		c._drainReadBuffer()
 	})
@@ -172,14 +172,14 @@ func (c *Conn) _drainReadBuffer() {
 	select {
 	case bs := <-c.readBuffer:
 		c.readCallback(bs)
-		c.RecvFrom(nil, c._drainReadBuffer) // In case there's more
+		c.Act(nil, c._drainReadBuffer) // In case there's more
 	default:
 	}
 }
 
 // Called by the session to pass a new message to the Conn
 func (c *Conn) recvMsg(from phony.Actor, msg []byte) {
-	c.RecvFrom(from, func() {
+	c.Act(from, func() {
 		if c.readCallback != nil {
 			c.readCallback(msg)
 		} else {
@@ -195,7 +195,7 @@ func (c *Conn) recvMsg(from phony.Actor, msg []byte) {
 func (c *Conn) ReadNoCopy() ([]byte, error) {
 	var cancel util.Cancellation
 	var doCancel bool
-	<-c.SyncExec(func() { cancel, doCancel = c._getDeadlineCancellation(c.readDeadline) })
+	phony.Block(c, func() { cancel, doCancel = c._getDeadlineCancellation(c.readDeadline) })
 	if doCancel {
 		defer cancel.Cancel(nil)
 	}
@@ -234,7 +234,7 @@ func (c *Conn) _write(msg FlowKeyMessage) error {
 	if len(msg.Message) > int(c.mtu) {
 		return ConnError{errors.New("packet too big"), true, false, false, int(c.mtu)}
 	}
-	c.session.RecvFrom(c, func() {
+	c.session.Act(c, func() {
 		// Send the packet
 		c.session._send(msg)
 		// Session keep-alive, while we wait for the crypto workers from send
@@ -258,7 +258,7 @@ func (c *Conn) _write(msg FlowKeyMessage) error {
 // This is used internaly by WriteNoCopy and Write.
 // If the callback is called with a non-nil value, then it is safe to reuse the argument FlowKeyMessage.
 func (c *Conn) WriteFrom(from phony.Actor, msg FlowKeyMessage, callback func(error)) {
-	c.RecvFrom(from, func() {
+	c.Act(from, func() {
 		callback(c._write(msg))
 	})
 }
@@ -268,7 +268,7 @@ func (c *Conn) WriteFrom(from phony.Actor, msg FlowKeyMessage, callback func(err
 func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error {
 	var cancel util.Cancellation
 	var doCancel bool
-	<-c.SyncExec(func() { cancel, doCancel = c._getDeadlineCancellation(c.writeDeadline) })
+	phony.Block(c, func() { cancel, doCancel = c._getDeadlineCancellation(c.writeDeadline) })
 	var err error
 	select {
 	case <-cancel.Finished():
@@ -299,7 +299,7 @@ func (c *Conn) Write(b []byte) (int, error) {
 }
 
 func (c *Conn) Close() (err error) {
-	<-c.SyncExec(func() {
+	phony.Block(c, func() {
 		if c.session != nil {
 			// Close the session, if it hasn't been closed already
 			if e := c.session.cancel.Cancel(errors.New("connection closed")); e != nil {
@@ -319,7 +319,7 @@ func (c *Conn) LocalAddr() crypto.NodeID {
 func (c *Conn) RemoteAddr() crypto.NodeID {
 	// TODO warn that this can block while waiting for the Conn actor to run, so don't call it from other actors...
 	var n crypto.NodeID
-	<-c.SyncExec(func() { n = *c.nodeID })
+	phony.Block(c, func() { n = *c.nodeID })
 	return n
 }
 
@@ -331,12 +331,12 @@ func (c *Conn) SetDeadline(t time.Time) error {
 
 func (c *Conn) SetReadDeadline(t time.Time) error {
 	// TODO warn that this can block while waiting for the Conn actor to run, so don't call it from other actors...
-	<-c.SyncExec(func() { c.readDeadline = &t })
+	phony.Block(c, func() { c.readDeadline = &t })
 	return nil
 }
 
 func (c *Conn) SetWriteDeadline(t time.Time) error {
 	// TODO warn that this can block while waiting for the Conn actor to run, so don't call it from other actors...
-	<-c.SyncExec(func() { c.writeDeadline = &t })
+	phony.Block(c, func() { c.writeDeadline = &t })
 	return nil
 }
diff --git a/src/yggdrasil/debug.go b/src/yggdrasil/debug.go
index 1cc7eae4..9d2199d0 100644
--- a/src/yggdrasil/debug.go
+++ b/src/yggdrasil/debug.go
@@ -29,6 +29,8 @@ import "github.com/yggdrasil-network/yggdrasil-go/src/config"
 import "github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 import "github.com/yggdrasil-network/yggdrasil-go/src/defaults"
 
+import "github.com/Arceliar/phony"
+
 // Start the profiler in debug builds, if the required environment variable is set.
 func init() {
 	envVarName := "PPROFLISTEN"
@@ -572,14 +574,14 @@ func DEBUG_simLinkPeers(p, q *peer) {
 					continue
 				case packet := <-send:
 					packets = append(packets, packet)
-					<-source.core.switchTable.SyncExec(func() {
+					phony.Block(&source.core.switchTable, func() {
 						source.core.switchTable._idleIn(source.port)
 					})
 					continue
 				default:
 				}
 				if len(packets) > 0 {
-					<-dest.SyncExec(func() { dest._handlePacket(packets[0]) })
+					phony.Block(dest, func() { dest._handlePacket(packets[0]) })
 					packets = packets[1:]
 					continue
 				}
@@ -588,7 +590,7 @@ func DEBUG_simLinkPeers(p, q *peer) {
 					packets = append(packets, packet)
 				case packet := <-send:
 					packets = append(packets, packet)
-					<-source.core.switchTable.SyncExec(func() {
+					phony.Block(&source.core.switchTable, func() {
 						source.core.switchTable._idleIn(source.port)
 					})
 				}
@@ -597,10 +599,10 @@ func DEBUG_simLinkPeers(p, q *peer) {
 	}
 	goWorkers(p, q)
 	goWorkers(q, p)
-	<-p.core.switchTable.SyncExec(func() {
+	phony.Block(&p.core.switchTable, func() {
 		p.core.switchTable._idleIn(p.port)
 	})
-	<-q.core.switchTable.SyncExec(func() {
+	phony.Block(&q.core.switchTable, func() {
 		q.core.switchTable._idleIn(q.port)
 	})
 }
diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index 6b7f446f..e65055bc 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -227,7 +227,7 @@ func (intf *linkInterface) handler() error {
 		strings.ToUpper(intf.info.linkType), themString, intf.info.local)
 	// Start things
 	go intf.peer.start()
-	intf.reader.RecvFrom(nil, intf.reader._read)
+	intf.reader.Act(nil, intf.reader._read)
 	// Wait for the reader to finish
 	err = <-intf.reader.err
 	if err != nil {
@@ -251,7 +251,7 @@ const (
 
 // notify the intf that we're currently sending
 func (intf *linkInterface) notifySending(size int, isLinkTraffic bool) {
-	intf.RecvFrom(&intf.writer, func() {
+	intf.Act(&intf.writer, func() {
 		if !isLinkTraffic {
 			intf.inSwitch = false
 		}
@@ -270,7 +270,7 @@ func (intf *linkInterface) _cancelStallTimer() {
 
 // called by an AfterFunc if we appear to have timed out
 func (intf *linkInterface) notifyBlockedSend() {
-	intf.RecvFrom(nil, func() { // Sent from a time.AfterFunc
+	intf.Act(nil, func() { // Sent from a time.AfterFunc
 		if intf.sendTimer != nil {
 			//As far as we know, we're still trying to send, and the timer fired.
 			intf.link.core.switchTable.blockPeer(intf.peer.port)
@@ -280,7 +280,7 @@ func (intf *linkInterface) notifyBlockedSend() {
 
 // notify the intf that we've finished sending, returning the peer to the switch
 func (intf *linkInterface) notifySent(size int, isLinkTraffic bool) {
-	intf.RecvFrom(&intf.writer, func() {
+	intf.Act(&intf.writer, func() {
 		intf.sendTimer.Stop()
 		intf.sendTimer = nil
 		if !isLinkTraffic {
@@ -296,7 +296,7 @@ func (intf *linkInterface) notifySent(size int, isLinkTraffic bool) {
 func (intf *linkInterface) _notifySwitch() {
 	if !intf.inSwitch && !intf.stalled {
 		intf.inSwitch = true
-		intf.link.core.switchTable.RecvFrom(intf, func() {
+		intf.link.core.switchTable.Act(intf, func() {
 			intf.link.core.switchTable._idleIn(intf.peer.port)
 		})
 	}
@@ -304,7 +304,7 @@ func (intf *linkInterface) _notifySwitch() {
 
 // Set the peer as stalled, to prevent them from returning to the switch until a read succeeds
 func (intf *linkInterface) notifyStalled() {
-	intf.RecvFrom(nil, func() { // Sent from a time.AfterFunc
+	intf.Act(nil, func() { // Sent from a time.AfterFunc
 		if intf.stallTimer != nil {
 			intf.stallTimer.Stop()
 			intf.stallTimer = nil
@@ -316,7 +316,7 @@ func (intf *linkInterface) notifyStalled() {
 
 // reset the close timer
 func (intf *linkInterface) notifyReading() {
-	intf.RecvFrom(&intf.reader, func() {
+	intf.Act(&intf.reader, func() {
 		if intf.closeTimer != nil {
 			intf.closeTimer.Stop()
 		}
@@ -326,7 +326,7 @@ func (intf *linkInterface) notifyReading() {
 
 // wake up the link if it was stalled, and (if size > 0) prepare to send keep-alive traffic
 func (intf *linkInterface) notifyRead(size int) {
-	intf.RecvFrom(&intf.reader, func() {
+	intf.Act(&intf.reader, func() {
 		if intf.stallTimer != nil {
 			intf.stallTimer.Stop()
 			intf.stallTimer = nil
@@ -341,7 +341,7 @@ func (intf *linkInterface) notifyRead(size int) {
 
 // We need to send keep-alive traffic now
 func (intf *linkInterface) notifyDoKeepAlive() {
-	intf.RecvFrom(nil, func() { // Sent from a time.AfterFunc
+	intf.Act(nil, func() { // Sent from a time.AfterFunc
 		if intf.stallTimer != nil {
 			intf.stallTimer.Stop()
 			intf.stallTimer = nil
@@ -358,7 +358,7 @@ type linkWriter struct {
 }
 
 func (w *linkWriter) sendFrom(from phony.Actor, bss [][]byte, isLinkTraffic bool) {
-	w.RecvFrom(from, func() {
+	w.Act(from, func() {
 		var size int
 		for _, bs := range bss {
 			size += len(bs)
@@ -396,5 +396,5 @@ func (r *linkReader) _read() {
 		return
 	}
 	// Now try to read again
-	r.RecvFrom(nil, r._read)
+	r.Act(nil, r._read)
 }
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 4cf0068b..d33c413a 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -178,7 +178,7 @@ func (ps *peers) sendSwitchMsgs(from phony.Actor) {
 		if p.port == 0 {
 			continue
 		}
-		p.RecvFrom(from, p._sendSwitchMsg)
+		p.Act(from, p._sendSwitchMsg)
 	}
 }
 
@@ -187,7 +187,7 @@ func (ps *peers) sendSwitchMsgs(from phony.Actor) {
 func (p *peer) start() {
 	var updateDHT func()
 	updateDHT = func() {
-		<-p.SyncExec(func() {
+		phony.Block(p, func() {
 			select {
 			case <-p.done:
 			default:
@@ -198,7 +198,7 @@ func (p *peer) start() {
 	}
 	updateDHT()
 	// Just for good measure, immediately send a switch message to this peer when we start
-	<-p.SyncExec(p._sendSwitchMsg)
+	p.Act(nil, p._sendSwitchMsg)
 }
 
 func (p *peer) _updateDHT() {
@@ -208,7 +208,7 @@ func (p *peer) _updateDHT() {
 }
 
 func (p *peer) handlePacketFrom(from phony.Actor, packet []byte) {
-	p.RecvFrom(from, func() {
+	p.Act(from, func() {
 		p._handlePacket(packet)
 	})
 }
@@ -246,7 +246,7 @@ func (p *peer) _handleTraffic(packet []byte) {
 }
 
 func (p *peer) sendPacketsFrom(from phony.Actor, packets [][]byte) {
-	p.RecvFrom(from, func() {
+	p.Act(from, func() {
 		p._sendPackets(packets)
 	})
 }
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index fec55b7c..bbc87c51 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -77,7 +77,7 @@ func (r *router) reconfigure(e chan error) {
 	defer close(e)
 	var errs []error
 	// Reconfigure the router
-	<-r.SyncExec(func() {
+	phony.Block(r, func() {
 		current := r.core.config.GetCurrent()
 		err := r.nodeinfo.setNodeInfo(current.NodeInfo, current.NodeInfoPrivacy)
 		if err != nil {
@@ -98,7 +98,7 @@ func (r *router) start() error {
 
 // In practice, the switch will call this with 1 packet
 func (r *router) handlePackets(from phony.Actor, packets [][]byte) {
-	r.RecvFrom(from, func() {
+	r.Act(from, func() {
 		for _, packet := range packets {
 			r._handlePacket(packet)
 		}
@@ -107,14 +107,14 @@ func (r *router) handlePackets(from phony.Actor, packets [][]byte) {
 
 // Insert a peer info into the dht, TODO? make the dht a separate actor
 func (r *router) insertPeer(from phony.Actor, info *dhtInfo) {
-	r.RecvFrom(from, func() {
+	r.Act(from, func() {
 		r.dht.insertPeer(info)
 	})
 }
 
 // Reset sessions and DHT after the switch sees our coords change
 func (r *router) reset(from phony.Actor) {
-	r.RecvFrom(from, func() {
+	r.Act(from, func() {
 		r.sessions.reset()
 		r.dht.reset()
 	})
@@ -123,7 +123,7 @@ func (r *router) reset(from phony.Actor) {
 // TODO remove reconfigure so this is just a ticker loop
 // and then find something better than a ticker loop to schedule things...
 func (r *router) doMaintenance() {
-	<-r.SyncExec(func() {
+	phony.Block(r, func() {
 		// Any periodic maintenance stuff goes here
 		r.core.switchTable.doMaintenance()
 		r.dht.doMaintenance()
@@ -252,7 +252,7 @@ func (r *router) _handleNodeInfo(bs []byte, fromKey *crypto.BoxPubKey) {
 	r.nodeinfo.handleNodeInfo(&req)
 }
 
-// TODO remove this, have things either be actors that send message or else call SyncExec directly
+// TODO remove this, have things either be actors that send message or else call Block directly
 func (r *router) doAdmin(f func()) {
-	<-r.SyncExec(f)
+	phony.Block(r, f)
 }
diff --git a/src/yggdrasil/search.go b/src/yggdrasil/search.go
index 9cf5f06e..ca357cc2 100644
--- a/src/yggdrasil/search.go
+++ b/src/yggdrasil/search.go
@@ -153,7 +153,7 @@ func (sinfo *searchInfo) continueSearch() {
 	// Note that this will spawn multiple parallel searches as time passes
 	// Any that die aren't restarted, but a new one will start later
 	time.AfterFunc(search_RETRY_TIME, func() {
-		sinfo.searches.router.RecvFrom(nil, func() {
+		sinfo.searches.router.Act(nil, func() {
 			// FIXME this keeps the search alive forever if not for the searches map, fix that
 			newSearchInfo := sinfo.searches.searches[sinfo.dest]
 			if newSearchInfo != sinfo {
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 0fc7ec80..06780e65 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -80,7 +80,7 @@ func (sinfo *sessionInfo) reconfigure(e chan error) {
 
 // TODO remove this, call SyncExec directly
 func (sinfo *sessionInfo) doFunc(f func()) {
-	<-sinfo.SyncExec(f)
+	phony.Block(sinfo, f)
 }
 
 // Represents a session ping/pong packet, andincludes information like public keys, a session handle, coords, a timestamp to prevent replays, and the tun/tap MTU.
@@ -164,7 +164,7 @@ func (ss *sessions) init(r *router) {
 func (ss *sessions) reconfigure(e chan error) {
 	defer close(e)
 	responses := make(map[crypto.Handle]chan error)
-	<-ss.router.SyncExec(func() {
+	phony.Block(ss.router, func() {
 		for index, session := range ss.sinfos {
 			responses[index] = make(chan error)
 			go session.reconfigure(responses[index])
@@ -287,7 +287,7 @@ func (ss *sessions) cleanup() {
 }
 
 func (sinfo *sessionInfo) doRemove() {
-	sinfo.sessions.router.RecvFrom(nil, func() {
+	sinfo.sessions.router.Act(nil, func() {
 		sinfo.sessions.removeSession(sinfo)
 	})
 }
@@ -341,7 +341,7 @@ func (ss *sessions) getSharedKey(myPriv *crypto.BoxPrivKey,
 
 // Sends a session ping by calling sendPingPong in ping mode.
 func (sinfo *sessionInfo) ping(from phony.Actor) {
-	sinfo.RecvFrom(from, func() {
+	sinfo.Act(from, func() {
 		sinfo._sendPingPong(false)
 	})
 }
@@ -362,14 +362,14 @@ func (sinfo *sessionInfo) _sendPingPong(isPong bool) {
 	}
 	packet := p.encode()
 	// TODO rewrite the below if/when the peer struct becomes an actor, to not go through the router first
-	sinfo.sessions.router.RecvFrom(sinfo, func() { sinfo.sessions.router.out(packet) })
+	sinfo.sessions.router.Act(sinfo, func() { sinfo.sessions.router.out(packet) })
 	if sinfo.pingTime.Before(sinfo.time) {
 		sinfo.pingTime = time.Now()
 	}
 }
 
 func (sinfo *sessionInfo) setConn(from phony.Actor, conn *Conn) {
-	sinfo.RecvFrom(from, func() {
+	sinfo.Act(from, func() {
 		sinfo.conn = conn
 		sinfo.conn.setMTU(sinfo, sinfo._getMTU())
 	})
@@ -404,7 +404,7 @@ func (ss *sessions) handlePing(ping *sessionPing) {
 		ss.listenerMutex.Unlock()
 	}
 	if sinfo != nil {
-		sinfo.RecvFrom(ss.router, func() {
+		sinfo.Act(ss.router, func() {
 			// Update the session
 			if !sinfo._update(ping) { /*panic("Should not happen in testing")*/
 				return
@@ -472,7 +472,7 @@ func (sinfo *sessionInfo) _updateNonce(theirNonce *crypto.BoxNonce) {
 // Called after coord changes, so attemtps to use a session will trigger a new ping and notify the remote end of the coord change.
 func (ss *sessions) reset() {
 	for _, sinfo := range ss.sinfos {
-		sinfo.RecvFrom(ss.router, func() {
+		sinfo.Act(ss.router, func() {
 			sinfo.reset = true
 		})
 	}
@@ -488,7 +488,7 @@ type FlowKeyMessage struct {
 }
 
 func (sinfo *sessionInfo) recv(from phony.Actor, packet *wire_trafficPacket) {
-	sinfo.RecvFrom(from, func() {
+	sinfo.Act(from, func() {
 		sinfo._recvPacket(packet)
 	})
 }
@@ -562,7 +562,7 @@ func (sinfo *sessionInfo) _send(msg FlowKeyMessage) {
 			util.PutBytes(p.Payload)
 			// Send the packet
 			// TODO replace this with a send to the peer struct if that becomes an actor
-			sinfo.sessions.router.RecvFrom(sinfo, func() {
+			sinfo.sessions.router.Act(sinfo, func() {
 				sinfo.sessions.router.out(packet)
 			})
 		}
@@ -574,7 +574,7 @@ func (sinfo *sessionInfo) _send(msg FlowKeyMessage) {
 }
 
 func (sinfo *sessionInfo) checkCallbacks() {
-	sinfo.RecvFrom(nil, func() {
+	sinfo.Act(nil, func() {
 		if len(sinfo.callbacks) > 0 {
 			select {
 			case callback := <-sinfo.callbacks[0]:
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index b30e59d5..93d79d29 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -192,7 +192,7 @@ func (t *switchTable) init(core *Core) {
 	t.updater.Store(&sync.Once{})
 	t.table.Store(lookupTable{})
 	t.drop = make(map[crypto.SigPubKey]int64)
-	<-t.SyncExec(func() {
+	phony.Block(t, func() {
 		t.queues.totalMaxSize = SwitchQueueTotalMinSize
 		t.queues.bufs = make(map[string]switch_buffer)
 		t.idle = make(map[switchPort]time.Time)
@@ -827,7 +827,7 @@ func (t *switchTable) _handleIdle(port switchPort) bool {
 }
 
 func (t *switchTable) packetInFrom(from phony.Actor, bytes []byte) {
-	t.RecvFrom(from, func() {
+	t.Act(from, func() {
 		t._packetIn(bytes)
 	})
 }
@@ -870,5 +870,5 @@ func (t *switchTable) _idleIn(port switchPort) {
 // Passed a function to call.
 // This will send the function to t.admin and block until it finishes.
 func (t *switchTable) doAdmin(f func()) {
-	<-t.SyncExec(f)
+	phony.Block(t, f)
 }

From a8b323acdd2236fb3f984cee6829395829196d14 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Tue, 27 Aug 2019 20:01:37 -0500
Subject: [PATCH 051/130] have an actor manage the crypto worker pool instead
 of each session trying to use it directly, this should result in a fairer
 round-robin behavior in cases where crypto congestion is the bottleneck

---
 src/yggdrasil/session.go | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 06780e65..9abfeca6 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -482,6 +482,18 @@ func (ss *sessions) reset() {
 //////////////////////////// Worker Functions Below ////////////////////////////
 ////////////////////////////////////////////////////////////////////////////////
 
+type sessionCryptoManager struct {
+	phony.Inbox
+}
+
+func (m *sessionCryptoManager) workerGo(from phony.Actor, f func()) {
+	m.Act(from, func() {
+		util.WorkerGo(f)
+	})
+}
+
+var manager = sessionCryptoManager{}
+
 type FlowKeyMessage struct {
 	FlowKey uint64
 	Message []byte
@@ -527,7 +539,7 @@ func (sinfo *sessionInfo) _recvPacket(p *wire_trafficPacket) {
 		sinfo.checkCallbacks()
 	}
 	sinfo.callbacks = append(sinfo.callbacks, ch)
-	util.WorkerGo(poolFunc)
+	manager.workerGo(sinfo, poolFunc)
 }
 
 func (sinfo *sessionInfo) _send(msg FlowKeyMessage) {
@@ -570,7 +582,7 @@ func (sinfo *sessionInfo) _send(msg FlowKeyMessage) {
 		sinfo.checkCallbacks()
 	}
 	sinfo.callbacks = append(sinfo.callbacks, ch)
-	util.WorkerGo(poolFunc)
+	manager.workerGo(sinfo, poolFunc)
 }
 
 func (sinfo *sessionInfo) checkCallbacks() {

From 5d7d84f82716c3689dca19d294ca239fa3c2f9f6 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 28 Aug 2019 12:17:19 +0100
Subject: [PATCH 052/130] Remove router.doAdmin and switchTable.doAdmin

---
 src/tuntap/ckr.go       |  3 +--
 src/yggdrasil/api.go    | 11 +++++------
 src/yggdrasil/conn.go   |  6 ++++--
 src/yggdrasil/core.go   |  3 ++-
 src/yggdrasil/debug.go  |  2 +-
 src/yggdrasil/peer.go   |  2 +-
 src/yggdrasil/router.go |  5 -----
 src/yggdrasil/switch.go |  8 +-------
 8 files changed, 15 insertions(+), 25 deletions(-)

diff --git a/src/tuntap/ckr.go b/src/tuntap/ckr.go
index ad8c89d4..229d2607 100644
--- a/src/tuntap/ckr.go
+++ b/src/tuntap/ckr.go
@@ -56,8 +56,7 @@ func (c *cryptokey) init(tun *TunAdapter) {
 	}
 }
 
-// Configure the CKR routes - this must only ever be called from the router
-// goroutine, e.g. through router.doAdmin
+// Configure the CKR routes.
 func (c *cryptokey) configure() error {
 	current := c.tun.config.GetCurrent()
 
diff --git a/src/yggdrasil/api.go b/src/yggdrasil/api.go
index 44cbad1d..2f684646 100644
--- a/src/yggdrasil/api.go
+++ b/src/yggdrasil/api.go
@@ -178,7 +178,7 @@ func (c *Core) GetDHT() []DHTEntry {
 			dhtentries = append(dhtentries, info)
 		}
 	}
-	c.router.doAdmin(getDHT)
+	phony.Block(c.router, getDHT)
 	return dhtentries
 }
 
@@ -205,9 +205,8 @@ func (c *Core) GetSwitchQueues() SwitchQueues {
 			}
 			switchqueues.Queues = append(switchqueues.Queues, queue)
 		}
-
 	}
-	c.switchTable.doAdmin(getSwitchQueues)
+	phony.Block(c.switchTable, getSwitchQueues)
 	return switchqueues
 }
 
@@ -244,7 +243,7 @@ func (c *Core) GetSessions() []Session {
 			sessions = append(sessions, session)
 		}
 	}
-	c.router.doAdmin(getSessions)
+	phony.Block(c.router, getSessions)
 	return sessions
 }
 
@@ -345,7 +344,7 @@ func (c *Core) GetNodeInfo(key crypto.BoxPubKey, coords []uint64, nocache bool)
 		})
 		c.router.nodeinfo.sendNodeInfo(key, wire_coordsUint64stoBytes(coords), false)
 	}
-	c.router.doAdmin(sendNodeInfoRequest)
+	phony.Block(c.router, sendNodeInfoRequest)
 	timer := time.AfterFunc(6*time.Second, func() { close(response) })
 	defer timer.Stop()
 	for res := range response {
@@ -455,7 +454,7 @@ func (c *Core) DHTPing(key crypto.BoxPubKey, coords []uint64, target *crypto.Nod
 		})
 		c.router.dht.ping(&info, &rq.dest)
 	}
-	c.router.doAdmin(sendPing)
+	phony.Block(c.router, sendPing)
 	// TODO: do something better than the below...
 	res := <-resCh
 	if res != nil {
diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index c56c2a90..4c222266 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -92,7 +92,9 @@ func (c *Conn) setMTU(from phony.Actor, mtu uint16) {
 func (c *Conn) search() error {
 	var sinfo *searchInfo
 	var isIn bool
-	c.core.router.doAdmin(func() { sinfo, isIn = c.core.router.searches.searches[*c.nodeID] })
+	phony.Block(c.core.router, func() {
+		sinfo, isIn = c.core.router.searches.searches[*c.nodeID]
+	})
 	if !isIn {
 		done := make(chan struct{}, 1)
 		var sess *sessionInfo
@@ -106,7 +108,7 @@ func (c *Conn) search() error {
 			default:
 			}
 		}
-		c.core.router.doAdmin(func() {
+		phony.Block(c.core.router, func() {
 			sinfo = c.core.router.searches.newIterSearch(c.nodeID, c.nodeMask, searchCompleted)
 			sinfo.continueSearch()
 		})
diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index ec530745..3719818b 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"time"
 
+	"github.com/Arceliar/phony"
 	"github.com/gologme/log"
 
 	"github.com/yggdrasil-network/yggdrasil-go/src/config"
@@ -175,7 +176,7 @@ func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState,
 
 	c.config.Mutex.RLock()
 	if c.config.Current.SwitchOptions.MaxTotalQueueSize >= SwitchQueueTotalMinSize {
-		c.switchTable.doAdmin(func() {
+		phony.Block(c.switchTable, func() {
 			c.switchTable.queues.totalMaxSize = c.config.Current.SwitchOptions.MaxTotalQueueSize
 		})
 	}
diff --git a/src/yggdrasil/debug.go b/src/yggdrasil/debug.go
index 9d2199d0..4502c11e 100644
--- a/src/yggdrasil/debug.go
+++ b/src/yggdrasil/debug.go
@@ -259,7 +259,7 @@ func DEBUG_wire_encode_coords(coords []byte) []byte {
 
 func (c *Core) DEBUG_getDHTSize() int {
 	var total int
-	c.router.doAdmin(func() {
+	phony.Block(c.router, func() {
 		total = len(c.router.dht.table)
 	})
 	return total
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index d33c413a..72ff2847 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -149,7 +149,7 @@ func (ps *peers) removePeer(port switchPort) {
 	if port == 0 {
 		return
 	} // Can't remove self peer
-	ps.core.router.doAdmin(func() {
+	phony.Block(ps.core.router, func() {
 		ps.core.switchTable.forgetPeer(port)
 	})
 	ps.mutex.Lock()
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index bbc87c51..d93ba433 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -251,8 +251,3 @@ func (r *router) _handleNodeInfo(bs []byte, fromKey *crypto.BoxPubKey) {
 	req.SendPermPub = *fromKey
 	r.nodeinfo.handleNodeInfo(&req)
 }
-
-// TODO remove this, have things either be actors that send message or else call Block directly
-func (r *router) doAdmin(f func()) {
-	phony.Block(r, f)
-}
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 93d79d29..5c613d8d 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -278,7 +278,7 @@ func (t *switchTable) blockPeer(port switchPort) {
 }
 
 // Removes a peer.
-// Must be called by the router mainLoop goroutine, e.g. call router.doAdmin with a lambda that calls this.
+// Must be called by the router actor with a lambda that calls this.
 // If the removed peer was this node's parent, it immediately tries to find a new parent.
 func (t *switchTable) forgetPeer(port switchPort) {
 	t.mutex.Lock()
@@ -866,9 +866,3 @@ func (t *switchTable) _idleIn(port switchPort) {
 		t.idle[port] = time.Now()
 	}
 }
-
-// Passed a function to call.
-// This will send the function to t.admin and block until it finishes.
-func (t *switchTable) doAdmin(f func()) {
-	phony.Block(t, f)
-}

From 607c9068201d10b49861a06b0d1fbd1cf2400582 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 28 Aug 2019 12:26:44 +0100
Subject: [PATCH 053/130] Pointer receivers for phony.Block

---
 src/yggdrasil/api.go  | 10 +++++-----
 src/yggdrasil/conn.go |  4 ++--
 src/yggdrasil/core.go |  2 +-
 src/yggdrasil/peer.go |  2 +-
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/yggdrasil/api.go b/src/yggdrasil/api.go
index 2f684646..9e82ece1 100644
--- a/src/yggdrasil/api.go
+++ b/src/yggdrasil/api.go
@@ -178,7 +178,7 @@ func (c *Core) GetDHT() []DHTEntry {
 			dhtentries = append(dhtentries, info)
 		}
 	}
-	phony.Block(c.router, getDHT)
+	phony.Block(&c.router, getDHT)
 	return dhtentries
 }
 
@@ -206,7 +206,7 @@ func (c *Core) GetSwitchQueues() SwitchQueues {
 			switchqueues.Queues = append(switchqueues.Queues, queue)
 		}
 	}
-	phony.Block(c.switchTable, getSwitchQueues)
+	phony.Block(&c.switchTable, getSwitchQueues)
 	return switchqueues
 }
 
@@ -243,7 +243,7 @@ func (c *Core) GetSessions() []Session {
 			sessions = append(sessions, session)
 		}
 	}
-	phony.Block(c.router, getSessions)
+	phony.Block(&c.router, getSessions)
 	return sessions
 }
 
@@ -344,7 +344,7 @@ func (c *Core) GetNodeInfo(key crypto.BoxPubKey, coords []uint64, nocache bool)
 		})
 		c.router.nodeinfo.sendNodeInfo(key, wire_coordsUint64stoBytes(coords), false)
 	}
-	phony.Block(c.router, sendNodeInfoRequest)
+	phony.Block(&c.router, sendNodeInfoRequest)
 	timer := time.AfterFunc(6*time.Second, func() { close(response) })
 	defer timer.Stop()
 	for res := range response {
@@ -454,7 +454,7 @@ func (c *Core) DHTPing(key crypto.BoxPubKey, coords []uint64, target *crypto.Nod
 		})
 		c.router.dht.ping(&info, &rq.dest)
 	}
-	phony.Block(c.router, sendPing)
+	phony.Block(&c.router, sendPing)
 	// TODO: do something better than the below...
 	res := <-resCh
 	if res != nil {
diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index 4c222266..0a4b84aa 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -92,7 +92,7 @@ func (c *Conn) setMTU(from phony.Actor, mtu uint16) {
 func (c *Conn) search() error {
 	var sinfo *searchInfo
 	var isIn bool
-	phony.Block(c.core.router, func() {
+	phony.Block(&c.core.router, func() {
 		sinfo, isIn = c.core.router.searches.searches[*c.nodeID]
 	})
 	if !isIn {
@@ -108,7 +108,7 @@ func (c *Conn) search() error {
 			default:
 			}
 		}
-		phony.Block(c.core.router, func() {
+		phony.Block(&c.core.router, func() {
 			sinfo = c.core.router.searches.newIterSearch(c.nodeID, c.nodeMask, searchCompleted)
 			sinfo.continueSearch()
 		})
diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index 3719818b..b466e3e8 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -176,7 +176,7 @@ func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState,
 
 	c.config.Mutex.RLock()
 	if c.config.Current.SwitchOptions.MaxTotalQueueSize >= SwitchQueueTotalMinSize {
-		phony.Block(c.switchTable, func() {
+		phony.Block(&c.switchTable, func() {
 			c.switchTable.queues.totalMaxSize = c.config.Current.SwitchOptions.MaxTotalQueueSize
 		})
 	}
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 72ff2847..8bd638c4 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -149,7 +149,7 @@ func (ps *peers) removePeer(port switchPort) {
 	if port == 0 {
 		return
 	} // Can't remove self peer
-	phony.Block(ps.core.router, func() {
+	phony.Block(&ps.core.router, func() {
 		ps.core.switchTable.forgetPeer(port)
 	})
 	ps.mutex.Lock()

From e553f3e013ecb5146f1fe9512a135446d8ea30ef Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 28 Aug 2019 12:46:12 +0100
Subject: [PATCH 054/130] Reconfigure functions now ran by actors

---
 src/yggdrasil/core.go    | 42 +++++++++++++++++++++++++---------------
 src/yggdrasil/link.go    |  2 +-
 src/yggdrasil/router.go  | 12 +++++-------
 src/yggdrasil/session.go | 10 ++++------
 4 files changed, 36 insertions(+), 30 deletions(-)

diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index b466e3e8..d48b6479 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -112,29 +112,39 @@ func (c *Core) addPeerLoop() {
 // config.NodeConfig and then signals the various module goroutines to
 // reconfigure themselves if needed.
 func (c *Core) UpdateConfig(config *config.NodeConfig) {
-	c.log.Debugln("Reloading node configuration...")
+	c.log.Infoln("Reloading node configuration...")
 
 	c.config.Replace(*config)
-
 	errors := 0
 
 	// Each reconfigure function should pass any errors to the channel, then close it
-	components := []func(chan error){
-		c.link.reconfigure,
-		c.peers.reconfigure,
-		c.router.reconfigure,
-		c.router.dht.reconfigure,
-		c.router.searches.reconfigure,
-		c.router.sessions.reconfigure,
-		c.switchTable.reconfigure,
+	components := map[phony.Actor][]func(chan error){
+		&c.router: []func(chan error){
+			c.router.reconfigure,
+			c.router.dht.reconfigure,
+			c.router.searches.reconfigure,
+			c.router.sessions.reconfigure,
+		},
+		&c.switchTable: []func(chan error){
+			c.switchTable.reconfigure,
+			c.link.reconfigure,
+			c.peers.reconfigure,
+		},
 	}
 
-	for _, component := range components {
-		response := make(chan error)
-		go component(response)
-		for err := range response {
-			c.log.Errorln(err)
-			errors++
+	// TODO: We count errors here but honestly that provides us with absolutely no
+	// benefit over components reporting errors themselves, so maybe we can use
+	// actor.Act() here instead and stop counting errors
+	for actor, functions := range components {
+		for _, function := range functions {
+			response := make(chan error)
+			phony.Block(actor, func() {
+				function(response)
+			})
+			for err := range response {
+				c.log.Errorln(err)
+				errors++
+			}
 		}
 	}
 
diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index e65055bc..ee4b9815 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -82,7 +82,7 @@ func (l *link) init(c *Core) error {
 func (l *link) reconfigure(e chan error) {
 	defer close(e)
 	tcpResponse := make(chan error)
-	go l.tcp.reconfigure(tcpResponse)
+	l.tcp.reconfigure(tcpResponse)
 	for err := range tcpResponse {
 		e <- err
 	}
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index d93ba433..5f894d9b 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -77,13 +77,11 @@ func (r *router) reconfigure(e chan error) {
 	defer close(e)
 	var errs []error
 	// Reconfigure the router
-	phony.Block(r, func() {
-		current := r.core.config.GetCurrent()
-		err := r.nodeinfo.setNodeInfo(current.NodeInfo, current.NodeInfoPrivacy)
-		if err != nil {
-			errs = append(errs, err)
-		}
-	})
+	current := r.core.config.GetCurrent()
+	err := r.nodeinfo.setNodeInfo(current.NodeInfo, current.NodeInfoPrivacy)
+	if err != nil {
+		errs = append(errs, err)
+	}
 	for _, err := range errs {
 		e <- err
 	}
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 9abfeca6..f6855b4d 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -164,12 +164,10 @@ func (ss *sessions) init(r *router) {
 func (ss *sessions) reconfigure(e chan error) {
 	defer close(e)
 	responses := make(map[crypto.Handle]chan error)
-	phony.Block(ss.router, func() {
-		for index, session := range ss.sinfos {
-			responses[index] = make(chan error)
-			go session.reconfigure(responses[index])
-		}
-	})
+	for index, session := range ss.sinfos {
+		responses[index] = make(chan error)
+		session.reconfigure(responses[index])
+	}
 	for _, response := range responses {
 		for err := range response {
 			e <- err

From 881d0a1ada34fa6fc57cf1c182995a7ca1686954 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 28 Aug 2019 12:46:49 +0100
Subject: [PATCH 055/130] Fix DEBUG_getDHTSize

---
 src/yggdrasil/debug.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/yggdrasil/debug.go b/src/yggdrasil/debug.go
index 4502c11e..8fdb2786 100644
--- a/src/yggdrasil/debug.go
+++ b/src/yggdrasil/debug.go
@@ -259,7 +259,7 @@ func DEBUG_wire_encode_coords(coords []byte) []byte {
 
 func (c *Core) DEBUG_getDHTSize() int {
 	var total int
-	phony.Block(c.router, func() {
+	phony.Block(&c.router, func() {
 		total = len(c.router.dht.table)
 	})
 	return total

From 764f9c8e117f8ee6a3db99fbbc5b8e5ba7f94ba7 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 28 Aug 2019 17:24:41 +0100
Subject: [PATCH 056/130] Remove legacy debug functions

---
 src/yggdrasil/debug.go | 606 -----------------------------------------
 1 file changed, 606 deletions(-)

diff --git a/src/yggdrasil/debug.go b/src/yggdrasil/debug.go
index 8fdb2786..b9bd5cfb 100644
--- a/src/yggdrasil/debug.go
+++ b/src/yggdrasil/debug.go
@@ -2,20 +2,7 @@
 
 package yggdrasil
 
-// These are functions that should not exist
-// They are (or were) used during development, to work around missing features
-// They're also used to configure things from the outside
-//  It would be better to define and export a few config functions elsewhere
-//  Or define some remote API and call it to send/request configuration info
-
-import _ "golang.org/x/net/ipv6" // TODO put this somewhere better
-
-//import "golang.org/x/net/proxy"
-
 import "fmt"
-import "net"
-import "regexp"
-import "encoding/hex"
 
 import _ "net/http/pprof"
 import "net/http"
@@ -24,13 +11,6 @@ import "os"
 
 import "github.com/gologme/log"
 
-import "github.com/yggdrasil-network/yggdrasil-go/src/address"
-import "github.com/yggdrasil-network/yggdrasil-go/src/config"
-import "github.com/yggdrasil-network/yggdrasil-go/src/crypto"
-import "github.com/yggdrasil-network/yggdrasil-go/src/defaults"
-
-import "github.com/Arceliar/phony"
-
 // Start the profiler in debug builds, if the required environment variable is set.
 func init() {
 	envVarName := "PPROFLISTEN"
@@ -51,589 +31,3 @@ func StartProfiler(log *log.Logger) error {
 	go func() { log.Println(http.ListenAndServe("localhost:6060", nil)) }()
 	return nil
 }
-
-// This function is only called by the simulator to set up a node with random
-// keys. It should not be used and may be removed in the future.
-func (c *Core) Init() {
-	bpub, bpriv := crypto.NewBoxKeys()
-	spub, spriv := crypto.NewSigKeys()
-	hbpub := hex.EncodeToString(bpub[:])
-	hbpriv := hex.EncodeToString(bpriv[:])
-	hspub := hex.EncodeToString(spub[:])
-	hspriv := hex.EncodeToString(spriv[:])
-	cfg := config.NodeConfig{
-		EncryptionPublicKey:  hbpub,
-		EncryptionPrivateKey: hbpriv,
-		SigningPublicKey:     hspub,
-		SigningPrivateKey:    hspriv,
-	}
-	c.config = config.NodeState{
-		Current:  cfg,
-		Previous: cfg,
-	}
-	c.init()
-	c.switchTable.start()
-	c.router.start()
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
-// Core
-
-func (c *Core) DEBUG_getSigningPublicKey() crypto.SigPubKey {
-	return (crypto.SigPubKey)(c.sigPub)
-}
-
-func (c *Core) DEBUG_getEncryptionPublicKey() crypto.BoxPubKey {
-	return (crypto.BoxPubKey)(c.boxPub)
-}
-
-/*
-func (c *Core) DEBUG_getSend() chan<- []byte {
-	return c.router.tun.send
-}
-
-func (c *Core) DEBUG_getRecv() <-chan []byte {
-	return c.router.tun.recv
-}
-*/
-
-// Peer
-
-func (c *Core) DEBUG_getPeers() *peers {
-	return &c.peers
-}
-
-func (ps *peers) DEBUG_newPeer(box crypto.BoxPubKey, sig crypto.SigPubKey, link crypto.BoxSharedKey) *peer {
-	sim := linkInterface{
-		name: "(simulator)",
-		info: linkInfo{
-			local:    "(simulator)",
-			remote:   "(simulator)",
-			linkType: "sim",
-		},
-	}
-	return ps.newPeer(&box, &sig, &link, &sim, nil)
-}
-
-/*
-func (ps *peers) DEBUG_startPeers() {
-  ps.mutex.RLock()
-  defer ps.mutex.RUnlock()
-  for _, p := range ps.ports {
-    if p == nil { continue }
-    go p.MainLoop()
-  }
-}
-*/
-
-func (ps *peers) DEBUG_hasPeer(key crypto.SigPubKey) bool {
-	ports := ps.ports.Load().(map[switchPort]*peer)
-	for _, p := range ports {
-		if p == nil {
-			continue
-		}
-		if p.sig == key {
-			return true
-		}
-	}
-	return false
-}
-
-func (ps *peers) DEBUG_getPorts() map[switchPort]*peer {
-	ports := ps.ports.Load().(map[switchPort]*peer)
-	newPeers := make(map[switchPort]*peer)
-	for port, p := range ports {
-		newPeers[port] = p
-	}
-	return newPeers
-}
-
-func (p *peer) DEBUG_getSigKey() crypto.SigPubKey {
-	return p.sig
-}
-
-func (p *peer) DEEBUG_getPort() switchPort {
-	return p.port
-}
-
-// Router
-
-func (c *Core) DEBUG_getSwitchTable() *switchTable {
-	return &c.switchTable
-}
-
-func (c *Core) DEBUG_getLocator() switchLocator {
-	return c.switchTable.getLocator()
-}
-
-func (l *switchLocator) DEBUG_getCoords() []byte {
-	return l.getCoords()
-}
-
-func (c *Core) DEBUG_switchLookup(dest []byte) switchPort {
-	return c.switchTable.DEBUG_lookup(dest)
-}
-
-// This does the switch layer lookups that decide how to route traffic.
-// Traffic uses greedy routing in a metric space, where the metric distance between nodes is equal to the distance between them on the tree.
-// Traffic must be routed to a node that is closer to the destination via the metric space distance.
-// In the event that two nodes are equally close, it gets routed to the one with the longest uptime (due to the order that things are iterated over).
-// The size of the outgoing packet queue is added to a node's tree distance when the cost of forwarding to a node, subject to the constraint that the real tree distance puts them closer to the destination than ourself.
-// Doing so adds a limited form of backpressure routing, based on local information, which allows us to forward traffic around *local* bottlenecks, provided that another greedy path exists.
-func (t *switchTable) DEBUG_lookup(dest []byte) switchPort {
-	table := t.getTable()
-	myDist := table.self.dist(dest)
-	if myDist == 0 {
-		return 0
-	}
-	// cost is in units of (expected distance) + (expected queue size), where expected distance is used as an approximation of the minimum backpressure gradient needed for packets to flow
-	ports := t.core.peers.getPorts()
-	var best switchPort
-	bestCost := int64(^uint64(0) >> 1)
-	for _, info := range table.elems {
-		dist := info.locator.dist(dest)
-		if !(dist < myDist) {
-			continue
-		}
-		//p, isIn := ports[info.port]
-		_, isIn := ports[info.port]
-		if !isIn {
-			continue
-		}
-		cost := int64(dist) // + p.getQueueSize()
-		if cost < bestCost {
-			best = info.port
-			bestCost = cost
-		}
-	}
-	return best
-}
-
-/*
-func (t *switchTable) DEBUG_isDirty() bool {
-  //data := t.data.Load().(*tabledata)
-  t.mutex.RLock()
-  defer t.mutex.RUnlock()
-  data := t.data
-  return data.dirty
-}
-*/
-
-func (t *switchTable) DEBUG_dumpTable() {
-	//data := t.data.Load().(*tabledata)
-	t.mutex.RLock()
-	defer t.mutex.RUnlock()
-	data := t.data
-	for _, peer := range data.peers {
-		//fmt.Println("DUMPTABLE:", t.treeID, peer.treeID, peer.port,
-		//            peer.locator.Root, peer.coords,
-		//            peer.reverse.Root, peer.reverse.Coords, peer.forward)
-		fmt.Println("DUMPTABLE:", t.key, peer.key, peer.locator.coords, peer.port /*, peer.forward*/)
-	}
-}
-
-func (t *switchTable) DEBUG_getReversePort(port switchPort) switchPort {
-	// Returns Port(0) if it cannot get the reverse peer for any reason
-	//data := t.data.Load().(*tabledata)
-	t.mutex.RLock()
-	defer t.mutex.RUnlock()
-	data := t.data
-	if port >= switchPort(len(data.peers)) {
-		return switchPort(0)
-	}
-	pinfo := data.peers[port]
-	if len(pinfo.locator.coords) < 1 {
-		return switchPort(0)
-	}
-	return pinfo.locator.coords[len(pinfo.locator.coords)-1]
-}
-
-// Wire
-
-func DEBUG_wire_encode_coords(coords []byte) []byte {
-	return wire_encode_coords(coords)
-}
-
-// DHT, via core
-
-func (c *Core) DEBUG_getDHTSize() int {
-	var total int
-	phony.Block(&c.router, func() {
-		total = len(c.router.dht.table)
-	})
-	return total
-}
-
-// TUN defaults
-
-func (c *Core) DEBUG_GetTUNDefaultIfName() string {
-	return defaults.GetDefaults().DefaultIfName
-}
-
-func (c *Core) DEBUG_GetTUNDefaultIfMTU() int {
-	return defaults.GetDefaults().DefaultIfMTU
-}
-
-func (c *Core) DEBUG_GetTUNDefaultIfTAPMode() bool {
-	return defaults.GetDefaults().DefaultIfTAPMode
-}
-
-// udpInterface
-//  FIXME udpInterface isn't exported
-//  So debug functions need to work differently...
-
-/*
-func (c *Core) DEBUG_setupLoopbackUDPInterface() {
-  iface := udpInterface{}
-  iface.init(c, "[::1]:0")
-  c.ifaces = append(c.ifaces[:0], &iface)
-}
-*/
-
-/*
-func (c *Core) DEBUG_getLoopbackAddr() net.Addr {
-  iface := c.ifaces[0]
-  return iface.sock.LocalAddr()
-}
-*/
-
-/*
-func (c *Core) DEBUG_addLoopbackPeer(addr *net.UDPAddr,
-                                     in (chan<- []byte),
-                                     out (<-chan []byte)) {
-  iface := c.ifaces[0]
-  iface.addPeer(addr, in, out)
-}
-*/
-
-/*
-func (c *Core) DEBUG_startLoopbackUDPInterface() {
-  iface := c.ifaces[0]
-  go iface.reader()
-  for addr, chs := range iface.peers {
-    udpAddr, err := net.ResolveUDPAddr("udp6", addr)
-    if err != nil { panic(err) }
-    go iface.writer(udpAddr, chs.out)
-  }
-}
-*/
-
-////////////////////////////////////////////////////////////////////////////////
-
-func (c *Core) DEBUG_getAddr() *address.Address {
-	return address.AddrForNodeID(&c.router.dht.nodeID)
-}
-
-/*
-func (c *Core) DEBUG_startTun(ifname string, iftapmode bool) {
-	c.DEBUG_startTunWithMTU(ifname, iftapmode, 1280)
-}
-
-func (c *Core) DEBUG_startTunWithMTU(ifname string, iftapmode bool, mtu int) {
-	addr := c.DEBUG_getAddr()
-	straddr := fmt.Sprintf("%s/%v", net.IP(addr[:]).String(), 8*len(address.GetPrefix()))
-	if ifname != "none" {
-		err := c.router.tun.setup(ifname, iftapmode, straddr, mtu)
-		if err != nil {
-			panic(err)
-		}
-		c.log.Println("Setup TUN/TAP:", c.router.tun.iface.Name(), straddr)
-		go func() { panic(c.router.tun.read()) }()
-	}
-	go func() { panic(c.router.tun.write()) }()
-}
-
-func (c *Core) DEBUG_stopTun() {
-	c.router.tun.close()
-}
-*/
-
-////////////////////////////////////////////////////////////////////////////////
-
-func (c *Core) DEBUG_newBoxKeys() (*crypto.BoxPubKey, *crypto.BoxPrivKey) {
-	return crypto.NewBoxKeys()
-}
-
-func (c *Core) DEBUG_getSharedKey(myPrivKey *crypto.BoxPrivKey, othersPubKey *crypto.BoxPubKey) *crypto.BoxSharedKey {
-	return crypto.GetSharedKey(myPrivKey, othersPubKey)
-}
-
-func (c *Core) DEBUG_newSigKeys() (*crypto.SigPubKey, *crypto.SigPrivKey) {
-	return crypto.NewSigKeys()
-}
-
-func (c *Core) DEBUG_getNodeID(pub *crypto.BoxPubKey) *crypto.NodeID {
-	return crypto.GetNodeID(pub)
-}
-
-func (c *Core) DEBUG_getTreeID(pub *crypto.SigPubKey) *crypto.TreeID {
-	return crypto.GetTreeID(pub)
-}
-
-func (c *Core) DEBUG_addrForNodeID(nodeID *crypto.NodeID) string {
-	return net.IP(address.AddrForNodeID(nodeID)[:]).String()
-}
-
-func (c *Core) DEBUG_init(bpub []byte,
-	bpriv []byte,
-	spub []byte,
-	spriv []byte) {
-	/*var boxPub crypto.BoxPubKey
-	var boxPriv crypto.BoxPrivKey
-	var sigPub crypto.SigPubKey
-	var sigPriv crypto.SigPrivKey
-	copy(boxPub[:], bpub)
-	copy(boxPriv[:], bpriv)
-	copy(sigPub[:], spub)
-	copy(sigPriv[:], spriv)
-	c.init(&boxPub, &boxPriv, &sigPub, &sigPriv)*/
-	hbpub := hex.EncodeToString(bpub[:])
-	hbpriv := hex.EncodeToString(bpriv[:])
-	hspub := hex.EncodeToString(spub[:])
-	hspriv := hex.EncodeToString(spriv[:])
-	cfg := config.NodeConfig{
-		EncryptionPublicKey:  hbpub,
-		EncryptionPrivateKey: hbpriv,
-		SigningPublicKey:     hspub,
-		SigningPrivateKey:    hspriv,
-	}
-	c.config = config.NodeState{
-		Current:  cfg,
-		Previous: cfg,
-	}
-	c.init()
-
-	if err := c.router.start(); err != nil {
-		panic(err)
-	}
-
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
-/*
-func (c *Core) DEBUG_setupAndStartGlobalUDPInterface(addrport string) {
-	if err := c.udp.init(c, addrport); err != nil {
-		c.log.Println("Failed to start UDP interface:", err)
-		panic(err)
-	}
-}
-
-func (c *Core) DEBUG_getGlobalUDPAddr() *net.UDPAddr {
-	return c.udp.sock.LocalAddr().(*net.UDPAddr)
-}
-
-func (c *Core) DEBUG_maybeSendUDPKeys(saddr string) {
-	udpAddr, err := net.ResolveUDPAddr("udp", saddr)
-	if err != nil {
-		panic(err)
-	}
-	var addr connAddr
-	addr.fromUDPAddr(udpAddr)
-	c.udp.mutex.RLock()
-	_, isIn := c.udp.conns[addr]
-	c.udp.mutex.RUnlock()
-	if !isIn {
-		c.udp.sendKeys(addr)
-	}
-}
-*/
-
-////////////////////////////////////////////////////////////////////////////////
-/*
-func (c *Core) DEBUG_addPeer(addr string) {
-	err := c.admin.addPeer(addr, "")
-	if err != nil {
-		panic(err)
-	}
-}
-*/
-/*
-func (c *Core) DEBUG_addSOCKSConn(socksaddr, peeraddr string) {
-	go func() {
-		dialer, err := proxy.SOCKS5("tcp", socksaddr, nil, proxy.Direct)
-		if err == nil {
-			conn, err := dialer.Dial("tcp", peeraddr)
-			if err == nil {
-				c.tcp.callWithConn(&wrappedConn{
-					c: conn,
-					raddr: &wrappedAddr{
-						network: "tcp",
-						addr:    peeraddr,
-					},
-				})
-			}
-		}
-	}()
-}
-*/
-
-/*
-func (c *Core) DEBUG_setupAndStartGlobalTCPInterface(addrport string) {
-	c.config.Listen = []string{addrport}
-	if err := c.link.init(c); err != nil {
-		c.log.Println("Failed to start interfaces:", err)
-		panic(err)
-	}
-}
-
-func (c *Core) DEBUG_getGlobalTCPAddr() *net.TCPAddr {
-	return c.link.tcp.getAddr()
-}
-
-func (c *Core) DEBUG_addTCPConn(saddr string) {
-	c.link.tcp.call(saddr, nil, "")
-}
-
-//*/
-
-/*
-func (c *Core) DEBUG_startSelfPeer() {
-  c.Peers.mutex.RLock()
-  defer c.Peers.mutex.RUnlock()
-  p := c.Peers.ports[0]
-  go p.MainLoop()
-}
-*/
-
-////////////////////////////////////////////////////////////////////////////////
-
-/*
-func (c *Core) DEBUG_setupAndStartGlobalKCPInterface(addrport string) {
-  iface := kcpInterface{}
-  iface.init(c, addrport)
-  c.kcp = &iface
-}
-
-func (c *Core) DEBUG_getGlobalKCPAddr() net.Addr {
-  return c.kcp.serv.Addr()
-}
-
-func (c *Core) DEBUG_addKCPConn(saddr string) {
-  c.kcp.call(saddr)
-}
-*/
-
-////////////////////////////////////////////////////////////////////////////////
-
-/*
-func (c *Core) DEBUG_setupAndStartAdminInterface(addrport string) {
-	a := admin{}
-	c.config.AdminListen = addrport
-	a.init()
-	c.admin = a
-}
-
-func (c *Core) DEBUG_setupAndStartMulticastInterface() {
-	m := multicast{}
-	m.init(c)
-	c.multicast = m
-	m.start()
-}
-*/
-
-////////////////////////////////////////////////////////////////////////////////
-
-func (c *Core) DEBUG_setLogger(log *log.Logger) {
-	c.log = log
-}
-
-func (c *Core) DEBUG_setIfceExpr(expr *regexp.Regexp) {
-	c.log.Println("DEBUG_setIfceExpr no longer implemented")
-}
-
-/*
-func (c *Core) DEBUG_addAllowedEncryptionPublicKey(boxStr string) {
-	err := c.admin.addAllowedEncryptionPublicKey(boxStr)
-	if err != nil {
-		panic(err)
-	}
-}
-*/
-////////////////////////////////////////////////////////////////////////////////
-
-func DEBUG_simLinkPeers(p, q *peer) {
-	// Sets q.out() to point to p and starts p.start()
-	goWorkers := func(source, dest *peer) {
-		linkOut := make(chan []byte, 1)
-		source.linkOut = func(bs []byte) { linkOut <- bs }
-		send := make(chan []byte, 1)
-		source.out = func(bss [][]byte) {
-			for _, bs := range bss {
-				send <- bs
-			}
-		}
-		go source.start()
-		go func() {
-			var packets [][]byte
-			for {
-				select {
-				case packet := <-linkOut:
-					packets = append(packets, packet)
-					continue
-				case packet := <-send:
-					packets = append(packets, packet)
-					phony.Block(&source.core.switchTable, func() {
-						source.core.switchTable._idleIn(source.port)
-					})
-					continue
-				default:
-				}
-				if len(packets) > 0 {
-					phony.Block(dest, func() { dest._handlePacket(packets[0]) })
-					packets = packets[1:]
-					continue
-				}
-				select {
-				case packet := <-linkOut:
-					packets = append(packets, packet)
-				case packet := <-send:
-					packets = append(packets, packet)
-					phony.Block(&source.core.switchTable, func() {
-						source.core.switchTable._idleIn(source.port)
-					})
-				}
-			}
-		}()
-	}
-	goWorkers(p, q)
-	goWorkers(q, p)
-	phony.Block(&p.core.switchTable, func() {
-		p.core.switchTable._idleIn(p.port)
-	})
-	phony.Block(&q.core.switchTable, func() {
-		q.core.switchTable._idleIn(q.port)
-	})
-}
-
-/*
-func (c *Core) DEBUG_simFixMTU() {
-	c.router.tun.mtu = 65535
-}
-*/
-
-////////////////////////////////////////////////////////////////////////////////
-
-func Util_testAddrIDMask() {
-	for idx := 0; idx < 16; idx++ {
-		var orig crypto.NodeID
-		orig[8] = 42
-		for bidx := 0; bidx < idx; bidx++ {
-			orig[bidx/8] |= (0x80 >> uint8(bidx%8))
-		}
-		addr := address.AddrForNodeID(&orig)
-		nid, mask := addr.GetNodeIDandMask()
-		for b := 0; b < len(mask); b++ {
-			nid[b] &= mask[b]
-			orig[b] &= mask[b]
-		}
-		if *nid != orig {
-			fmt.Println(orig)
-			fmt.Println(*addr)
-			fmt.Println(*nid)
-			fmt.Println(*mask)
-			panic(idx)
-		}
-	}
-}

From fc9a1c6c31b456fb55dd7b5c555975652573b44c Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 28 Aug 2019 19:31:04 +0100
Subject: [PATCH 057/130] Simplify reconfiguration

---
 cmd/yggdrasil/main.go      |  1 +
 src/multicast/multicast.go |  1 -
 src/tuntap/ckr.go          | 66 +++++++++++++++-----------------------
 src/tuntap/tun.go          | 25 +++------------
 src/yggdrasil/core.go      | 44 ++++---------------------
 src/yggdrasil/dht.go       |  3 +-
 src/yggdrasil/link.go      |  9 ++----
 src/yggdrasil/peer.go      |  3 +-
 src/yggdrasil/router.go    | 20 ++++++------
 src/yggdrasil/search.go    |  3 +-
 src/yggdrasil/session.go   | 17 +++-------
 src/yggdrasil/switch.go    |  5 +--
 src/yggdrasil/tcp.go       |  8 +++--
 13 files changed, 64 insertions(+), 141 deletions(-)

diff --git a/cmd/yggdrasil/main.go b/cmd/yggdrasil/main.go
index 2fa10720..e9a21c13 100644
--- a/cmd/yggdrasil/main.go
+++ b/cmd/yggdrasil/main.go
@@ -279,6 +279,7 @@ func main() {
 		case _ = <-r:
 			if *useconffile != "" {
 				cfg = readConfig(useconf, useconffile, normaliseconf)
+				logger.Infoln("Reloading configuration from", *useconffile)
 				n.core.UpdateConfig(cfg)
 				n.tuntap.UpdateConfig(cfg)
 				n.multicast.UpdateConfig(cfg)
diff --git a/src/multicast/multicast.go b/src/multicast/multicast.go
index 5ab9673e..22ac9356 100644
--- a/src/multicast/multicast.go
+++ b/src/multicast/multicast.go
@@ -83,7 +83,6 @@ func (m *Multicast) Stop() error {
 func (m *Multicast) UpdateConfig(config *config.NodeConfig) {
 	m.log.Debugln("Reloading multicast configuration...")
 	m.config.Replace(*config)
-	m.log.Infoln("Multicast configuration reloaded successfully")
 }
 
 // GetInterfaces returns the currently known/enabled multicast interfaces. It is
diff --git a/src/tuntap/ckr.go b/src/tuntap/ckr.go
index 229d2607..9af3564a 100644
--- a/src/tuntap/ckr.go
+++ b/src/tuntap/ckr.go
@@ -20,7 +20,6 @@ import (
 type cryptokey struct {
 	tun          *TunAdapter
 	enabled      atomic.Value // bool
-	reconfigure  chan chan error
 	ipv4remotes  []cryptokey_route
 	ipv6remotes  []cryptokey_route
 	ipv4cache    map[address.Address]cryptokey_route
@@ -40,24 +39,11 @@ type cryptokey_route struct {
 // Initialise crypto-key routing. This must be done before any other CKR calls.
 func (c *cryptokey) init(tun *TunAdapter) {
 	c.tun = tun
-	c.reconfigure = make(chan chan error, 1)
-	go func() {
-		for {
-			e := <-c.reconfigure
-			e <- nil
-		}
-	}()
-
-	c.tun.log.Debugln("Configuring CKR...")
-	if err := c.configure(); err != nil {
-		c.tun.log.Errorln("CKR configuration failed:", err)
-	} else {
-		c.tun.log.Debugln("CKR configured")
-	}
+	c.configure()
 }
 
-// Configure the CKR routes.
-func (c *cryptokey) configure() error {
+// Configure the CKR routes. This should only ever be ran by the TUN/TAP actor.
+func (c *cryptokey) configure() {
 	current := c.tun.config.GetCurrent()
 
 	// Set enabled/disabled state
@@ -72,14 +58,14 @@ func (c *cryptokey) configure() error {
 	// Add IPv6 routes
 	for ipv6, pubkey := range current.TunnelRouting.IPv6RemoteSubnets {
 		if err := c.addRemoteSubnet(ipv6, pubkey); err != nil {
-			return err
+			c.tun.log.Errorln("Error adding CKR IPv6 remote subnet:", err)
 		}
 	}
 
 	// Add IPv4 routes
 	for ipv4, pubkey := range current.TunnelRouting.IPv4RemoteSubnets {
 		if err := c.addRemoteSubnet(ipv4, pubkey); err != nil {
-			return err
+			c.tun.log.Errorln("Error adding CKR IPv4 remote subnet:", err)
 		}
 	}
 
@@ -93,7 +79,7 @@ func (c *cryptokey) configure() error {
 	c.ipv6locals = make([]net.IPNet, 0)
 	for _, source := range current.TunnelRouting.IPv6LocalSubnets {
 		if err := c.addLocalSubnet(source); err != nil {
-			return err
+			c.tun.log.Errorln("Error adding CKR IPv6 local subnet:", err)
 		}
 	}
 
@@ -101,7 +87,7 @@ func (c *cryptokey) configure() error {
 	c.ipv4locals = make([]net.IPNet, 0)
 	for _, source := range current.TunnelRouting.IPv4LocalSubnets {
 		if err := c.addLocalSubnet(source); err != nil {
-			return err
+			c.tun.log.Errorln("Error adding CKR IPv4 local subnet:", err)
 		}
 	}
 
@@ -110,8 +96,6 @@ func (c *cryptokey) configure() error {
 	c.ipv4cache = make(map[address.Address]cryptokey_route, 0)
 	c.ipv6cache = make(map[address.Address]cryptokey_route, 0)
 	c.mutexcaches.Unlock()
-
-	return nil
 }
 
 // Enable or disable crypto-key routing.
@@ -181,19 +165,19 @@ func (c *cryptokey) addLocalSubnet(cidr string) error {
 	} else if prefixsize == net.IPv4len*8 {
 		routingsources = &c.ipv4locals
 	} else {
-		return errors.New("Unexpected prefix size")
+		return errors.New("unexpected prefix size")
 	}
 
 	// Check if we already have this CIDR
 	for _, subnet := range *routingsources {
 		if subnet.String() == ipnet.String() {
-			return errors.New("Source subnet already configured")
+			return errors.New("local subnet already configured")
 		}
 	}
 
 	// Add the source subnet
 	*routingsources = append(*routingsources, *ipnet)
-	c.tun.log.Infoln("Added CKR source subnet", cidr)
+	c.tun.log.Infoln("Added CKR local subnet", cidr)
 	return nil
 }
 
@@ -226,7 +210,7 @@ func (c *cryptokey) addRemoteSubnet(cidr string, dest string) error {
 		routingtable = &c.ipv4remotes
 		routingcache = &c.ipv4cache
 	} else {
-		return errors.New("Unexpected prefix size")
+		return errors.New("unexpected prefix size")
 	}
 
 	// Is the route an Yggdrasil destination?
@@ -235,19 +219,19 @@ func (c *cryptokey) addRemoteSubnet(cidr string, dest string) error {
 	copy(addr[:], ipaddr)
 	copy(snet[:], ipnet.IP)
 	if addr.IsValid() || snet.IsValid() {
-		return errors.New("Can't specify Yggdrasil destination as crypto-key route")
+		return errors.New("can't specify Yggdrasil destination as crypto-key route")
 	}
 	// Do we already have a route for this subnet?
 	for _, route := range *routingtable {
 		if route.subnet.String() == ipnet.String() {
-			return errors.New(fmt.Sprintf("Route already exists for %s", cidr))
+			return fmt.Errorf("remote subnet already exists for %s", cidr)
 		}
 	}
 	// Decode the public key
 	if bpk, err := hex.DecodeString(dest); err != nil {
 		return err
 	} else if len(bpk) != crypto.BoxPubKeyLen {
-		return errors.New(fmt.Sprintf("Incorrect key length for %s", dest))
+		return fmt.Errorf("incorrect key length for %s", dest)
 	} else {
 		// Add the new crypto-key route
 		var key crypto.BoxPubKey
@@ -270,7 +254,7 @@ func (c *cryptokey) addRemoteSubnet(cidr string, dest string) error {
 			delete(*routingcache, k)
 		}
 
-		c.tun.log.Infoln("Added CKR destination subnet", cidr)
+		c.tun.log.Infoln("Added CKR remote subnet", cidr)
 		return nil
 	}
 }
@@ -284,7 +268,7 @@ func (c *cryptokey) getPublicKeyForAddress(addr address.Address, addrlen int) (c
 	// Check if the address is a valid Yggdrasil address - if so it
 	// is exempt from all CKR checking
 	if addr.IsValid() {
-		return crypto.BoxPubKey{}, errors.New("Cannot look up CKR for Yggdrasil addresses")
+		return crypto.BoxPubKey{}, errors.New("cannot look up CKR for Yggdrasil addresses")
 	}
 
 	// Build our references to the routing table and cache
@@ -297,7 +281,7 @@ func (c *cryptokey) getPublicKeyForAddress(addr address.Address, addrlen int) (c
 	} else if addrlen == net.IPv4len {
 		routingcache = &c.ipv4cache
 	} else {
-		return crypto.BoxPubKey{}, errors.New("Unexpected prefix size")
+		return crypto.BoxPubKey{}, errors.New("unexpected prefix size")
 	}
 
 	// Check if there's a cache entry for this addr
@@ -317,7 +301,7 @@ func (c *cryptokey) getPublicKeyForAddress(addr address.Address, addrlen int) (c
 	} else if addrlen == net.IPv4len {
 		routingtable = &c.ipv4remotes
 	} else {
-		return crypto.BoxPubKey{}, errors.New("Unexpected prefix size")
+		return crypto.BoxPubKey{}, errors.New("unexpected prefix size")
 	}
 
 	// No cache was found - start by converting the address into a net.IP
@@ -378,18 +362,18 @@ func (c *cryptokey) removeLocalSubnet(cidr string) error {
 	} else if prefixsize == net.IPv4len*8 {
 		routingsources = &c.ipv4locals
 	} else {
-		return errors.New("Unexpected prefix size")
+		return errors.New("unexpected prefix size")
 	}
 
 	// Check if we already have this CIDR
 	for idx, subnet := range *routingsources {
 		if subnet.String() == ipnet.String() {
 			*routingsources = append((*routingsources)[:idx], (*routingsources)[idx+1:]...)
-			c.tun.log.Infoln("Removed CKR source subnet", cidr)
+			c.tun.log.Infoln("Removed CKR local subnet", cidr)
 			return nil
 		}
 	}
-	return errors.New("Source subnet not found")
+	return errors.New("local subnet not found")
 }
 
 // Removes a destination route for the given CIDR to be tunnelled to the node
@@ -421,7 +405,7 @@ func (c *cryptokey) removeRemoteSubnet(cidr string, dest string) error {
 		routingtable = &c.ipv4remotes
 		routingcache = &c.ipv4cache
 	} else {
-		return errors.New("Unexpected prefix size")
+		return errors.New("unexpected prefix size")
 	}
 
 	// Decode the public key
@@ -429,7 +413,7 @@ func (c *cryptokey) removeRemoteSubnet(cidr string, dest string) error {
 	if err != nil {
 		return err
 	} else if len(bpk) != crypto.BoxPubKeyLen {
-		return errors.New(fmt.Sprintf("Incorrect key length for %s", dest))
+		return fmt.Errorf("incorrect key length for %s", dest)
 	}
 	netStr := ipnet.String()
 
@@ -439,9 +423,9 @@ func (c *cryptokey) removeRemoteSubnet(cidr string, dest string) error {
 			for k := range *routingcache {
 				delete(*routingcache, k)
 			}
-			c.tun.log.Infof("Removed CKR destination subnet %s via %s\n", cidr, dest)
+			c.tun.log.Infof("Removed CKR remote subnet %s via %s\n", cidr, dest)
 			return nil
 		}
 	}
-	return errors.New(fmt.Sprintf("Route does not exists for %s", cidr))
+	return fmt.Errorf("route does not exists for %s", cidr)
 }
diff --git a/src/tuntap/tun.go b/src/tuntap/tun.go
index 6317459c..8e1e5b0c 100644
--- a/src/tuntap/tun.go
+++ b/src/tuntap/tun.go
@@ -13,6 +13,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+
 	//"sync"
 
 	"github.com/Arceliar/phony"
@@ -200,29 +201,11 @@ func (tun *TunAdapter) _stop() error {
 func (tun *TunAdapter) UpdateConfig(config *config.NodeConfig) {
 	tun.log.Debugln("Reloading TUN/TAP configuration...")
 
+	// Replace the active configuration with the supplied one
 	tun.config.Replace(*config)
 
-	errors := 0
-
-	components := []chan chan error{
-		tun.reconfigure,
-		tun.ckr.reconfigure,
-	}
-
-	for _, component := range components {
-		response := make(chan error)
-		component <- response
-		if err := <-response; err != nil {
-			tun.log.Errorln(err)
-			errors++
-		}
-	}
-
-	if errors > 0 {
-		tun.log.Warnln(errors, "TUN/TAP module(s) reported errors during configuration reload")
-	} else {
-		tun.log.Infoln("TUN/TAP configuration reloaded successfully")
-	}
+	// Notify children about the configuration change
+	tun.Act(nil, tun.ckr.configure)
 }
 
 func (tun *TunAdapter) handler() error {
diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index d48b6479..831109dd 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -20,6 +20,7 @@ type Core struct {
 	// This is the main data structure that holds everything else for a node
 	// We're going to keep our own copy of the provided config - that way we can
 	// guarantee that it will be covered by the mutex
+	phony.Inbox
 	config      config.NodeState // Config
 	boxPub      crypto.BoxPubKey
 	boxPriv     crypto.BoxPrivKey
@@ -112,47 +113,14 @@ func (c *Core) addPeerLoop() {
 // config.NodeConfig and then signals the various module goroutines to
 // reconfigure themselves if needed.
 func (c *Core) UpdateConfig(config *config.NodeConfig) {
-	c.log.Infoln("Reloading node configuration...")
+	c.log.Debugln("Reloading node configuration...")
 
+	// Replace the active configuration with the supplied one
 	c.config.Replace(*config)
-	errors := 0
 
-	// Each reconfigure function should pass any errors to the channel, then close it
-	components := map[phony.Actor][]func(chan error){
-		&c.router: []func(chan error){
-			c.router.reconfigure,
-			c.router.dht.reconfigure,
-			c.router.searches.reconfigure,
-			c.router.sessions.reconfigure,
-		},
-		&c.switchTable: []func(chan error){
-			c.switchTable.reconfigure,
-			c.link.reconfigure,
-			c.peers.reconfigure,
-		},
-	}
-
-	// TODO: We count errors here but honestly that provides us with absolutely no
-	// benefit over components reporting errors themselves, so maybe we can use
-	// actor.Act() here instead and stop counting errors
-	for actor, functions := range components {
-		for _, function := range functions {
-			response := make(chan error)
-			phony.Block(actor, func() {
-				function(response)
-			})
-			for err := range response {
-				c.log.Errorln(err)
-				errors++
-			}
-		}
-	}
-
-	if errors > 0 {
-		c.log.Warnln(errors, "node module(s) reported errors during configuration reload")
-	} else {
-		c.log.Infoln("Node configuration reloaded successfully")
-	}
+	// Notify the router and switch about the new configuration
+	c.router.Act(c, c.router.reconfigure)
+	c.switchTable.Act(c, c.switchTable.reconfigure)
 }
 
 // Start starts up Yggdrasil using the provided config.NodeConfig, and outputs
diff --git a/src/yggdrasil/dht.go b/src/yggdrasil/dht.go
index 4f380363..575c8b1a 100644
--- a/src/yggdrasil/dht.go
+++ b/src/yggdrasil/dht.go
@@ -82,8 +82,7 @@ func (t *dht) init(r *router) {
 	t.reset()
 }
 
-func (t *dht) reconfigure(e chan error) {
-	defer close(e)
+func (t *dht) reconfigure() {
 	// This is where reconfiguration would go, if we had anything to do
 }
 
diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index ee4b9815..6e393514 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -79,13 +79,8 @@ func (l *link) init(c *Core) error {
 	return nil
 }
 
-func (l *link) reconfigure(e chan error) {
-	defer close(e)
-	tcpResponse := make(chan error)
-	l.tcp.reconfigure(tcpResponse)
-	for err := range tcpResponse {
-		e <- err
-	}
+func (l *link) reconfigure() {
+	l.tcp.reconfigure()
 }
 
 func (l *link) call(uri string, sintf string) error {
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 8bd638c4..381e6917 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -34,8 +34,7 @@ func (ps *peers) init(c *Core) {
 	ps.core = c
 }
 
-func (ps *peers) reconfigure(e chan error) {
-	defer close(e)
+func (ps *peers) reconfigure() {
 	// This is where reconfiguration would go, if we had anything to do
 }
 
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index 5f894d9b..64c81701 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -73,18 +73,20 @@ func (r *router) init(core *Core) {
 	r.sessions.init(r)
 }
 
-func (r *router) reconfigure(e chan error) {
-	defer close(e)
-	var errs []error
+// Reconfigures the router and any child modules. This should only ever be run
+// by the router actor.
+func (r *router) reconfigure() {
 	// Reconfigure the router
 	current := r.core.config.GetCurrent()
-	err := r.nodeinfo.setNodeInfo(current.NodeInfo, current.NodeInfoPrivacy)
-	if err != nil {
-		errs = append(errs, err)
-	}
-	for _, err := range errs {
-		e <- err
+	if err := r.nodeinfo.setNodeInfo(current.NodeInfo, current.NodeInfoPrivacy); err != nil {
+		r.core.log.Errorln("Error reloading NodeInfo:", err)
+	} else {
+		r.core.log.Infoln("NodeInfo updated")
 	}
+	// Reconfigure children
+	r.dht.reconfigure()
+	r.searches.reconfigure()
+	r.sessions.reconfigure()
 }
 
 // Starts the tickerLoop goroutine.
diff --git a/src/yggdrasil/search.go b/src/yggdrasil/search.go
index ca357cc2..c128175b 100644
--- a/src/yggdrasil/search.go
+++ b/src/yggdrasil/search.go
@@ -55,8 +55,7 @@ func (s *searches) init(r *router) {
 	s.searches = make(map[crypto.NodeID]*searchInfo)
 }
 
-func (s *searches) reconfigure(e chan error) {
-	defer close(e)
+func (s *searches) reconfigure() {
 	// This is where reconfiguration would go, if we had anything to do
 }
 
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index f6855b4d..0b55aac8 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -73,8 +73,7 @@ type sessionInfo struct {
 	callbacks      []chan func()                 // Finished work from crypto workers
 }
 
-func (sinfo *sessionInfo) reconfigure(e chan error) {
-	defer close(e)
+func (sinfo *sessionInfo) reconfigure() {
 	// This is where reconfiguration would go, if we had anything to do
 }
 
@@ -161,17 +160,9 @@ func (ss *sessions) init(r *router) {
 	ss.lastCleanup = time.Now()
 }
 
-func (ss *sessions) reconfigure(e chan error) {
-	defer close(e)
-	responses := make(map[crypto.Handle]chan error)
-	for index, session := range ss.sinfos {
-		responses[index] = make(chan error)
-		session.reconfigure(responses[index])
-	}
-	for _, response := range responses {
-		for err := range response {
-			e <- err
-		}
+func (ss *sessions) reconfigure() {
+	for _, session := range ss.sinfos {
+		session.reconfigure()
 	}
 }
 
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 5c613d8d..b6bd5b96 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -199,9 +199,10 @@ func (t *switchTable) init(core *Core) {
 	})
 }
 
-func (t *switchTable) reconfigure(e chan error) {
-	defer close(e)
+func (t *switchTable) reconfigure() {
 	// This is where reconfiguration would go, if we had anything useful to do.
+	t.core.link.reconfigure()
+	t.core.peers.reconfigure()
 }
 
 // Safely gets a copy of this node's locator.
diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go
index ccb488f9..cce352bd 100644
--- a/src/yggdrasil/tcp.go
+++ b/src/yggdrasil/tcp.go
@@ -95,8 +95,7 @@ func (t *tcp) init(l *link) error {
 	return nil
 }
 
-func (t *tcp) reconfigure(e chan error) {
-	defer close(e)
+func (t *tcp) reconfigure() {
 	t.link.core.config.Mutex.RLock()
 	added := util.Difference(t.link.core.config.Current.Listen, t.link.core.config.Previous.Listen)
 	deleted := util.Difference(t.link.core.config.Previous.Listen, t.link.core.config.Current.Listen)
@@ -107,7 +106,9 @@ func (t *tcp) reconfigure(e chan error) {
 				continue
 			}
 			if _, err := t.listen(a[6:]); err != nil {
-				e <- err
+				t.link.core.log.Errorln("Error adding TCP", a[6:], "listener:", err)
+			} else {
+				t.link.core.log.Infoln("Started TCP listener:", a[6:])
 			}
 		}
 		for _, d := range deleted {
@@ -118,6 +119,7 @@ func (t *tcp) reconfigure(e chan error) {
 			if listener, ok := t.listeners[d[6:]]; ok {
 				t.mutex.Unlock()
 				listener.Stop <- true
+				t.link.core.log.Infoln("Stopped TCP listener:", d[6:])
 			} else {
 				t.mutex.Unlock()
 			}

From aa0770546eae3b3931326d9df8fea9724715f95e Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 28 Aug 2019 19:39:23 +0100
Subject: [PATCH 058/130] Move responsibility for configuring max queue size
 into switch

---
 src/yggdrasil/core.go   | 8 --------
 src/yggdrasil/switch.go | 8 +++++++-
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index 831109dd..ba06899b 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -152,14 +152,6 @@ func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState,
 		return nil, err
 	}
 
-	c.config.Mutex.RLock()
-	if c.config.Current.SwitchOptions.MaxTotalQueueSize >= SwitchQueueTotalMinSize {
-		phony.Block(&c.switchTable, func() {
-			c.switchTable.queues.totalMaxSize = c.config.Current.SwitchOptions.MaxTotalQueueSize
-		})
-	}
-	c.config.Mutex.RUnlock()
-
 	if err := c.switchTable.start(); err != nil {
 		c.log.Errorln("Failed to start switch")
 		return nil, err
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index b6bd5b96..163c85cf 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -193,7 +193,13 @@ func (t *switchTable) init(core *Core) {
 	t.table.Store(lookupTable{})
 	t.drop = make(map[crypto.SigPubKey]int64)
 	phony.Block(t, func() {
-		t.queues.totalMaxSize = SwitchQueueTotalMinSize
+		core.config.Mutex.RLock()
+		if core.config.Current.SwitchOptions.MaxTotalQueueSize > SwitchQueueTotalMinSize {
+			t.queues.totalMaxSize = core.config.Current.SwitchOptions.MaxTotalQueueSize
+		} else {
+			t.queues.totalMaxSize = SwitchQueueTotalMinSize
+		}
+		core.config.Mutex.RUnlock()
 		t.queues.bufs = make(map[string]switch_buffer)
 		t.idle = make(map[switchPort]time.Time)
 	})

From 1f658cce76c9180104dcbdf93d3b04cedc05f5b1 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 28 Aug 2019 19:53:52 +0100
Subject: [PATCH 059/130] Add Core actor

---
 src/yggdrasil/core.go | 72 ++++++++++++++++++++++++++-----------------
 1 file changed, 43 insertions(+), 29 deletions(-)

diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index ba06899b..754d7d64 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -33,7 +33,7 @@ type Core struct {
 	log         *log.Logger
 }
 
-func (c *Core) init() error {
+func (c *Core) _init() error {
 	// TODO separate init and start functions
 	//  Init sets up structs
 	//  Start launches goroutines that depend on structs being set up
@@ -85,42 +85,44 @@ func (c *Core) init() error {
 // If any static peers were provided in the configuration above then we should
 // configure them. The loop ensures that disconnected peers will eventually
 // be reconnected with.
-func (c *Core) addPeerLoop() {
-	for {
-		//  the peers from the config - these could change!
-		current := c.config.GetCurrent()
+func (c *Core) _addPeerLoop() {
+	// Get the peers from the config - these could change!
+	current := c.config.GetCurrent()
 
-		// Add peers from the Peers section
-		for _, peer := range current.Peers {
-			go c.AddPeer(peer, "")
+	// Add peers from the Peers section
+	for _, peer := range current.Peers {
+		go c.AddPeer(peer, "") // TODO: this should be acted and not in a goroutine?
+		time.Sleep(time.Second)
+	}
+
+	// Add peers from the InterfacePeers section
+	for intf, intfpeers := range current.InterfacePeers {
+		for _, peer := range intfpeers {
+			go c.AddPeer(peer, intf) // TODO: this should be acted and not in a goroutine?
 			time.Sleep(time.Second)
 		}
-
-		// Add peers from the InterfacePeers section
-		for intf, intfpeers := range current.InterfacePeers {
-			for _, peer := range intfpeers {
-				go c.AddPeer(peer, intf)
-				time.Sleep(time.Second)
-			}
-		}
-
-		// Sit for a while
-		time.Sleep(time.Minute)
 	}
+
+	// Sit for a while
+	time.AfterFunc(time.Minute, func() {
+		c.Act(c, c._addPeerLoop)
+	})
 }
 
 // UpdateConfig updates the configuration in Core with the provided
 // config.NodeConfig and then signals the various module goroutines to
 // reconfigure themselves if needed.
 func (c *Core) UpdateConfig(config *config.NodeConfig) {
-	c.log.Debugln("Reloading node configuration...")
+	c.Act(nil, func() {
+		c.log.Debugln("Reloading node configuration...")
 
-	// Replace the active configuration with the supplied one
-	c.config.Replace(*config)
+		// Replace the active configuration with the supplied one
+		c.config.Replace(*config)
 
-	// Notify the router and switch about the new configuration
-	c.router.Act(c, c.router.reconfigure)
-	c.switchTable.Act(c, c.switchTable.reconfigure)
+		// Notify the router and switch about the new configuration
+		c.router.Act(c, c.router.reconfigure)
+		c.switchTable.Act(c, c.switchTable.reconfigure)
+	})
 }
 
 // Start starts up Yggdrasil using the provided config.NodeConfig, and outputs
@@ -128,7 +130,15 @@ func (c *Core) UpdateConfig(config *config.NodeConfig) {
 // TCP and UDP sockets, a multicast discovery socket, an admin socket, router,
 // switch and DHT node. A config.NodeState is returned which contains both the
 // current and previous configurations (from reconfigures).
-func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState, error) {
+func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) (conf *config.NodeState, err error) {
+	phony.Block(c, func() {
+		conf, err = c._start(nc, log)
+	})
+	return
+}
+
+// This function is unsafe and should only be ran by the core actor.
+func (c *Core) _start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState, error) {
 	c.log = log
 
 	c.config = config.NodeState{
@@ -144,8 +154,7 @@ func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState,
 	}
 
 	c.log.Infoln("Starting up...")
-
-	c.init()
+	c._init()
 
 	if err := c.link.init(c); err != nil {
 		c.log.Errorln("Failed to start link interfaces")
@@ -162,7 +171,7 @@ func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState,
 		return nil, err
 	}
 
-	go c.addPeerLoop()
+	c.Act(c, c._addPeerLoop)
 
 	c.log.Infoln("Startup complete")
 	return &c.config, nil
@@ -170,5 +179,10 @@ func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState,
 
 // Stop shuts down the Yggdrasil node.
 func (c *Core) Stop() {
+	phony.Block(c, c._stop)
+}
+
+// This function is unsafe and should only be ran by the core actor.
+func (c *Core) _stop() {
 	c.log.Infoln("Stopping...")
 }

From 7649ea0f9fe25f191018cc862a3c9341dac234ed Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Thu, 29 Aug 2019 21:59:28 -0500
Subject: [PATCH 060/130] remove sessionInfo.doFunc, have the api just use
 phony.Block instead

---
 src/yggdrasil/api.go     | 13 +------------
 src/yggdrasil/session.go |  5 -----
 2 files changed, 1 insertion(+), 17 deletions(-)

diff --git a/src/yggdrasil/api.go b/src/yggdrasil/api.go
index 9e82ece1..d1753b69 100644
--- a/src/yggdrasil/api.go
+++ b/src/yggdrasil/api.go
@@ -227,18 +227,7 @@ func (c *Core) GetSessions() []Session {
 				}
 				copy(session.PublicKey[:], sinfo.theirPermPub[:])
 			}
-			var skip bool
-			func() {
-				defer func() {
-					if recover() != nil {
-						skip = true
-					}
-				}()
-				sinfo.doFunc(workerFunc)
-			}()
-			if skip {
-				continue
-			}
+			phony.Block(sinfo, workerFunc)
 			// TODO? skipped known but timed out sessions?
 			sessions = append(sessions, session)
 		}
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 0b55aac8..d209a0de 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -77,11 +77,6 @@ func (sinfo *sessionInfo) reconfigure() {
 	// This is where reconfiguration would go, if we had anything to do
 }
 
-// TODO remove this, call SyncExec directly
-func (sinfo *sessionInfo) doFunc(f func()) {
-	phony.Block(sinfo, f)
-}
-
 // Represents a session ping/pong packet, andincludes information like public keys, a session handle, coords, a timestamp to prevent replays, and the tun/tap MTU.
 type sessionPing struct {
 	SendPermPub crypto.BoxPubKey // Sender's permanent key

From 9e4d4f33bab6186a6415801ef80e361c49765037 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Thu, 29 Aug 2019 23:30:39 -0500
Subject: [PATCH 061/130] upgrade to latest phony

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 9ff6031a..fc9549ff 100644
--- a/go.mod
+++ b/go.mod
@@ -1,7 +1,7 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
 require (
-	github.com/Arceliar/phony v0.0.0-20190828002416-0337564e2c44
+	github.com/Arceliar/phony v0.0.0-20190830042734-c3fdbc251992
 	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
 	github.com/hashicorp/go-syslog v1.0.0
 	github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222
diff --git a/go.sum b/go.sum
index 60388964..7499e926 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-github.com/Arceliar/phony v0.0.0-20190828002416-0337564e2c44 h1:8EiuIp65v8aLkLc4LWxtn4NTH+P2LwDmrKKWdBzn9cI=
-github.com/Arceliar/phony v0.0.0-20190828002416-0337564e2c44/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
+github.com/Arceliar/phony v0.0.0-20190830042734-c3fdbc251992 h1:sT4p8AJps69R/8Tio29ywFQSKcKGvrp3LvnvezxUDJU=
+github.com/Arceliar/phony v0.0.0-20190830042734-c3fdbc251992/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=

From 32633011efa20399d83852e5b64dfd71ed9c4084 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 30 Aug 2019 22:10:34 -0500
Subject: [PATCH 062/130] upgrade phony dependency

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index fc9549ff..44227a4f 100644
--- a/go.mod
+++ b/go.mod
@@ -1,7 +1,7 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
 require (
-	github.com/Arceliar/phony v0.0.0-20190830042734-c3fdbc251992
+	github.com/Arceliar/phony v0.0.0-20190831030740-7cac84315954
 	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
 	github.com/hashicorp/go-syslog v1.0.0
 	github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222
diff --git a/go.sum b/go.sum
index 7499e926..40977e6a 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-github.com/Arceliar/phony v0.0.0-20190830042734-c3fdbc251992 h1:sT4p8AJps69R/8Tio29ywFQSKcKGvrp3LvnvezxUDJU=
-github.com/Arceliar/phony v0.0.0-20190830042734-c3fdbc251992/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
+github.com/Arceliar/phony v0.0.0-20190831030740-7cac84315954 h1:kwhCKZy6GHYxuqUvmBbicaLlWsYXwqph8tkuaMrl5T4=
+github.com/Arceliar/phony v0.0.0-20190831030740-7cac84315954/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=

From 08f69de1e266c30fb0bdb8d24f8b9b4054903c73 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 31 Aug 2019 00:04:35 -0500
Subject: [PATCH 063/130] another phony update

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 44227a4f..3b0af5a4 100644
--- a/go.mod
+++ b/go.mod
@@ -1,7 +1,7 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
 require (
-	github.com/Arceliar/phony v0.0.0-20190831030740-7cac84315954
+	github.com/Arceliar/phony v0.0.0-20190831050304-94a6d3da9ba4
 	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
 	github.com/hashicorp/go-syslog v1.0.0
 	github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222
diff --git a/go.sum b/go.sum
index 40977e6a..df7ff630 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-github.com/Arceliar/phony v0.0.0-20190831030740-7cac84315954 h1:kwhCKZy6GHYxuqUvmBbicaLlWsYXwqph8tkuaMrl5T4=
-github.com/Arceliar/phony v0.0.0-20190831030740-7cac84315954/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
+github.com/Arceliar/phony v0.0.0-20190831050304-94a6d3da9ba4 h1:OePImnPRBqS6JiHuVVq4Rfvt2yyhqMpWCB63PrwGrJE=
+github.com/Arceliar/phony v0.0.0-20190831050304-94a6d3da9ba4/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=

From a64f7320d8ce8cd8c3f079e9bf7d7d92b7cb00b0 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 31 Aug 2019 16:27:36 -0500
Subject: [PATCH 064/130] update phony, add mobile versions of util bytes
 functions that don't try to store anything

---
 go.mod                   |  2 +-
 go.sum                   |  4 ++--
 src/util/bytes_mobile.go | 13 +++++++++++++
 src/util/bytes_other.go  | 18 ++++++++++++++++++
 src/util/util.go         | 14 --------------
 5 files changed, 34 insertions(+), 17 deletions(-)
 create mode 100644 src/util/bytes_mobile.go
 create mode 100644 src/util/bytes_other.go

diff --git a/go.mod b/go.mod
index 3b0af5a4..7a96c97e 100644
--- a/go.mod
+++ b/go.mod
@@ -1,7 +1,7 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
 require (
-	github.com/Arceliar/phony v0.0.0-20190831050304-94a6d3da9ba4
+	github.com/Arceliar/phony v0.0.0-20190831212216-7018ff05d824
 	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
 	github.com/hashicorp/go-syslog v1.0.0
 	github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222
diff --git a/go.sum b/go.sum
index df7ff630..b0b891e9 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-github.com/Arceliar/phony v0.0.0-20190831050304-94a6d3da9ba4 h1:OePImnPRBqS6JiHuVVq4Rfvt2yyhqMpWCB63PrwGrJE=
-github.com/Arceliar/phony v0.0.0-20190831050304-94a6d3da9ba4/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
+github.com/Arceliar/phony v0.0.0-20190831212216-7018ff05d824 h1:JY7qh6yR87H8xgPUTYrrqa5cajb7zynKsbAdjhsVkyU=
+github.com/Arceliar/phony v0.0.0-20190831212216-7018ff05d824/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=
diff --git a/src/util/bytes_mobile.go b/src/util/bytes_mobile.go
new file mode 100644
index 00000000..ceab763c
--- /dev/null
+++ b/src/util/bytes_mobile.go
@@ -0,0 +1,13 @@
+//+build mobile
+
+package util
+
+// On mobile, just return a nil slice.
+func GetBytes() []byte {
+	return nil
+}
+
+// On mobile, don't do anything.
+func PutBytes(bs []byte) {
+	return
+}
diff --git a/src/util/bytes_other.go b/src/util/bytes_other.go
new file mode 100644
index 00000000..41b8bec0
--- /dev/null
+++ b/src/util/bytes_other.go
@@ -0,0 +1,18 @@
+//+build !mobile
+
+package util
+
+import "sync"
+
+// This is used to buffer recently used slices of bytes, to prevent allocations in the hot loops.
+var byteStore = sync.Pool{New: func() interface{} { return []byte(nil) }}
+
+// Gets an empty slice from the byte store.
+func GetBytes() []byte {
+	return byteStore.Get().([]byte)[:0]
+}
+
+// Puts a slice in the store.
+func PutBytes(bs []byte) {
+	byteStore.Put(bs)
+}
diff --git a/src/util/util.go b/src/util/util.go
index a588a35c..97250122 100644
--- a/src/util/util.go
+++ b/src/util/util.go
@@ -6,7 +6,6 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 )
 
@@ -25,19 +24,6 @@ func UnlockThread() {
 	runtime.UnlockOSThread()
 }
 
-// This is used to buffer recently used slices of bytes, to prevent allocations in the hot loops.
-var byteStore = sync.Pool{New: func() interface{} { return []byte(nil) }}
-
-// Gets an empty slice from the byte store.
-func GetBytes() []byte {
-	return byteStore.Get().([]byte)[:0]
-}
-
-// Puts a slice in the store.
-func PutBytes(bs []byte) {
-	byteStore.Put(bs)
-}
-
 // Gets a slice of the appropriate length, reusing existing slice capacity when possible
 func ResizeBytes(bs []byte, length int) []byte {
 	if cap(bs) >= length {

From 0806f3e6eacc965a8019dbe04f93046eecd5809d Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 31 Aug 2019 16:49:13 -0500
Subject: [PATCH 065/130] upgrade phony

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 7a96c97e..d2ca6b7c 100644
--- a/go.mod
+++ b/go.mod
@@ -1,7 +1,7 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
 require (
-	github.com/Arceliar/phony v0.0.0-20190831212216-7018ff05d824
+	github.com/Arceliar/phony v0.0.0-20190831214819-9b642ea019ad
 	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
 	github.com/hashicorp/go-syslog v1.0.0
 	github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222
diff --git a/go.sum b/go.sum
index b0b891e9..f0fbacaf 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-github.com/Arceliar/phony v0.0.0-20190831212216-7018ff05d824 h1:JY7qh6yR87H8xgPUTYrrqa5cajb7zynKsbAdjhsVkyU=
-github.com/Arceliar/phony v0.0.0-20190831212216-7018ff05d824/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
+github.com/Arceliar/phony v0.0.0-20190831214819-9b642ea019ad h1:670inqspOp+tAnSvkOBgrKGOIT4605Jt+6KGi2j2/S8=
+github.com/Arceliar/phony v0.0.0-20190831214819-9b642ea019ad/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=

From cabdc27a54cee361ef6635295dd9632086df017b Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 31 Aug 2019 17:39:05 -0500
Subject: [PATCH 066/130] change how nonce is tracked, so we allow packets if
 we've recently received a highest nonce ever, but don't bother tracking all
 received nonce values, this means duplicate packets are possible but only for
 a small window of time (and significantly reduces memory usage per session)

---
 src/yggdrasil/session.go | 107 ++++++++++++---------------------------
 1 file changed, 33 insertions(+), 74 deletions(-)

diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index d209a0de..8a6d16fc 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -6,7 +6,6 @@ package yggdrasil
 
 import (
 	"bytes"
-	"container/heap"
 	"sync"
 	"time"
 
@@ -20,57 +19,40 @@ import (
 // Duration that we keep track of old nonces per session, to allow some out-of-order packet delivery
 const nonceWindow = time.Second
 
-// A heap of nonces, used with a map[nonce]time to allow out-of-order packets a little time to arrive without rejecting them
-type nonceHeap []crypto.BoxNonce
-
-func (h nonceHeap) Len() int            { return len(h) }
-func (h nonceHeap) Less(i, j int) bool  { return h[i].Minus(&h[j]) < 0 }
-func (h nonceHeap) Swap(i, j int)       { h[i], h[j] = h[j], h[i] }
-func (h *nonceHeap) Push(x interface{}) { *h = append(*h, x.(crypto.BoxNonce)) }
-func (h *nonceHeap) Pop() interface{} {
-	l := len(*h)
-	var n crypto.BoxNonce
-	n, *h = (*h)[l-1], (*h)[:l-1]
-	return n
-}
-func (h nonceHeap) peek() *crypto.BoxNonce { return &h[0] }
-
 // All the information we know about an active session.
 // This includes coords, permanent and ephemeral keys, handles and nonces, various sorts of timing information for timeout and maintenance, and some metadata for the admin API.
 type sessionInfo struct {
-	phony.Inbox                                  // Protects all of the below, use it any time you read/change the contents of a session
-	sessions       *sessions                     //
-	theirAddr      address.Address               //
-	theirSubnet    address.Subnet                //
-	theirPermPub   crypto.BoxPubKey              //
-	theirSesPub    crypto.BoxPubKey              //
-	mySesPub       crypto.BoxPubKey              //
-	mySesPriv      crypto.BoxPrivKey             //
-	sharedPermKey  crypto.BoxSharedKey           // used for session pings
-	sharedSesKey   crypto.BoxSharedKey           // derived from session keys
-	theirHandle    crypto.Handle                 //
-	myHandle       crypto.Handle                 //
-	theirNonce     crypto.BoxNonce               //
-	theirNonceHeap nonceHeap                     // priority queue to keep track of the lowest nonce we recently accepted
-	theirNonceMap  map[crypto.BoxNonce]time.Time // time we added each nonce to the heap
-	myNonce        crypto.BoxNonce               //
-	theirMTU       uint16                        //
-	myMTU          uint16                        //
-	wasMTUFixed    bool                          // Was the MTU fixed by a receive error?
-	timeOpened     time.Time                     // Time the sessino was opened
-	time           time.Time                     // Time we last received a packet
-	mtuTime        time.Time                     // time myMTU was last changed
-	pingTime       time.Time                     // time the first ping was sent since the last received packet
-	pingSend       time.Time                     // time the last ping was sent
-	coords         []byte                        // coords of destination
-	reset          bool                          // reset if coords change
-	tstamp         int64                         // ATOMIC - tstamp from their last session ping, replay attack mitigation
-	bytesSent      uint64                        // Bytes of real traffic sent in this session
-	bytesRecvd     uint64                        // Bytes of real traffic received in this session
-	init           chan struct{}                 // Closed when the first session pong arrives, used to signal that the session is ready for initial use
-	cancel         util.Cancellation             // Used to terminate workers
-	conn           *Conn                         // The associated Conn object
-	callbacks      []chan func()                 // Finished work from crypto workers
+	phony.Inbox                       // Protects all of the below, use it any time you read/change the contents of a session
+	sessions      *sessions           //
+	theirAddr     address.Address     //
+	theirSubnet   address.Subnet      //
+	theirPermPub  crypto.BoxPubKey    //
+	theirSesPub   crypto.BoxPubKey    //
+	mySesPub      crypto.BoxPubKey    //
+	mySesPriv     crypto.BoxPrivKey   //
+	sharedPermKey crypto.BoxSharedKey // used for session pings
+	sharedSesKey  crypto.BoxSharedKey // derived from session keys
+	theirHandle   crypto.Handle       //
+	myHandle      crypto.Handle       //
+	theirNonce    crypto.BoxNonce     //
+	myNonce       crypto.BoxNonce     //
+	theirMTU      uint16              //
+	myMTU         uint16              //
+	wasMTUFixed   bool                // Was the MTU fixed by a receive error?
+	timeOpened    time.Time           // Time the sessino was opened
+	time          time.Time           // Time we last received a packet
+	mtuTime       time.Time           // time myMTU was last changed
+	pingTime      time.Time           // time the first ping was sent since the last received packet
+	pingSend      time.Time           // time the last ping was sent
+	coords        []byte              // coords of destination
+	reset         bool                // reset if coords change
+	tstamp        int64               // ATOMIC - tstamp from their last session ping, replay attack mitigation
+	bytesSent     uint64              // Bytes of real traffic sent in this session
+	bytesRecvd    uint64              // Bytes of real traffic received in this session
+	init          chan struct{}       // Closed when the first session pong arrives, used to signal that the session is ready for initial use
+	cancel        util.Cancellation   // Used to terminate workers
+	conn          *Conn               // The associated Conn object
+	callbacks     []chan func()       // Finished work from crypto workers
 }
 
 func (sinfo *sessionInfo) reconfigure() {
@@ -105,8 +87,6 @@ func (s *sessionInfo) _update(p *sessionPing) bool {
 		s.theirHandle = p.Handle
 		s.sharedSesKey = *crypto.GetSharedKey(&s.mySesPriv, &s.theirSesPub)
 		s.theirNonce = crypto.BoxNonce{}
-		s.theirNonceHeap = nil
-		s.theirNonceMap = make(map[crypto.BoxNonce]time.Time)
 	}
 	if p.MTU >= 1280 || p.MTU == 0 {
 		s.theirMTU = p.MTU
@@ -420,36 +400,16 @@ func (sinfo *sessionInfo) _nonceIsOK(theirNonce *crypto.BoxNonce) bool {
 		// This is newer than the newest nonce we've seen
 		return true
 	}
-	if len(sinfo.theirNonceHeap) > 0 {
-		if theirNonce.Minus(sinfo.theirNonceHeap.peek()) > 0 {
-			if _, isIn := sinfo.theirNonceMap[*theirNonce]; !isIn {
-				// This nonce is recent enough that we keep track of older nonces, but it's not one we've seen yet
-				return true
-			}
-		}
-	}
-	return false
+	return time.Since(sinfo.time) < nonceWindow
 }
 
 // Updates the nonce mask by (possibly) shifting the bitmask and setting the bit corresponding to this nonce to 1, and then updating the most recent nonce
 func (sinfo *sessionInfo) _updateNonce(theirNonce *crypto.BoxNonce) {
-	// Start with some cleanup
-	for len(sinfo.theirNonceHeap) > 64 {
-		if time.Since(sinfo.theirNonceMap[*sinfo.theirNonceHeap.peek()]) < nonceWindow {
-			// This nonce is still fairly new, so keep it around
-			break
-		}
-		// TODO? reallocate the map in some cases, to free unused map space?
-		delete(sinfo.theirNonceMap, *sinfo.theirNonceHeap.peek())
-		heap.Pop(&sinfo.theirNonceHeap)
-	}
 	if theirNonce.Minus(&sinfo.theirNonce) > 0 {
 		// This nonce is the newest we've seen, so make a note of that
 		sinfo.theirNonce = *theirNonce
+		sinfo.time = time.Now()
 	}
-	// Add it to the heap/map so we know not to allow it again
-	heap.Push(&sinfo.theirNonceHeap, *theirNonce)
-	sinfo.theirNonceMap[*theirNonce] = time.Now()
 }
 
 // Resets all sessions to an uninitialized state.
@@ -515,7 +475,6 @@ func (sinfo *sessionInfo) _recvPacket(p *wire_trafficPacket) {
 				return
 			}
 			sinfo._updateNonce(&p.Nonce)
-			sinfo.time = time.Now()
 			sinfo.bytesRecvd += uint64(len(bs))
 			sinfo.conn.recvMsg(sinfo, bs)
 		}

From 3a493fe8945227b20ecbb101af014c847d15d912 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 1 Sep 2019 11:08:25 -0500
Subject: [PATCH 067/130] gc more often on mobile

---
 src/util/bytes_mobile.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/util/bytes_mobile.go b/src/util/bytes_mobile.go
index ceab763c..09e78050 100644
--- a/src/util/bytes_mobile.go
+++ b/src/util/bytes_mobile.go
@@ -2,6 +2,12 @@
 
 package util
 
+import "runtime/debug"
+
+func init() {
+	debug.SetGCPercent(25)
+}
+
 // On mobile, just return a nil slice.
 func GetBytes() []byte {
 	return nil

From e0ea845cdc26f47eeb2ac41e233d76b744f69918 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Sun, 1 Sep 2019 17:50:15 +0100
Subject: [PATCH 068/130] Update build

---
 build | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/build b/build
index c1e5f863..7ca5f4fa 100755
--- a/build
+++ b/build
@@ -34,13 +34,15 @@ if [ $IOS ]; then
   gomobile bind -target ios -tags mobile -ldflags="$LDFLAGS $STRIP" -gcflags="$GCFLAGS" \
     github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil \
     github.com/yggdrasil-network/yggdrasil-go/src/config \
-    github.com/yggdrasil-network/yggdrasil-extras/src/mobile
+    github.com/yggdrasil-network/yggdrasil-extras/src/mobile \
+    github.com/yggdrasil-network/yggdrasil-extras/src/dummy
 elif [ $ANDROID ]; then
   echo "Building aar for Android"
   gomobile bind -target android -tags mobile -ldflags="$LDFLAGS $STRIP" -gcflags="$GCFLAGS" \
     github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil \
     github.com/yggdrasil-network/yggdrasil-go/src/config \
-    github.com/yggdrasil-network/yggdrasil-extras/src/mobile
+    github.com/yggdrasil-network/yggdrasil-extras/src/mobile \
+    github.com/yggdrasil-network/yggdrasil-extras/src/dummy
 else
   for CMD in `ls cmd/` ; do
     echo "Building: $CMD"

From d08c2eb2375b50044b18c61c8dc83e5aa3e0b91a Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 1 Sep 2019 13:04:10 -0500
Subject: [PATCH 069/130] stop exporting ReadNoCopy and WriteNoCopy, since we
 use the actor functions / callbacks and everything else should use Read and
 Write instead...

---
 src/yggdrasil/conn.go | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index 0a4b84aa..352cf9d9 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -194,7 +194,7 @@ func (c *Conn) recvMsg(from phony.Actor, msg []byte) {
 }
 
 // Used internally by Read, the caller is responsible for util.PutBytes when they're done.
-func (c *Conn) ReadNoCopy() ([]byte, error) {
+func (c *Conn) readNoCopy() ([]byte, error) {
 	var cancel util.Cancellation
 	var doCancel bool
 	phony.Block(c, func() { cancel, doCancel = c._getDeadlineCancellation(c.readDeadline) })
@@ -216,7 +216,7 @@ func (c *Conn) ReadNoCopy() ([]byte, error) {
 
 // Implements net.Conn.Read
 func (c *Conn) Read(b []byte) (int, error) {
-	bs, err := c.ReadNoCopy()
+	bs, err := c.readNoCopy()
 	if err != nil {
 		return 0, err
 	}
@@ -257,7 +257,7 @@ func (c *Conn) _write(msg FlowKeyMessage) error {
 }
 
 // WriteFrom should be called by a phony.Actor, and tells the Conn to send a message.
-// This is used internaly by WriteNoCopy and Write.
+// This is used internaly by Write.
 // If the callback is called with a non-nil value, then it is safe to reuse the argument FlowKeyMessage.
 func (c *Conn) WriteFrom(from phony.Actor, msg FlowKeyMessage, callback func(error)) {
 	c.Act(from, func() {
@@ -265,9 +265,9 @@ func (c *Conn) WriteFrom(from phony.Actor, msg FlowKeyMessage, callback func(err
 	})
 }
 
-// WriteNoCopy is used internally by Write and makes use of WriteFrom under the hood.
+// writeNoCopy is used internally by Write and makes use of WriteFrom under the hood.
 // The caller must not reuse the argument FlowKeyMessage when a nil error is returned.
-func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error {
+func (c *Conn) writeNoCopy(msg FlowKeyMessage) error {
 	var cancel util.Cancellation
 	var doCancel bool
 	phony.Block(c, func() { cancel, doCancel = c._getDeadlineCancellation(c.writeDeadline) })
@@ -292,7 +292,7 @@ func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error {
 func (c *Conn) Write(b []byte) (int, error) {
 	written := len(b)
 	msg := FlowKeyMessage{Message: append(util.GetBytes(), b...)}
-	err := c.WriteNoCopy(msg)
+	err := c.writeNoCopy(msg)
 	if err != nil {
 		util.PutBytes(msg.Message)
 		written = 0

From c53831696b5aee61823c0da62a96055d3f0e02fd Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 1 Sep 2019 13:06:25 -0500
Subject: [PATCH 070/130] make tun stop check that iface is not nil, in case it
 wasn't set for some reason (windows bugs)

---
 src/tuntap/tun.go | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/tuntap/tun.go b/src/tuntap/tun.go
index 8e1e5b0c..74d055ee 100644
--- a/src/tuntap/tun.go
+++ b/src/tuntap/tun.go
@@ -189,9 +189,11 @@ func (tun *TunAdapter) Stop() error {
 
 func (tun *TunAdapter) _stop() error {
 	tun.isOpen = false
-	// TODO: we have nothing that cleanly stops all the various goroutines opened
 	// by TUN/TAP, e.g. readers/writers, sessions
-	tun.iface.Close()
+	if tun.iface != nil {
+		// Just in case we failed to start up the iface for some reason, this can apparently happen on Windows
+		tun.iface.Close()
+	}
 	return nil
 }
 

From 8d2c31d39cbeeb968976f94013933c904e233147 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 1 Sep 2019 13:20:48 -0500
Subject: [PATCH 071/130] add some artifical delay to windows netsh commands,
 since it seems like maybe they don't take effect immediately, and this was
 leading to races when setting MTU

---
 src/tuntap/tun_windows.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/tuntap/tun_windows.go b/src/tuntap/tun_windows.go
index a826c7ad..ea1515ff 100644
--- a/src/tuntap/tun_windows.go
+++ b/src/tuntap/tun_windows.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os/exec"
 	"strings"
+	"time"
 
 	water "github.com/yggdrasil-network/water"
 )
@@ -42,6 +43,7 @@ func (tun *TunAdapter) setup(ifname string, iftapmode bool, addr string, mtu int
 		tun.log.Traceln(string(output))
 		return err
 	}
+	time.Sleep(time.Second) // FIXME artifical delay to give netsh time to take effect
 	// Bring the interface back up
 	cmd = exec.Command("netsh", "interface", "set", "interface", iface.Name(), "admin=ENABLED")
 	tun.log.Debugln("netsh command:", strings.Join(cmd.Args, " "))
@@ -51,6 +53,7 @@ func (tun *TunAdapter) setup(ifname string, iftapmode bool, addr string, mtu int
 		tun.log.Traceln(string(output))
 		return err
 	}
+	time.Sleep(time.Second) // FIXME artifical delay to give netsh time to take effect
 	// Get a new iface
 	iface, err = water.New(config)
 	if err != nil {
@@ -86,6 +89,7 @@ func (tun *TunAdapter) setupMTU(mtu int) error {
 		tun.log.Traceln(string(output))
 		return err
 	}
+	time.Sleep(time.Second) // FIXME artifical delay to give netsh time to take effect
 	return nil
 }
 
@@ -106,5 +110,6 @@ func (tun *TunAdapter) setupAddress(addr string) error {
 		tun.log.Traceln(string(output))
 		return err
 	}
+	time.Sleep(time.Second) // FIXME artifical delay to give netsh time to take effect
 	return nil
 }

From 8c52ccadf9fe4aeae0e2f29c81905f9efb7e2716 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 1 Sep 2019 14:07:00 -0500
Subject: [PATCH 072/130] make dial fail if a session to the same node already
 exists, fixes race between simultaneous connections to a node's 200 address
 and one of its 300 addresses, should also fix races between a search and an
 accepted listen

---
 src/yggdrasil/search.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/yggdrasil/search.go b/src/yggdrasil/search.go
index c128175b..322131ed 100644
--- a/src/yggdrasil/search.go
+++ b/src/yggdrasil/search.go
@@ -204,6 +204,11 @@ func (sinfo *searchInfo) checkDHTRes(res *dhtRes) bool {
 		if !isIn {
 			panic("This should never happen")
 		}
+	} else {
+		sinfo.callback(nil, errors.New("session already exists"))
+		// Cleanup
+		delete(sinfo.searches.searches, res.Dest)
+		return true
 	}
 	// FIXME (!) replay attacks could mess with coords? Give it a handle (tstamp)?
 	sess.coords = res.Coords

From 174ebceaacae482df95e7bb96f0848bb1588a4ff Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Sun, 1 Sep 2019 21:32:40 +0100
Subject: [PATCH 073/130] Fix hjson-go import in go.mod/go.sum

---
 go.mod | 2 +-
 go.sum | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index d2ca6b7c..6703d5bd 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,7 @@ require (
 	github.com/Arceliar/phony v0.0.0-20190831214819-9b642ea019ad
 	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
 	github.com/hashicorp/go-syslog v1.0.0
-	github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222
+	github.com/hjson/hjson-go v3.0.1-0.20190209023717-9147687966d9+incompatible
 	github.com/kardianos/minwinsvc v0.0.0-20151122163309-cad6b2b879b0
 	github.com/mitchellh/mapstructure v1.1.2
 	github.com/songgao/packets v0.0.0-20160404182456-549a10cd4091
diff --git a/go.sum b/go.sum
index f0fbacaf..60e6090f 100644
--- a/go.sum
+++ b/go.sum
@@ -6,6 +6,8 @@ github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwM
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222 h1:xmvkbxXDeN1ffWq8kvrhyqVYAO2aXuRBsbpxVTR+JyU=
 github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
+github.com/hjson/hjson-go v3.0.1-0.20190209023717-9147687966d9+incompatible h1:bLQ2Ve+eW65id3b8xEMQiAwJT4qGZeywAEMLvXjznvw=
+github.com/hjson/hjson-go v3.0.1-0.20190209023717-9147687966d9+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
 github.com/kardianos/minwinsvc v0.0.0-20151122163309-cad6b2b879b0 h1:YnZmFjg0Nvk8851WTVWlqMC1ecJH07Ctz+Ezxx4u54g=
 github.com/kardianos/minwinsvc v0.0.0-20151122163309-cad6b2b879b0/go.mod h1:rUi0/YffDo1oXBOGn1KRq7Fr07LX48XEBecQnmwjsAo=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=

From 01517e5dc3b04ccdfd726fd7c011f437285e6df4 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Sun, 1 Sep 2019 22:43:27 +0100
Subject: [PATCH 074/130] Create doc.go for godoc preamble

---
 src/yggdrasil/doc.go | 156 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 156 insertions(+)
 create mode 100644 src/yggdrasil/doc.go

diff --git a/src/yggdrasil/doc.go b/src/yggdrasil/doc.go
new file mode 100644
index 00000000..3010765a
--- /dev/null
+++ b/src/yggdrasil/doc.go
@@ -0,0 +1,156 @@
+/*
+Package yggdrasil implements the core functionality of the Yggdrasil Network.
+
+Introduction
+
+Yggdrasil is a proof-of-concept mesh network which provides end-to-end encrypted
+communication between nodes in a decentralised fashion. The network is arranged
+using a globally-agreed spanning tree which provides each node with a locator
+(coordinates relative to the root) and a distributed hash table (DHT) mechanism
+for finding other nodes.
+
+Each node also implements a router, which is responsible for encryption of
+traffic, searches and connections, and a switch, which is responsible ultimately
+for forwarding traffic across the network.
+
+While many Yggdrasil nodes in existence today are IP nodes - that is, they are
+transporting IPv6 packets, like a kind of mesh VPN - it is also possible to
+integrate Yggdrasil into your own applications and use it as a generic data
+transport, similar to UDP.
+
+This library is what you need to integrate and use Yggdrasil in your own
+application.
+
+Basics
+
+In order to start an Yggdrasil node, you should start by generating node
+configuration, which amongst other things, includes encryption keypairs which
+are used to generate the node's identity, and supply a logger which Yggdrasil's
+output will be written to.
+
+This may look something like this:
+
+  import (
+    "os"
+    "github.com/gologme/log"
+    "github.com/yggdrasil-network/yggdrasil-go/src/config"
+    "github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil"
+  )
+
+  type node struct {
+    core   yggdrasil.Core
+    config *config.NodeConfig
+    log    *log.Logger
+  }
+
+You then can supply node configuration and a logger:
+
+  n := node{}
+  n.log = log.New(os.Stdout, "", log.Flags())
+  n.config = config.GenerateConfig()
+
+In the above example, we ask the config package to supply new configuration each
+time, which results in fresh encryption keys and therefore a new identity. It is
+normally preferable in most cases to persist node configuration onto the
+filesystem or into some configuration store so that the node's identity does not
+change each time that the program starts. Note that Yggdrasil will automatically
+fill in any missing configuration items with sane defaults.
+
+Once you have supplied a logger and some node configuration, you can then start
+the node:
+
+  n.core.Start(n.config, n.log)
+
+Add some peers to connect to the network:
+
+  n.core.AddPeer("tcp://some-host.net:54321", "")
+  n.core.AddPeer("tcp://[2001::1:2:3]:54321", "")
+  n.core.AddPeer("tcp://1.2.3.4:54321", "")
+
+You can also ask the API for information about our node:
+
+  n.log.Println("My node ID is", n.core.NodeID())
+  n.log.Println("My public key is", n.core.EncryptionPublicKey())
+  n.log.Println("My coords are", n.core.Coords())
+
+Incoming Connections
+
+Once your node is started, you can then listen for connections from other nodes
+by asking the API for a Listener:
+
+  listener, err := n.core.ConnListen()
+  if err != nil {
+    // ...
+  }
+
+The Listener has a blocking Accept function which will wait for incoming
+connections from remote nodes. It will return a Conn when a connection is
+received. If the node never receives any incoming connections then this function
+can block forever, so be prepared for that, perhaps by listening in a separate
+goroutine.
+
+Assuming that you have defined a myConnectionHandler function to deal with
+incoming connections:
+
+  for {
+    conn, err := listener.Accept()
+    if err != nil {
+      // ...
+    }
+
+    // We've got a new connection
+    go myConnectionHandler(conn)
+  }
+
+Outgoing Connections
+
+If you know the node ID of the remote node that you want to talk to, you can
+dial an outbound connection to it. To do this, you should first ask the API for
+a Dialer:
+
+  dialer, err := n.core.ConnDialer()
+  if err != nil {
+    // ...
+  }
+
+You can then dial using the 16-byte node ID in hexadecimal format, for example:
+
+  conn, err := dialer.Dial("nodeid", "24a58cfce691ec016b0f698f7be1bee983cea263781017e99ad3ef62b4ef710a45d6c1a072c5ce46131bd574b78818c9957042cafeeed13966f349e94eb771bf")
+  if err != nil {
+    // ...
+  }
+
+Using Connections
+
+Conn objects are implementations of io.ReadWriteCloser, and as such, you can
+Read, Write and Close them as necessary.
+
+For example, to write to the Conn from the supplied buffer:
+
+  buf := []byte{1, 2, 3, 4, 5}
+  w, err := conn.Write(buf)
+  if err != nil {
+    // ...
+  } else {
+    // written w bytes
+  }
+
+Reading from the Conn into the supplied buffer:
+
+  buf := make([]byte, 65535)
+  r, err := conn.Read(buf)
+  if err != nil {
+    // ...
+  } else {
+    // read r bytes
+  }
+
+When you are happy that a connection is no longer required, you can discard it:
+
+  err := conn.Close()
+  if err != nil {
+    // ...
+  }
+
+*/
+package yggdrasil

From 9e8e1c5a41cbceff8861479ec32e5ab723a26491 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Sun, 1 Sep 2019 23:10:46 +0100
Subject: [PATCH 075/130] Documentation updates

---
 src/yggdrasil/api.go | 84 +++++++++++++++++++++++++-------------------
 src/yggdrasil/doc.go | 20 +++++++++++
 2 files changed, 68 insertions(+), 36 deletions(-)

diff --git a/src/yggdrasil/api.go b/src/yggdrasil/api.go
index d1753b69..f69ef7e9 100644
--- a/src/yggdrasil/api.go
+++ b/src/yggdrasil/api.go
@@ -16,29 +16,37 @@ import (
 )
 
 // Peer represents a single peer object. This contains information from the
-// preferred switch port for this peer, although there may be more than one in
-// reality.
+// preferred switch port for this peer, although there may be more than one
+// active switch port connection to the peer in reality.
+//
+// This struct is informational only - you cannot manipulate peer connections
+// using instances of this struct. You should use the AddPeer or RemovePeer
+// functions instead.
 type Peer struct {
-	PublicKey  crypto.BoxPubKey
-	Endpoint   string
-	BytesSent  uint64
-	BytesRecvd uint64
-	Protocol   string
-	Port       uint64
-	Uptime     time.Duration
+	PublicKey  crypto.BoxPubKey // The public key of the remote node
+	Endpoint   string           // The connection string used to connect to the peer
+	BytesSent  uint64           // Number of bytes sent to this peer
+	BytesRecvd uint64           // Number of bytes received from this peer
+	Protocol   string           // The transport protocol that this peer is connected with, typically "tcp"
+	Port       uint64           // Switch port number for this peer connection
+	Uptime     time.Duration    // How long this peering has been active for
 }
 
 // SwitchPeer represents a switch connection to a peer. Note that there may be
 // multiple switch peers per actual peer, e.g. if there are multiple connections
 // to a given node.
+//
+// This struct is informational only - you cannot manipulate switch peer
+// connections using instances of this struct. You should use the AddPeer or
+// RemovePeer functions instead.
 type SwitchPeer struct {
-	PublicKey  crypto.BoxPubKey
-	Coords     []uint64
-	BytesSent  uint64
-	BytesRecvd uint64
-	Port       uint64
-	Protocol   string
-	Endpoint   string
+	PublicKey  crypto.BoxPubKey // The public key of the remote node
+	Coords     []uint64         // The coordinates of the remote node
+	BytesSent  uint64           // Number of bytes sent via this switch port
+	BytesRecvd uint64           // Number of bytes received via this switch port
+	Port       uint64           // Switch port number for this switch peer
+	Protocol   string           // The transport protocol that this switch port is connected with, typically "tcp"
+	Endpoint   string           // The connection string used to connect to the switch peer
 }
 
 // DHTEntry represents a single DHT entry that has been learned or cached from
@@ -64,32 +72,36 @@ type NodeInfoPayload []byte
 // congestion and a list of switch queues created in response to congestion on a
 // given link.
 type SwitchQueues struct {
-	Queues       []SwitchQueue
-	Count        uint64
-	Size         uint64
-	HighestCount uint64
-	HighestSize  uint64
-	MaximumSize  uint64
+	Queues       []SwitchQueue // An array of SwitchQueue objects containing information about individual queues
+	Count        uint64        // The current number of active switch queues
+	Size         uint64        // The current total size of active switch queues
+	HighestCount uint64        // The highest recorded number of switch queues so far
+	HighestSize  uint64        // The highest recorded total size of switch queues so far
+	MaximumSize  uint64        // The maximum allowed total size of switch queues, as specified by config
 }
 
-// SwitchQueue represents a single switch queue, which is created in response
-// to congestion on a given link.
+// SwitchQueue represents a single switch queue. Switch queues are only created
+// in response to congestion on a given link and represent how much data has
+// been temporarily cached for sending once the congestion has cleared.
 type SwitchQueue struct {
-	ID      string
-	Size    uint64
-	Packets uint64
-	Port    uint64
+	ID      string // The ID of the switch queue
+	Size    uint64 // The total size, in bytes, of the queue
+	Packets uint64 // The number of packets in the queue
+	Port    uint64 // The switch port to which the queue applies
 }
 
-// Session represents an open session with another node.
+// Session represents an open session with another node. Sessions are opened in
+// response to traffic being exchanged between two nodes using Conn objects.
+// Note that sessions will automatically be closed by Yggdrasil if no traffic is
+// exchanged for around two minutes.
 type Session struct {
-	PublicKey   crypto.BoxPubKey
-	Coords      []uint64
-	BytesSent   uint64
-	BytesRecvd  uint64
-	MTU         uint16
-	Uptime      time.Duration
-	WasMTUFixed bool
+	PublicKey   crypto.BoxPubKey // The public key of the remote node
+	Coords      []uint64         // The coordinates of the remote node
+	BytesSent   uint64           // Bytes sent to the session
+	BytesRecvd  uint64           // Bytes received from the session
+	MTU         uint16           // The maximum supported message size of the session
+	Uptime      time.Duration    // How long this session has been active for
+	WasMTUFixed bool             // This field is no longer used
 }
 
 // GetPeers returns one or more Peer objects containing information about active
diff --git a/src/yggdrasil/doc.go b/src/yggdrasil/doc.go
index 3010765a..44d39b66 100644
--- a/src/yggdrasil/doc.go
+++ b/src/yggdrasil/doc.go
@@ -125,6 +125,9 @@ Using Connections
 Conn objects are implementations of io.ReadWriteCloser, and as such, you can
 Read, Write and Close them as necessary.
 
+Each Read or Write operation can deal with a buffer with a maximum size of 65535
+bytes - any bigger than this and the operation will return an error.
+
 For example, to write to the Conn from the supplied buffer:
 
   buf := []byte{1, 2, 3, 4, 5}
@@ -152,5 +155,22 @@ When you are happy that a connection is no longer required, you can discard it:
     // ...
   }
 
+Limitations
+
+You should be aware of the following limitations when working with the Yggdrasil
+library:
+
+Individual messages written through Yggdrasil connections can not exceed 65535
+bytes in size. Yggdrasil has no concept of fragmentation, so if you try to send
+a message that exceeds 65535 bytes in size, it will be dropped altogether and
+an error will be returned.
+
+Yggdrasil connections are unreliable by nature. Messages are delivered on a
+best-effort basis, and employs congestion control where appropriate to ensure
+that congestion does not affect message transport, but Yggdrasil will not
+retransmit any messages that have been lost. If reliable delivery is important
+then you should manually implement acknowledgement and retransmission of
+messages.
+
 */
 package yggdrasil

From 935324efe12b485f135a90e75f8adbf4189e3628 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Sun, 1 Sep 2019 23:33:51 +0100
Subject: [PATCH 076/130] Update conn.go godoc

---
 src/yggdrasil/conn.go | 57 ++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 51 insertions(+), 6 deletions(-)

diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index 352cf9d9..a2dc1fa4 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -53,6 +53,9 @@ func (e *ConnError) Closed() bool {
 	return e.closed
 }
 
+// The Conn struct is a reference to an active connection session between the
+// local node and a remote node. Conn implements the io.ReadWriteCloser
+// interface and is used to send and receive traffic with a remote node.
 type Conn struct {
 	phony.Inbox
 	core          *Core
@@ -78,6 +81,11 @@ func newConn(core *Core, nodeID *crypto.NodeID, nodeMask *crypto.NodeID, session
 	return &conn
 }
 
+// String returns a string that uniquely identifies a connection. Currently this
+// takes a form similar to "conn=0x0000000", which contains a memory reference
+// to the Conn object. While this value should always be unique for each Conn
+// object, the format of this is not strictly defined and may change in the
+// future.
 func (c *Conn) String() string {
 	var s string
 	phony.Block(c, func() { s = fmt.Sprintf("conn=%p", c) })
@@ -159,7 +167,12 @@ func (c *Conn) _getDeadlineCancellation(t *time.Time) (util.Cancellation, bool)
 	}
 }
 
-// SetReadCallback sets a callback which will be called whenever a packet is received.
+// SetReadCallback allows you to specify a function that will be called whenever
+// a packet is received. This should be used if you wish to implement
+// asynchronous patterns for receiving data from the remote node.
+//
+// Note that if a read callback has been supplied, you should no longer attempt
+// to use the synchronous Read function.
 func (c *Conn) SetReadCallback(callback func([]byte)) {
 	c.Act(nil, func() {
 		c.readCallback = callback
@@ -214,7 +227,14 @@ func (c *Conn) readNoCopy() ([]byte, error) {
 	}
 }
 
-// Implements net.Conn.Read
+// Read allows you to read from the connection in a synchronous fashion. The
+// function will block up until the point that either new data is available, the
+// connection has been closed or the read deadline has been reached. If the
+// function succeeds, the number of bytes read from the connection will be
+// returned. Otherwise, an error condition will be returned.
+//
+// Note that you can also implement asynchronous reads by using SetReadCallback.
+// If you do that, you should no longer attempt to use the Read function.
 func (c *Conn) Read(b []byte) (int, error) {
 	bs, err := c.readNoCopy()
 	if err != nil {
@@ -256,9 +276,9 @@ func (c *Conn) _write(msg FlowKeyMessage) error {
 	return nil
 }
 
-// WriteFrom should be called by a phony.Actor, and tells the Conn to send a message.
-// This is used internaly by Write.
-// If the callback is called with a non-nil value, then it is safe to reuse the argument FlowKeyMessage.
+// WriteFrom should be called by a phony.Actor, and tells the Conn to send a
+// message. This is used internaly by Write. If the callback is called with a
+// non-nil value, then it is safe to reuse the argument FlowKeyMessage.
 func (c *Conn) WriteFrom(from phony.Actor, msg FlowKeyMessage, callback func(error)) {
 	c.Act(from, func() {
 		callback(c._write(msg))
@@ -288,7 +308,11 @@ func (c *Conn) writeNoCopy(msg FlowKeyMessage) error {
 	return err
 }
 
-// Write implement the Write function of a net.Conn, and makes use of WriteNoCopy under the hood.
+// Write allows you to write to the connection in a synchronous fashion. This
+// function may block until either the write has completed, the connection has
+// been closed or the write deadline has been reached. If the function succeeds,
+// the number of written bytes is returned. Otherwise, an error condition is
+// returned.
 func (c *Conn) Write(b []byte) (int, error) {
 	written := len(b)
 	msg := FlowKeyMessage{Message: append(util.GetBytes(), b...)}
@@ -300,6 +324,10 @@ func (c *Conn) Write(b []byte) (int, error) {
 	return written, err
 }
 
+// Close will close an open connection and any blocking operations on the
+// connection will unblock and return. From this point forward, the connection
+// can no longer be used and you should no longer attempt to Read or Write to
+// the connection.
 func (c *Conn) Close() (err error) {
 	phony.Block(c, func() {
 		if c.session != nil {
@@ -314,10 +342,13 @@ func (c *Conn) Close() (err error) {
 	return
 }
 
+// LocalAddr returns the complete node ID of the local side of the connection.
+// This is always going to return your own node's node ID.
 func (c *Conn) LocalAddr() crypto.NodeID {
 	return *crypto.GetNodeID(&c.core.boxPub)
 }
 
+// RemoteAddr returns the complete node ID of the remote side of the connection.
 func (c *Conn) RemoteAddr() crypto.NodeID {
 	// TODO warn that this can block while waiting for the Conn actor to run, so don't call it from other actors...
 	var n crypto.NodeID
@@ -325,18 +356,32 @@ func (c *Conn) RemoteAddr() crypto.NodeID {
 	return n
 }
 
+// SetDeadline is equivalent to calling both SetReadDeadline and
+// SetWriteDeadline with the same value, configuring the maximum amount of time
+// that synchronous Read and Write operations can block for. If no deadline is
+// configured, Read and Write operations can potentially block indefinitely.
 func (c *Conn) SetDeadline(t time.Time) error {
 	c.SetReadDeadline(t)
 	c.SetWriteDeadline(t)
 	return nil
 }
 
+// SetReadDeadline configures the maximum amount of time that a synchronous Read
+// operation can block for. A Read operation will unblock at the point that the
+// read deadline is reached if no other condition (such as data arrival or
+// connection closure) happens first. If no deadline is configured, Read
+// operations can potentially block indefinitely.
 func (c *Conn) SetReadDeadline(t time.Time) error {
 	// TODO warn that this can block while waiting for the Conn actor to run, so don't call it from other actors...
 	phony.Block(c, func() { c.readDeadline = &t })
 	return nil
 }
 
+// SetWriteDeadline configures the maximum amount of time that a synchronous
+// Write operation can block for. A Write operation will unblock at the point
+// that the read deadline is reached if no other condition (such as data sending
+// or connection closure) happens first. If no deadline is configured, Write
+// operations can potentially block indefinitely.
 func (c *Conn) SetWriteDeadline(t time.Time) error {
 	// TODO warn that this can block while waiting for the Conn actor to run, so don't call it from other actors...
 	phony.Block(c, func() { c.writeDeadline = &t })

From 903a8921fcf975c88eea561412d48c8da3ea7982 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Sun, 1 Sep 2019 23:47:47 +0100
Subject: [PATCH 077/130] Update api.go godoc

---
 src/yggdrasil/api.go | 58 +++++++++++++++++++++++++++++++++++---------
 1 file changed, 46 insertions(+), 12 deletions(-)

diff --git a/src/yggdrasil/api.go b/src/yggdrasil/api.go
index f69ef7e9..35269ebb 100644
--- a/src/yggdrasil/api.go
+++ b/src/yggdrasil/api.go
@@ -248,7 +248,10 @@ func (c *Core) GetSessions() []Session {
 	return sessions
 }
 
-// ConnListen returns a listener for Yggdrasil session connections.
+// ConnListen returns a listener for Yggdrasil session connections. You can only
+// call this function once as each Yggdrasil node can only have a single
+// ConnListener. Make sure to keep the reference to this for as long as it is
+// needed.
 func (c *Core) ConnListen() (*Listener, error) {
 	c.router.sessions.listenerMutex.Lock()
 	defer c.router.sessions.listenerMutex.Unlock()
@@ -263,7 +266,10 @@ func (c *Core) ConnListen() (*Listener, error) {
 	return c.router.sessions.listener, nil
 }
 
-// ConnDialer returns a dialer for Yggdrasil session connections.
+// ConnDialer returns a dialer for Yggdrasil session connections. Since
+// ConnDialers are stateless, you can request as many dialers as you like,
+// although ideally you should request only one and keep the reference to it for
+// as long as it is needed.
 func (c *Core) ConnDialer() (*Dialer, error) {
 	return &Dialer{
 		core: c,
@@ -277,48 +283,69 @@ func (c *Core) ListenTCP(uri string) (*TcpListener, error) {
 	return c.link.tcp.listen(uri)
 }
 
-// NodeID gets the node ID.
+// NodeID gets the node ID. This is derived from your router encryption keys.
+// Remote nodes wanting to open connections to your node will need to know your
+// node ID.
 func (c *Core) NodeID() *crypto.NodeID {
 	return crypto.GetNodeID(&c.boxPub)
 }
 
-// TreeID gets the tree ID.
+// TreeID gets the tree ID. This is derived from your switch signing keys. There
+// is typically no need to share this key.
 func (c *Core) TreeID() *crypto.TreeID {
 	return crypto.GetTreeID(&c.sigPub)
 }
 
-// SigningPublicKey gets the node's signing public key.
+// SigningPublicKey gets the node's signing public key, as used by the switch.
 func (c *Core) SigningPublicKey() string {
 	return hex.EncodeToString(c.sigPub[:])
 }
 
-// EncryptionPublicKey gets the node's encryption public key.
+// EncryptionPublicKey gets the node's encryption public key, as used by the
+// router.
 func (c *Core) EncryptionPublicKey() string {
 	return hex.EncodeToString(c.boxPub[:])
 }
 
-// Coords returns the current coordinates of the node.
+// Coords returns the current coordinates of the node. Note that these can
+// change at any time for a number of reasons, not limited to but including
+// changes to peerings (either yours or a parent nodes) or changes to the network
+// root.
+//
+// This function may return an empty array - this is normal behaviour if either
+// you are the root of the network that you are connected to, or you are not
+// connected to any other nodes (effectively making you the root of a
+// single-node network).
 func (c *Core) Coords() []uint64 {
 	table := c.switchTable.table.Load().(lookupTable)
 	return wire_coordsBytestoUint64s(table.self.getCoords())
 }
 
 // Address gets the IPv6 address of the Yggdrasil node. This is always a /128
-// address.
+// address. The IPv6 address is only relevant when the node is operating as an
+// IP router and often is meaningless when embedded into an application, unless
+// that application also implements either VPN functionality or deals with IP
+// packets specifically.
 func (c *Core) Address() net.IP {
 	address := net.IP(address.AddrForNodeID(c.NodeID())[:])
 	return address
 }
 
 // Subnet gets the routed IPv6 subnet of the Yggdrasil node. This is always a
-// /64 subnet.
+// /64 subnet. The IPv6 subnet is only relevant when the node is operating as an
+// IP router and often is meaningless when embedded into an application, unless
+// that application also implements either VPN functionality or deals with IP
+// packets specifically.
 func (c *Core) Subnet() net.IPNet {
 	subnet := address.SubnetForNodeID(c.NodeID())[:]
 	subnet = append(subnet, 0, 0, 0, 0, 0, 0, 0, 0)
 	return net.IPNet{IP: subnet, Mask: net.CIDRMask(64, 128)}
 }
 
-// MyNodeInfo gets the currently configured nodeinfo.
+// MyNodeInfo gets the currently configured nodeinfo. NodeInfo is typically
+// specified through the "NodeInfo" option in the node configuration or using
+// the SetNodeInfo function, although it may also contain other built-in values
+// such as "buildname", "buildversion" etc.
 func (c *Core) MyNodeInfo() NodeInfoPayload {
 	return c.router.nodeinfo.getNodeInfo()
 }
@@ -368,7 +395,9 @@ func (c *Core) SetSessionGatekeeper(f func(pubkey *crypto.BoxPubKey, initiator b
 }
 
 // SetLogger sets the output logger of the Yggdrasil node after startup. This
-// may be useful if you want to redirect the output later.
+// may be useful if you want to redirect the output later. Note that this
+// expects a Logger from the github.com/gologme/log package and not from Go's
+// built-in log package.
 func (c *Core) SetLogger(log *log.Logger) {
 	c.log = log
 }
@@ -418,12 +447,17 @@ func (c *Core) DisconnectPeer(port uint64) error {
 }
 
 // GetAllowedEncryptionPublicKeys returns the public keys permitted for incoming
-// peer connections.
+// peer connections. If this list is empty then all incoming peer connections
+// are accepted by default.
 func (c *Core) GetAllowedEncryptionPublicKeys() []string {
 	return c.peers.getAllowedEncryptionPublicKeys()
 }
 
 // AddAllowedEncryptionPublicKey whitelists a key for incoming peer connections.
+// By default all incoming peer connections are accepted, but adding public keys
+// to the whitelist using this function enables strict checking from that point
+// forward. Once the whitelist is enabled, only peer connections from
+// whitelisted public keys will be accepted.
 func (c *Core) AddAllowedEncryptionPublicKey(bstr string) (err error) {
 	c.peers.addAllowedEncryptionPublicKey(bstr)
 	return nil

From cd99d04bd41632c8541bfc6b5c1b62549f47db67 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 1 Sep 2019 18:53:45 -0500
Subject: [PATCH 078/130] document address, crypto, and util

---
 src/address/address.go   | 34 ++++++++++++------------
 src/crypto/crypto.go     | 57 +++++++++++++++++++++++++++++++++++++---
 src/util/bytes_mobile.go |  6 +++--
 src/util/bytes_other.go  |  4 +--
 src/util/cancellation.go | 20 +++++++++++---
 src/util/util.go         | 23 +++++++---------
 6 files changed, 104 insertions(+), 40 deletions(-)

diff --git a/src/address/address.go b/src/address/address.go
index 5c13257b..c30367e8 100644
--- a/src/address/address.go
+++ b/src/address/address.go
@@ -2,21 +2,21 @@ package address
 
 import "github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 
-// address represents an IPv6 address in the yggdrasil address range.
+// Address represents an IPv6 address in the yggdrasil address range.
 type Address [16]byte
 
-// subnet represents an IPv6 /64 subnet in the yggdrasil subnet range.
+// Subnet represents an IPv6 /64 subnet in the yggdrasil subnet range.
 type Subnet [8]byte
 
-// address_prefix is the prefix used for all addresses and subnets in the network.
+// GetPrefix returns the address prefix used by yggdrasil.
 // The current implementation requires this to be a muliple of 8 bits + 7 bits.
 // The 8th bit of the last byte is used to signal nodes (0) or /64 prefixes (1).
-// Nodes that configure this differently will be unable to communicate with eachother, though routing and the DHT machinery *should* still work.
+// Nodes that configure this differently will be unable to communicate with eachother using IP packets, though routing and the DHT machinery *should* still work.
 func GetPrefix() [1]byte {
 	return [...]byte{0x02}
 }
 
-// isValid returns true if an address falls within the range used by nodes in the network.
+// IsValid returns true if an address falls within the range used by nodes in the network.
 func (a *Address) IsValid() bool {
 	prefix := GetPrefix()
 	for idx := range prefix {
@@ -27,7 +27,7 @@ func (a *Address) IsValid() bool {
 	return true
 }
 
-// isValid returns true if a prefix falls within the range usable by the network.
+// IsValid returns true if a prefix falls within the range usable by the network.
 func (s *Subnet) IsValid() bool {
 	prefix := GetPrefix()
 	l := len(prefix)
@@ -39,8 +39,8 @@ func (s *Subnet) IsValid() bool {
 	return (*s)[l-1] == prefix[l-1]|0x01
 }
 
-// address_addrForNodeID takes a *NodeID as an argument and returns an *address.
-// This subnet begins with the address prefix, with the last bit set to 0 to indicate an address.
+// AddrForNodeID takes a *NodeID as an argument and returns an *Address.
+// This address begins with the contents of GetPrefix(), with the last bit set to 0 to indicate an address.
 // The following 8 bits are set to the number of leading 1 bits in the NodeID.
 // The NodeID, excluding the leading 1 bits and the first leading 0 bit, is truncated to the appropriate length and makes up the remainder of the address.
 func AddrForNodeID(nid *crypto.NodeID) *Address {
@@ -80,7 +80,7 @@ func AddrForNodeID(nid *crypto.NodeID) *Address {
 	return &addr
 }
 
-// address_subnetForNodeID takes a *NodeID as an argument and returns a *subnet.
+// SubnetForNodeID takes a *NodeID as an argument and returns an *Address.
 // This subnet begins with the address prefix, with the last bit set to 1 to indicate a prefix.
 // The following 8 bits are set to the number of leading 1 bits in the NodeID.
 // The NodeID, excluding the leading 1 bits and the first leading 0 bit, is truncated to the appropriate length and makes up the remainder of the subnet.
@@ -96,10 +96,10 @@ func SubnetForNodeID(nid *crypto.NodeID) *Subnet {
 	return &snet
 }
 
-// getNodeIDandMask returns two *NodeID.
-// The first is a NodeID with all the bits known from the address set to their correct values.
-// The second is a bitmask with 1 bit set for each bit that was known from the address.
-// This is used to look up NodeIDs in the DHT and tell if they match an address.
+// GetNodeIDandMask returns two *NodeID.
+// The first is a NodeID with all the bits known from the Address set to their correct values.
+// The second is a bitmask with 1 bit set for each bit that was known from the Address.
+// This is used to look up NodeIDs in the DHT and tell if they match an Address.
 func (a *Address) GetNodeIDandMask() (*crypto.NodeID, *crypto.NodeID) {
 	// Mask is a bitmask to mark the bits visible from the address
 	// This means truncated leading 1s, first leading 0, and visible part of addr
@@ -126,10 +126,10 @@ func (a *Address) GetNodeIDandMask() (*crypto.NodeID, *crypto.NodeID) {
 	return &nid, &mask
 }
 
-// getNodeIDandMask returns two *NodeID.
-// The first is a NodeID with all the bits known from the address set to their correct values.
-// The second is a bitmask with 1 bit set for each bit that was known from the subnet.
-// This is used to look up NodeIDs in the DHT and tell if they match a subnet.
+// GetNodeIDandMask returns two *NodeID.
+// The first is a NodeID with all the bits known from the Subnet set to their correct values.
+// The second is a bitmask with 1 bit set for each bit that was known from the Subnet.
+// This is used to look up NodeIDs in the DHT and tell if they match a Subnet.
 func (s *Subnet) GetNodeIDandMask() (*crypto.NodeID, *crypto.NodeID) {
 	// As with the address version, but visible parts of the subnet prefix instead
 	var nid crypto.NodeID
diff --git a/src/crypto/crypto.go b/src/crypto/crypto.go
index 75736ba7..55f26f74 100644
--- a/src/crypto/crypto.go
+++ b/src/crypto/crypto.go
@@ -26,12 +26,21 @@ import (
 
 // NodeID and TreeID
 
+// NodeIDLen is the length (in bytes) of a NodeID.
 const NodeIDLen = sha512.Size
+
+// TreeIDLen is the length (in bytes) of a TreeID.
 const TreeIDLen = sha512.Size
+
+// handleLen is the length (in bytes) of a Handle.
 const handleLen = 8
 
+// NodeID is how a yggdrasil node is identified in the DHT, and is used to derive IPv6 addresses and subnets in the main executable. It is a sha512sum hash of the node's BoxPubKey
 type NodeID [NodeIDLen]byte
+
+// TreeID is how a yggdrasil node is identified in the root selection algorithm used to construct the spanning tree.
 type TreeID [TreeIDLen]byte
+
 type Handle [handleLen]byte
 
 func (n *NodeID) String() string {
@@ -69,16 +78,19 @@ func (n *NodeID) PrefixLength() int {
 	return len
 }
 
+// GetNodeID returns the NodeID associated with a BoxPubKey.
 func GetNodeID(pub *BoxPubKey) *NodeID {
 	h := sha512.Sum512(pub[:])
 	return (*NodeID)(&h)
 }
 
+// GetTreeID returns the TreeID associated with a BoxPubKey
 func GetTreeID(pub *SigPubKey) *TreeID {
 	h := sha512.Sum512(pub[:])
 	return (*TreeID)(&h)
 }
 
+// NewHandle returns a new (cryptographically random) Handle, used by the session code to identify which session an incoming packet is associated with.
 func NewHandle() *Handle {
 	var h Handle
 	_, err := rand.Read(h[:])
@@ -92,14 +104,25 @@ func NewHandle() *Handle {
 
 // Signatures
 
+// SigPubKeyLen is the length of a SigPubKey in bytes.
 const SigPubKeyLen = ed25519.PublicKeySize
+
+// SigPrivKeyLen is the length of a SigPrivKey in bytes.
 const SigPrivKeyLen = ed25519.PrivateKeySize
+
+// SigLen is the length of SigBytes.
 const SigLen = ed25519.SignatureSize
 
+// SigPubKey is a public ed25519 signing key.
 type SigPubKey [SigPubKeyLen]byte
+
+// SigPrivKey is a private ed25519 signing key.
 type SigPrivKey [SigPrivKeyLen]byte
+
+// SigBytes is an ed25519 signature.
 type SigBytes [SigLen]byte
 
+// NewSigKeys generates a public/private ed25519 key pair.
 func NewSigKeys() (*SigPubKey, *SigPrivKey) {
 	var pub SigPubKey
 	var priv SigPrivKey
@@ -112,6 +135,7 @@ func NewSigKeys() (*SigPubKey, *SigPrivKey) {
 	return &pub, &priv
 }
 
+// Sign returns the SigBytes signing a message.
 func Sign(priv *SigPrivKey, msg []byte) *SigBytes {
 	var sig SigBytes
 	sigSlice := ed25519.Sign(priv[:], msg)
@@ -119,12 +143,14 @@ func Sign(priv *SigPrivKey, msg []byte) *SigBytes {
 	return &sig
 }
 
+// Verify returns true if the provided signature matches the key and message.
 func Verify(pub *SigPubKey, msg []byte, sig *SigBytes) bool {
 	// Should sig be an array instead of a slice?...
 	// It's fixed size, but
 	return ed25519.Verify(pub[:], msg, sig[:])
 }
 
+// Public returns the SigPubKey associated with this SigPrivKey.
 func (p SigPrivKey) Public() SigPubKey {
 	priv := make(ed25519.PrivateKey, ed25519.PrivateKeySize)
 	copy(priv[:], p[:])
@@ -138,17 +164,34 @@ func (p SigPrivKey) Public() SigPubKey {
 
 // NaCl-like crypto "box" (curve25519+xsalsa20+poly1305)
 
+// BoxPubKeyLen is the length of a BoxPubKey in bytes.
 const BoxPubKeyLen = 32
+
+// BoxPrivKeyLen is the length of a BoxPrivKey in bytes.
 const BoxPrivKeyLen = 32
+
+// BoxSharedKeyLen is the length of a BoxSharedKey in bytes.
 const BoxSharedKeyLen = 32
+
+// BoxNonceLen is the length of a BoxNonce in bytes.
 const BoxNonceLen = 24
+
+// BoxOverhead is the length of the overhead from boxing something.
 const BoxOverhead = box.Overhead
 
+// BoxPubKey is a NaCl-like "box" public key (curve25519+xsalsa20+poly1305).
 type BoxPubKey [BoxPubKeyLen]byte
+
+// BoxPrivKey is a NaCl-like "box" private key (curve25519+xsalsa20+poly1305).
 type BoxPrivKey [BoxPrivKeyLen]byte
+
+// BoxSharedKey is a NaCl-like "box" shared key (curve25519+xsalsa20+poly1305).
 type BoxSharedKey [BoxSharedKeyLen]byte
+
+// BoxNonce is the nonce used in NaCl-like crypto "box" operations (curve25519+xsalsa20+poly1305), and must not be reused for different messages encrypted using the same BoxSharedKey.
 type BoxNonce [BoxNonceLen]byte
 
+// NewBoxKeys generates a new pair of public/private crypto box keys.
 func NewBoxKeys() (*BoxPubKey, *BoxPrivKey) {
 	pubBytes, privBytes, err := box.GenerateKey(rand.Reader)
 	if err != nil {
@@ -159,6 +202,7 @@ func NewBoxKeys() (*BoxPubKey, *BoxPrivKey) {
 	return pub, priv
 }
 
+// GetSharedKey returns the shared key derived from your private key and the destination's public key.
 func GetSharedKey(myPrivKey *BoxPrivKey,
 	othersPubKey *BoxPubKey) *BoxSharedKey {
 	var shared [BoxSharedKeyLen]byte
@@ -168,6 +212,7 @@ func GetSharedKey(myPrivKey *BoxPrivKey,
 	return (*BoxSharedKey)(&shared)
 }
 
+// BoxOpen returns a message and true if it successfull opens a crypto box using the provided shared key and nonce.
 func BoxOpen(shared *BoxSharedKey,
 	boxed []byte,
 	nonce *BoxNonce) ([]byte, bool) {
@@ -178,6 +223,9 @@ func BoxOpen(shared *BoxSharedKey,
 	return unboxed, success
 }
 
+// BoxSeal seals a crypto box using the provided shared key, returning the box and the nonce needed to decrypt it.
+// If nonce is nil, a random BoxNonce will be used and returned.
+// If nonce is non-nil, then nonce.Increment() will be called before using it, and the incremented BoxNonce is what is returned.
 func BoxSeal(shared *BoxSharedKey, unboxed []byte, nonce *BoxNonce) ([]byte, *BoxNonce) {
 	if nonce == nil {
 		nonce = NewBoxNonce()
@@ -190,6 +238,7 @@ func BoxSeal(shared *BoxSharedKey, unboxed []byte, nonce *BoxNonce) ([]byte, *Bo
 	return boxed, nonce
 }
 
+// NewBoxNonce generates a (cryptographically) random BoxNonce.
 func NewBoxNonce() *BoxNonce {
 	var nonce BoxNonce
 	_, err := rand.Read(nonce[:])
@@ -204,6 +253,7 @@ func NewBoxNonce() *BoxNonce {
 	return &nonce
 }
 
+// Increment adds 2 to a BoxNonce, which is useful if one node intends to send only with odd BoxNonce values, and the other only with even BoxNonce values.
 func (n *BoxNonce) Increment() {
 	oldNonce := *n
 	n[len(n)-1] += 2
@@ -214,6 +264,7 @@ func (n *BoxNonce) Increment() {
 	}
 }
 
+// Public returns the BoxPubKey associated with this BoxPrivKey.
 func (p BoxPrivKey) Public() BoxPubKey {
 	var boxPub [BoxPubKeyLen]byte
 	var boxPriv [BoxPrivKeyLen]byte
@@ -222,9 +273,9 @@ func (p BoxPrivKey) Public() BoxPubKey {
 	return boxPub
 }
 
-// Used to subtract one nonce from another, staying in the range +- 64.
-// This is used by the nonce progression machinery to advance the bitmask of recently received packets (indexed by nonce), or to check the appropriate bit of the bitmask.
-// It's basically part of the machinery that prevents replays and duplicate packets.
+// Minus is the result of subtracting the provided BoNonce from this BoxNonce, bounded at +- 64.
+// It's primarily used to determine if a new BoxNonce is higher than the last known BoxNonce from a crypto session, and by how much.
+// This is used in the machinery that makes sure replayed packets can't keep a session open indefinitely or stuck using old/bad information about a node.
 func (n *BoxNonce) Minus(m *BoxNonce) int64 {
 	diff := int64(0)
 	for idx := range n {
diff --git a/src/util/bytes_mobile.go b/src/util/bytes_mobile.go
index 09e78050..f862c0cd 100644
--- a/src/util/bytes_mobile.go
+++ b/src/util/bytes_mobile.go
@@ -8,12 +8,14 @@ func init() {
 	debug.SetGCPercent(25)
 }
 
-// On mobile, just return a nil slice.
+// GetBytes always returns a nil slice on mobile platforms.
 func GetBytes() []byte {
 	return nil
 }
 
-// On mobile, don't do anything.
+// PutBytes does literally nothing on mobile platforms.
+// This is done rather than keeping a free list of bytes on platforms with memory constraints.
+// It's needed to help keep memory usage low enough to fall under the limits set for e.g. iOS NEPacketTunnelProvider apps.
 func PutBytes(bs []byte) {
 	return
 }
diff --git a/src/util/bytes_other.go b/src/util/bytes_other.go
index 41b8bec0..7c966087 100644
--- a/src/util/bytes_other.go
+++ b/src/util/bytes_other.go
@@ -7,12 +7,12 @@ import "sync"
 // This is used to buffer recently used slices of bytes, to prevent allocations in the hot loops.
 var byteStore = sync.Pool{New: func() interface{} { return []byte(nil) }}
 
-// Gets an empty slice from the byte store.
+// GetBytes returns a 0-length (possibly nil) slice of bytes from a free list, so it may have a larger capacity.
 func GetBytes() []byte {
 	return byteStore.Get().([]byte)[:0]
 }
 
-// Puts a slice in the store.
+// PutBytes stores a slice in a free list, where it can potentially be reused to prevent future allocations.
 func PutBytes(bs []byte) {
 	byteStore.Put(bs)
 }
diff --git a/src/util/cancellation.go b/src/util/cancellation.go
index af4721bb..6b2002c8 100644
--- a/src/util/cancellation.go
+++ b/src/util/cancellation.go
@@ -7,15 +7,22 @@ import (
 	"time"
 )
 
+// Cancellation is used to signal when things should shut down, such as signaling anything associated with a Conn to exit.
+// This is and is similar to a context, but with an error to specify the reason for the cancellation.
 type Cancellation interface {
-	Finished() <-chan struct{}
-	Cancel(error) error
-	Error() error
+	Finished() <-chan struct{} // Finished returns a channel which will be closed when Cancellation.Cancel is first called.
+	Cancel(error) error // Cancel closes the channel returned by Finished and sets the error returned by error, or else returns the existing error if the Cancellation has already run.
+	Error() error // Error returns the error provided to Cancel, or nil if no error has been provided.
 }
 
+// CancellationFinalized is an error returned if a cancellation object was garbage collected and the finalizer was run.
+// If you ever see this, then you're probably doing something wrong with your code.
 var CancellationFinalized = errors.New("finalizer called")
+
+// CancellationTimeoutError is used when a CancellationWithTimeout or CancellationWithDeadline is cancelled due to said timeout.
 var CancellationTimeoutError = errors.New("timeout")
 
+// CancellationFinalizer is set as a finalizer when creating a new cancellation with NewCancellation(), and generally shouldn't be needed by the user, but is included in case other implementations of the same interface want to make use of it.
 func CancellationFinalizer(c Cancellation) {
 	c.Cancel(CancellationFinalized)
 }
@@ -27,6 +34,7 @@ type cancellation struct {
 	done   bool
 }
 
+// NewCancellation returns a pointer to a struct satisfying the Cancellation interface.
 func NewCancellation() Cancellation {
 	c := cancellation{
 		cancel: make(chan struct{}),
@@ -35,10 +43,12 @@ func NewCancellation() Cancellation {
 	return &c
 }
 
+// Finished returns a channel which will be closed when Cancellation.Cancel is first called.
 func (c *cancellation) Finished() <-chan struct{} {
 	return c.cancel
 }
 
+// Cancel closes the channel returned by Finished and sets the error returned by error, or else returns the existing error if the Cancellation has already run.
 func (c *cancellation) Cancel(err error) error {
 	c.mutex.Lock()
 	defer c.mutex.Unlock()
@@ -52,6 +62,7 @@ func (c *cancellation) Cancel(err error) error {
 	}
 }
 
+// Error returns the error provided to Cancel, or nil if no error has been provided.
 func (c *cancellation) Error() error {
 	c.mutex.RLock()
 	err := c.err
@@ -59,6 +70,7 @@ func (c *cancellation) Error() error {
 	return err
 }
 
+// CancellationChild returns a new Cancellation which can be Cancelled independently of the parent, but which will also be Cancelled if the parent is Cancelled first.
 func CancellationChild(parent Cancellation) Cancellation {
 	child := NewCancellation()
 	go func() {
@@ -71,6 +83,7 @@ func CancellationChild(parent Cancellation) Cancellation {
 	return child
 }
 
+// CancellationWithTimeout returns a ChildCancellation that will automatically be Cancelled with a CancellationTimeoutError after the timeout.
 func CancellationWithTimeout(parent Cancellation, timeout time.Duration) Cancellation {
 	child := CancellationChild(parent)
 	go func() {
@@ -85,6 +98,7 @@ func CancellationWithTimeout(parent Cancellation, timeout time.Duration) Cancell
 	return child
 }
 
+// CancellationWithTimeout returns a ChildCancellation that will automatically be Cancelled with a CancellationTimeoutError after the specified deadline.
 func CancellationWithDeadline(parent Cancellation, deadline time.Time) Cancellation {
 	return CancellationWithTimeout(parent, deadline.Sub(time.Now()))
 }
diff --git a/src/util/util.go b/src/util/util.go
index 97250122..c73faf68 100644
--- a/src/util/util.go
+++ b/src/util/util.go
@@ -9,22 +9,22 @@ import (
 	"time"
 )
 
-// A wrapper around runtime.Gosched() so it doesn't need to be imported elsewhere.
+// Yield just executes runtime.Gosched(), and is included so we don't need to explicitly import runtime elsewhere.
 func Yield() {
 	runtime.Gosched()
 }
 
-// A wrapper around runtime.LockOSThread() so it doesn't need to be imported elsewhere.
+// LockThread executes runtime.LockOSThread(), and is included so we don't need to explicitly import runtime elsewhere.
 func LockThread() {
 	runtime.LockOSThread()
 }
 
-// A wrapper around runtime.UnlockOSThread() so it doesn't need to be imported elsewhere.
+// UnlockThread executes runtime.UnlockOSThread(), and is included so we don't need to explicitly import runtime elsewhere.
 func UnlockThread() {
 	runtime.UnlockOSThread()
 }
 
-// Gets a slice of the appropriate length, reusing existing slice capacity when possible
+// ResizeBytes returns a slice of the specified length. If the provided slice has sufficient capacity, it will be resized and returned rather than allocating a new slice.
 func ResizeBytes(bs []byte, length int) []byte {
 	if cap(bs) >= length {
 		return bs[:length]
@@ -33,7 +33,7 @@ func ResizeBytes(bs []byte, length int) []byte {
 	}
 }
 
-// This is a workaround to go's broken timer implementation
+// TimerStop stops a timer and makes sure the channel is drained, returns true if the timer was stopped before firing.
 func TimerStop(t *time.Timer) bool {
 	stopped := t.Stop()
 	select {
@@ -43,10 +43,8 @@ func TimerStop(t *time.Timer) bool {
 	return stopped
 }
 
-// Run a blocking function with a timeout.
-// Returns true if the function returns.
-// Returns false if the timer fires.
-// The blocked function remains blocked--the caller is responsible for somehow killing it.
+// FuncTimeout runs the provided function in a separate goroutine, and returns true if the function finishes executing before the timeout passes, or false if the timeout passes.
+// It includes no mechanism to stop the function if the timeout fires, so the user is expected to do so on their own (such as with a Cancellation or a context).
 func FuncTimeout(f func(), timeout time.Duration) bool {
 	success := make(chan struct{})
 	go func() {
@@ -63,9 +61,8 @@ func FuncTimeout(f func(), timeout time.Duration) bool {
 	}
 }
 
-// This calculates the difference between two arrays and returns items
-// that appear in A but not in B - useful somewhat when reconfiguring
-// and working out what configuration items changed
+// Difference loops over two strings and returns the elements of A which do not appear in B.
+// This is somewhat useful when needing to determine which elements of a configuration file have changed.
 func Difference(a, b []string) []string {
 	ab := []string{}
 	mb := map[string]bool{}
@@ -93,7 +90,7 @@ func DecodeCoordString(in string) (out []uint64) {
 	return out
 }
 
-// GetFlowLabel takes an IP packet as an argument and returns some information about the traffic flow.
+// GetFlowKey takes an IP packet as an argument and returns some information about the traffic flow.
 // For IPv4 packets, this is derived from the source and destination protocol and port numbers.
 // For IPv6 packets, this is derived from the FlowLabel field of the packet if this was set, otherwise it's handled like IPv4.
 // The FlowKey is then used internally by Yggdrasil for congestion control.

From b3361d4bbc15fa78a9fb08a271a2fb731518dd68 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 1 Sep 2019 19:01:33 -0500
Subject: [PATCH 079/130] package level documentation for address/crypto/util

---
 src/address/address.go | 2 ++
 src/crypto/crypto.go   | 3 +++
 src/util/util.go       | 2 ++
 3 files changed, 7 insertions(+)

diff --git a/src/address/address.go b/src/address/address.go
index c30367e8..3960b783 100644
--- a/src/address/address.go
+++ b/src/address/address.go
@@ -1,3 +1,5 @@
+// Package address contains the types used by yggdrasil to represent IPv6 addresses or prefixes, as well as functions for working with these types.
+// Of particular importance are the functions used to derive addresses or subnets from a NodeID, or to get the NodeID and bitmask of the bits visible from an address, which is needed for DHT searches.
 package address
 
 import "github.com/yggdrasil-network/yggdrasil-go/src/crypto"
diff --git a/src/crypto/crypto.go b/src/crypto/crypto.go
index 55f26f74..6c10a2ef 100644
--- a/src/crypto/crypto.go
+++ b/src/crypto/crypto.go
@@ -1,3 +1,6 @@
+// Package crypto is a wrapper around packages under golang.org/x/crypto/, particulaly curve25519, ed25519, and nacl/box.
+// This is used to avoid explicitly importing and using these packages throughout yggdrasil.
+// It also includes the all-important NodeID and TreeID types, which are used to identify nodes in the DHT and in the spanning tree's root selection algorithm, respectively.
 package crypto
 
 /*
diff --git a/src/util/util.go b/src/util/util.go
index c73faf68..a7a54562 100644
--- a/src/util/util.go
+++ b/src/util/util.go
@@ -1,3 +1,5 @@
+// Package util contains miscellaneous utilities used by yggdrasil.
+// In particular, this includes a crypto worker pool, Cancellation machinery, and a sync.Pool used to reuse []byte.
 package util
 
 // These are misc. utility functions that didn't really fit anywhere else

From af3dcb44d88cb7d70023a9af77a07eb2d5ab1bb1 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 2 Sep 2019 09:45:11 +0100
Subject: [PATCH 080/130] Update config.go godoc

---
 src/config/config.go | 61 +++++++++++++++++++++++++++++---------------
 1 file changed, 40 insertions(+), 21 deletions(-)

diff --git a/src/config/config.go b/src/config/config.go
index 6c127552..ac88bfc5 100644
--- a/src/config/config.go
+++ b/src/config/config.go
@@ -1,3 +1,19 @@
+/*
+The config package contains structures related to the configuration of an
+Yggdrasil node.
+
+The configuration contains, amongst other things, encryption keys which are used
+to derive a node's identity, information about peerings and node information
+that is shared with the network. There are also some module-specific options
+related to TUN/TAP, multicast and the admin socket.
+
+In order for a node to maintain the same identity across restarts, you should
+persist the configuration onto the filesystem or into some configuration storage
+so that the encryption keys (and therefore the node ID) do not change.
+
+Note that Yggdrasil will automatically populate sane defaults for any
+configuration option that is not provided.
+*/
 package config
 
 import (
@@ -8,30 +24,30 @@ import (
 	"github.com/yggdrasil-network/yggdrasil-go/src/defaults"
 )
 
-// NodeState represents the active and previous configuration of the node and
-// protects it with a mutex
+// NodeState represents the active and previous configuration of an Yggdrasil
+// node. A NodeState object is returned when starting an Yggdrasil node. Note
+// that this structure and related functions are likely to disappear soon.
 type NodeState struct {
 	Current  NodeConfig
 	Previous NodeConfig
 	Mutex    sync.RWMutex
 }
 
-// Current returns the current node config
+// Current returns the active node configuration.
 func (s *NodeState) GetCurrent() NodeConfig {
 	s.Mutex.RLock()
 	defer s.Mutex.RUnlock()
 	return s.Current
 }
 
-// Previous returns the previous node config
+// Previous returns the previous node configuration.
 func (s *NodeState) GetPrevious() NodeConfig {
 	s.Mutex.RLock()
 	defer s.Mutex.RUnlock()
 	return s.Previous
 }
 
-// Replace the node configuration with new configuration. This method returns
-// both the new and the previous node configs
+// Replace the node configuration with new configuration.
 func (s *NodeState) Replace(n NodeConfig) {
 	s.Mutex.Lock()
 	defer s.Mutex.Unlock()
@@ -39,7 +55,9 @@ func (s *NodeState) Replace(n NodeConfig) {
 	s.Current = n
 }
 
-// NodeConfig defines all configuration values needed to run a signle yggdrasil node
+// NodeConfig is the main configuration structure, containing configuration
+// options that are necessary for an Yggdrasil node to run. You will need to
+// supply one of these structs to the Yggdrasil core when starting a node.
 type NodeConfig struct {
 	Peers                       []string               `comment:"List of connection strings for outbound peer connections in URI format,\ne.g. tcp://a.b.c.d:e or socks://a.b.c.d:e/f.g.h.i:j. These connections\nwill obey the operating system routing table, therefore you should\nuse this section when you may connect via different interfaces."`
 	InterfacePeers              map[string][]string    `comment:"List of connection strings for outbound peer connections in URI format,\narranged by source interface, e.g. { \"eth0\": [ tcp://a.b.c.d:e ] }.\nNote that SOCKS peerings will NOT be affected by this option and should\ngo in the \"Peers\" section instead."`
@@ -62,7 +80,7 @@ type NodeConfig struct {
 	NodeInfo                    map[string]interface{} `comment:"Optional node info. This must be a { \"key\": \"value\", ... } map\nor set as null. This is entirely optional but, if set, is visible\nto the whole network on request."`
 }
 
-// SessionFirewall controls the session firewall configuration
+// SessionFirewall controls the session firewall configuration.
 type SessionFirewall struct {
 	Enable                        bool     `comment:"Enable or disable the session firewall. If disabled, network traffic\nfrom any node will be allowed. If enabled, the below rules apply."`
 	AllowFromDirect               bool     `comment:"Allow network traffic from directly connected peers."`
@@ -72,7 +90,8 @@ type SessionFirewall struct {
 	BlacklistEncryptionPublicKeys []string `comment:"List of public keys from which network traffic is always rejected,\nregardless of the whitelist, AllowFromDirect or AllowFromRemote."`
 }
 
-// TunnelRouting contains the crypto-key routing tables for tunneling
+// TunnelRouting contains the crypto-key routing tables for tunneling regular
+// IPv4 or IPv6 subnets across the Yggdrasil network.
 type TunnelRouting struct {
 	Enable            bool              `comment:"Enable or disable tunnel routing."`
 	IPv6RemoteSubnets map[string]string `comment:"IPv6 subnets belonging to remote nodes, mapped to the node's public\nkey, e.g. { \"aaaa:bbbb:cccc::/e\": \"boxpubkey\", ... }"`
@@ -81,18 +100,15 @@ type TunnelRouting struct {
 	IPv4LocalSubnets  []string          `comment:"IPv4 subnets belonging to this node's end of the tunnels. Only traffic\nfrom these ranges will be tunnelled."`
 }
 
-// SwitchOptions contains tuning options for the switch
+// SwitchOptions contains tuning options for the switch. These are advanced
+// options and shouldn't be changed unless necessary.
 type SwitchOptions struct {
 	MaxTotalQueueSize uint64 `comment:"Maximum size of all switch queues combined (in bytes)."`
 }
 
-// Generates default configuration. This is used when outputting the -genconf
-// parameter and also when using -autoconf. The isAutoconf flag is used to
-// determine whether the operating system should select a free port by itself
-// (which guarantees that there will not be a conflict with any other services)
-// or whether to generate a random port number. The only side effect of setting
-// isAutoconf is that the TCP and UDP ports will likely end up with different
-// port numbers.
+// Generates default configuration and returns a pointer to the resulting
+// NodeConfig. This is used when outputting the -genconf parameter and also when
+// using -autoconf.
 func GenerateConfig() *NodeConfig {
 	// Generate encryption keys.
 	bpub, bpriv := crypto.NewBoxKeys()
@@ -122,16 +138,19 @@ func GenerateConfig() *NodeConfig {
 	return &cfg
 }
 
-// NewEncryptionKeys generates a new encryption keypair. The encryption keys are
-// used to encrypt traffic and to derive the IPv6 address/subnet of the node.
+// NewEncryptionKeys replaces the encryption keypair in the NodeConfig with a
+// new encryption keypair. The encryption keys are used by the router to encrypt
+// traffic and to derive the node ID and IPv6 address/subnet of the node, so
+// this is equivalent to discarding the node's identity on the network.
 func (cfg *NodeConfig) NewEncryptionKeys() {
 	bpub, bpriv := crypto.NewBoxKeys()
 	cfg.EncryptionPublicKey = hex.EncodeToString(bpub[:])
 	cfg.EncryptionPrivateKey = hex.EncodeToString(bpriv[:])
 }
 
-// NewSigningKeys generates a new signing keypair. The signing keys are used to
-// derive the structure of the spanning tree.
+// NewSigningKeys replaces the signing keypair in the NodeConfig with a new
+// signing keypair. The signing keys are used by the switch to derive the
+// structure of the spanning tree.
 func (cfg *NodeConfig) NewSigningKeys() {
 	spub, spriv := crypto.NewSigKeys()
 	cfg.SigningPublicKey = hex.EncodeToString(spub[:])

From 2426a87ccccc36141d3a6f31aeec8a4126494f12 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Tue, 3 Sep 2019 19:03:12 -0500
Subject: [PATCH 081/130] really finish initializing the session before
 returning it / giving up control of the router, in the Conn.search function
 used by Dial

---
 src/yggdrasil/conn.go | 70 ++++++++++++++++++++++---------------------
 1 file changed, 36 insertions(+), 34 deletions(-)

diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index 352cf9d9..cb28a6f2 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -90,45 +90,47 @@ func (c *Conn) setMTU(from phony.Actor, mtu uint16) {
 
 // This should never be called from the router goroutine, used in the dial functions
 func (c *Conn) search() error {
-	var sinfo *searchInfo
-	var isIn bool
+	var err error
+	done := make(chan struct{})
 	phony.Block(&c.core.router, func() {
-		sinfo, isIn = c.core.router.searches.searches[*c.nodeID]
-	})
-	if !isIn {
-		done := make(chan struct{}, 1)
-		var sess *sessionInfo
-		var err error
-		searchCompleted := func(sinfo *sessionInfo, e error) {
-			sess = sinfo
-			err = e
-			// FIXME close can be called multiple times, do a non-blocking send instead
-			select {
-			case done <- struct{}{}:
-			default:
+		_, isIn := c.core.router.searches.searches[*c.nodeID]
+		if !isIn {
+			searchCompleted := func(sinfo *sessionInfo, e error) {
+				select {
+				case <-done:
+					// Somehow this was called multiple times, TODO don't let that happen
+					if sinfo != nil {
+						// Need to clean up to avoid a session leak
+						sinfo.cancel.Cancel(nil)
+					}
+				default:
+					if sinfo != nil {
+						// Finish initializing the session
+						sinfo.conn = c
+					}
+					c.session = sinfo
+					err = e
+					close(done)
+				}
 			}
-		}
-		phony.Block(&c.core.router, func() {
-			sinfo = c.core.router.searches.newIterSearch(c.nodeID, c.nodeMask, searchCompleted)
+			sinfo := c.core.router.searches.newIterSearch(c.nodeID, c.nodeMask, searchCompleted)
 			sinfo.continueSearch()
-		})
-		<-done
-		c.session = sess
-		if c.session == nil && err == nil {
-			panic("search failed but returned no error")
+		} else {
+			err = errors.New("search already exists")
+			close(done)
 		}
-		if c.session != nil {
-			c.nodeID = crypto.GetNodeID(&c.session.theirPermPub)
-			for i := range c.nodeMask {
-				c.nodeMask[i] = 0xFF
-			}
-			c.session.conn = c
-		}
-		return err
-	} else {
-		return errors.New("search already exists")
+	})
+	<-done
+	if c.session == nil && err == nil {
+		panic("search failed but returned no error")
 	}
-	return nil
+	if c.session != nil {
+		c.nodeID = crypto.GetNodeID(&c.session.theirPermPub)
+		for i := range c.nodeMask {
+			c.nodeMask[i] = 0xFF
+		}
+	}
+	return err
 }
 
 // Used in session keep-alive traffic

From eec055313dffba04633db0e21a8639e9590a8786 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 6 Sep 2019 22:20:36 -0500
Subject: [PATCH 082/130] update phony dependency

---
 go.mod | 2 +-
 go.sum | 6 ++----
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/go.mod b/go.mod
index 6703d5bd..d86101bf 100644
--- a/go.mod
+++ b/go.mod
@@ -1,7 +1,7 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
 require (
-	github.com/Arceliar/phony v0.0.0-20190831214819-9b642ea019ad
+	github.com/Arceliar/phony v0.0.0-20190907031509-af5bdbeecab6
 	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
 	github.com/hashicorp/go-syslog v1.0.0
 	github.com/hjson/hjson-go v3.0.1-0.20190209023717-9147687966d9+incompatible
diff --git a/go.sum b/go.sum
index 60e6090f..cdabc402 100644
--- a/go.sum
+++ b/go.sum
@@ -1,11 +1,9 @@
-github.com/Arceliar/phony v0.0.0-20190831214819-9b642ea019ad h1:670inqspOp+tAnSvkOBgrKGOIT4605Jt+6KGi2j2/S8=
-github.com/Arceliar/phony v0.0.0-20190831214819-9b642ea019ad/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
+github.com/Arceliar/phony v0.0.0-20190907031509-af5bdbeecab6 h1:zMj5Q1V0yF4WNfV/FpXG6iXfPJ965Xc5asR2vHXanXc=
+github.com/Arceliar/phony v0.0.0-20190907031509-af5bdbeecab6/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
-github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222 h1:xmvkbxXDeN1ffWq8kvrhyqVYAO2aXuRBsbpxVTR+JyU=
-github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
 github.com/hjson/hjson-go v3.0.1-0.20190209023717-9147687966d9+incompatible h1:bLQ2Ve+eW65id3b8xEMQiAwJT4qGZeywAEMLvXjznvw=
 github.com/hjson/hjson-go v3.0.1-0.20190209023717-9147687966d9+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
 github.com/kardianos/minwinsvc v0.0.0-20151122163309-cad6b2b879b0 h1:YnZmFjg0Nvk8851WTVWlqMC1ecJH07Ctz+Ezxx4u54g=

From 10a828af2c42b42fdd9f35563b0badfe1039f6af Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Mon, 9 Sep 2019 19:20:46 -0500
Subject: [PATCH 083/130] when forwarding traffic, break distance ties by
 favoring the link that sent the most recent switch update the fastest

---
 src/yggdrasil/switch.go | 36 ++++++++++++++++++++++++------------
 1 file changed, 24 insertions(+), 12 deletions(-)

diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 163c85cf..5ed70323 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -143,6 +143,7 @@ type switchPort uint64
 type tableElem struct {
 	port    switchPort
 	locator switchLocator
+	time    time.Time
 }
 
 // This is the subset of the information about all peers needed to make routing decisions, and it stored separately in an atomically accessed table, which gets hammered in the "hot loop" of the routing logic (see: peer.handleTraffic in peers.go).
@@ -562,6 +563,7 @@ func (t *switchTable) updateTable() {
 		newTable.elems[pinfo.port] = tableElem{
 			locator: loc,
 			port:    pinfo.port,
+			time:    pinfo.time,
 		}
 	}
 	t.table.Store(newTable)
@@ -581,7 +583,7 @@ func (t *switchTable) start() error {
 }
 
 type closerInfo struct {
-	port switchPort
+	elem tableElem
 	dist int
 }
 
@@ -598,7 +600,7 @@ func (t *switchTable) getCloser(dest []byte) []closerInfo {
 	for _, info := range table.elems {
 		dist := info.locator.dist(dest)
 		if dist < myDist {
-			t.queues.closer = append(t.queues.closer, closerInfo{info.port, dist})
+			t.queues.closer = append(t.queues.closer, closerInfo{info, dist})
 		}
 	}
 	return t.queues.closer
@@ -671,13 +673,12 @@ func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]time.Time) bo
 		self.sendPacketsFrom(t, [][]byte{packet})
 		return true
 	}
-	var best *peer
-	var bestDist int
+	var best *closerInfo
 	var bestTime time.Time
 	ports := t.core.peers.getPorts()
 	for _, cinfo := range closer {
-		to := ports[cinfo.port]
-		thisTime, isIdle := idle[cinfo.port]
+		to := ports[cinfo.elem.port]
+		thisTime, isIdle := idle[cinfo.elem.port]
 		var update bool
 		switch {
 		case to == nil:
@@ -688,13 +689,24 @@ func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]time.Time) bo
 			// this is the first idle port we've found, so select it until we find a
 			// better candidate port to use instead
 			update = true
-		case cinfo.dist < bestDist:
+		case cinfo.dist < best.dist:
 			// the port takes a shorter path/is more direct than our current
 			// candidate, so select that instead
 			update = true
-		case cinfo.dist > bestDist:
+		case cinfo.dist > best.dist:
 			// the port takes a longer path/is less direct than our current candidate,
 			// ignore it
+		case cinfo.elem.locator.tstamp > best.elem.locator.tstamp:
+			// has a newer tstamp from the root, so presumably a better path
+			update = true
+		case cinfo.elem.locator.tstamp < best.elem.locator.tstamp:
+			// has a n older tstamp, so presumably a worse path
+		case cinfo.elem.time.Before(best.elem.time):
+			// same tstamp, but got it earlier, so presumably a better path
+			update = true
+		case cinfo.elem.time.After(best.elem.time):
+			// same tstamp, but got it later, so presumably a worse path
+			// I do not expect the remaining cases to ever be reached... TODO cleanup
 		case thisTime.After(bestTime):
 			// all else equal, this port was used more recently than our current
 			// candidate, so choose that instead. this should mean that, in low
@@ -705,15 +717,15 @@ func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]time.Time) bo
 			// the search for a port has finished
 		}
 		if update {
-			best = to
-			bestDist = cinfo.dist
+			b := cinfo // because cinfo gets mutated by the iteration
+			best = &b
 			bestTime = thisTime
 		}
 	}
 	if best != nil {
 		// Send to the best idle next hop
-		delete(idle, best.port)
-		best.sendPacketsFrom(t, [][]byte{packet})
+		delete(idle, best.elem.port)
+		ports[best.elem.port].sendPacketsFrom(t, [][]byte{packet})
 		return true
 	}
 	// Didn't find anyone idle to send it to

From 0141180279cc896f70f8f8b1b80d3e895c24b16a Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Mon, 9 Sep 2019 19:25:10 -0500
Subject: [PATCH 084/130] cleanup

---
 src/yggdrasil/switch.go | 21 +++++----------------
 1 file changed, 5 insertions(+), 16 deletions(-)

diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 5ed70323..f4df60d5 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -176,7 +176,7 @@ type switchTable struct {
 	table       atomic.Value               // lookupTable
 	phony.Inbox                            // Owns the below
 	queues      switch_buffers             // Queues - not atomic so ONLY use through the actor
-	idle        map[switchPort]time.Time   // idle peers - not atomic so ONLY use through the actor
+	idle        map[switchPort]struct{}    // idle peers - not atomic so ONLY use through the actor
 }
 
 // Minimum allowed total size of switch queues.
@@ -202,7 +202,7 @@ func (t *switchTable) init(core *Core) {
 		}
 		core.config.Mutex.RUnlock()
 		t.queues.bufs = make(map[string]switch_buffer)
-		t.idle = make(map[switchPort]time.Time)
+		t.idle = make(map[switchPort]struct{})
 	})
 }
 
@@ -664,7 +664,7 @@ func (t *switchTable) bestPortForCoords(coords []byte) switchPort {
 // Handle an incoming packet
 // Either send it to ourself, or to the first idle peer that's free
 // Returns true if the packet has been handled somehow, false if it should be queued
-func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]time.Time) bool {
+func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]struct{}) bool {
 	coords := switch_getPacketCoords(packet)
 	closer := t.getCloser(coords)
 	if len(closer) == 0 {
@@ -674,11 +674,10 @@ func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]time.Time) bo
 		return true
 	}
 	var best *closerInfo
-	var bestTime time.Time
 	ports := t.core.peers.getPorts()
 	for _, cinfo := range closer {
 		to := ports[cinfo.elem.port]
-		thisTime, isIdle := idle[cinfo.elem.port]
+		_, isIdle := idle[cinfo.elem.port]
 		var update bool
 		switch {
 		case to == nil:
@@ -704,22 +703,12 @@ func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]time.Time) bo
 		case cinfo.elem.time.Before(best.elem.time):
 			// same tstamp, but got it earlier, so presumably a better path
 			update = true
-		case cinfo.elem.time.After(best.elem.time):
-			// same tstamp, but got it later, so presumably a worse path
-			// I do not expect the remaining cases to ever be reached... TODO cleanup
-		case thisTime.After(bestTime):
-			// all else equal, this port was used more recently than our current
-			// candidate, so choose that instead. this should mean that, in low
-			// traffic scenarios, we consistently pick the same link which helps with
-			// packet ordering
-			update = true
 		default:
 			// the search for a port has finished
 		}
 		if update {
 			b := cinfo // because cinfo gets mutated by the iteration
 			best = &b
-			bestTime = thisTime
 		}
 	}
 	if best != nil {
@@ -882,6 +871,6 @@ func (t *switchTable) _idleIn(port switchPort) {
 	// Try to find something to send to this peer
 	if !t._handleIdle(port) {
 		// Didn't find anything ready to send yet, so stay idle
-		t.idle[port] = time.Now()
+		t.idle[port] = struct{}{}
 	}
 }

From 8ca11874517b032c446f434792ae0e41a75263df Mon Sep 17 00:00:00 2001
From: William Fleurant <william@netblazr.com>
Date: Wed, 11 Sep 2019 06:52:03 -0400
Subject: [PATCH 085/130] README: update platforms link

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ec7fe74c..07b202f1 100644
--- a/README.md
+++ b/README.md
@@ -37,7 +37,7 @@ some of the below:
 - NetBSD
 - OpenWrt
 
-Please see our [Platforms](https://yggdrasil-network.github.io/) pages for more
+Please see our [Platforms](https://yggdrasil-network.github.io/platforms.html) pages for more
 specific information about each of our supported platforms, including
 installation steps and caveats.
 

From 80ba24d51255c3751e2b25aceee52b20d59ff746 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Tue, 17 Sep 2019 19:42:07 -0500
Subject: [PATCH 086/130] force things to buffer in the switch if the best link
 is currently busy. note that other links can end up sending if they become
 non-idle for other reasons. this is a temporary workaround to packet
 reordering, until we can figure out a better solution

---
 src/yggdrasil/switch.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index f4df60d5..4a1999a5 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -677,13 +677,10 @@ func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]struct{}) boo
 	ports := t.core.peers.getPorts()
 	for _, cinfo := range closer {
 		to := ports[cinfo.elem.port]
-		_, isIdle := idle[cinfo.elem.port]
 		var update bool
 		switch {
 		case to == nil:
 			// no port was found, ignore it
-		case !isIdle:
-			// the port is busy, ignore it
 		case best == nil:
 			// this is the first idle port we've found, so select it until we find a
 			// better candidate port to use instead
@@ -713,6 +710,9 @@ func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]struct{}) boo
 	}
 	if best != nil {
 		// Send to the best idle next hop
+		if _, isIdle := idle[best.elem.port]; !isIdle {
+			return false
+		}
 		delete(idle, best.elem.port)
 		ports[best.elem.port].sendPacketsFrom(t, [][]byte{packet})
 		return true

From be35675d0f01aa5e22571f9c71ed36ba4a87b8ba Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 18 Sep 2019 13:37:01 +0100
Subject: [PATCH 087/130] Catch a nil pointer when sending a session packet to
 a conn, this shouldn't happen but it's caused multiple crashes in
 conn.recvMsg

---
 src/yggdrasil/session.go | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 8a6d16fc..49808623 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -470,7 +470,14 @@ func (sinfo *sessionInfo) _recvPacket(p *wire_trafficPacket) {
 		callback := func() {
 			util.PutBytes(p.Payload)
 			if !isOK || k != sinfo.sharedSesKey || !sinfo._nonceIsOK(&p.Nonce) {
-				// Either we failed to decrypt, or the session was updated, or we received this packet in the mean time
+				// Either we failed to decrypt, or the session was updated, or we
+				// received this packet in the mean time
+				util.PutBytes(bs)
+				return
+			}
+			if sinfo.conn == nil {
+				// There's no connection associated with this session for some reason
+				// TODO: Figure out why this happens sometimes, it shouldn't
 				util.PutBytes(bs)
 				return
 			}

From 40204caab6c9a29a2d56ba9c18f8c75ce4850bed Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 18 Sep 2019 14:03:31 +0100
Subject: [PATCH 088/130] Try to fix race condition in sessions.reset

---
 src/yggdrasil/session.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 8a6d16fc..778fa2a9 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -414,11 +414,10 @@ func (sinfo *sessionInfo) _updateNonce(theirNonce *crypto.BoxNonce) {
 
 // Resets all sessions to an uninitialized state.
 // Called after coord changes, so attemtps to use a session will trigger a new ping and notify the remote end of the coord change.
+// Only call this from the router actor.
 func (ss *sessions) reset() {
 	for _, sinfo := range ss.sinfos {
-		sinfo.Act(ss.router, func() {
-			sinfo.reset = true
-		})
+		sinfo.reset = true
 	}
 }
 

From c3016e680c77e1dfb018d22422c7d6621f04505a Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 18 Sep 2019 14:05:18 +0100
Subject: [PATCH 089/130] Fix panic where slice goes out of bounds because
 iface.Read returns less than zero (which might happen when the TUN/TAP
 interface is closed)

---
 src/tuntap/iface.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tuntap/iface.go b/src/tuntap/iface.go
index 92ba36ab..753a8d02 100644
--- a/src/tuntap/iface.go
+++ b/src/tuntap/iface.go
@@ -111,7 +111,7 @@ func (r *tunReader) _read() {
 	recvd := util.ResizeBytes(util.GetBytes(), 65535+tun_ETHER_HEADER_LENGTH)
 	// Wait for a packet to be delivered to us through the TUN/TAP adapter
 	n, err := r.tun.iface.Read(recvd)
-	if n == 0 {
+	if n <= 0 {
 		util.PutBytes(recvd)
 	} else {
 		r.tun.handlePacketFrom(r, recvd[:n], err)

From e9bacda0b328a67871b3b66c9328778348b1a268 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 18 Sep 2019 13:37:01 +0100
Subject: [PATCH 090/130] Catch a nil pointer when sending a session packet to
 a conn, this shouldn't happen but it's caused multiple crashes in
 conn.recvMsg

---
 src/yggdrasil/session.go | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 778fa2a9..0b3947e2 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -469,7 +469,14 @@ func (sinfo *sessionInfo) _recvPacket(p *wire_trafficPacket) {
 		callback := func() {
 			util.PutBytes(p.Payload)
 			if !isOK || k != sinfo.sharedSesKey || !sinfo._nonceIsOK(&p.Nonce) {
-				// Either we failed to decrypt, or the session was updated, or we received this packet in the mean time
+				// Either we failed to decrypt, or the session was updated, or we
+				// received this packet in the mean time
+				util.PutBytes(bs)
+				return
+			}
+			if sinfo.conn == nil {
+				// There's no connection associated with this session for some reason
+				// TODO: Figure out why this happens sometimes, it shouldn't
 				util.PutBytes(bs)
 				return
 			}

From 200b3623b21cbf05e1254a1ca122d301166e3e71 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 18 Sep 2019 14:32:28 +0100
Subject: [PATCH 091/130] Fix #539

---
 src/yggdrasil/tcp.go | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go
index cce352bd..5ac921c4 100644
--- a/src/yggdrasil/tcp.go
+++ b/src/yggdrasil/tcp.go
@@ -249,7 +249,7 @@ func (t *tcp) call(saddr string, options interface{}, sintf string) {
 			if err != nil {
 				return
 			}
-			t.handler(conn, false, dialerdst.String())
+			t.handler(conn, false, saddr)
 		} else {
 			dst, err := net.ResolveTCPAddr("tcp", saddr)
 			if err != nil {
@@ -322,18 +322,19 @@ func (t *tcp) handler(sock net.Conn, incoming bool, options interface{}) {
 	t.setExtraOptions(sock)
 	stream := stream{}
 	stream.init(sock)
-	local, _, _ := net.SplitHostPort(sock.LocalAddr().String())
-	remote, _, _ := net.SplitHostPort(sock.RemoteAddr().String())
-	force := net.ParseIP(strings.Split(remote, "%")[0]).IsLinkLocalUnicast()
-	var name string
-	var proto string
+	var name, proto, local, remote string
 	if socksaddr, issocks := options.(string); issocks {
-		name = "socks://" + socksaddr + "/" + sock.RemoteAddr().String()
+		name = "socks://" + sock.RemoteAddr().String() + "/" + socksaddr
 		proto = "socks"
+		local, _, _ = net.SplitHostPort(sock.LocalAddr().String())
+		remote, _, _ = net.SplitHostPort(socksaddr)
 	} else {
 		name = "tcp://" + sock.RemoteAddr().String()
 		proto = "tcp"
+		local, _, _ = net.SplitHostPort(sock.LocalAddr().String())
+		remote, _, _ = net.SplitHostPort(sock.RemoteAddr().String())
 	}
+	force := net.ParseIP(strings.Split(remote, "%")[0]).IsLinkLocalUnicast()
 	link, err := t.link.core.link.create(&stream, name, proto, local, remote, incoming, force)
 	if err != nil {
 		t.link.core.log.Println(err)

From 27158d7b44993fea0ba2bff666c4ef8192aa99b1 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 18 Sep 2019 14:35:11 +0100
Subject: [PATCH 092/130] Fix #509

---
 build | 2 --
 1 file changed, 2 deletions(-)

diff --git a/build b/build
index 7ca5f4fa..d11f991d 100755
--- a/build
+++ b/build
@@ -1,7 +1,5 @@
 #!/bin/sh
 
-set -ef
-
 PKGSRC=${PKGSRC:-github.com/yggdrasil-network/yggdrasil-go/src/version}
 PKGNAME=${PKGNAME:-$(sh contrib/semver/name.sh)}
 PKGVER=${PKGVER:-$(sh contrib/semver/version.sh --bare)}

From a62e029e21eadf1b40fd0c0931727e97c6c5999b Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 18 Sep 2019 14:37:25 +0100
Subject: [PATCH 093/130] Update apt before trying to pull in RPM dependencies

---
 .circleci/config.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index f9cd0720..5778ffc9 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -29,6 +29,7 @@ jobs:
       - run:
           name: Install RPM utilities
           command: |
+              sudo apt-get update
               sudo apt-get install -y rpm file
               mkdir -p ~/rpmbuild/BUILD ~/rpmbuild/RPMS ~/rpmbuild/SOURCES ~/rpmbuild/SPECS ~/rpmbuild/SRPMS
 

From 846df4789a0f3e74c13a6f2c6cf302e761e46ad5 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 18 Sep 2019 15:01:19 +0100
Subject: [PATCH 094/130] Be more verbose when a peer or listener is badly
 formatted

---
 src/yggdrasil/core.go | 14 ++++++++++----
 src/yggdrasil/link.go |  4 ++--
 src/yggdrasil/tcp.go  |  3 +++
 3 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index 754d7d64..4dcd16fd 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -91,15 +91,21 @@ func (c *Core) _addPeerLoop() {
 
 	// Add peers from the Peers section
 	for _, peer := range current.Peers {
-		go c.AddPeer(peer, "") // TODO: this should be acted and not in a goroutine?
-		time.Sleep(time.Second)
+		go func() {
+			if err := c.AddPeer(peer, ""); err != nil {
+				c.log.Errorln("Failed to add peer:", err)
+			}
+		}() // TODO: this should be acted and not in a goroutine?
 	}
 
 	// Add peers from the InterfacePeers section
 	for intf, intfpeers := range current.InterfacePeers {
 		for _, peer := range intfpeers {
-			go c.AddPeer(peer, intf) // TODO: this should be acted and not in a goroutine?
-			time.Sleep(time.Second)
+			go func() {
+				if err := c.AddPeer(peer, intf); err != nil {
+					c.log.Errorln("Failed to add peer:", err)
+				}
+			}() // TODO: this should be acted and not in a goroutine?
 		}
 	}
 
diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index 6e393514..bdf15547 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -86,7 +86,7 @@ func (l *link) reconfigure() {
 func (l *link) call(uri string, sintf string) error {
 	u, err := url.Parse(uri)
 	if err != nil {
-		return err
+		return fmt.Errorf("peer %s is not correctly formatted (%s)", uri, err)
 	}
 	pathtokens := strings.Split(strings.Trim(u.Path, "/"), "/")
 	switch u.Scheme {
@@ -103,7 +103,7 @@ func (l *link) call(uri string, sintf string) error {
 func (l *link) listen(uri string) error {
 	u, err := url.Parse(uri)
 	if err != nil {
-		return err
+		return fmt.Errorf("listener %s is not correctly formatted (%s)", uri, err)
 	}
 	switch u.Scheme {
 	case "tcp":
diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go
index 5ac921c4..93e39e40 100644
--- a/src/yggdrasil/tcp.go
+++ b/src/yggdrasil/tcp.go
@@ -85,6 +85,7 @@ func (t *tcp) init(l *link) error {
 	defer t.link.core.config.Mutex.RUnlock()
 	for _, listenaddr := range t.link.core.config.Current.Listen {
 		if listenaddr[:6] != "tcp://" {
+			t.link.core.log.Errorln("Failed to add listener: listener", listenaddr, "is not correctly formatted, ignoring")
 			continue
 		}
 		if _, err := t.listen(listenaddr[6:]); err != nil {
@@ -103,6 +104,7 @@ func (t *tcp) reconfigure() {
 	if len(added) > 0 || len(deleted) > 0 {
 		for _, a := range added {
 			if a[:6] != "tcp://" {
+				t.link.core.log.Errorln("Failed to add listener: listener", a, "is not correctly formatted, ignoring")
 				continue
 			}
 			if _, err := t.listen(a[6:]); err != nil {
@@ -113,6 +115,7 @@ func (t *tcp) reconfigure() {
 		}
 		for _, d := range deleted {
 			if d[:6] != "tcp://" {
+				t.link.core.log.Errorln("Failed to delete listener: listener", d, "is not correctly formatted, ignoring")
 				continue
 			}
 			t.mutex.Lock()

From 00a972b74ed838c2d7b8f634ec316a39eabbac10 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 18 Sep 2019 15:22:17 +0100
Subject: [PATCH 095/130] Disconnect peers when stopping, stop modules before
 core

---
 cmd/yggdrasil/main.go |  2 +-
 src/yggdrasil/core.go | 31 +++++++++++++++++++------------
 2 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/cmd/yggdrasil/main.go b/cmd/yggdrasil/main.go
index e9a21c13..3b7c9220 100644
--- a/cmd/yggdrasil/main.go
+++ b/cmd/yggdrasil/main.go
@@ -292,10 +292,10 @@ exit:
 }
 
 func (n *node) shutdown() {
-	n.core.Stop()
 	n.admin.Stop()
 	n.multicast.Stop()
 	n.tuntap.Stop()
+	n.core.Stop()
 	os.Exit(0)
 }
 
diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index 4dcd16fd..2857de1d 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -21,16 +21,17 @@ type Core struct {
 	// We're going to keep our own copy of the provided config - that way we can
 	// guarantee that it will be covered by the mutex
 	phony.Inbox
-	config      config.NodeState // Config
-	boxPub      crypto.BoxPubKey
-	boxPriv     crypto.BoxPrivKey
-	sigPub      crypto.SigPubKey
-	sigPriv     crypto.SigPrivKey
-	switchTable switchTable
-	peers       peers
-	router      router
-	link        link
-	log         *log.Logger
+	config       config.NodeState // Config
+	boxPub       crypto.BoxPubKey
+	boxPriv      crypto.BoxPrivKey
+	sigPub       crypto.SigPubKey
+	sigPriv      crypto.SigPrivKey
+	switchTable  switchTable
+	peers        peers
+	router       router
+	link         link
+	log          *log.Logger
+	addPeerTimer *time.Timer
 }
 
 func (c *Core) _init() error {
@@ -110,7 +111,7 @@ func (c *Core) _addPeerLoop() {
 	}
 
 	// Sit for a while
-	time.AfterFunc(time.Minute, func() {
+	c.addPeerTimer = time.AfterFunc(time.Minute, func() {
 		c.Act(c, c._addPeerLoop)
 	})
 }
@@ -177,7 +178,9 @@ func (c *Core) _start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState
 		return nil, err
 	}
 
-	c.Act(c, c._addPeerLoop)
+	c.addPeerTimer = time.AfterFunc(time.Second, func() {
+		c.Act(c, c._addPeerLoop)
+	})
 
 	c.log.Infoln("Startup complete")
 	return &c.config, nil
@@ -191,4 +194,8 @@ func (c *Core) Stop() {
 // This function is unsafe and should only be ran by the core actor.
 func (c *Core) _stop() {
 	c.log.Infoln("Stopping...")
+	c.addPeerTimer.Stop()
+	for _, peer := range c.GetPeers() {
+		c.DisconnectPeer(peer.Port)
+	}
 }

From 366fe7e772376a51b7937bc451f63cd7c6aa899a Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 18 Sep 2019 15:31:43 +0100
Subject: [PATCH 096/130] Allow multicast to be shut down more sanely

---
 src/multicast/multicast.go        | 212 +++++++++++++++---------------
 src/multicast/multicast_darwin.go |  19 +--
 2 files changed, 113 insertions(+), 118 deletions(-)

diff --git a/src/multicast/multicast.go b/src/multicast/multicast.go
index 22ac9356..2102a4d9 100644
--- a/src/multicast/multicast.go
+++ b/src/multicast/multicast.go
@@ -19,14 +19,16 @@ import (
 // configured multicast interface, Yggdrasil will attempt to peer with that node
 // automatically.
 type Multicast struct {
-	core       *yggdrasil.Core
-	config     *config.NodeState
-	log        *log.Logger
-	sock       *ipv6.PacketConn
-	groupAddr  string
-	listeners  map[string]*yggdrasil.TcpListener
-	listenPort uint16
-	isOpen     bool
+	core            *yggdrasil.Core
+	config          *config.NodeState
+	log             *log.Logger
+	sock            *ipv6.PacketConn
+	groupAddr       string
+	listeners       map[string]*yggdrasil.TcpListener
+	listenPort      uint16
+	isOpen          bool
+	announcer       *time.Timer
+	platformhandler *time.Timer
 }
 
 // Init prepares the multicast interface for use.
@@ -63,9 +65,9 @@ func (m *Multicast) Start() error {
 	}
 
 	m.isOpen = true
-	go m.multicastStarted()
 	go m.listen()
-	go m.announce()
+	m.multicastStarted()
+	m.announce()
 
 	return nil
 }
@@ -73,6 +75,8 @@ func (m *Multicast) Start() error {
 // Stop is not implemented for multicast yet.
 func (m *Multicast) Stop() error {
 	m.isOpen = false
+	m.announcer.Stop()
+	m.platformhandler.Stop()
 	m.sock.Close()
 	return nil
 }
@@ -136,108 +140,106 @@ func (m *Multicast) announce() {
 	if err != nil {
 		panic(err)
 	}
-	for {
-		interfaces := m.Interfaces()
-		// There might be interfaces that we configured listeners for but are no
-		// longer up - if that's the case then we should stop the listeners
-		for name, listener := range m.listeners {
-			// Prepare our stop function!
-			stop := func() {
-				listener.Stop <- true
-				delete(m.listeners, name)
-				m.log.Debugln("No longer multicasting on", name)
-			}
-			// If the interface is no longer visible on the system then stop the
-			// listener, as another one will be started further down
-			if _, ok := interfaces[name]; !ok {
-				stop()
-				continue
-			}
-			// It's possible that the link-local listener address has changed so if
-			// that is the case then we should clean up the interface listener
-			found := false
-			listenaddr, err := net.ResolveTCPAddr("tcp6", listener.Listener.Addr().String())
-			if err != nil {
-				stop()
-				continue
-			}
-			// Find the interface that matches the listener
-			if intf, err := net.InterfaceByName(name); err == nil {
-				if addrs, err := intf.Addrs(); err == nil {
-					// Loop through the addresses attached to that listener and see if any
-					// of them match the current address of the listener
-					for _, addr := range addrs {
-						if ip, _, err := net.ParseCIDR(addr.String()); err == nil {
-							// Does the interface address match our listener address?
-							if ip.Equal(listenaddr.IP) {
-								found = true
-								break
-							}
+	interfaces := m.Interfaces()
+	// There might be interfaces that we configured listeners for but are no
+	// longer up - if that's the case then we should stop the listeners
+	for name, listener := range m.listeners {
+		// Prepare our stop function!
+		stop := func() {
+			listener.Stop <- true
+			delete(m.listeners, name)
+			m.log.Debugln("No longer multicasting on", name)
+		}
+		// If the interface is no longer visible on the system then stop the
+		// listener, as another one will be started further down
+		if _, ok := interfaces[name]; !ok {
+			stop()
+			continue
+		}
+		// It's possible that the link-local listener address has changed so if
+		// that is the case then we should clean up the interface listener
+		found := false
+		listenaddr, err := net.ResolveTCPAddr("tcp6", listener.Listener.Addr().String())
+		if err != nil {
+			stop()
+			continue
+		}
+		// Find the interface that matches the listener
+		if intf, err := net.InterfaceByName(name); err == nil {
+			if addrs, err := intf.Addrs(); err == nil {
+				// Loop through the addresses attached to that listener and see if any
+				// of them match the current address of the listener
+				for _, addr := range addrs {
+					if ip, _, err := net.ParseCIDR(addr.String()); err == nil {
+						// Does the interface address match our listener address?
+						if ip.Equal(listenaddr.IP) {
+							found = true
+							break
 						}
 					}
 				}
 			}
-			// If the address has not been found on the adapter then we should stop
-			// and clean up the TCP listener. A new one will be created below if a
-			// suitable link-local address is found
-			if !found {
-				stop()
-			}
 		}
-		// Now that we have a list of valid interfaces from the operating system,
-		// we can start checking if we can send multicasts on them
-		for _, iface := range interfaces {
-			// Find interface addresses
-			addrs, err := iface.Addrs()
-			if err != nil {
-				panic(err)
-			}
-			for _, addr := range addrs {
-				addrIP, _, _ := net.ParseCIDR(addr.String())
-				// Ignore IPv4 addresses
-				if addrIP.To4() != nil {
-					continue
-				}
-				// Ignore non-link-local addresses
-				if !addrIP.IsLinkLocalUnicast() {
-					continue
-				}
-				// Join the multicast group
-				m.sock.JoinGroup(&iface, groupAddr)
-				// Try and see if we already have a TCP listener for this interface
-				var listener *yggdrasil.TcpListener
-				if l, ok := m.listeners[iface.Name]; !ok || l.Listener == nil {
-					// No listener was found - let's create one
-					listenaddr := fmt.Sprintf("[%s%%%s]:%d", addrIP, iface.Name, m.listenPort)
-					if li, err := m.core.ListenTCP(listenaddr); err == nil {
-						m.log.Debugln("Started multicasting on", iface.Name)
-						// Store the listener so that we can stop it later if needed
-						m.listeners[iface.Name] = li
-						listener = li
-					} else {
-						m.log.Warnln("Not multicasting on", iface.Name, "due to error:", err)
-					}
-				} else {
-					// An existing listener was found
-					listener = m.listeners[iface.Name]
-				}
-				// Make sure nothing above failed for some reason
-				if listener == nil {
-					continue
-				}
-				// Get the listener details and construct the multicast beacon
-				lladdr := listener.Listener.Addr().String()
-				if a, err := net.ResolveTCPAddr("tcp6", lladdr); err == nil {
-					a.Zone = ""
-					destAddr.Zone = iface.Name
-					msg := []byte(a.String())
-					m.sock.WriteTo(msg, nil, destAddr)
-				}
-				break
-			}
+		// If the address has not been found on the adapter then we should stop
+		// and clean up the TCP listener. A new one will be created below if a
+		// suitable link-local address is found
+		if !found {
+			stop()
 		}
-		time.Sleep(time.Second * 15)
 	}
+	// Now that we have a list of valid interfaces from the operating system,
+	// we can start checking if we can send multicasts on them
+	for _, iface := range interfaces {
+		// Find interface addresses
+		addrs, err := iface.Addrs()
+		if err != nil {
+			panic(err)
+		}
+		for _, addr := range addrs {
+			addrIP, _, _ := net.ParseCIDR(addr.String())
+			// Ignore IPv4 addresses
+			if addrIP.To4() != nil {
+				continue
+			}
+			// Ignore non-link-local addresses
+			if !addrIP.IsLinkLocalUnicast() {
+				continue
+			}
+			// Join the multicast group
+			m.sock.JoinGroup(&iface, groupAddr)
+			// Try and see if we already have a TCP listener for this interface
+			var listener *yggdrasil.TcpListener
+			if l, ok := m.listeners[iface.Name]; !ok || l.Listener == nil {
+				// No listener was found - let's create one
+				listenaddr := fmt.Sprintf("[%s%%%s]:%d", addrIP, iface.Name, m.listenPort)
+				if li, err := m.core.ListenTCP(listenaddr); err == nil {
+					m.log.Debugln("Started multicasting on", iface.Name)
+					// Store the listener so that we can stop it later if needed
+					m.listeners[iface.Name] = li
+					listener = li
+				} else {
+					m.log.Warnln("Not multicasting on", iface.Name, "due to error:", err)
+				}
+			} else {
+				// An existing listener was found
+				listener = m.listeners[iface.Name]
+			}
+			// Make sure nothing above failed for some reason
+			if listener == nil {
+				continue
+			}
+			// Get the listener details and construct the multicast beacon
+			lladdr := listener.Listener.Addr().String()
+			if a, err := net.ResolveTCPAddr("tcp6", lladdr); err == nil {
+				a.Zone = ""
+				destAddr.Zone = iface.Name
+				msg := []byte(a.String())
+				m.sock.WriteTo(msg, nil, destAddr)
+			}
+			break
+		}
+	}
+	m.announcer = time.AfterFunc(time.Second*15, m.announce)
 }
 
 func (m *Multicast) listen() {
diff --git a/src/multicast/multicast_darwin.go b/src/multicast/multicast_darwin.go
index c88b4a81..6fdccb2b 100644
--- a/src/multicast/multicast_darwin.go
+++ b/src/multicast/multicast_darwin.go
@@ -32,21 +32,14 @@ import (
 var awdlGoroutineStarted bool
 
 func (m *Multicast) multicastStarted() {
-	if awdlGoroutineStarted {
-		return
-	}
-	awdlGoroutineStarted = true
-	for {
-		C.StopAWDLBrowsing()
-		for intf := range m.Interfaces() {
-			if intf == "awdl0" {
-				m.log.Infoln("Multicast discovery is using AWDL discovery")
-				C.StartAWDLBrowsing()
-				break
-			}
+	C.StopAWDLBrowsing()
+	for intf := range m.Interfaces() {
+		if intf == "awdl0" {
+			C.StartAWDLBrowsing()
+			break
 		}
-		time.Sleep(time.Minute)
 	}
+	m.platformhandler = time.AfterFunc(time.Minute, m.multicastStarted)
 }
 
 func (m *Multicast) multicastReuse(network string, address string, c syscall.RawConn) error {

From c78a4cb28fa1fe572adb0770539633b02ee41674 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 18 Sep 2019 15:34:26 +0100
Subject: [PATCH 097/130] Only stop timers if they are running

---
 src/multicast/multicast.go | 8 ++++++--
 src/yggdrasil/core.go      | 9 ++++-----
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/src/multicast/multicast.go b/src/multicast/multicast.go
index 2102a4d9..08b7180e 100644
--- a/src/multicast/multicast.go
+++ b/src/multicast/multicast.go
@@ -75,8 +75,12 @@ func (m *Multicast) Start() error {
 // Stop is not implemented for multicast yet.
 func (m *Multicast) Stop() error {
 	m.isOpen = false
-	m.announcer.Stop()
-	m.platformhandler.Stop()
+	if m.announcer != nil {
+		m.announcer.Stop()
+	}
+	if m.platformhandler != nil {
+		m.platformhandler.Stop()
+	}
 	m.sock.Close()
 	return nil
 }
diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index 2857de1d..4cdcd8e4 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -110,7 +110,6 @@ func (c *Core) _addPeerLoop() {
 		}
 	}
 
-	// Sit for a while
 	c.addPeerTimer = time.AfterFunc(time.Minute, func() {
 		c.Act(c, c._addPeerLoop)
 	})
@@ -178,9 +177,7 @@ func (c *Core) _start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState
 		return nil, err
 	}
 
-	c.addPeerTimer = time.AfterFunc(time.Second, func() {
-		c.Act(c, c._addPeerLoop)
-	})
+	c.Act(c, c._addPeerLoop)
 
 	c.log.Infoln("Startup complete")
 	return &c.config, nil
@@ -194,7 +191,9 @@ func (c *Core) Stop() {
 // This function is unsafe and should only be ran by the core actor.
 func (c *Core) _stop() {
 	c.log.Infoln("Stopping...")
-	c.addPeerTimer.Stop()
+	if c.addPeerTimer != nil {
+		c.addPeerTimer.Stop()
+	}
 	for _, peer := range c.GetPeers() {
 		c.DisconnectPeer(peer.Port)
 	}

From b0df9e2f31020373b62d15600410672fe3a2a34c Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 18 Sep 2019 16:15:33 +0100
Subject: [PATCH 098/130] Fix race when adding peers

---
 src/yggdrasil/core.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index 4cdcd8e4..598d8121 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -92,21 +92,21 @@ func (c *Core) _addPeerLoop() {
 
 	// Add peers from the Peers section
 	for _, peer := range current.Peers {
-		go func() {
-			if err := c.AddPeer(peer, ""); err != nil {
+		go func(peer, intf string) {
+			if err := c.AddPeer(peer, intf); err != nil {
 				c.log.Errorln("Failed to add peer:", err)
 			}
-		}() // TODO: this should be acted and not in a goroutine?
+		}(peer, "") // TODO: this should be acted and not in a goroutine?
 	}
 
 	// Add peers from the InterfacePeers section
 	for intf, intfpeers := range current.InterfacePeers {
 		for _, peer := range intfpeers {
-			go func() {
+			go func(peer, intf string) {
 				if err := c.AddPeer(peer, intf); err != nil {
 					c.log.Errorln("Failed to add peer:", err)
 				}
-			}() // TODO: this should be acted and not in a goroutine?
+			}(peer, intf) // TODO: this should be acted and not in a goroutine?
 		}
 	}
 

From b959f53fee55783a9b2c821efb08b67eca064fb3 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 18 Sep 2019 16:32:22 +0100
Subject: [PATCH 099/130] Shut down listeners when stopping

---
 src/yggdrasil/core.go | 2 ++
 src/yggdrasil/link.go | 7 +++++++
 src/yggdrasil/tcp.go  | 7 +++++++
 3 files changed, 16 insertions(+)

diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index 598d8121..cdc8ad44 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -194,7 +194,9 @@ func (c *Core) _stop() {
 	if c.addPeerTimer != nil {
 		c.addPeerTimer.Stop()
 	}
+	c.link.stop()
 	for _, peer := range c.GetPeers() {
 		c.DisconnectPeer(peer.Port)
 	}
+	c.log.Infoln("Stopped")
 }
diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index bdf15547..df73cc4c 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -134,6 +134,13 @@ func (l *link) create(msgIO linkInterfaceMsgIO, name, linkType, local, remote st
 	return &intf, nil
 }
 
+func (l *link) stop() error {
+	if err := l.tcp.stop(); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (intf *linkInterface) handler() error {
 	// TODO split some of this into shorter functions, so it's easier to read, and for the FIXME duplicate peer issue mentioned later
 	myLinkPub, myLinkPriv := crypto.NewBoxKeys()
diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go
index 93e39e40..a8c23623 100644
--- a/src/yggdrasil/tcp.go
+++ b/src/yggdrasil/tcp.go
@@ -96,6 +96,13 @@ func (t *tcp) init(l *link) error {
 	return nil
 }
 
+func (t *tcp) stop() error {
+	for _, listener := range t.listeners {
+		close(listener.Stop)
+	}
+	return nil
+}
+
 func (t *tcp) reconfigure() {
 	t.link.core.config.Mutex.RLock()
 	added := util.Difference(t.link.core.config.Current.Listen, t.link.core.config.Previous.Listen)

From 2dc136f94a64fdce5a1c08c04c51ba66b24374f0 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 18 Sep 2019 16:51:46 +0100
Subject: [PATCH 100/130] Multicast actor to prevent races

---
 src/multicast/multicast.go        | 10 +++++++---
 src/multicast/multicast_darwin.go |  4 +++-
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/multicast/multicast.go b/src/multicast/multicast.go
index 08b7180e..63156d82 100644
--- a/src/multicast/multicast.go
+++ b/src/multicast/multicast.go
@@ -7,6 +7,7 @@ import (
 	"regexp"
 	"time"
 
+	"github.com/Arceliar/phony"
 	"github.com/gologme/log"
 
 	"github.com/yggdrasil-network/yggdrasil-go/src/config"
@@ -19,6 +20,7 @@ import (
 // configured multicast interface, Yggdrasil will attempt to peer with that node
 // automatically.
 type Multicast struct {
+	phony.Inbox
 	core            *yggdrasil.Core
 	config          *config.NodeState
 	log             *log.Logger
@@ -66,8 +68,8 @@ func (m *Multicast) Start() error {
 
 	m.isOpen = true
 	go m.listen()
-	m.multicastStarted()
-	m.announce()
+	m.Act(m, m.multicastStarted)
+	m.Act(m, m.announce)
 
 	return nil
 }
@@ -243,7 +245,9 @@ func (m *Multicast) announce() {
 			break
 		}
 	}
-	m.announcer = time.AfterFunc(time.Second*15, m.announce)
+	m.announcer = time.AfterFunc(time.Second*15, func() {
+		m.Act(m, m.announce)
+	})
 }
 
 func (m *Multicast) listen() {
diff --git a/src/multicast/multicast_darwin.go b/src/multicast/multicast_darwin.go
index 6fdccb2b..4cfef9e9 100644
--- a/src/multicast/multicast_darwin.go
+++ b/src/multicast/multicast_darwin.go
@@ -39,7 +39,9 @@ func (m *Multicast) multicastStarted() {
 			break
 		}
 	}
-	m.platformhandler = time.AfterFunc(time.Minute, m.multicastStarted)
+	m.platformhandler = time.AfterFunc(time.Minute, func() {
+		m.Act(m, m.multicastStarted)
+	})
 }
 
 func (m *Multicast) multicastReuse(network string, address string, c syscall.RawConn) error {

From ae0b2672ff819dbea4a556525f9972556ca51768 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 18 Sep 2019 14:32:28 +0100
Subject: [PATCH 101/130] Fix #539

---
 src/yggdrasil/tcp.go | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go
index cce352bd..5ac921c4 100644
--- a/src/yggdrasil/tcp.go
+++ b/src/yggdrasil/tcp.go
@@ -249,7 +249,7 @@ func (t *tcp) call(saddr string, options interface{}, sintf string) {
 			if err != nil {
 				return
 			}
-			t.handler(conn, false, dialerdst.String())
+			t.handler(conn, false, saddr)
 		} else {
 			dst, err := net.ResolveTCPAddr("tcp", saddr)
 			if err != nil {
@@ -322,18 +322,19 @@ func (t *tcp) handler(sock net.Conn, incoming bool, options interface{}) {
 	t.setExtraOptions(sock)
 	stream := stream{}
 	stream.init(sock)
-	local, _, _ := net.SplitHostPort(sock.LocalAddr().String())
-	remote, _, _ := net.SplitHostPort(sock.RemoteAddr().String())
-	force := net.ParseIP(strings.Split(remote, "%")[0]).IsLinkLocalUnicast()
-	var name string
-	var proto string
+	var name, proto, local, remote string
 	if socksaddr, issocks := options.(string); issocks {
-		name = "socks://" + socksaddr + "/" + sock.RemoteAddr().String()
+		name = "socks://" + sock.RemoteAddr().String() + "/" + socksaddr
 		proto = "socks"
+		local, _, _ = net.SplitHostPort(sock.LocalAddr().String())
+		remote, _, _ = net.SplitHostPort(socksaddr)
 	} else {
 		name = "tcp://" + sock.RemoteAddr().String()
 		proto = "tcp"
+		local, _, _ = net.SplitHostPort(sock.LocalAddr().String())
+		remote, _, _ = net.SplitHostPort(sock.RemoteAddr().String())
 	}
+	force := net.ParseIP(strings.Split(remote, "%")[0]).IsLinkLocalUnicast()
 	link, err := t.link.core.link.create(&stream, name, proto, local, remote, incoming, force)
 	if err != nil {
 		t.link.core.log.Println(err)

From 368f499f1df53df85a1fc9ede1f662635b32f3e2 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 18 Sep 2019 14:37:25 +0100
Subject: [PATCH 102/130] Update apt before trying to pull in RPM dependencies

---
 .circleci/config.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index f9cd0720..5778ffc9 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -29,6 +29,7 @@ jobs:
       - run:
           name: Install RPM utilities
           command: |
+              sudo apt-get update
               sudo apt-get install -y rpm file
               mkdir -p ~/rpmbuild/BUILD ~/rpmbuild/RPMS ~/rpmbuild/SOURCES ~/rpmbuild/SPECS ~/rpmbuild/SRPMS
 

From 94cf2854a93575db75aed221ec4ecb24a609a85b Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 18 Sep 2019 14:05:18 +0100
Subject: [PATCH 103/130] Fix panic where slice goes out of bounds because
 iface.Read returns less than zero (which might happen when the TUN/TAP
 interface is closed)

---
 src/tuntap/iface.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tuntap/iface.go b/src/tuntap/iface.go
index 92ba36ab..753a8d02 100644
--- a/src/tuntap/iface.go
+++ b/src/tuntap/iface.go
@@ -111,7 +111,7 @@ func (r *tunReader) _read() {
 	recvd := util.ResizeBytes(util.GetBytes(), 65535+tun_ETHER_HEADER_LENGTH)
 	// Wait for a packet to be delivered to us through the TUN/TAP adapter
 	n, err := r.tun.iface.Read(recvd)
-	if n == 0 {
+	if n <= 0 {
 		util.PutBytes(recvd)
 	} else {
 		r.tun.handlePacketFrom(r, recvd[:n], err)

From ddaaa865cb264b375ff836b91549be4768569fe7 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 18 Sep 2019 15:01:19 +0100
Subject: [PATCH 104/130] Be more verbose when a peer or listener is badly
 formatted

---
 src/yggdrasil/core.go | 14 ++++++++++----
 src/yggdrasil/link.go |  4 ++--
 src/yggdrasil/tcp.go  |  3 +++
 3 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index 754d7d64..4dcd16fd 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -91,15 +91,21 @@ func (c *Core) _addPeerLoop() {
 
 	// Add peers from the Peers section
 	for _, peer := range current.Peers {
-		go c.AddPeer(peer, "") // TODO: this should be acted and not in a goroutine?
-		time.Sleep(time.Second)
+		go func() {
+			if err := c.AddPeer(peer, ""); err != nil {
+				c.log.Errorln("Failed to add peer:", err)
+			}
+		}() // TODO: this should be acted and not in a goroutine?
 	}
 
 	// Add peers from the InterfacePeers section
 	for intf, intfpeers := range current.InterfacePeers {
 		for _, peer := range intfpeers {
-			go c.AddPeer(peer, intf) // TODO: this should be acted and not in a goroutine?
-			time.Sleep(time.Second)
+			go func() {
+				if err := c.AddPeer(peer, intf); err != nil {
+					c.log.Errorln("Failed to add peer:", err)
+				}
+			}() // TODO: this should be acted and not in a goroutine?
 		}
 	}
 
diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index 6e393514..bdf15547 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -86,7 +86,7 @@ func (l *link) reconfigure() {
 func (l *link) call(uri string, sintf string) error {
 	u, err := url.Parse(uri)
 	if err != nil {
-		return err
+		return fmt.Errorf("peer %s is not correctly formatted (%s)", uri, err)
 	}
 	pathtokens := strings.Split(strings.Trim(u.Path, "/"), "/")
 	switch u.Scheme {
@@ -103,7 +103,7 @@ func (l *link) call(uri string, sintf string) error {
 func (l *link) listen(uri string) error {
 	u, err := url.Parse(uri)
 	if err != nil {
-		return err
+		return fmt.Errorf("listener %s is not correctly formatted (%s)", uri, err)
 	}
 	switch u.Scheme {
 	case "tcp":
diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go
index 5ac921c4..93e39e40 100644
--- a/src/yggdrasil/tcp.go
+++ b/src/yggdrasil/tcp.go
@@ -85,6 +85,7 @@ func (t *tcp) init(l *link) error {
 	defer t.link.core.config.Mutex.RUnlock()
 	for _, listenaddr := range t.link.core.config.Current.Listen {
 		if listenaddr[:6] != "tcp://" {
+			t.link.core.log.Errorln("Failed to add listener: listener", listenaddr, "is not correctly formatted, ignoring")
 			continue
 		}
 		if _, err := t.listen(listenaddr[6:]); err != nil {
@@ -103,6 +104,7 @@ func (t *tcp) reconfigure() {
 	if len(added) > 0 || len(deleted) > 0 {
 		for _, a := range added {
 			if a[:6] != "tcp://" {
+				t.link.core.log.Errorln("Failed to add listener: listener", a, "is not correctly formatted, ignoring")
 				continue
 			}
 			if _, err := t.listen(a[6:]); err != nil {
@@ -113,6 +115,7 @@ func (t *tcp) reconfigure() {
 		}
 		for _, d := range deleted {
 			if d[:6] != "tcp://" {
+				t.link.core.log.Errorln("Failed to delete listener: listener", d, "is not correctly formatted, ignoring")
 				continue
 			}
 			t.mutex.Lock()

From d44a7faa04375190f1b72c9f55c8f1a721df2319 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 18 Sep 2019 20:09:53 +0100
Subject: [PATCH 105/130] semver: Don't return failure codes when git history
 is not present

---
 contrib/semver/name.sh    | 2 +-
 contrib/semver/version.sh | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/contrib/semver/name.sh b/contrib/semver/name.sh
index 1fa2ce07..308e07bc 100644
--- a/contrib/semver/name.sh
+++ b/contrib/semver/name.sh
@@ -6,7 +6,7 @@ BRANCH=$(git symbolic-ref --short HEAD 2>/dev/null)
 # Complain if the git history is not available
 if [ $? != 0 ] || [ -z "$BRANCH" ]; then
   printf "yggdrasil"
-  exit 1
+  exit 0
 fi
 
 # Remove "/" characters from the branch name if present
diff --git a/contrib/semver/version.sh b/contrib/semver/version.sh
index 3052094a..db96e339 100644
--- a/contrib/semver/version.sh
+++ b/contrib/semver/version.sh
@@ -6,7 +6,7 @@ TAG=$(git describe --abbrev=0 --tags --match="v[0-9]*\.[0-9]*\.[0-9]*" 2>/dev/nu
 # Did getting the tag succeed?
 if [ $? != 0 ] || [ -z "$TAG" ]; then
   printf -- "unknown"
-  exit 1
+  exit 0
 fi
 
 # Get the current branch
@@ -36,7 +36,7 @@ if [ "$BRANCH" != "master" ]; then
   # Did getting the count of commits since the tag succeed?
   if [ $? != 0 ] || [ -z "$BUILD" ]; then
     printf -- "-unknown"
-    exit 1
+    exit 0
   fi
 
   # Is the build greater than zero?

From 0a12e4b1c1d50c910defbf0607ccb86165c16791 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 18 Sep 2019 20:26:06 +0100
Subject: [PATCH 106/130] Revert "Catch a nil pointer when sending a session
 packet to a conn, this shouldn't happen but it's caused multiple crashes in
 conn.recvMsg"

This reverts commit be35675d0f01aa5e22571f9c71ed36ba4a87b8ba.
---
 src/yggdrasil/session.go | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 49808623..8a6d16fc 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -470,14 +470,7 @@ func (sinfo *sessionInfo) _recvPacket(p *wire_trafficPacket) {
 		callback := func() {
 			util.PutBytes(p.Payload)
 			if !isOK || k != sinfo.sharedSesKey || !sinfo._nonceIsOK(&p.Nonce) {
-				// Either we failed to decrypt, or the session was updated, or we
-				// received this packet in the mean time
-				util.PutBytes(bs)
-				return
-			}
-			if sinfo.conn == nil {
-				// There's no connection associated with this session for some reason
-				// TODO: Figure out why this happens sometimes, it shouldn't
+				// Either we failed to decrypt, or the session was updated, or we received this packet in the mean time
 				util.PutBytes(bs)
 				return
 			}

From 909e4e29a8f1de08b33afdd1e97c731a21211b37 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 18 Sep 2019 23:44:28 +0100
Subject: [PATCH 107/130] Don't spawn goroutines for addPeerLoop, TCP connect
 timeout of 5 seconds for now

---
 src/yggdrasil/core.go | 16 ++++++----------
 src/yggdrasil/link.go |  2 +-
 src/yggdrasil/tcp.go  |  1 +
 3 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index 4dcd16fd..884ef9e0 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -91,21 +91,17 @@ func (c *Core) _addPeerLoop() {
 
 	// Add peers from the Peers section
 	for _, peer := range current.Peers {
-		go func() {
-			if err := c.AddPeer(peer, ""); err != nil {
-				c.log.Errorln("Failed to add peer:", err)
-			}
-		}() // TODO: this should be acted and not in a goroutine?
+		if err := c.AddPeer(peer, ""); err != nil {
+			c.log.Errorln("Failed to add peer:", err)
+		}
 	}
 
 	// Add peers from the InterfacePeers section
 	for intf, intfpeers := range current.InterfacePeers {
 		for _, peer := range intfpeers {
-			go func() {
-				if err := c.AddPeer(peer, intf); err != nil {
-					c.log.Errorln("Failed to add peer:", err)
-				}
-			}() // TODO: this should be acted and not in a goroutine?
+			if err := c.AddPeer(peer, intf); err != nil {
+				c.log.Errorln("Failed to add peer:", err)
+			}
 		}
 	}
 
diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index bdf15547..3686ab9c 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -86,7 +86,7 @@ func (l *link) reconfigure() {
 func (l *link) call(uri string, sintf string) error {
 	u, err := url.Parse(uri)
 	if err != nil {
-		return fmt.Errorf("peer %s is not correctly formatted (%s)", uri, err)
+		return fmt.Errorf("peer %s is not correctly formatted", uri)
 	}
 	pathtokens := strings.Split(strings.Trim(u.Path, "/"), "/")
 	switch u.Scheme {
diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go
index 93e39e40..f81e49ad 100644
--- a/src/yggdrasil/tcp.go
+++ b/src/yggdrasil/tcp.go
@@ -266,6 +266,7 @@ func (t *tcp) call(saddr string, options interface{}, sintf string) {
 			}
 			dialer := net.Dialer{
 				Control: t.tcpContext,
+				Timeout: time.Second * 5,
 			}
 			if sintf != "" {
 				ief, err := net.InterfaceByName(sintf)

From 2d64a6380ab26f91080267254d167ec2ba6074b5 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Wed, 18 Sep 2019 18:33:51 -0500
Subject: [PATCH 108/130] misc other fixes

---
 src/yggdrasil/conn.go    |  3 ++-
 src/yggdrasil/session.go | 13 +++++--------
 src/yggdrasil/tcp.go     | 10 ++++------
 3 files changed, 11 insertions(+), 15 deletions(-)

diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index cb28a6f2..9af5d240 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -102,11 +102,12 @@ func (c *Conn) search() error {
 					if sinfo != nil {
 						// Need to clean up to avoid a session leak
 						sinfo.cancel.Cancel(nil)
+						sinfo.sessions.removeSession(sinfo)
 					}
 				default:
 					if sinfo != nil {
 						// Finish initializing the session
-						sinfo.conn = c
+						sinfo.setConn(nil, c)
 					}
 					c.session = sinfo
 					err = e
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 0b3947e2..2f5e0af3 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -416,8 +416,11 @@ func (sinfo *sessionInfo) _updateNonce(theirNonce *crypto.BoxNonce) {
 // Called after coord changes, so attemtps to use a session will trigger a new ping and notify the remote end of the coord change.
 // Only call this from the router actor.
 func (ss *sessions) reset() {
-	for _, sinfo := range ss.sinfos {
-		sinfo.reset = true
+	for _, _sinfo := range ss.sinfos {
+		sinfo := _sinfo // So we can safely put it in a closure
+		sinfo.Act(ss.router, func() {
+			sinfo.reset = true
+		})
 	}
 }
 
@@ -474,12 +477,6 @@ func (sinfo *sessionInfo) _recvPacket(p *wire_trafficPacket) {
 				util.PutBytes(bs)
 				return
 			}
-			if sinfo.conn == nil {
-				// There's no connection associated with this session for some reason
-				// TODO: Figure out why this happens sometimes, it shouldn't
-				util.PutBytes(bs)
-				return
-			}
 			sinfo._updateNonce(&p.Nonce)
 			sinfo.bytesRecvd += uint64(len(bs))
 			sinfo.conn.recvMsg(sinfo, bs)
diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go
index a8c23623..ffda6b76 100644
--- a/src/yggdrasil/tcp.go
+++ b/src/yggdrasil/tcp.go
@@ -207,11 +207,12 @@ func (t *tcp) listener(l *TcpListener, listenaddr string) {
 }
 
 // Checks if we already are calling this address
-func (t *tcp) isAlreadyCalling(saddr string) bool {
+func (t *tcp) startCalling(saddr string) bool {
 	t.mutex.Lock()
 	defer t.mutex.Unlock()
 	_, isIn := t.calls[saddr]
-	return isIn
+	t.calls[saddr] = struct{}{}
+	return !isIn
 }
 
 // Checks if a connection already exists.
@@ -225,12 +226,9 @@ func (t *tcp) call(saddr string, options interface{}, sintf string) {
 		if sintf != "" {
 			callname = fmt.Sprintf("%s/%s", saddr, sintf)
 		}
-		if t.isAlreadyCalling(callname) {
+		if !t.startCalling(callname) {
 			return
 		}
-		t.mutex.Lock()
-		t.calls[callname] = struct{}{}
-		t.mutex.Unlock()
 		defer func() {
 			// Block new calls for a little while, to mitigate livelock scenarios
 			time.Sleep(default_timeout)

From 995d67cca8fda11457a28e22cd5380656935c16f Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Wed, 18 Sep 2019 18:46:03 -0500
Subject: [PATCH 109/130] fix leak in _addPeerLoop

---
 src/yggdrasil/core.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index cdc8ad44..98a5c6e1 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -93,7 +93,7 @@ func (c *Core) _addPeerLoop() {
 	// Add peers from the Peers section
 	for _, peer := range current.Peers {
 		go func(peer, intf string) {
-			if err := c.AddPeer(peer, intf); err != nil {
+			if err := c.CallPeer(peer, intf); err != nil {
 				c.log.Errorln("Failed to add peer:", err)
 			}
 		}(peer, "") // TODO: this should be acted and not in a goroutine?
@@ -103,7 +103,7 @@ func (c *Core) _addPeerLoop() {
 	for intf, intfpeers := range current.InterfacePeers {
 		for _, peer := range intfpeers {
 			go func(peer, intf string) {
-				if err := c.AddPeer(peer, intf); err != nil {
+				if err := c.CallPeer(peer, intf); err != nil {
 					c.log.Errorln("Failed to add peer:", err)
 				}
 			}(peer, intf) // TODO: this should be acted and not in a goroutine?
@@ -111,7 +111,7 @@ func (c *Core) _addPeerLoop() {
 	}
 
 	c.addPeerTimer = time.AfterFunc(time.Minute, func() {
-		c.Act(c, c._addPeerLoop)
+		c.Act(nil, c._addPeerLoop)
 	})
 }
 

From 5a382e7e0bf2c86195a689419c7a7ab186c53f6e Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 19 Sep 2019 08:55:55 +0100
Subject: [PATCH 110/130] Cherrypick fixes for _addPeerLoop memory leak for now

---
 src/yggdrasil/api.go  | 23 ++++++++++++++++++++++-
 src/yggdrasil/core.go | 28 +++++++++++++++-------------
 2 files changed, 37 insertions(+), 14 deletions(-)

diff --git a/src/yggdrasil/api.go b/src/yggdrasil/api.go
index d1753b69..b54d7a11 100644
--- a/src/yggdrasil/api.go
+++ b/src/yggdrasil/api.go
@@ -368,13 +368,34 @@ func (c *Core) SetLogger(log *log.Logger) {
 // connection drops.
 func (c *Core) AddPeer(addr string, sintf string) error {
 	if err := c.CallPeer(addr, sintf); err != nil {
+		// TODO: We maybe want this to write the peer to the persistent
+		// configuration even if a connection attempt fails, but first we'll need to
+		// move the code to check the peer URI so that we don't deliberately save a
+		// peer with a known bad URI. Loading peers from config should really do the
+		// same thing too but I don't think that happens today
 		return err
 	}
 	c.config.Mutex.Lock()
 	if sintf == "" {
+		for _, peer := range c.config.Current.Peers {
+			if peer == addr {
+				return errors.New("peer already added")
+			}
+		}
 		c.config.Current.Peers = append(c.config.Current.Peers, addr)
 	} else {
-		c.config.Current.InterfacePeers[sintf] = append(c.config.Current.InterfacePeers[sintf], addr)
+		if _, ok := c.config.Current.InterfacePeers[sintf]; ok {
+			for _, peer := range c.config.Current.InterfacePeers[sintf] {
+				if peer == addr {
+					return errors.New("peer already added")
+				}
+			}
+		}
+		if _, ok := c.config.Current.InterfacePeers[sintf]; !ok {
+			c.config.Current.InterfacePeers[sintf] = []string{addr}
+		} else {
+			c.config.Current.InterfacePeers[sintf] = append(c.config.Current.InterfacePeers[sintf], addr)
+		}
 	}
 	c.config.Mutex.Unlock()
 	return nil
diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index 884ef9e0..907db699 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -21,16 +21,17 @@ type Core struct {
 	// We're going to keep our own copy of the provided config - that way we can
 	// guarantee that it will be covered by the mutex
 	phony.Inbox
-	config      config.NodeState // Config
-	boxPub      crypto.BoxPubKey
-	boxPriv     crypto.BoxPrivKey
-	sigPub      crypto.SigPubKey
-	sigPriv     crypto.SigPrivKey
-	switchTable switchTable
-	peers       peers
-	router      router
-	link        link
-	log         *log.Logger
+	config       config.NodeState // Config
+	boxPub       crypto.BoxPubKey
+	boxPriv      crypto.BoxPrivKey
+	sigPub       crypto.SigPubKey
+	sigPriv      crypto.SigPrivKey
+	switchTable  switchTable
+	peers        peers
+	router       router
+	link         link
+	log          *log.Logger
+	addPeerTimer *time.Timer
 }
 
 func (c *Core) _init() error {
@@ -91,7 +92,7 @@ func (c *Core) _addPeerLoop() {
 
 	// Add peers from the Peers section
 	for _, peer := range current.Peers {
-		if err := c.AddPeer(peer, ""); err != nil {
+		if err := c.CallPeer(peer, ""); err != nil {
 			c.log.Errorln("Failed to add peer:", err)
 		}
 	}
@@ -99,14 +100,14 @@ func (c *Core) _addPeerLoop() {
 	// Add peers from the InterfacePeers section
 	for intf, intfpeers := range current.InterfacePeers {
 		for _, peer := range intfpeers {
-			if err := c.AddPeer(peer, intf); err != nil {
+			if err := c.CallPeer(peer, intf); err != nil {
 				c.log.Errorln("Failed to add peer:", err)
 			}
 		}
 	}
 
 	// Sit for a while
-	time.AfterFunc(time.Minute, func() {
+	c.addPeerTimer = time.AfterFunc(time.Minute, func() {
 		c.Act(c, c._addPeerLoop)
 	})
 }
@@ -187,4 +188,5 @@ func (c *Core) Stop() {
 // This function is unsafe and should only be ran by the core actor.
 func (c *Core) _stop() {
 	c.log.Infoln("Stopping...")
+	c.addPeerTimer.Stop()
 }

From 7b1678a11d6261e4adc9c7c62489bf644f75a3fa Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 19 Sep 2019 09:04:25 +0100
Subject: [PATCH 111/130] Goroutines in _addPeerLoop from bugfixes

---
 src/yggdrasil/core.go | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index 907db699..42910aa7 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -92,17 +92,21 @@ func (c *Core) _addPeerLoop() {
 
 	// Add peers from the Peers section
 	for _, peer := range current.Peers {
-		if err := c.CallPeer(peer, ""); err != nil {
-			c.log.Errorln("Failed to add peer:", err)
-		}
+		go func(peer, intf string) {
+			if err := c.CallPeer(peer, intf); err != nil {
+				c.log.Errorln("Failed to add peer:", err)
+			}
+		}(peer, "")
 	}
 
 	// Add peers from the InterfacePeers section
 	for intf, intfpeers := range current.InterfacePeers {
 		for _, peer := range intfpeers {
-			if err := c.CallPeer(peer, intf); err != nil {
-				c.log.Errorln("Failed to add peer:", err)
-			}
+			go func(peer, intf string) {
+				if err := c.CallPeer(peer, intf); err != nil {
+					c.log.Errorln("Failed to add peer:", err)
+				}
+			}(peer, intf)
 		}
 	}
 

From 39461cb60363d8d1c0b5139a4a54e2c9b3986439 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 19 Sep 2019 09:56:27 +0100
Subject: [PATCH 112/130] Don't os.Exit

---
 build                 | 2 ++
 cmd/yggdrasil/main.go | 1 -
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/build b/build
index d11f991d..7ca5f4fa 100755
--- a/build
+++ b/build
@@ -1,5 +1,7 @@
 #!/bin/sh
 
+set -ef
+
 PKGSRC=${PKGSRC:-github.com/yggdrasil-network/yggdrasil-go/src/version}
 PKGNAME=${PKGNAME:-$(sh contrib/semver/name.sh)}
 PKGVER=${PKGVER:-$(sh contrib/semver/version.sh --bare)}
diff --git a/cmd/yggdrasil/main.go b/cmd/yggdrasil/main.go
index 3b7c9220..33a8769d 100644
--- a/cmd/yggdrasil/main.go
+++ b/cmd/yggdrasil/main.go
@@ -296,7 +296,6 @@ func (n *node) shutdown() {
 	n.multicast.Stop()
 	n.tuntap.Stop()
 	n.core.Stop()
-	os.Exit(0)
 }
 
 func (n *node) sessionFirewall(pubkey *crypto.BoxPubKey, initiator bool) bool {

From 93e81867fdd986f2812d72923d28348bc7bf9e8a Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Thu, 19 Sep 2019 19:15:59 -0500
Subject: [PATCH 113/130] have link.stop signal active links to close, have
 tcp.stop wait for all listeners and active connections to close

---
 src/yggdrasil/link.go | 14 ++++++++++++++
 src/yggdrasil/tcp.go  | 11 +++++++++++
 2 files changed, 25 insertions(+)

diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index df73cc4c..a4a41e7f 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -25,6 +25,7 @@ type link struct {
 	mutex      sync.RWMutex // protects interfaces below
 	interfaces map[linkInfo]*linkInterface
 	tcp        tcp // TCP interface support
+	stopped    chan struct{}
 	// TODO timeout (to remove from switch), read from config.ReadTimeout
 }
 
@@ -70,6 +71,7 @@ func (l *link) init(c *Core) error {
 	l.mutex.Lock()
 	l.interfaces = make(map[linkInfo]*linkInterface)
 	l.mutex.Unlock()
+	l.stopped = make(chan struct{})
 
 	if err := l.tcp.init(l); err != nil {
 		c.log.Errorln("Failed to start TCP interface")
@@ -135,6 +137,7 @@ func (l *link) create(msgIO linkInterfaceMsgIO, name, linkType, local, remote st
 }
 
 func (l *link) stop() error {
+	close(l.stopped)
 	if err := l.tcp.stop(); err != nil {
 		return err
 	}
@@ -231,7 +234,18 @@ func (intf *linkInterface) handler() error {
 	go intf.peer.start()
 	intf.reader.Act(nil, intf.reader._read)
 	// Wait for the reader to finish
+	// TODO find a way to do this without keeping live goroutines around
+	done := make(chan struct{})
+	defer close(done)
+	go func() {
+		select {
+		case <-intf.link.stopped:
+			intf.msgIO.close()
+		case <-done:
+		}
+	}()
 	err = <-intf.reader.err
+	// TODO don't report an error if it's just a 'use of closed network connection'
 	if err != nil {
 		intf.link.core.log.Infof("Disconnected %s: %s, source %s; error: %s",
 			strings.ToUpper(intf.info.linkType), themString, intf.info.local, err)
diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go
index 8389ecc6..36d80589 100644
--- a/src/yggdrasil/tcp.go
+++ b/src/yggdrasil/tcp.go
@@ -34,6 +34,7 @@ const tcp_ping_interval = (default_timeout * 2 / 3)
 // The TCP listener and information about active TCP connections, to avoid duplication.
 type tcp struct {
 	link      *link
+	waitgroup sync.WaitGroup
 	mutex     sync.Mutex // Protecting the below
 	listeners map[string]*TcpListener
 	calls     map[string]struct{}
@@ -97,9 +98,12 @@ func (t *tcp) init(l *link) error {
 }
 
 func (t *tcp) stop() error {
+	t.mutex.Lock()
 	for _, listener := range t.listeners {
 		close(listener.Stop)
 	}
+	t.mutex.Unlock()
+	t.waitgroup.Wait()
 	return nil
 }
 
@@ -150,6 +154,7 @@ func (t *tcp) listen(listenaddr string) (*TcpListener, error) {
 			Listener: listener,
 			Stop:     make(chan bool),
 		}
+		t.waitgroup.Add(1)
 		go t.listener(&l, listenaddr)
 		return &l, nil
 	}
@@ -159,6 +164,7 @@ func (t *tcp) listen(listenaddr string) (*TcpListener, error) {
 
 // Runs the listener, which spawns off goroutines for incoming connections.
 func (t *tcp) listener(l *TcpListener, listenaddr string) {
+	defer t.waitgroup.Done()
 	if l == nil {
 		return
 	}
@@ -199,8 +205,10 @@ func (t *tcp) listener(l *TcpListener, listenaddr string) {
 				t.link.core.log.Errorln("Failed to accept connection:", err)
 				return
 			}
+			t.waitgroup.Add(1)
 			go t.handler(sock, true, nil)
 		case <-l.Stop:
+			// FIXME this races with the goroutine that Accepts a TCP connection, may leak connections when a listener is removed
 			return
 		}
 	}
@@ -257,6 +265,7 @@ func (t *tcp) call(saddr string, options interface{}, sintf string) {
 			if err != nil {
 				return
 			}
+			t.waitgroup.Add(1)
 			t.handler(conn, false, saddr)
 		} else {
 			dst, err := net.ResolveTCPAddr("tcp", saddr)
@@ -321,12 +330,14 @@ func (t *tcp) call(saddr string, options interface{}, sintf string) {
 				t.link.core.log.Debugln("Failed to dial TCP:", err)
 				return
 			}
+			t.waitgroup.Add(1)
 			t.handler(conn, false, nil)
 		}
 	}()
 }
 
 func (t *tcp) handler(sock net.Conn, incoming bool, options interface{}) {
+	defer t.waitgroup.Done() // Happens after sock.close
 	defer sock.Close()
 	t.setExtraOptions(sock)
 	stream := stream{}

From eeb34ce4e4f0cdab87b68d44d6caa524f30993d4 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Thu, 19 Sep 2019 19:45:17 -0500
Subject: [PATCH 114/130] modify TcpListener

---
 src/multicast/multicast.go |  2 +-
 src/yggdrasil/tcp.go       | 15 ++++++++++-----
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/multicast/multicast.go b/src/multicast/multicast.go
index 63156d82..7f2f5915 100644
--- a/src/multicast/multicast.go
+++ b/src/multicast/multicast.go
@@ -152,7 +152,7 @@ func (m *Multicast) announce() {
 	for name, listener := range m.listeners {
 		// Prepare our stop function!
 		stop := func() {
-			listener.Stop <- true
+			listener.Stop()
 			delete(m.listeners, name)
 			m.log.Debugln("No longer multicasting on", name)
 		}
diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go
index 36d80589..01185e54 100644
--- a/src/yggdrasil/tcp.go
+++ b/src/yggdrasil/tcp.go
@@ -47,7 +47,12 @@ type tcp struct {
 // multicast interfaces.
 type TcpListener struct {
 	Listener net.Listener
-	Stop     chan bool
+	stop     chan struct{}
+}
+
+func (l *TcpListener) Stop() {
+	defer func() { recover() }()
+	close(l.stop)
 }
 
 // Wrapper function to set additional options for specific connection types.
@@ -100,7 +105,7 @@ func (t *tcp) init(l *link) error {
 func (t *tcp) stop() error {
 	t.mutex.Lock()
 	for _, listener := range t.listeners {
-		close(listener.Stop)
+		listener.Stop()
 	}
 	t.mutex.Unlock()
 	t.waitgroup.Wait()
@@ -132,7 +137,7 @@ func (t *tcp) reconfigure() {
 			t.mutex.Lock()
 			if listener, ok := t.listeners[d[6:]]; ok {
 				t.mutex.Unlock()
-				listener.Stop <- true
+				listener.Stop()
 				t.link.core.log.Infoln("Stopped TCP listener:", d[6:])
 			} else {
 				t.mutex.Unlock()
@@ -152,7 +157,7 @@ func (t *tcp) listen(listenaddr string) (*TcpListener, error) {
 	if err == nil {
 		l := TcpListener{
 			Listener: listener,
-			Stop:     make(chan bool),
+			stop:     make(chan struct{}),
 		}
 		t.waitgroup.Add(1)
 		go t.listener(&l, listenaddr)
@@ -207,7 +212,7 @@ func (t *tcp) listener(l *TcpListener, listenaddr string) {
 			}
 			t.waitgroup.Add(1)
 			go t.handler(sock, true, nil)
-		case <-l.Stop:
+		case <-l.stop:
 			// FIXME this races with the goroutine that Accepts a TCP connection, may leak connections when a listener is removed
 			return
 		}

From f9163a56b64e90a7085afc20e9123027960b7042 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Thu, 19 Sep 2019 19:50:45 -0500
Subject: [PATCH 115/130] fix race between listener accepting and shutting down

---
 src/yggdrasil/tcp.go | 31 ++++++++++---------------------
 1 file changed, 10 insertions(+), 21 deletions(-)

diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go
index 01185e54..ed8f7b9b 100644
--- a/src/yggdrasil/tcp.go
+++ b/src/yggdrasil/tcp.go
@@ -184,7 +184,6 @@ func (t *tcp) listener(l *TcpListener, listenaddr string) {
 		t.mutex.Unlock()
 	}
 	// And here we go!
-	accepted := make(chan bool)
 	defer func() {
 		t.link.core.log.Infoln("Stopping TCP listener on:", l.Listener.Addr().String())
 		l.Listener.Close()
@@ -193,29 +192,19 @@ func (t *tcp) listener(l *TcpListener, listenaddr string) {
 		t.mutex.Unlock()
 	}()
 	t.link.core.log.Infoln("Listening for TCP on:", l.Listener.Addr().String())
+	go func() {
+		<-l.stop
+		l.Listener.Close()
+	}()
+	defer l.Stop()
 	for {
-		var sock net.Conn
-		var err error
-		// Listen in a separate goroutine, as that way it does not block us from
-		// receiving "stop" events
-		go func() {
-			sock, err = l.Listener.Accept()
-			accepted <- true
-		}()
-		// Wait for either an accepted connection, or a message telling us to stop
-		// the TCP listener
-		select {
-		case <-accepted:
-			if err != nil {
-				t.link.core.log.Errorln("Failed to accept connection:", err)
-				return
-			}
-			t.waitgroup.Add(1)
-			go t.handler(sock, true, nil)
-		case <-l.stop:
-			// FIXME this races with the goroutine that Accepts a TCP connection, may leak connections when a listener is removed
+		sock, err := l.Listener.Accept()
+		if err != nil {
+			t.link.core.log.Errorln("Failed to accept connection:", err)
 			return
 		}
+		t.waitgroup.Add(1)
+		go t.handler(sock, true, nil)
 	}
 }
 

From 1cd4b6e8ddc8d72b67bb6fadf3896f2246e57425 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 20 Sep 2019 10:08:41 +0100
Subject: [PATCH 116/130] Increase multicast interval at startup from 1s to 15s

---
 src/multicast/multicast.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/multicast/multicast.go b/src/multicast/multicast.go
index 7f2f5915..cc7b9af5 100644
--- a/src/multicast/multicast.go
+++ b/src/multicast/multicast.go
@@ -29,6 +29,7 @@ type Multicast struct {
 	listeners       map[string]*yggdrasil.TcpListener
 	listenPort      uint16
 	isOpen          bool
+	interval        time.Duration
 	announcer       *time.Timer
 	platformhandler *time.Timer
 }
@@ -42,6 +43,7 @@ func (m *Multicast) Init(core *yggdrasil.Core, state *config.NodeState, log *log
 	current := m.config.GetCurrent()
 	m.listenPort = current.LinkLocalTCPPort
 	m.groupAddr = "[ff02::114]:9001"
+	m.interval = time.Second
 	return nil
 }
 
@@ -245,9 +247,12 @@ func (m *Multicast) announce() {
 			break
 		}
 	}
-	m.announcer = time.AfterFunc(time.Second*15, func() {
+	m.announcer = time.AfterFunc(m.interval, func() {
 		m.Act(m, m.announce)
 	})
+	if m.interval.Seconds() < 15 {
+		m.interval += time.Second
+	}
 }
 
 func (m *Multicast) listen() {

From 8003ea0f3ef61ac7c60651a5cef361d3d21804e8 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 20 Sep 2019 17:42:42 -0500
Subject: [PATCH 117/130] use a separate multicast beacon interval per
 multicast interface

---
 src/multicast/multicast.go | 43 ++++++++++++++++++++++----------------
 1 file changed, 25 insertions(+), 18 deletions(-)

diff --git a/src/multicast/multicast.go b/src/multicast/multicast.go
index cc7b9af5..7044eaa2 100644
--- a/src/multicast/multicast.go
+++ b/src/multicast/multicast.go
@@ -26,24 +26,28 @@ type Multicast struct {
 	log             *log.Logger
 	sock            *ipv6.PacketConn
 	groupAddr       string
-	listeners       map[string]*yggdrasil.TcpListener
+	listeners       map[string]*listenerInfo
 	listenPort      uint16
 	isOpen          bool
-	interval        time.Duration
 	announcer       *time.Timer
 	platformhandler *time.Timer
 }
 
+type listenerInfo struct {
+	listener *yggdrasil.TcpListener
+	time     time.Time
+	interval time.Duration
+}
+
 // Init prepares the multicast interface for use.
 func (m *Multicast) Init(core *yggdrasil.Core, state *config.NodeState, log *log.Logger, options interface{}) error {
 	m.core = core
 	m.config = state
 	m.log = log
-	m.listeners = make(map[string]*yggdrasil.TcpListener)
+	m.listeners = make(map[string]*listenerInfo)
 	current := m.config.GetCurrent()
 	m.listenPort = current.LinkLocalTCPPort
 	m.groupAddr = "[ff02::114]:9001"
-	m.interval = time.Second
 	return nil
 }
 
@@ -151,10 +155,10 @@ func (m *Multicast) announce() {
 	interfaces := m.Interfaces()
 	// There might be interfaces that we configured listeners for but are no
 	// longer up - if that's the case then we should stop the listeners
-	for name, listener := range m.listeners {
+	for name, info := range m.listeners {
 		// Prepare our stop function!
 		stop := func() {
-			listener.Stop()
+			info.listener.Stop()
 			delete(m.listeners, name)
 			m.log.Debugln("No longer multicasting on", name)
 		}
@@ -167,7 +171,7 @@ func (m *Multicast) announce() {
 		// It's possible that the link-local listener address has changed so if
 		// that is the case then we should clean up the interface listener
 		found := false
-		listenaddr, err := net.ResolveTCPAddr("tcp6", listener.Listener.Addr().String())
+		listenaddr, err := net.ResolveTCPAddr("tcp6", info.listener.Listener.Addr().String())
 		if err != nil {
 			stop()
 			continue
@@ -216,43 +220,46 @@ func (m *Multicast) announce() {
 			// Join the multicast group
 			m.sock.JoinGroup(&iface, groupAddr)
 			// Try and see if we already have a TCP listener for this interface
-			var listener *yggdrasil.TcpListener
-			if l, ok := m.listeners[iface.Name]; !ok || l.Listener == nil {
+			var info *listenerInfo
+			if nfo, ok := m.listeners[iface.Name]; !ok || nfo.listener.Listener == nil {
 				// No listener was found - let's create one
 				listenaddr := fmt.Sprintf("[%s%%%s]:%d", addrIP, iface.Name, m.listenPort)
 				if li, err := m.core.ListenTCP(listenaddr); err == nil {
 					m.log.Debugln("Started multicasting on", iface.Name)
 					// Store the listener so that we can stop it later if needed
-					m.listeners[iface.Name] = li
-					listener = li
+					info = &listenerInfo{listener: li, time: time.Now()}
+					m.listeners[iface.Name] = info
 				} else {
 					m.log.Warnln("Not multicasting on", iface.Name, "due to error:", err)
 				}
 			} else {
 				// An existing listener was found
-				listener = m.listeners[iface.Name]
+				info = m.listeners[iface.Name]
 			}
 			// Make sure nothing above failed for some reason
-			if listener == nil {
+			if info == nil {
+				continue
+			}
+			if time.Since(info.time) < info.interval {
 				continue
 			}
 			// Get the listener details and construct the multicast beacon
-			lladdr := listener.Listener.Addr().String()
+			lladdr := info.listener.Listener.Addr().String()
 			if a, err := net.ResolveTCPAddr("tcp6", lladdr); err == nil {
 				a.Zone = ""
 				destAddr.Zone = iface.Name
 				msg := []byte(a.String())
 				m.sock.WriteTo(msg, nil, destAddr)
 			}
+			if info.interval.Seconds() < 15 {
+				info.interval += time.Second
+			}
 			break
 		}
 	}
-	m.announcer = time.AfterFunc(m.interval, func() {
+	m.announcer = time.AfterFunc(time.Second, func() {
 		m.Act(m, m.announce)
 	})
-	if m.interval.Seconds() < 15 {
-		m.interval += time.Second
-	}
 }
 
 func (m *Multicast) listen() {

From 87658f83e941fa574eaa7c0ea5a828480b6310b6 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 20 Sep 2019 23:09:12 -0500
Subject: [PATCH 118/130] Revert "force things to buffer in the switch if the
 best link is currently busy. note that other links can end up sending if they
 become non-idle for other reasons. this is a temporary workaround to packet
 reordering, until we can figure out a better solution"

This reverts commit 80ba24d51255c3751e2b25aceee52b20d59ff746.
---
 src/yggdrasil/switch.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 4a1999a5..f4df60d5 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -677,10 +677,13 @@ func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]struct{}) boo
 	ports := t.core.peers.getPorts()
 	for _, cinfo := range closer {
 		to := ports[cinfo.elem.port]
+		_, isIdle := idle[cinfo.elem.port]
 		var update bool
 		switch {
 		case to == nil:
 			// no port was found, ignore it
+		case !isIdle:
+			// the port is busy, ignore it
 		case best == nil:
 			// this is the first idle port we've found, so select it until we find a
 			// better candidate port to use instead
@@ -710,9 +713,6 @@ func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]struct{}) boo
 	}
 	if best != nil {
 		// Send to the best idle next hop
-		if _, isIdle := idle[best.elem.port]; !isIdle {
-			return false
-		}
 		delete(idle, best.elem.port)
 		ports[best.elem.port].sendPacketsFrom(t, [][]byte{packet})
 		return true

From 691192ff5ae5a3f012f5f616327645b01e4c1462 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 21 Sep 2019 14:33:45 -0500
Subject: [PATCH 119/130] weird scheduler hack, seems to tend to make things
 more stable without actually locking streams to any particular link

---
 src/yggdrasil/switch.go | 11 ++++++++---
 src/yggdrasil/tcp.go    |  5 +++--
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index f4df60d5..ece2fa26 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -712,10 +712,15 @@ func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]struct{}) boo
 		}
 	}
 	if best != nil {
-		// Send to the best idle next hop
 		delete(idle, best.elem.port)
-		ports[best.elem.port].sendPacketsFrom(t, [][]byte{packet})
-		return true
+		// Tell ourselves to send to this node later
+		// If another (e.g. even better) hop becomes idle in the mean time, it'll take the packet instead
+		// FIXME this is just a hack, but seems to help with stability...
+		go t.Act(nil, func() {
+			t._idleIn(best.elem.port)
+		})
+		//ports[best.elem.port].sendPacketsFrom(t, [][]byte{packet})
+		//return true
 	}
 	// Didn't find anyone idle to send it to
 	return false
diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go
index ed8f7b9b..66f708c2 100644
--- a/src/yggdrasil/tcp.go
+++ b/src/yggdrasil/tcp.go
@@ -233,8 +233,9 @@ func (t *tcp) call(saddr string, options interface{}, sintf string) {
 		}
 		defer func() {
 			// Block new calls for a little while, to mitigate livelock scenarios
-			time.Sleep(default_timeout)
-			time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond)
+			rand.Seed(time.Now().UnixNano())
+			delay := default_timeout + time.Duration(rand.Intn(10000))*time.Millisecond
+			time.Sleep(delay)
 			t.mutex.Lock()
 			delete(t.calls, callname)
 			t.mutex.Unlock()

From 606d9ac97baec8018dd5e21289f32d03dad52202 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 24 Sep 2019 22:06:12 +0100
Subject: [PATCH 120/130] Build VyOS amd64/i386 Vyatta packages as well as
 EdgeRouter packages

---
 .circleci/config.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 5778ffc9..1456332a 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -68,13 +68,15 @@ jobs:
               find ~/rpmbuild/SRPMS/ -name '*.rpm' -exec mv {} /tmp/upload \;
 
       - run:
-          name: Build for EdgeRouter
+          name: Build for EdgeRouter and VyOS
           command: |
               rm -f {yggdrasil,yggdrasilctl}
               git clone https://github.com/neilalexander/vyatta-yggdrasil /tmp/vyatta-yggdrasil;
               cd /tmp/vyatta-yggdrasil;
               BUILDDIR_YGG=$CIRCLE_WORKING_DIRECTORY ./build-edgerouter-x $CIRCLE_BRANCH;
               BUILDDIR_YGG=$CIRCLE_WORKING_DIRECTORY ./build-edgerouter-lite $CIRCLE_BRANCH;
+              BUILDDIR_YGG=$CIRCLE_WORKING_DIRECTORY ./build-vyos-i386 $CIRCLE_BRANCH
+              BUILDDIR_YGG=$CIRCLE_WORKING_DIRECTORY ./build-vyos-amd64 $CIRCLE_BRANCH
               mv *.deb /tmp/upload;
 
       - persist_to_workspace:

From 8c64e6fa093a06f166fd654450a8958191b9c6cd Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Tue, 24 Sep 2019 18:01:35 -0500
Subject: [PATCH 121/130] explicitly notify the switch when a link appears to
 be blocked in a send instead of assuming this is the case for all idle links.
 how we decide when it's really blocked still needs testing/optimizing

---
 src/yggdrasil/link.go   | 17 +++++++++++++++++
 src/yggdrasil/switch.go | 34 +++++++++++++++++++++++++---------
 2 files changed, 42 insertions(+), 9 deletions(-)

diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index a4a41e7f..ece69caf 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -64,6 +64,8 @@ type linkInterface struct {
 	closeTimer     *time.Timer // Fires when the link has been idle so long we need to close it
 	inSwitch       bool        // True if the switch is tracking this link
 	stalled        bool        // True if we haven't been receiving any response traffic
+	sendSeqSent    uint        // Incremented each time we start sending
+	sendSeqRecv    uint        // Incremented each time we finish sending
 }
 
 func (l *link) init(c *Core) error {
@@ -273,9 +275,23 @@ func (intf *linkInterface) notifySending(size int, isLinkTraffic bool) {
 		}
 		intf.sendTimer = time.AfterFunc(sendTime, intf.notifyBlockedSend)
 		intf._cancelStallTimer()
+		intf.sendSeqSent++
+		seq := intf.sendSeqSent
+		intf.Act(nil, func() {
+			intf._checkSending(seq)
+		})
 	})
 }
 
+// If check if we're still sending
+func (intf *linkInterface) _checkSending(seq uint) {
+	if intf.sendSeqRecv != seq {
+		intf.link.core.switchTable.Act(intf, func() {
+			intf.link.core.switchTable._sendingIn(intf.peer.port)
+		})
+	}
+}
+
 // we just sent something, so cancel any pending timer to send keep-alive traffic
 func (intf *linkInterface) _cancelStallTimer() {
 	if intf.stallTimer != nil {
@@ -305,6 +321,7 @@ func (intf *linkInterface) notifySent(size int, isLinkTraffic bool) {
 		if size > 0 && intf.stallTimer == nil {
 			intf.stallTimer = time.AfterFunc(stallTime, intf.notifyStalled)
 		}
+		intf.sendSeqRecv++
 	})
 }
 
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index ece2fa26..0150e173 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -177,6 +177,7 @@ type switchTable struct {
 	phony.Inbox                            // Owns the below
 	queues      switch_buffers             // Queues - not atomic so ONLY use through the actor
 	idle        map[switchPort]struct{}    // idle peers - not atomic so ONLY use through the actor
+	sending     map[switchPort]struct{}    // peers known to be blocked in a send (somehow)
 }
 
 // Minimum allowed total size of switch queues.
@@ -203,6 +204,7 @@ func (t *switchTable) init(core *Core) {
 		core.config.Mutex.RUnlock()
 		t.queues.bufs = make(map[string]switch_buffer)
 		t.idle = make(map[switchPort]struct{})
+		t.sending = make(map[switchPort]struct{})
 	})
 }
 
@@ -527,7 +529,7 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort, rep
 		t.parent = sender.port
 		t.core.peers.sendSwitchMsgs(t)
 	}
-	if doUpdate {
+	if true || doUpdate {
 		t.updater.Store(&sync.Once{})
 	}
 	return
@@ -664,7 +666,7 @@ func (t *switchTable) bestPortForCoords(coords []byte) switchPort {
 // Handle an incoming packet
 // Either send it to ourself, or to the first idle peer that's free
 // Returns true if the packet has been handled somehow, false if it should be queued
-func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]struct{}) bool {
+func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]struct{}, sending map[switchPort]struct{}) bool {
 	coords := switch_getPacketCoords(packet)
 	closer := t.getCloser(coords)
 	if len(closer) == 0 {
@@ -677,12 +679,13 @@ func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]struct{}) boo
 	ports := t.core.peers.getPorts()
 	for _, cinfo := range closer {
 		to := ports[cinfo.elem.port]
-		_, isIdle := idle[cinfo.elem.port]
+		//_, isIdle := idle[cinfo.elem.port]
+		_, isSending := sending[cinfo.elem.port]
 		var update bool
 		switch {
 		case to == nil:
 			// no port was found, ignore it
-		case !isIdle:
+		case isSending:
 			// the port is busy, ignore it
 		case best == nil:
 			// this is the first idle port we've found, so select it until we find a
@@ -702,6 +705,7 @@ func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]struct{}) boo
 			// has a n older tstamp, so presumably a worse path
 		case cinfo.elem.time.Before(best.elem.time):
 			// same tstamp, but got it earlier, so presumably a better path
+			//t.core.log.Println("DEBUG new best:", best.elem.time, cinfo.elem.time)
 			update = true
 		default:
 			// the search for a port has finished
@@ -712,13 +716,18 @@ func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]struct{}) boo
 		}
 	}
 	if best != nil {
-		delete(idle, best.elem.port)
+		if _, isIdle := idle[best.elem.port]; isIdle {
+			delete(idle, best.elem.port)
+			ports[best.elem.port].sendPacketsFrom(t, [][]byte{packet})
+			return true
+		}
+		//delete(idle, best.elem.port)
 		// Tell ourselves to send to this node later
 		// If another (e.g. even better) hop becomes idle in the mean time, it'll take the packet instead
 		// FIXME this is just a hack, but seems to help with stability...
-		go t.Act(nil, func() {
-			t._idleIn(best.elem.port)
-		})
+		//go t.Act(nil, func() {
+		//	t._idleIn(best.elem.port)
+		//})
 		//ports[best.elem.port].sendPacketsFrom(t, [][]byte{packet})
 		//return true
 	}
@@ -847,7 +856,7 @@ func (t *switchTable) packetInFrom(from phony.Actor, bytes []byte) {
 
 func (t *switchTable) _packetIn(bytes []byte) {
 	// Try to send it somewhere (or drop it if it's corrupt or at a dead end)
-	if !t._handleIn(bytes, t.idle) {
+	if !t._handleIn(bytes, t.idle, t.sending) {
 		// There's nobody free to take it right now, so queue it for later
 		packet := switch_packetInfo{bytes, time.Now()}
 		streamID := switch_getPacketStreamID(packet.bytes)
@@ -874,8 +883,15 @@ func (t *switchTable) _packetIn(bytes []byte) {
 
 func (t *switchTable) _idleIn(port switchPort) {
 	// Try to find something to send to this peer
+	delete(t.sending, port)
 	if !t._handleIdle(port) {
 		// Didn't find anything ready to send yet, so stay idle
 		t.idle[port] = struct{}{}
 	}
 }
+
+func (t *switchTable) _sendingIn(port switchPort) {
+	if _, isIn := t.idle[port]; !isIn {
+		t.sending[port] = struct{}{}
+	}
+}

From b9e74f34ec2663b043374d1e0163aa55338792f5 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Tue, 24 Sep 2019 18:28:13 -0500
Subject: [PATCH 122/130] replace the send-to-self with a timer and an
 arbitrary timeout; i don't really like this but it seems to work better (1 ms
 is fast by human standards but an eternity for a syscall or the scheduler, so
 i think that's reasonable)

---
 src/yggdrasil/link.go | 28 +++++++++++++---------------
 1 file changed, 13 insertions(+), 15 deletions(-)

diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index ece69caf..875fde5e 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -64,8 +64,6 @@ type linkInterface struct {
 	closeTimer     *time.Timer // Fires when the link has been idle so long we need to close it
 	inSwitch       bool        // True if the switch is tracking this link
 	stalled        bool        // True if we haven't been receiving any response traffic
-	sendSeqSent    uint        // Incremented each time we start sending
-	sendSeqRecv    uint        // Incremented each time we finish sending
 }
 
 func (l *link) init(c *Core) error {
@@ -275,21 +273,14 @@ func (intf *linkInterface) notifySending(size int, isLinkTraffic bool) {
 		}
 		intf.sendTimer = time.AfterFunc(sendTime, intf.notifyBlockedSend)
 		intf._cancelStallTimer()
-		intf.sendSeqSent++
-		seq := intf.sendSeqSent
-		intf.Act(nil, func() {
-			intf._checkSending(seq)
-		})
 	})
 }
 
-// If check if we're still sending
-func (intf *linkInterface) _checkSending(seq uint) {
-	if intf.sendSeqRecv != seq {
-		intf.link.core.switchTable.Act(intf, func() {
-			intf.link.core.switchTable._sendingIn(intf.peer.port)
-		})
-	}
+// called by an AfterFunc if we seem to be blocked in a send syscall for a long time
+func (intf *linkInterface) _notifySyscall() {
+	intf.link.core.switchTable.Act(intf, func() {
+		intf.link.core.switchTable._sendingIn(intf.peer.port)
+	})
 }
 
 // we just sent something, so cancel any pending timer to send keep-alive traffic
@@ -321,7 +312,6 @@ func (intf *linkInterface) notifySent(size int, isLinkTraffic bool) {
 		if size > 0 && intf.stallTimer == nil {
 			intf.stallTimer = time.AfterFunc(stallTime, intf.notifyStalled)
 		}
-		intf.sendSeqRecv++
 	})
 }
 
@@ -397,7 +387,15 @@ func (w *linkWriter) sendFrom(from phony.Actor, bss [][]byte, isLinkTraffic bool
 			size += len(bs)
 		}
 		w.intf.notifySending(size, isLinkTraffic)
+		var once sync.Once
+		timer := time.AfterFunc(time.Millisecond, func() {
+			once.Do(func() {
+				w.intf.Act(nil, w.intf._notifySyscall)
+			})
+		})
 		w.intf.msgIO.writeMsgs(bss)
+		// Make sure we either stop the timer from doing anything or wait until it's done
+		once.Do(func() { timer.Stop() })
 		w.intf.notifySent(size, isLinkTraffic)
 		// Cleanup
 		for _, bs := range bss {

From ac58c3586eacda562b3e71477a0f8353de460be5 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Wed, 25 Sep 2019 17:53:25 -0500
Subject: [PATCH 123/130] cleanup/comments

---
 src/yggdrasil/link.go   | 10 ++++++++--
 src/yggdrasil/switch.go | 10 +---------
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index 875fde5e..98c080c7 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -291,9 +291,11 @@ func (intf *linkInterface) _cancelStallTimer() {
 	}
 }
 
-// called by an AfterFunc if we appear to have timed out
+// This gets called from a time.AfterFunc, and notifies the switch that we appear
+// to have gotten blocked on a write, so the switch should start routing traffic
+// through other links, if alternatives exist
 func (intf *linkInterface) notifyBlockedSend() {
-	intf.Act(nil, func() { // Sent from a time.AfterFunc
+	intf.Act(nil, func() {
 		if intf.sendTimer != nil {
 			//As far as we know, we're still trying to send, and the timer fired.
 			intf.link.core.switchTable.blockPeer(intf.peer.port)
@@ -387,8 +389,12 @@ func (w *linkWriter) sendFrom(from phony.Actor, bss [][]byte, isLinkTraffic bool
 			size += len(bs)
 		}
 		w.intf.notifySending(size, isLinkTraffic)
+		// start a timer that will fire if we get stuck in writeMsgs for an oddly long time
 		var once sync.Once
 		timer := time.AfterFunc(time.Millisecond, func() {
+			// 1 ms is kind of arbitrary
+			// the rationale is that this should be very long compared to a syscall
+			// but it's still short compared to end-to-end latency or human perception
 			once.Do(func() {
 				w.intf.Act(nil, w.intf._notifySyscall)
 			})
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 0150e173..ba30758c 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -721,15 +721,6 @@ func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]struct{}, sen
 			ports[best.elem.port].sendPacketsFrom(t, [][]byte{packet})
 			return true
 		}
-		//delete(idle, best.elem.port)
-		// Tell ourselves to send to this node later
-		// If another (e.g. even better) hop becomes idle in the mean time, it'll take the packet instead
-		// FIXME this is just a hack, but seems to help with stability...
-		//go t.Act(nil, func() {
-		//	t._idleIn(best.elem.port)
-		//})
-		//ports[best.elem.port].sendPacketsFrom(t, [][]byte{packet})
-		//return true
 	}
 	// Didn't find anyone idle to send it to
 	return false
@@ -799,6 +790,7 @@ func (b *switch_buffers) _cleanup(t *switchTable) {
 // Loops over packets and sends the newest one that's OK for this peer to send
 // Returns true if the peer is no longer idle, false if it should be added to the idle list
 func (t *switchTable) _handleIdle(port switchPort) bool {
+	// TODO? only send packets for which this is the best next hop that isn't currently blocked sending
 	to := t.core.peers.getPorts()[port]
 	if to == nil {
 		return true

From 19c2a573aab101a38a8f978fbbface47abda98b0 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 26 Sep 2019 22:56:45 +0100
Subject: [PATCH 124/130] Update changelog for v0.3.9

---
 CHANGELOG.md | 33 +++++++++++++++++++++++++++++++--
 1 file changed, 31 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b8133a29..3945bf01 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -25,6 +25,33 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - in case of vulnerabilities.
 -->
 
+## [0.3.9] - 2019-09-27
+### Added
+- Yggdrasil will now complain more verbosely when a peer URI is incorrectly formatted
+- Soft-shutdown methods have been added, allowing a node to shut down gracefully when terminated
+- New multicast interval logic which sends multicast beacons more often when Yggdrasil is first started to increase the chance of finding nearby nodes quickly after startup
+
+### Changed
+- The switch now buffers packets more eagerly in an attempt to give the best link a chance to send, which appears to reduce packet reordering when crossing aggregate sets of peerings
+- Substantial amounts of the codebase have been refactored to use the actor model, which should substantially reduce the chance of deadlocks
+- Nonce tracking in sessions has been modified so that memory usage is reduced whilst still only allowing duplicate packets within a small window
+- Soft-reconfiguration support has been simplified using new actor functions
+- The garbage collector threshold has been adjusted for mobile builds
+- The maximum queue size is now managed exclusively by the switch rather than by the core
+
+### Fixed
+- The broken `hjson-go` dependency which affected builds of the previous version has now been resolved in the module manifest
+- Some minor memory leaks in the switch have been fixed, which improves memory usage on mobile builds
+- A memory leak in the add-peer loop has been fixed
+- The admin socket now reports the correct URI strings for SOCKS peers in `getPeers`
+- A race condition when dialling a remote node by both the node address and routed prefix simultaneously has been fixed
+- A race condition between the router and the dial code resulting in a panic has been fixed
+- A panic which could occur when the TUN/TAP interface disappears (e.g. during soft-shutdown) has been fixed
+- A bug in the semantic versioning script which accompanies Yggdrasil for builds has been fixed
+
+### Removed
+- A number of legacy debug functions have now been removed and a number of exported API functions are now better documented
+
 ## [0.3.8] - 2019-08-21
 ### Changed
 - Yggdrasil can now send multiple packets from the switch at once, which results in improved throughput with smaller packets or lower MTUs
@@ -39,10 +66,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - New nonce tracking should help to reduce the number of packets dropped as a result of multiple/aggregate paths or congestion control in the switch
 
 ### Fixed
-- **Security vulnerability**: Address verification was not strict enough, which could result in a malicious session sending traffic with unexpected or spoofed source or destination addresses which Yggdrasil could fail to reject
+- A deadlock was fixed in the session code which could result in Yggdrasil failing to pass traffic after some time
+
+### Security
+- Address verification was not strict enough, which could result in a malicious session sending traffic with unexpected or spoofed source or destination addresses which Yggdrasil could fail to reject
   - Versions `0.3.6` and `0.3.7` are vulnerable - users of these versions should upgrade as soon as possible
   - Versions `0.3.5` and earlier are not affected
-- A deadlock was fixed in the session code which could result in Yggdrasil failing to pass traffic after some time
 
 ## [0.3.7] - 2019-08-14
 ### Changed

From e16d3efb0aa973c7a5591ebbf6baeba6d7bb55d6 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Thu, 26 Sep 2019 18:11:58 -0500
Subject: [PATCH 125/130] check packet length before checking if it's an ipv6
 packet, and add some trace level logging whenever a packet is rejected for
 being too short to parse

---
 src/tuntap/iface.go      | 7 +++++++
 src/util/cancellation.go | 4 ++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/tuntap/iface.go b/src/tuntap/iface.go
index 753a8d02..5db9f1ca 100644
--- a/src/tuntap/iface.go
+++ b/src/tuntap/iface.go
@@ -148,6 +148,11 @@ func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
 	// Offset the buffer from now on so that we can ignore ethernet frames if
 	// they are present
 	bs := recvd[offset:]
+	// Check if the packet is long enough to detect if it's an ICMP packet or not
+	if len(bs) < 7 {
+		tun.log.Traceln("TUN/TAP iface read undersized unknown packet, length:", len(bs))
+		return
+	}
 	// If we detect an ICMP packet then hand it to the ICMPv6 module
 	if bs[6] == 58 {
 		// Found an ICMPv6 packet - we need to make sure to give ICMPv6 the full
@@ -175,6 +180,7 @@ func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
 	if bs[0]&0xf0 == 0x60 {
 		// Check if we have a fully-sized IPv6 header
 		if len(bs) < 40 {
+			tun.log.Traceln("TUN/TAP iface read undersized ipv6 packet, length:", len(bs))
 			return
 		}
 		// Check the packet size
@@ -188,6 +194,7 @@ func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
 	} else if bs[0]&0xf0 == 0x40 {
 		// Check if we have a fully-sized IPv4 header
 		if len(bs) < 20 {
+			tun.log.Traceln("TUN/TAP iface read undersized ipv6 packet, length:", len(bs))
 			return
 		}
 		// Check the packet size
diff --git a/src/util/cancellation.go b/src/util/cancellation.go
index 6b2002c8..1f6d1658 100644
--- a/src/util/cancellation.go
+++ b/src/util/cancellation.go
@@ -11,8 +11,8 @@ import (
 // This is and is similar to a context, but with an error to specify the reason for the cancellation.
 type Cancellation interface {
 	Finished() <-chan struct{} // Finished returns a channel which will be closed when Cancellation.Cancel is first called.
-	Cancel(error) error // Cancel closes the channel returned by Finished and sets the error returned by error, or else returns the existing error if the Cancellation has already run.
-	Error() error // Error returns the error provided to Cancel, or nil if no error has been provided.
+	Cancel(error) error        // Cancel closes the channel returned by Finished and sets the error returned by error, or else returns the existing error if the Cancellation has already run.
+	Error() error              // Error returns the error provided to Cancel, or nil if no error has been provided.
 }
 
 // CancellationFinalized is an error returned if a cancellation object was garbage collected and the finalizer was run.

From 0f99d590a1c2592e333a51a71b4536aebaea5cc0 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Thu, 26 Sep 2019 18:15:26 -0500
Subject: [PATCH 126/130] typo, ipv6->ipv4

---
 src/tuntap/iface.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tuntap/iface.go b/src/tuntap/iface.go
index 5db9f1ca..0da99631 100644
--- a/src/tuntap/iface.go
+++ b/src/tuntap/iface.go
@@ -194,7 +194,7 @@ func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
 	} else if bs[0]&0xf0 == 0x40 {
 		// Check if we have a fully-sized IPv4 header
 		if len(bs) < 20 {
-			tun.log.Traceln("TUN/TAP iface read undersized ipv6 packet, length:", len(bs))
+			tun.log.Traceln("TUN/TAP iface read undersized ipv4 packet, length:", len(bs))
 			return
 		}
 		// Check the packet size

From 94f4d6e2866aa49e1752e5fbcec67db2456b8816 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Thu, 26 Sep 2019 18:21:35 -0500
Subject: [PATCH 127/130] Update CHANGELOG.md

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3945bf01..fdac254f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -48,6 +48,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - A race condition between the router and the dial code resulting in a panic has been fixed
 - A panic which could occur when the TUN/TAP interface disappears (e.g. during soft-shutdown) has been fixed
 - A bug in the semantic versioning script which accompanies Yggdrasil for builds has been fixed
+- A panic which could occur when the TUN/TAP interface reads an undersized/corrupted packet has been fixed
 
 ### Removed
 - A number of legacy debug functions have now been removed and a number of exported API functions are now better documented

From d6ee20580de03e8f44d993ab274dbf9f962bc25d Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 27 Sep 2019 09:37:34 +0100
Subject: [PATCH 128/130] Set TimeoutStopSec for systemd service

---
 contrib/systemd/yggdrasil.service | 1 +
 1 file changed, 1 insertion(+)

diff --git a/contrib/systemd/yggdrasil.service b/contrib/systemd/yggdrasil.service
index bd52ec9f..37859e79 100644
--- a/contrib/systemd/yggdrasil.service
+++ b/contrib/systemd/yggdrasil.service
@@ -16,6 +16,7 @@ ExecStartPre=/bin/sh -ec "if ! test -s /etc/yggdrasil.conf; \
 ExecStart=/usr/bin/yggdrasil -useconffile /etc/yggdrasil.conf
 ExecReload=/bin/kill -HUP $MAINPID
 Restart=always
+TimeoutStopSec=5
 
 [Install]
 WantedBy=multi-user.target

From 6ead31fb87d38cfe26568b997aea3ae0e1d22220 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 27 Sep 2019 09:44:55 +0100
Subject: [PATCH 129/130] Remove RPM spec from contrib as it is now in
 yggdrasil-network/yggdrasil-package-rpm

---
 contrib/rpm/yggdrasil.spec | 47 --------------------------------------
 1 file changed, 47 deletions(-)
 delete mode 100644 contrib/rpm/yggdrasil.spec

diff --git a/contrib/rpm/yggdrasil.spec b/contrib/rpm/yggdrasil.spec
deleted file mode 100644
index bab50906..00000000
--- a/contrib/rpm/yggdrasil.spec
+++ /dev/null
@@ -1,47 +0,0 @@
-Name:           yggdrasil
-Version:        0.3.0
-Release:        1%{?dist}
-Summary:        End-to-end encrypted IPv6 networking
-
-License:        GPLv3
-URL:            https://yggdrasil-network.github.io
-Source0:        https://codeload.github.com/yggdrasil-network/yggdrasil-go/tar.gz/v0.3.0
-
-%{?systemd_requires}
-BuildRequires:  systemd golang >= 1.11
-
-%description
-Yggdrasil is a proof-of-concept to explore a wholly different approach to
-network routing. Whereas current computer networks depend heavily on very
-centralised design and configuration, Yggdrasil breaks this mould by making
-use of a global spanning tree to form a scalable IPv6 encrypted mesh network.
-
-%prep
-%setup -qn yggdrasil-go-%{version}
-
-%build
-./build -t -l "-linkmode=external"
-
-%install
-rm -rf %{buildroot}
-mkdir -p %{buildroot}/%{_bindir}
-mkdir -p %{buildroot}/%{_sysconfdir}/systemd/system
-install -m 0755 yggdrasil %{buildroot}/%{_bindir}/yggdrasil
-install -m 0755 yggdrasilctl %{buildroot}/%{_bindir}/yggdrasilctl
-install -m 0755 contrib/systemd/yggdrasil.service %{buildroot}/%{_sysconfdir}/systemd/system/yggdrasil.service
-install -m 0755 contrib/systemd/yggdrasil-resume.service %{buildroot}/%{_sysconfdir}/systemd/system/yggdrasil-resume.service
-
-%files
-%{_bindir}/yggdrasil
-%{_bindir}/yggdrasilctl
-%{_sysconfdir}/systemd/system/yggdrasil.service
-%{_sysconfdir}/systemd/system/yggdrasil-resume.service
-
-%post
-%systemd_post yggdrasil.service
-
-%preun
-%systemd_preun yggdrasil.service
-
-%postun
-%systemd_postun_with_restart yggdrasil.service

From 5c3f7df77c9265f3e33cd24bf3c5286d7f5c4ed7 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 27 Sep 2019 09:49:19 +0100
Subject: [PATCH 130/130] Update submodule doc/yggdrasil-network.github.io

---
 doc/yggdrasil-network.github.io | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/yggdrasil-network.github.io b/doc/yggdrasil-network.github.io
index 10672210..c876890a 160000
--- a/doc/yggdrasil-network.github.io
+++ b/doc/yggdrasil-network.github.io
@@ -1 +1 @@
-Subproject commit 10672210f2fdce97dd5c301dfeed47284d4a28f2
+Subproject commit c876890a51d9140e68d5cec7fbeb2146c2562792