From 9d7e7288c68093f8e857ab83bc44203102a72ca0 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 18:47:15 -0500
Subject: [PATCH 01/62] start migrating the router to an actor

---
 go.mod                  |  1 +
 go.sum                  |  2 +
 src/yggdrasil/conn.go   |  2 +-
 src/yggdrasil/dht.go    |  2 -
 src/yggdrasil/peer.go   |  2 +-
 src/yggdrasil/router.go | 84 +++++++++++++++++++++--------------------
 src/yggdrasil/search.go |  2 +-
 src/yggdrasil/switch.go | 10 +----
 8 files changed, 51 insertions(+), 54 deletions(-)

diff --git a/go.mod b/go.mod
index 5569496c..3f5cae88 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,7 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
 require (
+	github.com/Arceliar/phony v0.0.0-20190821233739-c7f353f14438
 	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
 	github.com/hashicorp/go-syslog v1.0.0
 	github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222
diff --git a/go.sum b/go.sum
index 756ed5af..22276c2c 100644
--- a/go.sum
+++ b/go.sum
@@ -1,3 +1,5 @@
+github.com/Arceliar/phony v0.0.0-20190821233739-c7f353f14438 h1:t4tRgrItIq2ap4O31yOuWm17lUiyzf8gf/P+bEfgmrw=
+github.com/Arceliar/phony v0.0.0-20190821233739-c7f353f14438/go.mod h1:2Q9yJvg2PlMrnOEa3RTEy9hElWAICo/D8HTUDqAHUAo=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=
diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index 25330aae..c77e19d6 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -134,7 +134,7 @@ func (c *Conn) doSearch() {
 			sinfo.continueSearch()
 		}
 	}
-	go func() { c.core.router.admin <- routerWork }()
+	go c.core.router.doAdmin(routerWork)
 }
 
 func (c *Conn) getDeadlineCancellation(value *atomic.Value) (util.Cancellation, bool) {
diff --git a/src/yggdrasil/dht.go b/src/yggdrasil/dht.go
index b53e29c9..d35d3aaf 100644
--- a/src/yggdrasil/dht.go
+++ b/src/yggdrasil/dht.go
@@ -68,7 +68,6 @@ type dht struct {
 	core        *Core
 	reconfigure chan chan error
 	nodeID      crypto.NodeID
-	peers       chan *dhtInfo                    // other goroutines put incoming dht updates here
 	reqs        map[dhtReqKey]time.Time          // Keeps track of recent outstanding requests
 	callbacks   map[dhtReqKey][]dht_callbackInfo // Search and admin lookup callbacks
 	// These next two could be replaced by a single linked list or similar...
@@ -87,7 +86,6 @@ func (t *dht) init(c *Core) {
 		}
 	}()
 	t.nodeID = *t.core.NodeID()
-	t.peers = make(chan *dhtInfo, 1024)
 	t.callbacks = make(map[dhtReqKey][]dht_callbackInfo)
 	t.reset()
 }
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 379ca85b..fcd9364c 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -210,7 +210,7 @@ func (p *peer) linkLoop() {
 		case dinfo = <-p.dinfo:
 		case _ = <-tick.C:
 			if dinfo != nil {
-				p.core.dht.peers <- dinfo
+				p.core.router.insertPeer(&p.core.router, dinfo)
 			}
 		}
 	}
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index bdead848..4ce6b7ed 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -30,19 +30,19 @@ import (
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
+
+	"github.com/Arceliar/phony"
 )
 
 // The router struct has channels to/from the adapter device and a self peer (0), which is how messages are passed between this node and the peers/switch layer.
 // The router's mainLoop goroutine is responsible for managing all information related to the dht, searches, and crypto sessions.
 type router struct {
+	phony.Actor
 	core        *Core
 	reconfigure chan chan error
 	addr        address.Address
 	subnet      address.Subnet
-	in          <-chan [][]byte // packets we received from the network, link to peer's "out"
-	out         func([]byte)    // packets we're sending to the network, link to peer's "in"
-	reset       chan struct{}   // signal that coords changed (re-init sessions/dht)
-	admin       chan func()     // pass a lambda for the admin socket to query stuff
+	out         func([]byte) // packets we're sending to the network, link to peer's "in"
 	nodeinfo    nodeinfo
 }
 
@@ -52,7 +52,6 @@ func (r *router) init(core *Core) {
 	r.reconfigure = make(chan chan error, 1)
 	r.addr = *address.AddrForNodeID(&r.core.dht.nodeID)
 	r.subnet = *address.SubnetForNodeID(&r.core.dht.nodeID)
-	in := make(chan [][]byte, 1) // TODO something better than this...
 	self := linkInterface{
 		name: "(self)",
 		info: linkInfo{
@@ -62,8 +61,10 @@ func (r *router) init(core *Core) {
 		},
 	}
 	p := r.core.peers.newPeer(&r.core.boxPub, &r.core.sigPub, &crypto.BoxSharedKey{}, &self, nil)
-	p.out = func(packets [][]byte) { in <- packets }
-	r.in = in
+	p.out = func(packets [][]byte) {
+		// TODO make peers and/or the switch into actors, have them pass themselves as the from field
+		r.handlePackets(r, packets)
+	}
 	out := make(chan []byte, 32)
 	go func() {
 		for packet := range out {
@@ -90,8 +91,6 @@ func (r *router) init(core *Core) {
 		}
 	}()
 	r.out = func(packet []byte) { out2 <- packet }
-	r.reset = make(chan struct{}, 1)
-	r.admin = make(chan func(), 32)
 	r.nodeinfo.init(r.core)
 	r.core.config.Mutex.RLock()
 	r.nodeinfo.setNodeInfo(r.core.config.Current.NodeInfo, r.core.config.Current.NodeInfoPrivacy)
@@ -105,42 +104,55 @@ func (r *router) start() error {
 	return nil
 }
 
-// Takes traffic from the adapter and passes it to router.send, or from r.in and handles incoming traffic.
-// Also adds new peer info to the DHT.
-// Also resets the DHT and sesssions in the event of a coord change.
-// Also does periodic maintenance stuff.
+// In practice, the switch will call this with 1 packet
+func (r *router) handlePackets(from phony.IActor, packets [][]byte) {
+	r.EnqueueFrom(from, func() {
+		for _, packet := range packets {
+			r.handlePacket(packet)
+		}
+	})
+}
+
+// Insert a peer info into the dht, TODO? make the dht a separate actor
+func (r *router) insertPeer(from phony.IActor, info *dhtInfo) {
+	r.EnqueueFrom(from, func() {
+		r.core.dht.insertPeer(info)
+	})
+}
+
+// Reset sessions and DHT after the switch sees our coords change
+func (r *router) reset(from phony.IActor) {
+	r.EnqueueFrom(from, func() {
+		r.core.sessions.reset()
+		r.core.dht.reset()
+	})
+}
+
+// TODO remove reconfigure so this is just a ticker loop
+// and then find something better than a ticker loop to schedule things...
 func (r *router) mainLoop() {
 	ticker := time.NewTicker(time.Second)
 	defer ticker.Stop()
 	for {
 		select {
-		case ps := <-r.in:
-			for _, p := range ps {
-				r.handleIn(p)
-			}
-		case info := <-r.core.dht.peers:
-			r.core.dht.insertPeer(info)
-		case <-r.reset:
-			r.core.sessions.reset()
-			r.core.dht.reset()
 		case <-ticker.C:
-			{
+			r.SyncExec(func() {
 				// Any periodic maintenance stuff goes here
 				r.core.switchTable.doMaintenance()
 				r.core.dht.doMaintenance()
 				r.core.sessions.cleanup()
-			}
-		case f := <-r.admin:
-			f()
+			})
 		case e := <-r.reconfigure:
-			current := r.core.config.GetCurrent()
-			e <- r.nodeinfo.setNodeInfo(current.NodeInfo, current.NodeInfoPrivacy)
+			r.SyncExec(func() {
+				current := r.core.config.GetCurrent()
+				e <- r.nodeinfo.setNodeInfo(current.NodeInfo, current.NodeInfoPrivacy)
+			})
 		}
 	}
 }
 
 // Checks incoming traffic type and passes it to the appropriate handler.
-func (r *router) handleIn(packet []byte) {
+func (r *router) handlePacket(packet []byte) {
 	pType, pTypeLen := wire_decode_uint64(packet)
 	if pTypeLen == 0 {
 		return
@@ -263,17 +275,7 @@ func (r *router) handleNodeInfo(bs []byte, fromKey *crypto.BoxPubKey) {
 	r.nodeinfo.handleNodeInfo(&req)
 }
 
-// Passed a function to call.
-// This will send the function to r.admin and block until it finishes.
-// It's used by the admin socket to ask the router mainLoop goroutine about information in the session or dht structs, which cannot be read safely from outside that goroutine.
+// TODO remove this, have things either be actors that send message or else call SyncExec directly
 func (r *router) doAdmin(f func()) {
-	// Pass this a function that needs to be run by the router's main goroutine
-	// It will pass the function to the router and wait for the router to finish
-	done := make(chan struct{})
-	newF := func() {
-		f()
-		close(done)
-	}
-	r.admin <- newF
-	<-done
+	r.SyncExec(f)
 }
diff --git a/src/yggdrasil/search.go b/src/yggdrasil/search.go
index b970fe55..c035e72f 100644
--- a/src/yggdrasil/search.go
+++ b/src/yggdrasil/search.go
@@ -165,7 +165,7 @@ func (sinfo *searchInfo) continueSearch() {
 	}
 	go func() {
 		time.Sleep(search_RETRY_TIME)
-		sinfo.core.router.admin <- retryLater
+		sinfo.core.router.doAdmin(retryLater)
 	}()
 }
 
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index cc316d16..86ae102b 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -245,10 +245,7 @@ func (t *switchTable) cleanRoot() {
 		if t.data.locator.root != t.key {
 			t.data.seq++
 			t.updater.Store(&sync.Once{})
-			select {
-			case t.core.router.reset <- struct{}{}:
-			default:
-			}
+			t.core.router.reset(&t.core.router)
 		}
 		t.data.locator = switchLocator{root: t.key, tstamp: now.Unix()}
 		t.core.peers.sendSwitchMsgs()
@@ -511,10 +508,7 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort, rep
 		if !equiv(&sender.locator, &t.data.locator) {
 			doUpdate = true
 			t.data.seq++
-			select {
-			case t.core.router.reset <- struct{}{}:
-			default:
-			}
+			t.core.router.reset(&t.core.router)
 		}
 		if t.data.locator.tstamp != sender.locator.tstamp {
 			t.time = now

From 232e6d3cb3e6f63100e4af1aa9fa05f21c7cb78d Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 18:55:41 -0500
Subject: [PATCH 02/62] more router migration

---
 src/yggdrasil/router.go | 27 +--------------------------
 1 file changed, 1 insertion(+), 26 deletions(-)

diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index 4ce6b7ed..34dcb913 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -65,32 +65,7 @@ func (r *router) init(core *Core) {
 		// TODO make peers and/or the switch into actors, have them pass themselves as the from field
 		r.handlePackets(r, packets)
 	}
-	out := make(chan []byte, 32)
-	go func() {
-		for packet := range out {
-			p.handlePacket(packet)
-		}
-	}()
-	out2 := make(chan []byte, 32)
-	go func() {
-		// This worker makes sure r.out never blocks
-		// It will buffer traffic long enough for the switch worker to take it
-		// If (somehow) you can send faster than the switch can receive, then this would use unbounded memory
-		// But crypto slows sends enough that the switch should always be able to take the packets...
-		var buf [][]byte
-		for {
-			buf = append(buf, <-out2)
-			for len(buf) > 0 {
-				select {
-				case bs := <-out2:
-					buf = append(buf, bs)
-				case out <- buf[0]:
-					buf = buf[1:]
-				}
-			}
-		}
-	}()
-	r.out = func(packet []byte) { out2 <- packet }
+	r.out = p.handlePacket // TODO if the peer becomes its own actor, then send a message here
 	r.nodeinfo.init(r.core)
 	r.core.config.Mutex.RLock()
 	r.nodeinfo.setNodeInfo(r.core.config.Current.NodeInfo, r.core.config.Current.NodeInfoPrivacy)

From 8e89816099b842f73fa7c25cd441d0e54400231e Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 18:59:34 -0500
Subject: [PATCH 03/62] more router migration: rename functions that should
 only be called internally by the actor

---
 src/yggdrasil/router.go | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index 34dcb913..b24400da 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -75,7 +75,7 @@ func (r *router) init(core *Core) {
 // Starts the mainLoop goroutine.
 func (r *router) start() error {
 	r.core.log.Infoln("Starting router")
-	go r.mainLoop()
+	go r._mainLoop()
 	return nil
 }
 
@@ -83,7 +83,7 @@ func (r *router) start() error {
 func (r *router) handlePackets(from phony.IActor, packets [][]byte) {
 	r.EnqueueFrom(from, func() {
 		for _, packet := range packets {
-			r.handlePacket(packet)
+			r._handlePacket(packet)
 		}
 	})
 }
@@ -105,7 +105,7 @@ func (r *router) reset(from phony.IActor) {
 
 // TODO remove reconfigure so this is just a ticker loop
 // and then find something better than a ticker loop to schedule things...
-func (r *router) mainLoop() {
+func (r *router) _mainLoop() {
 	ticker := time.NewTicker(time.Second)
 	defer ticker.Stop()
 	for {
@@ -127,23 +127,23 @@ func (r *router) mainLoop() {
 }
 
 // Checks incoming traffic type and passes it to the appropriate handler.
-func (r *router) handlePacket(packet []byte) {
+func (r *router) _handlePacket(packet []byte) {
 	pType, pTypeLen := wire_decode_uint64(packet)
 	if pTypeLen == 0 {
 		return
 	}
 	switch pType {
 	case wire_Traffic:
-		r.handleTraffic(packet)
+		r._handleTraffic(packet)
 	case wire_ProtocolTraffic:
-		r.handleProto(packet)
+		r._handleProto(packet)
 	default:
 	}
 }
 
 // Handles incoming traffic, i.e. encapuslated ordinary IPv6 packets.
 // Passes them to the crypto session worker to be decrypted and sent to the adapter.
-func (r *router) handleTraffic(packet []byte) {
+func (r *router) _handleTraffic(packet []byte) {
 	defer util.PutBytes(packet)
 	p := wire_trafficPacket{}
 	if !p.decode(packet) {
@@ -162,7 +162,7 @@ func (r *router) handleTraffic(packet []byte) {
 }
 
 // Handles protocol traffic by decrypting it, checking its type, and passing it to the appropriate handler for that traffic type.
-func (r *router) handleProto(packet []byte) {
+func (r *router) _handleProto(packet []byte) {
 	// First parse the packet
 	p := wire_protoTrafficPacket{}
 	if !p.decode(packet) {
@@ -189,24 +189,24 @@ func (r *router) handleProto(packet []byte) {
 	}
 	switch bsType {
 	case wire_SessionPing:
-		r.handlePing(bs, &p.FromKey)
+		r._handlePing(bs, &p.FromKey)
 	case wire_SessionPong:
-		r.handlePong(bs, &p.FromKey)
+		r._handlePong(bs, &p.FromKey)
 	case wire_NodeInfoRequest:
 		fallthrough
 	case wire_NodeInfoResponse:
-		r.handleNodeInfo(bs, &p.FromKey)
+		r._handleNodeInfo(bs, &p.FromKey)
 	case wire_DHTLookupRequest:
-		r.handleDHTReq(bs, &p.FromKey)
+		r._handleDHTReq(bs, &p.FromKey)
 	case wire_DHTLookupResponse:
-		r.handleDHTRes(bs, &p.FromKey)
+		r._handleDHTRes(bs, &p.FromKey)
 	default:
 		util.PutBytes(packet)
 	}
 }
 
 // Decodes session pings from wire format and passes them to sessions.handlePing where they either create or update a session.
-func (r *router) handlePing(bs []byte, fromKey *crypto.BoxPubKey) {
+func (r *router) _handlePing(bs []byte, fromKey *crypto.BoxPubKey) {
 	ping := sessionPing{}
 	if !ping.decode(bs) {
 		return
@@ -216,12 +216,12 @@ func (r *router) handlePing(bs []byte, fromKey *crypto.BoxPubKey) {
 }
 
 // Handles session pongs (which are really pings with an extra flag to prevent acknowledgement).
-func (r *router) handlePong(bs []byte, fromKey *crypto.BoxPubKey) {
-	r.handlePing(bs, fromKey)
+func (r *router) _handlePong(bs []byte, fromKey *crypto.BoxPubKey) {
+	r._handlePing(bs, fromKey)
 }
 
 // Decodes dht requests and passes them to dht.handleReq to trigger a lookup/response.
-func (r *router) handleDHTReq(bs []byte, fromKey *crypto.BoxPubKey) {
+func (r *router) _handleDHTReq(bs []byte, fromKey *crypto.BoxPubKey) {
 	req := dhtReq{}
 	if !req.decode(bs) {
 		return
@@ -231,7 +231,7 @@ func (r *router) handleDHTReq(bs []byte, fromKey *crypto.BoxPubKey) {
 }
 
 // Decodes dht responses and passes them to dht.handleRes to update the DHT table and further pass them to the search code (if applicable).
-func (r *router) handleDHTRes(bs []byte, fromKey *crypto.BoxPubKey) {
+func (r *router) _handleDHTRes(bs []byte, fromKey *crypto.BoxPubKey) {
 	res := dhtRes{}
 	if !res.decode(bs) {
 		return
@@ -241,7 +241,7 @@ func (r *router) handleDHTRes(bs []byte, fromKey *crypto.BoxPubKey) {
 }
 
 // Decodes nodeinfo request
-func (r *router) handleNodeInfo(bs []byte, fromKey *crypto.BoxPubKey) {
+func (r *router) _handleNodeInfo(bs []byte, fromKey *crypto.BoxPubKey) {
 	req := nodeinfoReqRes{}
 	if !req.decode(bs) {
 		return

From bbcbbaf3b14d5b8ceb6f06ca33fc3f7969fc950b Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 20:05:18 -0500
Subject: [PATCH 04/62] start migrating sessionInfo to be an actor

---
 src/yggdrasil/api.go     |  2 +-
 src/yggdrasil/conn.go    |  8 ++---
 src/yggdrasil/router.go  |  8 ++---
 src/yggdrasil/search.go  |  2 +-
 src/yggdrasil/session.go | 71 ++++++++++++++++++++++------------------
 5 files changed, 49 insertions(+), 42 deletions(-)

diff --git a/src/yggdrasil/api.go b/src/yggdrasil/api.go
index 57ee5c65..785a5f72 100644
--- a/src/yggdrasil/api.go
+++ b/src/yggdrasil/api.go
@@ -213,7 +213,7 @@ func (c *Core) GetSessions() []Session {
 			workerFunc := func() {
 				session = Session{
 					Coords:      append([]uint64{}, wire_coordsBytestoUint64s(sinfo.coords)...),
-					MTU:         sinfo.getMTU(),
+					MTU:         sinfo._getMTU(),
 					BytesSent:   sinfo.bytesSent,
 					BytesRecvd:  sinfo.bytesRecvd,
 					Uptime:      time.Now().Sub(sinfo.timeOpened),
diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index c77e19d6..51dd9509 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -190,8 +190,8 @@ func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error {
 	var err error
 	sessionFunc := func() {
 		// Does the packet exceed the permitted size for the session?
-		if uint16(len(msg.Message)) > c.session.getMTU() {
-			err = ConnError{errors.New("packet too big"), true, false, false, int(c.session.getMTU())}
+		if uint16(len(msg.Message)) > c.session._getMTU() {
+			err = ConnError{errors.New("packet too big"), true, false, false, int(c.session._getMTU())}
 			return
 		}
 		// The rest of this work is session keep-alive traffic
@@ -201,10 +201,10 @@ func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error {
 				// TODO double check that the above condition is correct
 				c.doSearch()
 			} else {
-				c.core.sessions.ping(c.session)
+				c.session.ping(c.session) // TODO send from self if this becomes an actor
 			}
 		case c.session.reset && c.session.pingTime.Before(c.session.time):
-			c.core.sessions.ping(c.session)
+			c.session.ping(c.session) // TODO send from self if this becomes an actor
 		default: // Don't do anything, to keep traffic throttled
 		}
 	}
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index b24400da..95521841 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -98,7 +98,7 @@ func (r *router) insertPeer(from phony.IActor, info *dhtInfo) {
 // Reset sessions and DHT after the switch sees our coords change
 func (r *router) reset(from phony.IActor) {
 	r.EnqueueFrom(from, func() {
-		r.core.sessions.reset()
+		r.core.sessions.reset(r)
 		r.core.dht.reset()
 	})
 }
@@ -111,14 +111,14 @@ func (r *router) _mainLoop() {
 	for {
 		select {
 		case <-ticker.C:
-			r.SyncExec(func() {
+			<-r.SyncExec(func() {
 				// Any periodic maintenance stuff goes here
 				r.core.switchTable.doMaintenance()
 				r.core.dht.doMaintenance()
 				r.core.sessions.cleanup()
 			})
 		case e := <-r.reconfigure:
-			r.SyncExec(func() {
+			<-r.SyncExec(func() {
 				current := r.core.config.GetCurrent()
 				e <- r.nodeinfo.setNodeInfo(current.NodeInfo, current.NodeInfoPrivacy)
 			})
@@ -252,5 +252,5 @@ func (r *router) _handleNodeInfo(bs []byte, fromKey *crypto.BoxPubKey) {
 
 // TODO remove this, have things either be actors that send message or else call SyncExec directly
 func (r *router) doAdmin(f func()) {
-	r.SyncExec(f)
+	<-r.SyncExec(f)
 }
diff --git a/src/yggdrasil/search.go b/src/yggdrasil/search.go
index c035e72f..56adda80 100644
--- a/src/yggdrasil/search.go
+++ b/src/yggdrasil/search.go
@@ -213,7 +213,7 @@ func (sinfo *searchInfo) checkDHTRes(res *dhtRes) bool {
 	}
 	// FIXME (!) replay attacks could mess with coords? Give it a handle (tstamp)?
 	sess.coords = res.Coords
-	sinfo.core.sessions.ping(sess)
+	sess.ping(&sinfo.core.router)
 	sinfo.callback(sess, nil)
 	// Cleanup
 	delete(sinfo.core.searches.searches, res.Dest)
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 7cd92db9..f28e014a 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -14,6 +14,8 @@ import (
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
+
+	"github.com/Arceliar/phony"
 )
 
 // Duration that we keep track of old nonces per session, to allow some out-of-order packet delivery
@@ -37,7 +39,7 @@ func (h nonceHeap) peek() *crypto.BoxNonce { return &h[len(h)-1] }
 // All the information we know about an active session.
 // This includes coords, permanent and ephemeral keys, handles and nonces, various sorts of timing information for timeout and maintenance, and some metadata for the admin API.
 type sessionInfo struct {
-	mutex          sync.Mutex                    // Protects all of the below, use it any time you read/chance the contents of a session
+	phony.Actor                                  // Protects all of the below, use it any time you read/change the contents of a session
 	core           *Core                         //
 	reconfigure    chan chan error               //
 	theirAddr      address.Address               //
@@ -46,6 +48,7 @@ type sessionInfo struct {
 	theirSesPub    crypto.BoxPubKey              //
 	mySesPub       crypto.BoxPubKey              //
 	mySesPriv      crypto.BoxPrivKey             //
+	sharedPermKey  crypto.BoxSharedKey           // used for session pings
 	sharedSesKey   crypto.BoxSharedKey           // derived from session keys
 	theirHandle    crypto.Handle                 //
 	myHandle       crypto.Handle                 //
@@ -73,10 +76,9 @@ type sessionInfo struct {
 	send           chan FlowKeyMessage           // Packets with optional flow key go here, to be encrypted and sent
 }
 
+// TODO remove this, call SyncExec directly
 func (sinfo *sessionInfo) doFunc(f func()) {
-	sinfo.mutex.Lock()
-	defer sinfo.mutex.Unlock()
-	f()
+	<-sinfo.SyncExec(f)
 }
 
 // Represents a session ping/pong packet, andincludes information like public keys, a session handle, coords, a timestamp to prevent replays, and the tun/tap MTU.
@@ -92,7 +94,7 @@ type sessionPing struct {
 
 // Updates session info in response to a ping, after checking that the ping is OK.
 // Returns true if the session was updated, or false otherwise.
-func (s *sessionInfo) update(p *sessionPing) bool {
+func (s *sessionInfo) _update(p *sessionPing) bool {
 	if !(p.Tstamp > s.tstamp) {
 		// To protect against replay attacks
 		return false
@@ -214,6 +216,7 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 	sinfo.core = ss.core
 	sinfo.reconfigure = make(chan chan error, 1)
 	sinfo.theirPermPub = *theirPermKey
+	sinfo.sharedPermKey = *ss.getSharedKey(&ss.core.boxPriv, &sinfo.theirPermPub)
 	pub, priv := crypto.NewBoxKeys()
 	sinfo.mySesPub = *pub
 	sinfo.mySesPriv = *priv
@@ -257,7 +260,9 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 	go func() {
 		// Run cleanup when the session is canceled
 		<-sinfo.cancel.Finished()
-		sinfo.core.router.doAdmin(sinfo.close)
+		sinfo.core.router.doAdmin(func() {
+			sinfo.core.sessions.removeSession(&sinfo)
+		})
 	}()
 	go sinfo.startWorkers()
 	return &sinfo
@@ -292,7 +297,7 @@ func (ss *sessions) cleanup() {
 }
 
 // Closes a session, removing it from sessions maps.
-func (sinfo *sessionInfo) close() {
+func (ss *sessions) removeSession(sinfo *sessionInfo) {
 	if s := sinfo.core.sessions.sinfos[sinfo.myHandle]; s == sinfo {
 		delete(sinfo.core.sessions.sinfos, sinfo.myHandle)
 		delete(sinfo.core.sessions.byTheirPerm, sinfo.theirPermPub)
@@ -300,11 +305,11 @@ func (sinfo *sessionInfo) close() {
 }
 
 // Returns a session ping appropriate for the given session info.
-func (ss *sessions) getPing(sinfo *sessionInfo) sessionPing {
-	loc := ss.core.switchTable.getLocator()
+func (sinfo *sessionInfo) _getPing() sessionPing {
+	loc := sinfo.core.switchTable.getLocator()
 	coords := loc.getCoords()
-	ref := sessionPing{
-		SendPermPub: ss.core.boxPub,
+	ping := sessionPing{
+		SendPermPub: sinfo.core.boxPub,
 		Handle:      sinfo.myHandle,
 		SendSesPub:  sinfo.mySesPub,
 		Tstamp:      time.Now().Unix(),
@@ -312,7 +317,7 @@ func (ss *sessions) getPing(sinfo *sessionInfo) sessionPing {
 		MTU:         sinfo.myMTU,
 	}
 	sinfo.myNonce.Increment()
-	return ref
+	return ping
 }
 
 // Gets the shared key for a pair of box keys.
@@ -339,27 +344,29 @@ func (ss *sessions) getSharedKey(myPriv *crypto.BoxPrivKey,
 }
 
 // Sends a session ping by calling sendPingPong in ping mode.
-func (ss *sessions) ping(sinfo *sessionInfo) {
-	ss.sendPingPong(sinfo, false)
+func (sinfo *sessionInfo) ping(from phony.IActor) {
+	sinfo.EnqueueFrom(from, func() {
+		sinfo._sendPingPong(false)
+	})
 }
 
 // Calls getPing, sets the appropriate ping/pong flag, encodes to wire format, and send it.
 // Updates the time the last ping was sent in the session info.
-func (ss *sessions) sendPingPong(sinfo *sessionInfo, isPong bool) {
-	ping := ss.getPing(sinfo)
+func (sinfo *sessionInfo) _sendPingPong(isPong bool) {
+	ping := sinfo._getPing()
 	ping.IsPong = isPong
 	bs := ping.encode()
-	shared := ss.getSharedKey(&ss.core.boxPriv, &sinfo.theirPermPub)
-	payload, nonce := crypto.BoxSeal(shared, bs, nil)
+	payload, nonce := crypto.BoxSeal(&sinfo.sharedPermKey, bs, nil)
 	p := wire_protoTrafficPacket{
 		Coords:  sinfo.coords,
 		ToKey:   sinfo.theirPermPub,
-		FromKey: ss.core.boxPub,
+		FromKey: sinfo.core.boxPub,
 		Nonce:   *nonce,
 		Payload: payload,
 	}
 	packet := p.encode()
-	ss.core.router.out(packet)
+	// TODO rewrite the below if/when the peer struct becomes an actor, to not go through the router first
+	sinfo.core.router.EnqueueFrom(sinfo, func() { sinfo.core.router.out(packet) })
 	if sinfo.pingTime.Before(sinfo.time) {
 		sinfo.pingTime = time.Now()
 	}
@@ -371,9 +378,9 @@ func (ss *sessions) handlePing(ping *sessionPing) {
 	// Get the corresponding session (or create a new session)
 	sinfo, isIn := ss.getByTheirPerm(&ping.SendPermPub)
 	switch {
+	case ping.IsPong: // This is a response, not an initial ping, so ignore it.
 	case isIn: // Session already exists
 	case !ss.isSessionAllowed(&ping.SendPermPub, false): // Session is not allowed
-	case ping.IsPong: // This is a response, not an initial ping, so ignore it.
 	default:
 		ss.listenerMutex.Lock()
 		if ss.listener != nil {
@@ -393,13 +400,13 @@ func (ss *sessions) handlePing(ping *sessionPing) {
 		ss.listenerMutex.Unlock()
 	}
 	if sinfo != nil {
-		sinfo.doFunc(func() {
+		sinfo.EnqueueFrom(&ss.core.router, func() {
 			// Update the session
-			if !sinfo.update(ping) { /*panic("Should not happen in testing")*/
+			if !sinfo._update(ping) { /*panic("Should not happen in testing")*/
 				return
 			}
 			if !ping.IsPong {
-				ss.sendPingPong(sinfo, true)
+				sinfo._sendPingPong(true)
 			}
 		})
 	}
@@ -408,7 +415,7 @@ func (ss *sessions) handlePing(ping *sessionPing) {
 // Get the MTU of the session.
 // Will be equal to the smaller of this node's MTU or the remote node's MTU.
 // If sending over links with a maximum message size (this was a thing with the old UDP code), it could be further lowered, to a minimum of 1280.
-func (sinfo *sessionInfo) getMTU() uint16 {
+func (sinfo *sessionInfo) _getMTU() uint16 {
 	if sinfo.theirMTU == 0 || sinfo.myMTU == 0 {
 		return 0
 	}
@@ -419,7 +426,7 @@ func (sinfo *sessionInfo) getMTU() uint16 {
 }
 
 // Checks if a packet's nonce is recent enough to fall within the window of allowed packets, and not already received.
-func (sinfo *sessionInfo) nonceIsOK(theirNonce *crypto.BoxNonce) bool {
+func (sinfo *sessionInfo) _nonceIsOK(theirNonce *crypto.BoxNonce) bool {
 	// The bitmask is to allow for some non-duplicate out-of-order packets
 	if theirNonce.Minus(&sinfo.theirNonce) > 0 {
 		// This is newer than the newest nonce we've seen
@@ -437,7 +444,7 @@ func (sinfo *sessionInfo) nonceIsOK(theirNonce *crypto.BoxNonce) bool {
 }
 
 // Updates the nonce mask by (possibly) shifting the bitmask and setting the bit corresponding to this nonce to 1, and then updating the most recent nonce
-func (sinfo *sessionInfo) updateNonce(theirNonce *crypto.BoxNonce) {
+func (sinfo *sessionInfo) _updateNonce(theirNonce *crypto.BoxNonce) {
 	// Start with some cleanup
 	for len(sinfo.theirNonceHeap) > 64 {
 		if time.Since(sinfo.theirNonceMap[*sinfo.theirNonceHeap.peek()]) < nonceWindow {
@@ -459,9 +466,9 @@ func (sinfo *sessionInfo) updateNonce(theirNonce *crypto.BoxNonce) {
 
 // Resets all sessions to an uninitialized state.
 // Called after coord changes, so attemtps to use a session will trigger a new ping and notify the remote end of the coord change.
-func (ss *sessions) reset() {
+func (ss *sessions) reset(from phony.IActor) {
 	for _, sinfo := range ss.sinfos {
-		sinfo.doFunc(func() {
+		sinfo.EnqueueFrom(from, func() {
 			sinfo.reset = true
 		})
 	}
@@ -492,7 +499,7 @@ func (sinfo *sessionInfo) recvWorker() {
 		var err error
 		var k crypto.BoxSharedKey
 		sessionFunc := func() {
-			if !sinfo.nonceIsOK(&p.Nonce) {
+			if !sinfo._nonceIsOK(&p.Nonce) {
 				err = ConnError{errors.New("packet dropped due to invalid nonce"), false, true, false, 0}
 				return
 			}
@@ -514,12 +521,12 @@ func (sinfo *sessionInfo) recvWorker() {
 					return
 				}
 				sessionFunc = func() {
-					if k != sinfo.sharedSesKey || !sinfo.nonceIsOK(&p.Nonce) {
+					if k != sinfo.sharedSesKey || !sinfo._nonceIsOK(&p.Nonce) {
 						// The session updated in the mean time, so return an error
 						err = ConnError{errors.New("session updated during crypto operation"), false, true, false, 0}
 						return
 					}
-					sinfo.updateNonce(&p.Nonce)
+					sinfo._updateNonce(&p.Nonce)
 					sinfo.time = time.Now()
 					sinfo.bytesRecvd += uint64(len(bs))
 				}

From 9835c638180dc7a493c10ecb0a5e4772bacc145c Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 20:26:15 -0500
Subject: [PATCH 05/62] refactor things the router owns (dht, sessions,
 searches) into that struct, to make the ownership more explicit

---
 src/yggdrasil/api.go      | 26 +++++++++++++-------------
 src/yggdrasil/conn.go     |  8 ++++----
 src/yggdrasil/core.go     | 15 ++++++---------
 src/yggdrasil/dht.go      |  4 ++--
 src/yggdrasil/listener.go |  4 ++--
 src/yggdrasil/nodeinfo.go |  2 +-
 src/yggdrasil/router.go   | 27 +++++++++++++++------------
 src/yggdrasil/search.go   | 20 ++++++++++----------
 src/yggdrasil/session.go  | 12 ++++++------
 9 files changed, 59 insertions(+), 59 deletions(-)

diff --git a/src/yggdrasil/api.go b/src/yggdrasil/api.go
index 785a5f72..f50c8ceb 100644
--- a/src/yggdrasil/api.go
+++ b/src/yggdrasil/api.go
@@ -156,11 +156,11 @@ func (c *Core) GetDHT() []DHTEntry {
 	getDHT := func() {
 		now := time.Now()
 		var dhtentry []*dhtInfo
-		for _, v := range c.dht.table {
+		for _, v := range c.router.dht.table {
 			dhtentry = append(dhtentry, v)
 		}
 		sort.SliceStable(dhtentry, func(i, j int) bool {
-			return dht_ordered(&c.dht.nodeID, dhtentry[i].getNodeID(), dhtentry[j].getNodeID())
+			return dht_ordered(&c.router.dht.nodeID, dhtentry[i].getNodeID(), dhtentry[j].getNodeID())
 		})
 		for _, v := range dhtentry {
 			info := DHTEntry{
@@ -208,7 +208,7 @@ func (c *Core) GetSwitchQueues() SwitchQueues {
 func (c *Core) GetSessions() []Session {
 	var sessions []Session
 	getSessions := func() {
-		for _, sinfo := range c.sessions.sinfos {
+		for _, sinfo := range c.router.sessions.sinfos {
 			var session Session
 			workerFunc := func() {
 				session = Session{
@@ -243,17 +243,17 @@ func (c *Core) GetSessions() []Session {
 
 // ConnListen returns a listener for Yggdrasil session connections.
 func (c *Core) ConnListen() (*Listener, error) {
-	c.sessions.listenerMutex.Lock()
-	defer c.sessions.listenerMutex.Unlock()
-	if c.sessions.listener != nil {
+	c.router.sessions.listenerMutex.Lock()
+	defer c.router.sessions.listenerMutex.Unlock()
+	if c.router.sessions.listener != nil {
 		return nil, errors.New("a listener already exists")
 	}
-	c.sessions.listener = &Listener{
+	c.router.sessions.listener = &Listener{
 		core:  c,
 		conn:  make(chan *Conn),
 		close: make(chan interface{}),
 	}
-	return c.sessions.listener, nil
+	return c.router.sessions.listener, nil
 }
 
 // ConnDialer returns a dialer for Yggdrasil session connections.
@@ -356,10 +356,10 @@ func (c *Core) GetNodeInfo(key crypto.BoxPubKey, coords []uint64, nocache bool)
 // received an incoming session request. The function should return true to
 // allow the session or false to reject it.
 func (c *Core) SetSessionGatekeeper(f func(pubkey *crypto.BoxPubKey, initiator bool) bool) {
-	c.sessions.isAllowedMutex.Lock()
-	defer c.sessions.isAllowedMutex.Unlock()
+	c.router.sessions.isAllowedMutex.Lock()
+	defer c.router.sessions.isAllowedMutex.Unlock()
 
-	c.sessions.isAllowedHandler = f
+	c.router.sessions.isAllowedHandler = f
 }
 
 // SetLogger sets the output logger of the Yggdrasil node after startup. This
@@ -445,10 +445,10 @@ func (c *Core) DHTPing(key crypto.BoxPubKey, coords []uint64, target *crypto.Nod
 	}
 	rq := dhtReqKey{info.key, *target}
 	sendPing := func() {
-		c.dht.addCallback(&rq, func(res *dhtRes) {
+		c.router.dht.addCallback(&rq, func(res *dhtRes) {
 			resCh <- res
 		})
-		c.dht.ping(&info, &rq.dest)
+		c.router.dht.ping(&info, &rq.dest)
 	}
 	c.router.doAdmin(sendPing)
 	// TODO: do something better than the below...
diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index 51dd9509..b4ce71f9 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -84,7 +84,7 @@ func (c *Conn) String() string {
 func (c *Conn) search() error {
 	var sinfo *searchInfo
 	var isIn bool
-	c.core.router.doAdmin(func() { sinfo, isIn = c.core.searches.searches[*c.nodeID] })
+	c.core.router.doAdmin(func() { sinfo, isIn = c.core.router.searches.searches[*c.nodeID] })
 	if !isIn {
 		done := make(chan struct{}, 1)
 		var sess *sessionInfo
@@ -99,7 +99,7 @@ func (c *Conn) search() error {
 			}
 		}
 		c.core.router.doAdmin(func() {
-			sinfo = c.core.searches.newIterSearch(c.nodeID, c.nodeMask, searchCompleted)
+			sinfo = c.core.router.searches.newIterSearch(c.nodeID, c.nodeMask, searchCompleted)
 			sinfo.continueSearch()
 		})
 		<-done
@@ -124,11 +124,11 @@ func (c *Conn) search() error {
 func (c *Conn) doSearch() {
 	routerWork := func() {
 		// Check to see if there is a search already matching the destination
-		sinfo, isIn := c.core.searches.searches[*c.nodeID]
+		sinfo, isIn := c.core.router.searches.searches[*c.nodeID]
 		if !isIn {
 			// Nothing was found, so create a new search
 			searchCompleted := func(sinfo *sessionInfo, e error) {}
-			sinfo = c.core.searches.newIterSearch(c.nodeID, c.nodeMask, searchCompleted)
+			sinfo = c.core.router.searches.newIterSearch(c.nodeID, c.nodeMask, searchCompleted)
 			c.core.log.Debugf("%s DHT search started: %p", c.String(), sinfo)
 			// Start the search
 			sinfo.continueSearch()
diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index 0921ab9f..870597a9 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -26,10 +26,7 @@ type Core struct {
 	sigPriv     crypto.SigPrivKey
 	switchTable switchTable
 	peers       peers
-	sessions    sessions
 	router      router
-	dht         dht
-	searches    searches
 	link        link
 	log         *log.Logger
 }
@@ -76,9 +73,9 @@ func (c *Core) init() error {
 		c.log.Warnln("SigningPublicKey in config is incorrect, should be", sp)
 	}
 
-	c.searches.init(c)
-	c.dht.init(c)
-	c.sessions.init(c)
+	c.router.searches.init(c)
+	c.router.dht.init(c)
+	c.router.sessions.init(c)
 	c.peers.init(c)
 	c.router.init(c)
 	c.switchTable.init(c) // TODO move before peers? before router?
@@ -124,9 +121,9 @@ func (c *Core) UpdateConfig(config *config.NodeConfig) {
 	errors := 0
 
 	components := []chan chan error{
-		c.searches.reconfigure,
-		c.dht.reconfigure,
-		c.sessions.reconfigure,
+		c.router.searches.reconfigure,
+		c.router.dht.reconfigure,
+		c.router.sessions.reconfigure,
 		c.peers.reconfigure,
 		c.router.reconfigure,
 		c.switchTable.reconfigure,
diff --git a/src/yggdrasil/dht.go b/src/yggdrasil/dht.go
index d35d3aaf..d0d98558 100644
--- a/src/yggdrasil/dht.go
+++ b/src/yggdrasil/dht.go
@@ -221,7 +221,7 @@ func (t *dht) handleReq(req *dhtReq) {
 func (t *dht) sendRes(res *dhtRes, req *dhtReq) {
 	// Send a reply for a dhtReq
 	bs := res.encode()
-	shared := t.core.sessions.getSharedKey(&t.core.boxPriv, &req.Key)
+	shared := t.core.router.sessions.getSharedKey(&t.core.boxPriv, &req.Key)
 	payload, nonce := crypto.BoxSeal(shared, bs, nil)
 	p := wire_protoTrafficPacket{
 		Coords:  req.Coords,
@@ -285,7 +285,7 @@ func (t *dht) handleRes(res *dhtRes) {
 func (t *dht) sendReq(req *dhtReq, dest *dhtInfo) {
 	// Send a dhtReq to the node in dhtInfo
 	bs := req.encode()
-	shared := t.core.sessions.getSharedKey(&t.core.boxPriv, &dest.key)
+	shared := t.core.router.sessions.getSharedKey(&t.core.boxPriv, &dest.key)
 	payload, nonce := crypto.BoxSeal(shared, bs, nil)
 	p := wire_protoTrafficPacket{
 		Coords:  dest.coords,
diff --git a/src/yggdrasil/listener.go b/src/yggdrasil/listener.go
index 62225412..fec543f4 100644
--- a/src/yggdrasil/listener.go
+++ b/src/yggdrasil/listener.go
@@ -31,8 +31,8 @@ func (l *Listener) Close() (err error) {
 		recover()
 		err = errors.New("already closed")
 	}()
-	if l.core.sessions.listener == l {
-		l.core.sessions.listener = nil
+	if l.core.router.sessions.listener == l {
+		l.core.router.sessions.listener = nil
 	}
 	close(l.close)
 	close(l.conn)
diff --git a/src/yggdrasil/nodeinfo.go b/src/yggdrasil/nodeinfo.go
index 73d4e115..50f5bf9c 100644
--- a/src/yggdrasil/nodeinfo.go
+++ b/src/yggdrasil/nodeinfo.go
@@ -172,7 +172,7 @@ func (m *nodeinfo) sendNodeInfo(key crypto.BoxPubKey, coords []byte, isResponse
 		NodeInfo:   m.getNodeInfo(),
 	}
 	bs := nodeinfo.encode()
-	shared := m.core.sessions.getSharedKey(&m.core.boxPriv, &key)
+	shared := m.core.router.sessions.getSharedKey(&m.core.boxPriv, &key)
 	payload, nonce := crypto.BoxSeal(shared, bs, nil)
 	p := wire_protoTrafficPacket{
 		Coords:  coords,
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index 95521841..ed919983 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -43,15 +43,18 @@ type router struct {
 	addr        address.Address
 	subnet      address.Subnet
 	out         func([]byte) // packets we're sending to the network, link to peer's "in"
+	dht         dht
 	nodeinfo    nodeinfo
+	searches    searches
+	sessions    sessions
 }
 
 // Initializes the router struct, which includes setting up channels to/from the adapter.
 func (r *router) init(core *Core) {
 	r.core = core
 	r.reconfigure = make(chan chan error, 1)
-	r.addr = *address.AddrForNodeID(&r.core.dht.nodeID)
-	r.subnet = *address.SubnetForNodeID(&r.core.dht.nodeID)
+	r.addr = *address.AddrForNodeID(&r.dht.nodeID)
+	r.subnet = *address.SubnetForNodeID(&r.dht.nodeID)
 	self := linkInterface{
 		name: "(self)",
 		info: linkInfo{
@@ -91,15 +94,15 @@ func (r *router) handlePackets(from phony.IActor, packets [][]byte) {
 // Insert a peer info into the dht, TODO? make the dht a separate actor
 func (r *router) insertPeer(from phony.IActor, info *dhtInfo) {
 	r.EnqueueFrom(from, func() {
-		r.core.dht.insertPeer(info)
+		r.dht.insertPeer(info)
 	})
 }
 
 // Reset sessions and DHT after the switch sees our coords change
 func (r *router) reset(from phony.IActor) {
 	r.EnqueueFrom(from, func() {
-		r.core.sessions.reset(r)
-		r.core.dht.reset()
+		r.sessions.reset()
+		r.dht.reset()
 	})
 }
 
@@ -114,8 +117,8 @@ func (r *router) _mainLoop() {
 			<-r.SyncExec(func() {
 				// Any periodic maintenance stuff goes here
 				r.core.switchTable.doMaintenance()
-				r.core.dht.doMaintenance()
-				r.core.sessions.cleanup()
+				r.dht.doMaintenance()
+				r.sessions.cleanup()
 			})
 		case e := <-r.reconfigure:
 			<-r.SyncExec(func() {
@@ -149,7 +152,7 @@ func (r *router) _handleTraffic(packet []byte) {
 	if !p.decode(packet) {
 		return
 	}
-	sinfo, isIn := r.core.sessions.getSessionForHandle(&p.Handle)
+	sinfo, isIn := r.sessions.getSessionForHandle(&p.Handle)
 	if !isIn {
 		util.PutBytes(p.Payload)
 		return
@@ -172,7 +175,7 @@ func (r *router) _handleProto(packet []byte) {
 	var sharedKey *crypto.BoxSharedKey
 	if p.ToKey == r.core.boxPub {
 		// Try to open using our permanent key
-		sharedKey = r.core.sessions.getSharedKey(&r.core.boxPriv, &p.FromKey)
+		sharedKey = r.sessions.getSharedKey(&r.core.boxPriv, &p.FromKey)
 	} else {
 		return
 	}
@@ -212,7 +215,7 @@ func (r *router) _handlePing(bs []byte, fromKey *crypto.BoxPubKey) {
 		return
 	}
 	ping.SendPermPub = *fromKey
-	r.core.sessions.handlePing(&ping)
+	r.sessions.handlePing(&ping)
 }
 
 // Handles session pongs (which are really pings with an extra flag to prevent acknowledgement).
@@ -227,7 +230,7 @@ func (r *router) _handleDHTReq(bs []byte, fromKey *crypto.BoxPubKey) {
 		return
 	}
 	req.Key = *fromKey
-	r.core.dht.handleReq(&req)
+	r.dht.handleReq(&req)
 }
 
 // Decodes dht responses and passes them to dht.handleRes to update the DHT table and further pass them to the search code (if applicable).
@@ -237,7 +240,7 @@ func (r *router) _handleDHTRes(bs []byte, fromKey *crypto.BoxPubKey) {
 		return
 	}
 	res.Key = *fromKey
-	r.core.dht.handleRes(&res)
+	r.dht.handleRes(&res)
 }
 
 // Decodes nodeinfo request
diff --git a/src/yggdrasil/search.go b/src/yggdrasil/search.go
index 56adda80..5751043f 100644
--- a/src/yggdrasil/search.go
+++ b/src/yggdrasil/search.go
@@ -100,7 +100,7 @@ func (sinfo *searchInfo) addToSearch(res *dhtRes) {
 	from := dhtInfo{key: res.Key, coords: res.Coords}
 	sinfo.visited[*from.getNodeID()] = true
 	for _, info := range res.Infos {
-		if *info.getNodeID() == sinfo.core.dht.nodeID || sinfo.visited[*info.getNodeID()] {
+		if *info.getNodeID() == sinfo.core.router.dht.nodeID || sinfo.visited[*info.getNodeID()] {
 			continue
 		}
 		if dht_ordered(&sinfo.dest, info.getNodeID(), from.getNodeID()) {
@@ -134,7 +134,7 @@ func (sinfo *searchInfo) doSearchStep() {
 	if len(sinfo.toVisit) == 0 {
 		if time.Since(sinfo.time) > search_RETRY_TIME {
 			// Dead end and no response in too long, do cleanup
-			delete(sinfo.core.searches.searches, sinfo.dest)
+			delete(sinfo.core.router.searches.searches, sinfo.dest)
 			sinfo.callback(nil, errors.New("search reached dead end"))
 		}
 		return
@@ -143,8 +143,8 @@ func (sinfo *searchInfo) doSearchStep() {
 	var next *dhtInfo
 	next, sinfo.toVisit = sinfo.toVisit[0], sinfo.toVisit[1:]
 	rq := dhtReqKey{next.key, sinfo.dest}
-	sinfo.core.dht.addCallback(&rq, sinfo.handleDHTRes)
-	sinfo.core.dht.ping(next, &sinfo.dest)
+	sinfo.core.router.dht.addCallback(&rq, sinfo.handleDHTRes)
+	sinfo.core.router.dht.ping(next, &sinfo.dest)
 	sinfo.time = time.Now()
 }
 
@@ -157,7 +157,7 @@ func (sinfo *searchInfo) continueSearch() {
 	// Any that die aren't restarted, but a new one will start later
 	retryLater := func() {
 		// FIXME this keeps the search alive forever if not for the searches map, fix that
-		newSearchInfo := sinfo.core.searches.searches[sinfo.dest]
+		newSearchInfo := sinfo.core.router.searches.searches[sinfo.dest]
 		if newSearchInfo != sinfo {
 			return
 		}
@@ -196,17 +196,17 @@ func (sinfo *searchInfo) checkDHTRes(res *dhtRes) bool {
 		return false
 	}
 	// They match, so create a session and send a sessionRequest
-	sess, isIn := sinfo.core.sessions.getByTheirPerm(&res.Key)
+	sess, isIn := sinfo.core.router.sessions.getByTheirPerm(&res.Key)
 	if !isIn {
-		sess = sinfo.core.sessions.createSession(&res.Key)
+		sess = sinfo.core.router.sessions.createSession(&res.Key)
 		if sess == nil {
 			// nil if the DHT search finished but the session wasn't allowed
 			sinfo.callback(nil, errors.New("session not allowed"))
 			// Cleanup
-			delete(sinfo.core.searches.searches, res.Dest)
+			delete(sinfo.core.router.searches.searches, res.Dest)
 			return true
 		}
-		_, isIn := sinfo.core.sessions.getByTheirPerm(&res.Key)
+		_, isIn := sinfo.core.router.sessions.getByTheirPerm(&res.Key)
 		if !isIn {
 			panic("This should never happen")
 		}
@@ -216,6 +216,6 @@ func (sinfo *searchInfo) checkDHTRes(res *dhtRes) bool {
 	sess.ping(&sinfo.core.router)
 	sinfo.callback(sess, nil)
 	// Cleanup
-	delete(sinfo.core.searches.searches, res.Dest)
+	delete(sinfo.core.router.searches.searches, res.Dest)
 	return true
 }
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index f28e014a..02d4dc82 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -261,7 +261,7 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 		// Run cleanup when the session is canceled
 		<-sinfo.cancel.Finished()
 		sinfo.core.router.doAdmin(func() {
-			sinfo.core.sessions.removeSession(&sinfo)
+			sinfo.core.router.sessions.removeSession(&sinfo)
 		})
 	}()
 	go sinfo.startWorkers()
@@ -298,9 +298,9 @@ func (ss *sessions) cleanup() {
 
 // Closes a session, removing it from sessions maps.
 func (ss *sessions) removeSession(sinfo *sessionInfo) {
-	if s := sinfo.core.sessions.sinfos[sinfo.myHandle]; s == sinfo {
-		delete(sinfo.core.sessions.sinfos, sinfo.myHandle)
-		delete(sinfo.core.sessions.byTheirPerm, sinfo.theirPermPub)
+	if s := sinfo.core.router.sessions.sinfos[sinfo.myHandle]; s == sinfo {
+		delete(sinfo.core.router.sessions.sinfos, sinfo.myHandle)
+		delete(sinfo.core.router.sessions.byTheirPerm, sinfo.theirPermPub)
 	}
 }
 
@@ -466,9 +466,9 @@ func (sinfo *sessionInfo) _updateNonce(theirNonce *crypto.BoxNonce) {
 
 // Resets all sessions to an uninitialized state.
 // Called after coord changes, so attemtps to use a session will trigger a new ping and notify the remote end of the coord change.
-func (ss *sessions) reset(from phony.IActor) {
+func (ss *sessions) reset() {
 	for _, sinfo := range ss.sinfos {
-		sinfo.EnqueueFrom(from, func() {
+		sinfo.EnqueueFrom(&ss.core.router, func() {
 			sinfo.reset = true
 		})
 	}

From ebd806f27a9396d5184345425e4cf7d409f66582 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 20:29:16 -0500
Subject: [PATCH 06/62] move router member initialization into router.init

---
 src/yggdrasil/core.go   | 3 ---
 src/yggdrasil/router.go | 3 +++
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index 870597a9..fb2142c0 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -73,9 +73,6 @@ func (c *Core) init() error {
 		c.log.Warnln("SigningPublicKey in config is incorrect, should be", sp)
 	}
 
-	c.router.searches.init(c)
-	c.router.dht.init(c)
-	c.router.sessions.init(c)
 	c.peers.init(c)
 	c.router.init(c)
 	c.switchTable.init(c) // TODO move before peers? before router?
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index ed919983..29bdf1d4 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -73,6 +73,9 @@ func (r *router) init(core *Core) {
 	r.core.config.Mutex.RLock()
 	r.nodeinfo.setNodeInfo(r.core.config.Current.NodeInfo, r.core.config.Current.NodeInfoPrivacy)
 	r.core.config.Mutex.RUnlock()
+	r.dht.init(r.core)
+	r.searches.init(r.core)
+	r.sessions.init(r.core)
 }
 
 // Starts the mainLoop goroutine.

From e7024a00e712c7097ed8c194a937546566d80f10 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 20:35:54 -0500
Subject: [PATCH 07/62] have dht store a pointer to router instead of core

---
 src/yggdrasil/dht.go    | 34 +++++++++++++++++-----------------
 src/yggdrasil/router.go |  2 +-
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/src/yggdrasil/dht.go b/src/yggdrasil/dht.go
index d0d98558..adfc40e7 100644
--- a/src/yggdrasil/dht.go
+++ b/src/yggdrasil/dht.go
@@ -65,7 +65,7 @@ type dhtReqKey struct {
 
 // The main DHT struct.
 type dht struct {
-	core        *Core
+	router      *router
 	reconfigure chan chan error
 	nodeID      crypto.NodeID
 	reqs        map[dhtReqKey]time.Time          // Keeps track of recent outstanding requests
@@ -76,8 +76,8 @@ type dht struct {
 }
 
 // Initializes the DHT.
-func (t *dht) init(c *Core) {
-	t.core = c
+func (t *dht) init(r *router) {
+	t.router = r
 	t.reconfigure = make(chan chan error, 1)
 	go func() {
 		for {
@@ -85,7 +85,7 @@ func (t *dht) init(c *Core) {
 			e <- nil
 		}
 	}()
-	t.nodeID = *t.core.NodeID()
+	t.nodeID = *t.router.core.NodeID()
 	t.callbacks = make(map[dhtReqKey][]dht_callbackInfo)
 	t.reset()
 }
@@ -190,10 +190,10 @@ func dht_ordered(first, second, third *crypto.NodeID) bool {
 // Update info about the node that sent the request.
 func (t *dht) handleReq(req *dhtReq) {
 	// Send them what they asked for
-	loc := t.core.switchTable.getLocator()
+	loc := t.router.core.switchTable.getLocator()
 	coords := loc.getCoords()
 	res := dhtRes{
-		Key:    t.core.boxPub,
+		Key:    t.router.core.boxPub,
 		Coords: coords,
 		Dest:   req.Dest,
 		Infos:  t.lookup(&req.Dest, false),
@@ -221,17 +221,17 @@ func (t *dht) handleReq(req *dhtReq) {
 func (t *dht) sendRes(res *dhtRes, req *dhtReq) {
 	// Send a reply for a dhtReq
 	bs := res.encode()
-	shared := t.core.router.sessions.getSharedKey(&t.core.boxPriv, &req.Key)
+	shared := t.router.sessions.getSharedKey(&t.router.core.boxPriv, &req.Key)
 	payload, nonce := crypto.BoxSeal(shared, bs, nil)
 	p := wire_protoTrafficPacket{
 		Coords:  req.Coords,
 		ToKey:   req.Key,
-		FromKey: t.core.boxPub,
+		FromKey: t.router.core.boxPub,
 		Nonce:   *nonce,
 		Payload: payload,
 	}
 	packet := p.encode()
-	t.core.router.out(packet)
+	t.router.out(packet)
 }
 
 type dht_callbackInfo struct {
@@ -285,17 +285,17 @@ func (t *dht) handleRes(res *dhtRes) {
 func (t *dht) sendReq(req *dhtReq, dest *dhtInfo) {
 	// Send a dhtReq to the node in dhtInfo
 	bs := req.encode()
-	shared := t.core.router.sessions.getSharedKey(&t.core.boxPriv, &dest.key)
+	shared := t.router.sessions.getSharedKey(&t.router.core.boxPriv, &dest.key)
 	payload, nonce := crypto.BoxSeal(shared, bs, nil)
 	p := wire_protoTrafficPacket{
 		Coords:  dest.coords,
 		ToKey:   dest.key,
-		FromKey: t.core.boxPub,
+		FromKey: t.router.core.boxPub,
 		Nonce:   *nonce,
 		Payload: payload,
 	}
 	packet := p.encode()
-	t.core.router.out(packet)
+	t.router.out(packet)
 	rq := dhtReqKey{dest.key, req.Dest}
 	t.reqs[rq] = time.Now()
 }
@@ -306,10 +306,10 @@ func (t *dht) ping(info *dhtInfo, target *crypto.NodeID) {
 	if target == nil {
 		target = &t.nodeID
 	}
-	loc := t.core.switchTable.getLocator()
+	loc := t.router.core.switchTable.getLocator()
 	coords := loc.getCoords()
 	req := dhtReq{
-		Key:    t.core.boxPub,
+		Key:    t.router.core.boxPub,
 		Coords: coords,
 		Dest:   *target,
 	}
@@ -384,7 +384,7 @@ func (t *dht) getImportant() []*dhtInfo {
 		})
 		// Keep the ones that are no further than the closest seen so far
 		minDist := ^uint64(0)
-		loc := t.core.switchTable.getLocator()
+		loc := t.router.core.switchTable.getLocator()
 		important := infos[:0]
 		for _, info := range infos {
 			dist := uint64(loc.dist(info.coords))
@@ -413,12 +413,12 @@ func (t *dht) getImportant() []*dhtInfo {
 
 // Returns true if this is a node we need to keep track of for the DHT to work.
 func (t *dht) isImportant(ninfo *dhtInfo) bool {
-	if ninfo.key == t.core.boxPub {
+	if ninfo.key == t.router.core.boxPub {
 		return false
 	}
 	important := t.getImportant()
 	// Check if ninfo is of equal or greater importance to what we already know
-	loc := t.core.switchTable.getLocator()
+	loc := t.router.core.switchTable.getLocator()
 	ndist := uint64(loc.dist(ninfo.coords))
 	minDist := ^uint64(0)
 	for _, info := range important {
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index 29bdf1d4..bd23768b 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -73,7 +73,7 @@ func (r *router) init(core *Core) {
 	r.core.config.Mutex.RLock()
 	r.nodeinfo.setNodeInfo(r.core.config.Current.NodeInfo, r.core.config.Current.NodeInfoPrivacy)
 	r.core.config.Mutex.RUnlock()
-	r.dht.init(r.core)
+	r.dht.init(r)
 	r.searches.init(r.core)
 	r.sessions.init(r.core)
 }

From 5bb85cf07b7bc7d75f6a1afd4648e3d88b10d676 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 20:42:38 -0500
Subject: [PATCH 08/62] refactor searches to store a pointer to router instead
 of core

---
 src/yggdrasil/router.go |  2 +-
 src/yggdrasil/search.go | 38 +++++++++++++++++++-------------------
 2 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index bd23768b..ad9bc15a 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -74,7 +74,7 @@ func (r *router) init(core *Core) {
 	r.nodeinfo.setNodeInfo(r.core.config.Current.NodeInfo, r.core.config.Current.NodeInfoPrivacy)
 	r.core.config.Mutex.RUnlock()
 	r.dht.init(r)
-	r.searches.init(r.core)
+	r.searches.init(r)
 	r.sessions.init(r.core)
 }
 
diff --git a/src/yggdrasil/search.go b/src/yggdrasil/search.go
index 5751043f..397c28a9 100644
--- a/src/yggdrasil/search.go
+++ b/src/yggdrasil/search.go
@@ -33,7 +33,7 @@ const search_RETRY_TIME = time.Second
 // Information about an ongoing search.
 // Includes the target NodeID, the bitmask to match it to an IP, and the list of nodes to visit / already visited.
 type searchInfo struct {
-	core     *Core
+	searches *searches
 	dest     crypto.NodeID
 	mask     crypto.NodeID
 	time     time.Time
@@ -45,14 +45,14 @@ type searchInfo struct {
 
 // This stores a map of active searches.
 type searches struct {
-	core        *Core
+	router      *router
 	reconfigure chan chan error
 	searches    map[crypto.NodeID]*searchInfo
 }
 
 // Initializes the searches struct.
-func (s *searches) init(core *Core) {
-	s.core = core
+func (s *searches) init(r *router) {
+	s.router = r
 	s.reconfigure = make(chan chan error, 1)
 	go func() {
 		for {
@@ -66,7 +66,7 @@ func (s *searches) init(core *Core) {
 // Creates a new search info, adds it to the searches struct, and returns a pointer to the info.
 func (s *searches) createSearch(dest *crypto.NodeID, mask *crypto.NodeID, callback func(*sessionInfo, error)) *searchInfo {
 	info := searchInfo{
-		core:     s.core,
+		searches: s,
 		dest:     *dest,
 		mask:     *mask,
 		time:     time.Now(),
@@ -100,7 +100,7 @@ func (sinfo *searchInfo) addToSearch(res *dhtRes) {
 	from := dhtInfo{key: res.Key, coords: res.Coords}
 	sinfo.visited[*from.getNodeID()] = true
 	for _, info := range res.Infos {
-		if *info.getNodeID() == sinfo.core.router.dht.nodeID || sinfo.visited[*info.getNodeID()] {
+		if *info.getNodeID() == sinfo.searches.router.dht.nodeID || sinfo.visited[*info.getNodeID()] {
 			continue
 		}
 		if dht_ordered(&sinfo.dest, info.getNodeID(), from.getNodeID()) {
@@ -134,7 +134,7 @@ func (sinfo *searchInfo) doSearchStep() {
 	if len(sinfo.toVisit) == 0 {
 		if time.Since(sinfo.time) > search_RETRY_TIME {
 			// Dead end and no response in too long, do cleanup
-			delete(sinfo.core.router.searches.searches, sinfo.dest)
+			delete(sinfo.searches.searches, sinfo.dest)
 			sinfo.callback(nil, errors.New("search reached dead end"))
 		}
 		return
@@ -143,8 +143,8 @@ func (sinfo *searchInfo) doSearchStep() {
 	var next *dhtInfo
 	next, sinfo.toVisit = sinfo.toVisit[0], sinfo.toVisit[1:]
 	rq := dhtReqKey{next.key, sinfo.dest}
-	sinfo.core.router.dht.addCallback(&rq, sinfo.handleDHTRes)
-	sinfo.core.router.dht.ping(next, &sinfo.dest)
+	sinfo.searches.router.dht.addCallback(&rq, sinfo.handleDHTRes)
+	sinfo.searches.router.dht.ping(next, &sinfo.dest)
 	sinfo.time = time.Now()
 }
 
@@ -157,7 +157,7 @@ func (sinfo *searchInfo) continueSearch() {
 	// Any that die aren't restarted, but a new one will start later
 	retryLater := func() {
 		// FIXME this keeps the search alive forever if not for the searches map, fix that
-		newSearchInfo := sinfo.core.router.searches.searches[sinfo.dest]
+		newSearchInfo := sinfo.searches.searches[sinfo.dest]
 		if newSearchInfo != sinfo {
 			return
 		}
@@ -165,7 +165,7 @@ func (sinfo *searchInfo) continueSearch() {
 	}
 	go func() {
 		time.Sleep(search_RETRY_TIME)
-		sinfo.core.router.doAdmin(retryLater)
+		sinfo.searches.router.doAdmin(retryLater)
 	}()
 }
 
@@ -173,9 +173,9 @@ func (sinfo *searchInfo) continueSearch() {
 func (s *searches) newIterSearch(dest *crypto.NodeID, mask *crypto.NodeID, callback func(*sessionInfo, error)) *searchInfo {
 	sinfo := s.createSearch(dest, mask, callback)
 	sinfo.visited = make(map[crypto.NodeID]bool)
-	loc := s.core.switchTable.getLocator()
+	loc := s.router.core.switchTable.getLocator()
 	sinfo.toVisit = append(sinfo.toVisit, &dhtInfo{
-		key:    s.core.boxPub,
+		key:    s.router.core.boxPub,
 		coords: loc.getCoords(),
 	}) // Start the search by asking ourself, useful if we're the destination
 	return sinfo
@@ -196,26 +196,26 @@ func (sinfo *searchInfo) checkDHTRes(res *dhtRes) bool {
 		return false
 	}
 	// They match, so create a session and send a sessionRequest
-	sess, isIn := sinfo.core.router.sessions.getByTheirPerm(&res.Key)
+	sess, isIn := sinfo.searches.router.sessions.getByTheirPerm(&res.Key)
 	if !isIn {
-		sess = sinfo.core.router.sessions.createSession(&res.Key)
+		sess = sinfo.searches.router.sessions.createSession(&res.Key)
 		if sess == nil {
 			// nil if the DHT search finished but the session wasn't allowed
 			sinfo.callback(nil, errors.New("session not allowed"))
 			// Cleanup
-			delete(sinfo.core.router.searches.searches, res.Dest)
+			delete(sinfo.searches.searches, res.Dest)
 			return true
 		}
-		_, isIn := sinfo.core.router.sessions.getByTheirPerm(&res.Key)
+		_, isIn := sinfo.searches.router.sessions.getByTheirPerm(&res.Key)
 		if !isIn {
 			panic("This should never happen")
 		}
 	}
 	// FIXME (!) replay attacks could mess with coords? Give it a handle (tstamp)?
 	sess.coords = res.Coords
-	sess.ping(&sinfo.core.router)
+	sess.ping(sinfo.searches.router)
 	sinfo.callback(sess, nil)
 	// Cleanup
-	delete(sinfo.core.router.searches.searches, res.Dest)
+	delete(sinfo.searches.searches, res.Dest)
 	return true
 }

From 436c84ca3311cf9c57eb8eec6901a0edd408645f Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 20:53:00 -0500
Subject: [PATCH 09/62] refactor sessions to store a pointer to router instead
 of core

---
 src/yggdrasil/conn.go    |  2 +-
 src/yggdrasil/router.go  |  2 +-
 src/yggdrasil/session.go | 53 +++++++++++++++++++++-------------------
 3 files changed, 30 insertions(+), 27 deletions(-)

diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index b4ce71f9..4ba0b2aa 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -252,7 +252,7 @@ func (c *Conn) Close() (err error) {
 }
 
 func (c *Conn) LocalAddr() crypto.NodeID {
-	return *crypto.GetNodeID(&c.session.core.boxPub)
+	return *crypto.GetNodeID(&c.core.boxPub)
 }
 
 func (c *Conn) RemoteAddr() crypto.NodeID {
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index ad9bc15a..25f8e800 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -75,7 +75,7 @@ func (r *router) init(core *Core) {
 	r.core.config.Mutex.RUnlock()
 	r.dht.init(r)
 	r.searches.init(r)
-	r.sessions.init(r.core)
+	r.sessions.init(r)
 }
 
 // Starts the mainLoop goroutine.
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 02d4dc82..13f64424 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -40,7 +40,7 @@ func (h nonceHeap) peek() *crypto.BoxNonce { return &h[len(h)-1] }
 // This includes coords, permanent and ephemeral keys, handles and nonces, various sorts of timing information for timeout and maintenance, and some metadata for the admin API.
 type sessionInfo struct {
 	phony.Actor                                  // Protects all of the below, use it any time you read/change the contents of a session
-	core           *Core                         //
+	sessions       *sessions                     //
 	reconfigure    chan chan error               //
 	theirAddr      address.Address               //
 	theirSubnet    address.Subnet                //
@@ -136,7 +136,7 @@ func (s *sessionInfo) _update(p *sessionPing) bool {
 // Sessions are indexed by handle.
 // Additionally, stores maps of address/subnet onto keys, and keys onto handles.
 type sessions struct {
-	core             *Core
+	router           *router
 	listener         *Listener
 	listenerMutex    sync.Mutex
 	reconfigure      chan chan error
@@ -149,8 +149,8 @@ type sessions struct {
 }
 
 // Initializes the session struct.
-func (ss *sessions) init(core *Core) {
-	ss.core = core
+func (ss *sessions) init(r *router) {
+	ss.router = r
 	ss.reconfigure = make(chan chan error, 1)
 	go func() {
 		for {
@@ -213,18 +213,18 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 		return nil
 	}
 	sinfo := sessionInfo{}
-	sinfo.core = ss.core
+	sinfo.sessions = ss
 	sinfo.reconfigure = make(chan chan error, 1)
 	sinfo.theirPermPub = *theirPermKey
-	sinfo.sharedPermKey = *ss.getSharedKey(&ss.core.boxPriv, &sinfo.theirPermPub)
+	sinfo.sharedPermKey = *ss.getSharedKey(&ss.router.core.boxPriv, &sinfo.theirPermPub)
 	pub, priv := crypto.NewBoxKeys()
 	sinfo.mySesPub = *pub
 	sinfo.mySesPriv = *priv
 	sinfo.myNonce = *crypto.NewBoxNonce()
 	sinfo.theirMTU = 1280
-	ss.core.config.Mutex.RLock()
-	sinfo.myMTU = uint16(ss.core.config.Current.IfMTU)
-	ss.core.config.Mutex.RUnlock()
+	ss.router.core.config.Mutex.RLock()
+	sinfo.myMTU = uint16(ss.router.core.config.Current.IfMTU)
+	ss.router.core.config.Mutex.RUnlock()
 	now := time.Now()
 	sinfo.timeOpened = now
 	sinfo.time = now
@@ -234,11 +234,11 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 	sinfo.init = make(chan struct{})
 	sinfo.cancel = util.NewCancellation()
 	higher := false
-	for idx := range ss.core.boxPub {
-		if ss.core.boxPub[idx] > sinfo.theirPermPub[idx] {
+	for idx := range ss.router.core.boxPub {
+		if ss.router.core.boxPub[idx] > sinfo.theirPermPub[idx] {
 			higher = true
 			break
-		} else if ss.core.boxPub[idx] < sinfo.theirPermPub[idx] {
+		} else if ss.router.core.boxPub[idx] < sinfo.theirPermPub[idx] {
 			break
 		}
 	}
@@ -260,8 +260,8 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 	go func() {
 		// Run cleanup when the session is canceled
 		<-sinfo.cancel.Finished()
-		sinfo.core.router.doAdmin(func() {
-			sinfo.core.router.sessions.removeSession(&sinfo)
+		sinfo.sessions.router.doAdmin(func() {
+			sinfo.sessions.removeSession(&sinfo)
 		})
 	}()
 	go sinfo.startWorkers()
@@ -298,18 +298,18 @@ func (ss *sessions) cleanup() {
 
 // Closes a session, removing it from sessions maps.
 func (ss *sessions) removeSession(sinfo *sessionInfo) {
-	if s := sinfo.core.router.sessions.sinfos[sinfo.myHandle]; s == sinfo {
-		delete(sinfo.core.router.sessions.sinfos, sinfo.myHandle)
-		delete(sinfo.core.router.sessions.byTheirPerm, sinfo.theirPermPub)
+	if s := sinfo.sessions.sinfos[sinfo.myHandle]; s == sinfo {
+		delete(sinfo.sessions.sinfos, sinfo.myHandle)
+		delete(sinfo.sessions.byTheirPerm, sinfo.theirPermPub)
 	}
 }
 
 // Returns a session ping appropriate for the given session info.
 func (sinfo *sessionInfo) _getPing() sessionPing {
-	loc := sinfo.core.switchTable.getLocator()
+	loc := sinfo.sessions.router.core.switchTable.getLocator()
 	coords := loc.getCoords()
 	ping := sessionPing{
-		SendPermPub: sinfo.core.boxPub,
+		SendPermPub: sinfo.sessions.router.core.boxPub,
 		Handle:      sinfo.myHandle,
 		SendSesPub:  sinfo.mySesPub,
 		Tstamp:      time.Now().Unix(),
@@ -360,13 +360,13 @@ func (sinfo *sessionInfo) _sendPingPong(isPong bool) {
 	p := wire_protoTrafficPacket{
 		Coords:  sinfo.coords,
 		ToKey:   sinfo.theirPermPub,
-		FromKey: sinfo.core.boxPub,
+		FromKey: sinfo.sessions.router.core.boxPub,
 		Nonce:   *nonce,
 		Payload: payload,
 	}
 	packet := p.encode()
 	// TODO rewrite the below if/when the peer struct becomes an actor, to not go through the router first
-	sinfo.core.router.EnqueueFrom(sinfo, func() { sinfo.core.router.out(packet) })
+	sinfo.sessions.router.EnqueueFrom(sinfo, func() { sinfo.sessions.router.out(packet) })
 	if sinfo.pingTime.Before(sinfo.time) {
 		sinfo.pingTime = time.Now()
 	}
@@ -390,7 +390,7 @@ func (ss *sessions) handlePing(ping *sessionPing) {
 			if s, _ := ss.getByTheirPerm(&ping.SendPermPub); s != sinfo {
 				panic("This should not happen")
 			}
-			conn := newConn(ss.core, crypto.GetNodeID(&sinfo.theirPermPub), &crypto.NodeID{}, sinfo)
+			conn := newConn(ss.router.core, crypto.GetNodeID(&sinfo.theirPermPub), &crypto.NodeID{}, sinfo)
 			for i := range conn.nodeMask {
 				conn.nodeMask[i] = 0xFF
 			}
@@ -400,7 +400,7 @@ func (ss *sessions) handlePing(ping *sessionPing) {
 		ss.listenerMutex.Unlock()
 	}
 	if sinfo != nil {
-		sinfo.EnqueueFrom(&ss.core.router, func() {
+		sinfo.EnqueueFrom(ss.router, func() {
 			// Update the session
 			if !sinfo._update(ping) { /*panic("Should not happen in testing")*/
 				return
@@ -468,7 +468,7 @@ func (sinfo *sessionInfo) _updateNonce(theirNonce *crypto.BoxNonce) {
 // Called after coord changes, so attemtps to use a session will trigger a new ping and notify the remote end of the coord change.
 func (ss *sessions) reset() {
 	for _, sinfo := range ss.sinfos {
-		sinfo.EnqueueFrom(&ss.core.router, func() {
+		sinfo.EnqueueFrom(ss.router, func() {
 			sinfo.reset = true
 		})
 	}
@@ -639,7 +639,10 @@ func (sinfo *sessionInfo) sendWorker() {
 				util.PutBytes(msg.Message)
 				util.PutBytes(p.Payload)
 				// Send the packet
-				sinfo.core.router.out(packet)
+				// TODO replace this with a send to the peer struct if that becomes an actor
+				sinfo.sessions.router.EnqueueFrom(sinfo, func() {
+					sinfo.sessions.router.out(packet)
+				})
 			}
 			ch <- callback
 		}

From 533da351f9b0f0b1ffa6a2d459aeece7c05d2b2c Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 22:23:01 -0500
Subject: [PATCH 10/62] fix actor EnqueueFrom stack overflow (use nil now to
 send from self) and replace session send/recv workers with actor functions

---
 go.mod                   |   2 +-
 go.sum                   |   4 +-
 src/yggdrasil/conn.go    |   4 +-
 src/yggdrasil/peer.go    |   2 +-
 src/yggdrasil/router.go  |   4 +-
 src/yggdrasil/session.go | 129 ++++++++++++++++++++++++++++++++++++---
 src/yggdrasil/switch.go  |   4 +-
 7 files changed, 132 insertions(+), 17 deletions(-)

diff --git a/go.mod b/go.mod
index 3f5cae88..fff4ae6f 100644
--- a/go.mod
+++ b/go.mod
@@ -1,7 +1,7 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
 require (
-	github.com/Arceliar/phony v0.0.0-20190821233739-c7f353f14438
+	github.com/Arceliar/phony v0.0.0-20190824031448-b53e115f69b5
 	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
 	github.com/hashicorp/go-syslog v1.0.0
 	github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222
diff --git a/go.sum b/go.sum
index 22276c2c..29854cbf 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-github.com/Arceliar/phony v0.0.0-20190821233739-c7f353f14438 h1:t4tRgrItIq2ap4O31yOuWm17lUiyzf8gf/P+bEfgmrw=
-github.com/Arceliar/phony v0.0.0-20190821233739-c7f353f14438/go.mod h1:2Q9yJvg2PlMrnOEa3RTEy9hElWAICo/D8HTUDqAHUAo=
+github.com/Arceliar/phony v0.0.0-20190824031448-b53e115f69b5 h1:D2Djqo/q7mftrtHLCpW4Rpplm8jj+Edc9jNz8Ll6E0A=
+github.com/Arceliar/phony v0.0.0-20190824031448-b53e115f69b5/go.mod h1:2Q9yJvg2PlMrnOEa3RTEy9hElWAICo/D8HTUDqAHUAo=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=
diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index 4ba0b2aa..1828b556 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -162,7 +162,7 @@ func (c *Conn) ReadNoCopy() ([]byte, error) {
 		} else {
 			return nil, ConnError{errors.New("session closed"), false, false, true, 0}
 		}
-	case bs := <-c.session.recv:
+	case bs := <-c.session.toConn:
 		return bs, nil
 	}
 }
@@ -221,7 +221,7 @@ func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error {
 			} else {
 				err = ConnError{errors.New("session closed"), false, false, true, 0}
 			}
-		case c.session.send <- msg:
+		case <-c.session.SyncExec(func() { c.session._send(msg) }):
 		}
 	}
 	return err
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index fcd9364c..8869bd2a 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -210,7 +210,7 @@ func (p *peer) linkLoop() {
 		case dinfo = <-p.dinfo:
 		case _ = <-tick.C:
 			if dinfo != nil {
-				p.core.router.insertPeer(&p.core.router, dinfo)
+				p.core.router.insertPeer(nil, dinfo)
 			}
 		}
 	}
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index 25f8e800..b5df107e 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -66,7 +66,7 @@ func (r *router) init(core *Core) {
 	p := r.core.peers.newPeer(&r.core.boxPub, &r.core.sigPub, &crypto.BoxSharedKey{}, &self, nil)
 	p.out = func(packets [][]byte) {
 		// TODO make peers and/or the switch into actors, have them pass themselves as the from field
-		r.handlePackets(r, packets)
+		r.handlePackets(nil, packets)
 	}
 	r.out = p.handlePacket // TODO if the peer becomes its own actor, then send a message here
 	r.nodeinfo.init(r.core)
@@ -160,6 +160,8 @@ func (r *router) _handleTraffic(packet []byte) {
 		util.PutBytes(p.Payload)
 		return
 	}
+	sinfo.recv(r, &p)
+	return
 	select {
 	case sinfo.fromRouter <- p:
 	case <-sinfo.cancel.Finished():
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 13f64424..84ea92af 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -72,8 +72,9 @@ type sessionInfo struct {
 	init           chan struct{}                 // Closed when the first session pong arrives, used to signal that the session is ready for initial use
 	cancel         util.Cancellation             // Used to terminate workers
 	fromRouter     chan wire_trafficPacket       // Received packets go here, to be decrypted by the session
-	recv           chan []byte                   // Decrypted packets go here, picked up by the associated Conn
-	send           chan FlowKeyMessage           // Packets with optional flow key go here, to be encrypted and sent
+	toConn         chan []byte                   // Decrypted packets go here, picked up by the associated Conn
+	fromConn       chan FlowKeyMessage           // Packets with optional flow key go here, to be encrypted and sent
+	callbacks      []chan func()                 // Finished work from crypto workers
 }
 
 // TODO remove this, call SyncExec directly
@@ -253,8 +254,8 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 	sinfo.theirAddr = *address.AddrForNodeID(crypto.GetNodeID(&sinfo.theirPermPub))
 	sinfo.theirSubnet = *address.SubnetForNodeID(crypto.GetNodeID(&sinfo.theirPermPub))
 	sinfo.fromRouter = make(chan wire_trafficPacket, 1)
-	sinfo.recv = make(chan []byte, 32)
-	sinfo.send = make(chan FlowKeyMessage, 32)
+	sinfo.toConn = make(chan []byte, 32)
+	sinfo.fromConn = make(chan FlowKeyMessage, 32)
 	ss.sinfos[sinfo.myHandle] = &sinfo
 	ss.byTheirPerm[sinfo.theirPermPub] = &sinfo.myHandle
 	go func() {
@@ -264,7 +265,7 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 			sinfo.sessions.removeSession(&sinfo)
 		})
 	}()
-	go sinfo.startWorkers()
+	//go sinfo.startWorkers()
 	return &sinfo
 }
 
@@ -539,7 +540,7 @@ func (sinfo *sessionInfo) recvWorker() {
 					select {
 					case <-sinfo.cancel.Finished():
 						util.PutBytes(bs)
-					case sinfo.recv <- bs:
+					case sinfo.toConn <- bs:
 					}
 				}
 			}
@@ -664,15 +665,127 @@ func (sinfo *sessionInfo) sendWorker() {
 				f()
 			case <-sinfo.cancel.Finished():
 				return
-			case msg := <-sinfo.send:
+			case msg := <-sinfo.fromConn:
 				doSend(msg)
 			}
 		}
 		select {
 		case <-sinfo.cancel.Finished():
 			return
-		case bs := <-sinfo.send:
+		case bs := <-sinfo.fromConn:
 			doSend(bs)
 		}
 	}
 }
+
+////////////////////////////////////////////////////////////////////////////////
+
+func (sinfo *sessionInfo) recv(from phony.IActor, packet *wire_trafficPacket) {
+	sinfo.EnqueueFrom(from, func() {
+		sinfo._recvPacket(packet)
+	})
+}
+
+func (sinfo *sessionInfo) _recvPacket(p *wire_trafficPacket) {
+	select {
+	case <-sinfo.init:
+	default:
+		// TODO find a better way to drop things until initialized
+		util.PutBytes(p.Payload)
+		return
+	}
+	switch {
+	case sinfo._nonceIsOK(&p.Nonce):
+	case len(sinfo.toConn) < cap(sinfo.toConn):
+	default:
+		// We're either full or don't like this nonce
+		util.PutBytes(p.Payload)
+		return
+	}
+
+	k := sinfo.sharedSesKey
+	var isOK bool
+	var bs []byte
+	ch := make(chan func(), 1)
+	poolFunc := func() {
+		bs, isOK = crypto.BoxOpen(&k, p.Payload, &p.Nonce)
+		callback := func() {
+			util.PutBytes(p.Payload)
+			if !isOK || k != sinfo.sharedSesKey || !sinfo._nonceIsOK(&p.Nonce) {
+				// Either we failed to decrypt, or the session was updated, or we received this packet in the mean time
+				util.PutBytes(bs)
+				return
+			}
+			sinfo._updateNonce(&p.Nonce)
+			sinfo.time = time.Now()
+			sinfo.bytesRecvd += uint64(len(bs))
+			select {
+			case sinfo.toConn <- bs:
+			default:
+				// We seem to have filled up the buffer in the mean time, so drop it
+				util.PutBytes(bs)
+			}
+		}
+		ch <- callback
+		sinfo.checkCallbacks()
+	}
+	sinfo.callbacks = append(sinfo.callbacks, ch)
+	util.WorkerGo(poolFunc)
+}
+
+func (sinfo *sessionInfo) _send(msg FlowKeyMessage) {
+	select {
+	case <-sinfo.init:
+	default:
+		// TODO find a better way to drop things until initialized
+		util.PutBytes(msg.Message)
+		return
+	}
+	sinfo.bytesSent += uint64(len(msg.Message))
+	coords := append([]byte(nil), sinfo.coords...)
+	if msg.FlowKey != 0 {
+		coords = append(coords, 0)
+		coords = append(coords, wire_encode_uint64(msg.FlowKey)...)
+	}
+	p := wire_trafficPacket{
+		Coords: coords,
+		Handle: sinfo.theirHandle,
+		Nonce:  sinfo.myNonce,
+	}
+	sinfo.myNonce.Increment()
+	k := sinfo.sharedSesKey
+	ch := make(chan func(), 1)
+	poolFunc := func() {
+		p.Payload, _ = crypto.BoxSeal(&k, msg.Message, &p.Nonce)
+		callback := func() {
+			// Encoding may block on a util.GetBytes(), so kept out of the worker pool
+			packet := p.encode()
+			// Cleanup
+			util.PutBytes(msg.Message)
+			util.PutBytes(p.Payload)
+			// Send the packet
+			// TODO replace this with a send to the peer struct if that becomes an actor
+			sinfo.sessions.router.EnqueueFrom(sinfo, func() {
+				sinfo.sessions.router.out(packet)
+			})
+		}
+		ch <- callback
+		sinfo.checkCallbacks()
+	}
+	sinfo.callbacks = append(sinfo.callbacks, ch)
+	util.WorkerGo(poolFunc)
+}
+
+func (sinfo *sessionInfo) checkCallbacks() {
+	sinfo.EnqueueFrom(nil, func() {
+		if len(sinfo.callbacks) > 0 {
+			select {
+			case callback := <-sinfo.callbacks[0]:
+				sinfo.callbacks = sinfo.callbacks[1:]
+				callback()
+				sinfo.checkCallbacks()
+			default:
+			}
+		}
+	})
+}
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 86ae102b..1fa75a6c 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -245,7 +245,7 @@ func (t *switchTable) cleanRoot() {
 		if t.data.locator.root != t.key {
 			t.data.seq++
 			t.updater.Store(&sync.Once{})
-			t.core.router.reset(&t.core.router)
+			t.core.router.reset(nil)
 		}
 		t.data.locator = switchLocator{root: t.key, tstamp: now.Unix()}
 		t.core.peers.sendSwitchMsgs()
@@ -508,7 +508,7 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort, rep
 		if !equiv(&sender.locator, &t.data.locator) {
 			doUpdate = true
 			t.data.seq++
-			t.core.router.reset(&t.core.router)
+			t.core.router.reset(nil)
 		}
 		if t.data.locator.tstamp != sender.locator.tstamp {
 			t.time = now

From e3603c0462fba858e330d7bc69d1c41d1016a98d Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 22:25:40 -0500
Subject: [PATCH 11/62] clean up unused session code

---
 src/yggdrasil/router.go  |   6 --
 src/yggdrasil/session.go | 201 ---------------------------------------
 2 files changed, 207 deletions(-)

diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index b5df107e..464a4778 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -161,12 +161,6 @@ func (r *router) _handleTraffic(packet []byte) {
 		return
 	}
 	sinfo.recv(r, &p)
-	return
-	select {
-	case sinfo.fromRouter <- p:
-	case <-sinfo.cancel.Finished():
-		util.PutBytes(p.Payload)
-	}
 }
 
 // Handles protocol traffic by decrypting it, checking its type, and passing it to the appropriate handler for that traffic type.
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 84ea92af..d17b119f 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -7,7 +7,6 @@ package yggdrasil
 import (
 	"bytes"
 	"container/heap"
-	"errors"
 	"sync"
 	"time"
 
@@ -71,9 +70,7 @@ type sessionInfo struct {
 	bytesRecvd     uint64                        // Bytes of real traffic received in this session
 	init           chan struct{}                 // Closed when the first session pong arrives, used to signal that the session is ready for initial use
 	cancel         util.Cancellation             // Used to terminate workers
-	fromRouter     chan wire_trafficPacket       // Received packets go here, to be decrypted by the session
 	toConn         chan []byte                   // Decrypted packets go here, picked up by the associated Conn
-	fromConn       chan FlowKeyMessage           // Packets with optional flow key go here, to be encrypted and sent
 	callbacks      []chan func()                 // Finished work from crypto workers
 }
 
@@ -253,9 +250,7 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 	sinfo.myHandle = *crypto.NewHandle()
 	sinfo.theirAddr = *address.AddrForNodeID(crypto.GetNodeID(&sinfo.theirPermPub))
 	sinfo.theirSubnet = *address.SubnetForNodeID(crypto.GetNodeID(&sinfo.theirPermPub))
-	sinfo.fromRouter = make(chan wire_trafficPacket, 1)
 	sinfo.toConn = make(chan []byte, 32)
-	sinfo.fromConn = make(chan FlowKeyMessage, 32)
 	ss.sinfos[sinfo.myHandle] = &sinfo
 	ss.byTheirPerm[sinfo.theirPermPub] = &sinfo.myHandle
 	go func() {
@@ -479,207 +474,11 @@ func (ss *sessions) reset() {
 //////////////////////////// Worker Functions Below ////////////////////////////
 ////////////////////////////////////////////////////////////////////////////////
 
-func (sinfo *sessionInfo) startWorkers() {
-	go sinfo.recvWorker()
-	go sinfo.sendWorker()
-}
-
 type FlowKeyMessage struct {
 	FlowKey uint64
 	Message []byte
 }
 
-func (sinfo *sessionInfo) recvWorker() {
-	// TODO move theirNonce etc into a struct that gets stored here, passed in over a channel
-	//  Since there's no reason for anywhere else in the session code to need to *read* it...
-	//  Only needs to be updated from the outside if a ping resets it...
-	//  That would get rid of the need to take a mutex for the sessionFunc
-	var callbacks []chan func()
-	doRecv := func(p wire_trafficPacket) {
-		var bs []byte
-		var err error
-		var k crypto.BoxSharedKey
-		sessionFunc := func() {
-			if !sinfo._nonceIsOK(&p.Nonce) {
-				err = ConnError{errors.New("packet dropped due to invalid nonce"), false, true, false, 0}
-				return
-			}
-			k = sinfo.sharedSesKey
-		}
-		sinfo.doFunc(sessionFunc)
-		if err != nil {
-			util.PutBytes(p.Payload)
-			return
-		}
-		var isOK bool
-		ch := make(chan func(), 1)
-		poolFunc := func() {
-			bs, isOK = crypto.BoxOpen(&k, p.Payload, &p.Nonce)
-			callback := func() {
-				util.PutBytes(p.Payload)
-				if !isOK {
-					util.PutBytes(bs)
-					return
-				}
-				sessionFunc = func() {
-					if k != sinfo.sharedSesKey || !sinfo._nonceIsOK(&p.Nonce) {
-						// The session updated in the mean time, so return an error
-						err = ConnError{errors.New("session updated during crypto operation"), false, true, false, 0}
-						return
-					}
-					sinfo._updateNonce(&p.Nonce)
-					sinfo.time = time.Now()
-					sinfo.bytesRecvd += uint64(len(bs))
-				}
-				sinfo.doFunc(sessionFunc)
-				if err != nil {
-					// Not sure what else to do with this packet, I guess just drop it
-					util.PutBytes(bs)
-				} else {
-					// Pass the packet to the buffer for Conn.Read
-					select {
-					case <-sinfo.cancel.Finished():
-						util.PutBytes(bs)
-					case sinfo.toConn <- bs:
-					}
-				}
-			}
-			ch <- callback
-		}
-		// Send to the worker and wait for it to finish
-		util.WorkerGo(poolFunc)
-		callbacks = append(callbacks, ch)
-	}
-	fromHelper := make(chan wire_trafficPacket, 1)
-	go func() {
-		var buf []wire_trafficPacket
-		for {
-			for len(buf) > 0 {
-				select {
-				case <-sinfo.cancel.Finished():
-					return
-				case p := <-sinfo.fromRouter:
-					buf = append(buf, p)
-					for len(buf) > 64 { // Based on nonce window size
-						util.PutBytes(buf[0].Payload)
-						buf = buf[1:]
-					}
-				case fromHelper <- buf[0]:
-					buf = buf[1:]
-				}
-			}
-			select {
-			case <-sinfo.cancel.Finished():
-				return
-			case p := <-sinfo.fromRouter:
-				buf = append(buf, p)
-			}
-		}
-	}()
-	select {
-	case <-sinfo.cancel.Finished():
-		return
-	case <-sinfo.init:
-		// Wait until the session has finished initializing before processing any packets
-	}
-	for {
-		for len(callbacks) > 0 {
-			select {
-			case f := <-callbacks[0]:
-				callbacks = callbacks[1:]
-				f()
-			case <-sinfo.cancel.Finished():
-				return
-			case p := <-fromHelper:
-				doRecv(p)
-			}
-		}
-		select {
-		case <-sinfo.cancel.Finished():
-			return
-		case p := <-fromHelper:
-			doRecv(p)
-		}
-	}
-}
-
-func (sinfo *sessionInfo) sendWorker() {
-	// TODO move info that this worker needs here, send updates via a channel
-	//  Otherwise we need to take a mutex to avoid races with update()
-	var callbacks []chan func()
-	doSend := func(msg FlowKeyMessage) {
-		var p wire_trafficPacket
-		var k crypto.BoxSharedKey
-		sessionFunc := func() {
-			sinfo.bytesSent += uint64(len(msg.Message))
-			p = wire_trafficPacket{
-				Coords: append([]byte(nil), sinfo.coords...),
-				Handle: sinfo.theirHandle,
-				Nonce:  sinfo.myNonce,
-			}
-			if msg.FlowKey != 0 {
-				// Helps ensure that traffic from this flow ends up in a separate queue from other flows
-				// The zero padding relies on the fact that the self-peer is always on port 0
-				p.Coords = append(p.Coords, 0)
-				p.Coords = wire_put_uint64(msg.FlowKey, p.Coords)
-			}
-			sinfo.myNonce.Increment()
-			k = sinfo.sharedSesKey
-		}
-		// Get the mutex-protected info needed to encrypt the packet
-		sinfo.doFunc(sessionFunc)
-		ch := make(chan func(), 1)
-		poolFunc := func() {
-			// Encrypt the packet
-			p.Payload, _ = crypto.BoxSeal(&k, msg.Message, &p.Nonce)
-			// The callback will send the packet
-			callback := func() {
-				// Encoding may block on a util.GetBytes(), so kept out of the worker pool
-				packet := p.encode()
-				// Cleanup
-				util.PutBytes(msg.Message)
-				util.PutBytes(p.Payload)
-				// Send the packet
-				// TODO replace this with a send to the peer struct if that becomes an actor
-				sinfo.sessions.router.EnqueueFrom(sinfo, func() {
-					sinfo.sessions.router.out(packet)
-				})
-			}
-			ch <- callback
-		}
-		// Send to the worker and wait for it to finish
-		util.WorkerGo(poolFunc)
-		callbacks = append(callbacks, ch)
-	}
-	select {
-	case <-sinfo.cancel.Finished():
-		return
-	case <-sinfo.init:
-		// Wait until the session has finished initializing before processing any packets
-	}
-	for {
-		for len(callbacks) > 0 {
-			select {
-			case f := <-callbacks[0]:
-				callbacks = callbacks[1:]
-				f()
-			case <-sinfo.cancel.Finished():
-				return
-			case msg := <-sinfo.fromConn:
-				doSend(msg)
-			}
-		}
-		select {
-		case <-sinfo.cancel.Finished():
-			return
-		case bs := <-sinfo.fromConn:
-			doSend(bs)
-		}
-	}
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
 func (sinfo *sessionInfo) recv(from phony.IActor, packet *wire_trafficPacket) {
 	sinfo.EnqueueFrom(from, func() {
 		sinfo._recvPacket(packet)

From cf9880464bdf100ea6c88aebd58dc02afef35e07 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 22:36:59 -0500
Subject: [PATCH 12/62] explicitly consider the session finished case, and make
 a note that we could fix the packet drop situation by making the Conn into an
 actor too

---
 src/yggdrasil/session.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index d17b119f..3f5e913e 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -520,8 +520,12 @@ func (sinfo *sessionInfo) _recvPacket(p *wire_trafficPacket) {
 			sinfo.bytesRecvd += uint64(len(bs))
 			select {
 			case sinfo.toConn <- bs:
+			case <-sinfo.cancel.Finished():
+				util.PutBytes(bs)
 			default:
-				// We seem to have filled up the buffer in the mean time, so drop it
+				// We seem to have filled up the buffer in the mean time
+				// Since we need to not block, but the conn isn't an actor, we need to drop this packet
+				// TODO find some nicer way to interact with the Conn...
 				util.PutBytes(bs)
 			}
 		}

From cac3444d9ae0a2e45aabe142430461843f91948b Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 22:40:13 -0500
Subject: [PATCH 13/62] fix debug builds

---
 src/yggdrasil/debug.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/yggdrasil/debug.go b/src/yggdrasil/debug.go
index 9f7707e2..d90bf1f4 100644
--- a/src/yggdrasil/debug.go
+++ b/src/yggdrasil/debug.go
@@ -258,7 +258,7 @@ func DEBUG_wire_encode_coords(coords []byte) []byte {
 func (c *Core) DEBUG_getDHTSize() int {
 	var total int
 	c.router.doAdmin(func() {
-		total = len(c.dht.table)
+		total = len(c.router.dht.table)
 	})
 	return total
 }
@@ -320,7 +320,7 @@ func (c *Core) DEBUG_startLoopbackUDPInterface() {
 ////////////////////////////////////////////////////////////////////////////////
 
 func (c *Core) DEBUG_getAddr() *address.Address {
-	return address.AddrForNodeID(&c.dht.nodeID)
+	return address.AddrForNodeID(&c.router.dht.nodeID)
 }
 
 /*

From 6ecbc439f0c0755597ab56fe1fe1b364fcea9be4 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 23 Aug 2019 23:36:00 -0500
Subject: [PATCH 14/62] start migrating Conn to be an actor

---
 src/yggdrasil/conn.go | 54 +++++++++++++++++++++++--------------------
 1 file changed, 29 insertions(+), 25 deletions(-)

diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index 1828b556..fa05fff0 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -3,12 +3,12 @@ package yggdrasil
 import (
 	"errors"
 	"fmt"
-	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
+
+	"github.com/Arceliar/phony"
 )
 
 // ConnError implements the net.Error interface
@@ -54,10 +54,10 @@ func (e *ConnError) Closed() bool {
 }
 
 type Conn struct {
+	phony.Actor
 	core          *Core
-	readDeadline  atomic.Value // time.Time // TODO timer
-	writeDeadline atomic.Value // time.Time // TODO timer
-	mutex         sync.RWMutex // protects the below
+	readDeadline  *time.Time
+	writeDeadline *time.Time
 	nodeID        *crypto.NodeID
 	nodeMask      *crypto.NodeID
 	session       *sessionInfo
@@ -75,9 +75,9 @@ func newConn(core *Core, nodeID *crypto.NodeID, nodeMask *crypto.NodeID, session
 }
 
 func (c *Conn) String() string {
-	c.mutex.RLock()
-	defer c.mutex.RUnlock()
-	return fmt.Sprintf("conn=%p", c)
+	var s string
+	<-c.SyncExec(func() { s = fmt.Sprintf("conn=%p", c) })
+	return s
 }
 
 // This should never be called from the router goroutine, used in the dial functions
@@ -137,10 +137,10 @@ func (c *Conn) doSearch() {
 	go c.core.router.doAdmin(routerWork)
 }
 
-func (c *Conn) getDeadlineCancellation(value *atomic.Value) (util.Cancellation, bool) {
-	if deadline, ok := value.Load().(time.Time); ok {
+func (c *Conn) _getDeadlineCancellation(t *time.Time) (util.Cancellation, bool) {
+	if t != nil {
 		// A deadline is set, so return a Cancellation that uses it
-		c := util.CancellationWithDeadline(c.session.cancel, deadline)
+		c := util.CancellationWithDeadline(c.session.cancel, *t)
 		return c, true
 	} else {
 		// No deadline was set, so just return the existinc cancellation and a dummy value
@@ -150,7 +150,9 @@ func (c *Conn) getDeadlineCancellation(value *atomic.Value) (util.Cancellation,
 
 // Used internally by Read, the caller is responsible for util.PutBytes when they're done.
 func (c *Conn) ReadNoCopy() ([]byte, error) {
-	cancel, doCancel := c.getDeadlineCancellation(&c.readDeadline)
+	var cancel util.Cancellation
+	var doCancel bool
+	<-c.SyncExec(func() { cancel, doCancel = c._getDeadlineCancellation(c.readDeadline) })
 	if doCancel {
 		defer cancel.Cancel(nil)
 	}
@@ -210,7 +212,9 @@ func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error {
 	}
 	c.session.doFunc(sessionFunc)
 	if err == nil {
-		cancel, doCancel := c.getDeadlineCancellation(&c.writeDeadline)
+		var cancel util.Cancellation
+		var doCancel bool
+		<-c.SyncExec(func() { cancel, doCancel = c._getDeadlineCancellation(c.writeDeadline) })
 		if doCancel {
 			defer cancel.Cancel(nil)
 		}
@@ -240,14 +244,14 @@ func (c *Conn) Write(b []byte) (int, error) {
 }
 
 func (c *Conn) Close() (err error) {
-	c.mutex.Lock()
-	defer c.mutex.Unlock()
-	if c.session != nil {
-		// Close the session, if it hasn't been closed already
-		if e := c.session.cancel.Cancel(errors.New("connection closed")); e != nil {
-			err = ConnError{errors.New("close failed, session already closed"), false, false, true, 0}
+	<-c.SyncExec(func() {
+		if c.session != nil {
+			// Close the session, if it hasn't been closed already
+			if e := c.session.cancel.Cancel(errors.New("connection closed")); e != nil {
+				err = ConnError{errors.New("close failed, session already closed"), false, false, true, 0}
+			}
 		}
-	}
+	})
 	return
 }
 
@@ -256,9 +260,9 @@ func (c *Conn) LocalAddr() crypto.NodeID {
 }
 
 func (c *Conn) RemoteAddr() crypto.NodeID {
-	c.mutex.RLock()
-	defer c.mutex.RUnlock()
-	return *c.nodeID
+	var n crypto.NodeID
+	<-c.SyncExec(func() { n = *c.nodeID })
+	return n
 }
 
 func (c *Conn) SetDeadline(t time.Time) error {
@@ -268,11 +272,11 @@ func (c *Conn) SetDeadline(t time.Time) error {
 }
 
 func (c *Conn) SetReadDeadline(t time.Time) error {
-	c.readDeadline.Store(t)
+	<-c.SyncExec(func() { c.readDeadline = &t })
 	return nil
 }
 
 func (c *Conn) SetWriteDeadline(t time.Time) error {
-	c.writeDeadline.Store(t)
+	<-c.SyncExec(func() { c.writeDeadline = &t })
 	return nil
 }

From da9f7151e3da50547b7320a6bbe9e9bbede23694 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 00:17:37 -0500
Subject: [PATCH 15/62] more conn migration

---
 src/yggdrasil/conn.go    | 63 ++++++++++++++++++++++------------------
 src/yggdrasil/dialer.go  |  1 +
 src/yggdrasil/session.go | 12 ++++++++
 3 files changed, 47 insertions(+), 29 deletions(-)

diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index fa05fff0..7121fc15 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -61,6 +61,7 @@ type Conn struct {
 	nodeID        *crypto.NodeID
 	nodeMask      *crypto.NodeID
 	session       *sessionInfo
+	mtu           uint16
 }
 
 // TODO func NewConn() that initializes additional fields as needed
@@ -80,6 +81,10 @@ func (c *Conn) String() string {
 	return s
 }
 
+func (c *Conn) setMTU(from phony.IActor, mtu uint16) {
+	c.EnqueueFrom(from, func() { c.mtu = mtu })
+}
+
 // This should never be called from the router goroutine, used in the dial functions
 func (c *Conn) search() error {
 	var sinfo *searchInfo
@@ -112,6 +117,7 @@ func (c *Conn) search() error {
 			for i := range c.nodeMask {
 				c.nodeMask[i] = 0xFF
 			}
+			c.session.conn = c
 		}
 		return err
 	} else {
@@ -120,7 +126,7 @@ func (c *Conn) search() error {
 	return nil
 }
 
-// Used in session keep-alive traffic in Conn.Write
+// Used in session keep-alive traffic
 func (c *Conn) doSearch() {
 	routerWork := func() {
 		// Check to see if there is a search already matching the destination
@@ -134,7 +140,7 @@ func (c *Conn) doSearch() {
 			sinfo.continueSearch()
 		}
 	}
-	go c.core.router.doAdmin(routerWork)
+	c.core.router.EnqueueFrom(c.session, routerWork)
 }
 
 func (c *Conn) _getDeadlineCancellation(t *time.Time) (util.Cancellation, bool) {
@@ -187,16 +193,14 @@ func (c *Conn) Read(b []byte) (int, error) {
 	return n, err
 }
 
-// Used internally by Write, the caller must not reuse the argument bytes when no error occurs
-func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error {
-	var err error
-	sessionFunc := func() {
-		// Does the packet exceed the permitted size for the session?
-		if uint16(len(msg.Message)) > c.session._getMTU() {
-			err = ConnError{errors.New("packet too big"), true, false, false, int(c.session._getMTU())}
-			return
-		}
-		// The rest of this work is session keep-alive traffic
+func (c *Conn) _write(msg FlowKeyMessage) error {
+	if len(msg.Message) > int(c.mtu) {
+		return ConnError{errors.New("packet too big"), true, false, false, int(c.mtu)}
+	}
+	c.session.EnqueueFrom(c, func() {
+		// Send the packet
+		c.session._send(msg)
+		// Session keep-alive, while we wait for the crypto workers from send
 		switch {
 		case time.Since(c.session.time) > 6*time.Second:
 			if c.session.time.Before(c.session.pingTime) && time.Since(c.session.pingTime) > 6*time.Second {
@@ -209,24 +213,25 @@ func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error {
 			c.session.ping(c.session) // TODO send from self if this becomes an actor
 		default: // Don't do anything, to keep traffic throttled
 		}
-	}
-	c.session.doFunc(sessionFunc)
-	if err == nil {
-		var cancel util.Cancellation
-		var doCancel bool
-		<-c.SyncExec(func() { cancel, doCancel = c._getDeadlineCancellation(c.writeDeadline) })
-		if doCancel {
-			defer cancel.Cancel(nil)
-		}
-		select {
-		case <-cancel.Finished():
-			if cancel.Error() == util.CancellationTimeoutError {
-				err = ConnError{errors.New("write timeout"), true, false, false, 0}
-			} else {
-				err = ConnError{errors.New("session closed"), false, false, true, 0}
-			}
-		case <-c.session.SyncExec(func() { c.session._send(msg) }):
+	})
+	return nil
+}
+
+// Used internally by Write, the caller must not reuse the argument bytes when no error occurs
+func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error {
+	var cancel util.Cancellation
+	var doCancel bool
+	<-c.SyncExec(func() { cancel, doCancel = c._getDeadlineCancellation(c.writeDeadline) })
+	var err error
+	select {
+	case <-cancel.Finished():
+		if cancel.Error() == util.CancellationTimeoutError {
+			err = ConnError{errors.New("write timeout"), true, false, false, 0}
+		} else {
+			err = ConnError{errors.New("session closed"), false, false, true, 0}
 		}
+	default:
+		<-c.SyncExec(func() { err = c._write(msg) })
 	}
 	return err
 }
diff --git a/src/yggdrasil/dialer.go b/src/yggdrasil/dialer.go
index 6b24cfb4..04410855 100644
--- a/src/yggdrasil/dialer.go
+++ b/src/yggdrasil/dialer.go
@@ -65,6 +65,7 @@ func (d *Dialer) DialByNodeIDandMask(nodeID, nodeMask *crypto.NodeID) (*Conn, er
 		conn.Close()
 		return nil, err
 	}
+	conn.session.setConn(nil, conn)
 	t := time.NewTimer(6 * time.Second) // TODO use a context instead
 	defer t.Stop()
 	select {
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 3f5e913e..854eb24d 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -71,6 +71,7 @@ type sessionInfo struct {
 	init           chan struct{}                 // Closed when the first session pong arrives, used to signal that the session is ready for initial use
 	cancel         util.Cancellation             // Used to terminate workers
 	toConn         chan []byte                   // Decrypted packets go here, picked up by the associated Conn
+	conn           *Conn                         // The associated Conn object
 	callbacks      []chan func()                 // Finished work from crypto workers
 }
 
@@ -112,6 +113,9 @@ func (s *sessionInfo) _update(p *sessionPing) bool {
 	}
 	if p.MTU >= 1280 || p.MTU == 0 {
 		s.theirMTU = p.MTU
+		if s.conn != nil {
+			s.conn.setMTU(s, s._getMTU())
+		}
 	}
 	if !bytes.Equal(s.coords, p.Coords) {
 		// allocate enough space for additional coords
@@ -368,6 +372,13 @@ func (sinfo *sessionInfo) _sendPingPong(isPong bool) {
 	}
 }
 
+func (sinfo *sessionInfo) setConn(from phony.IActor, conn *Conn) {
+	sinfo.EnqueueFrom(from, func() {
+		sinfo.conn = conn
+		sinfo.conn.setMTU(sinfo, sinfo._getMTU())
+	})
+}
+
 // Handles a session ping, creating a session if needed and calling update, then possibly responding with a pong if the ping was in ping mode and the update was successful.
 // If the session has a packet cached (common when first setting up a session), it will be sent.
 func (ss *sessions) handlePing(ping *sessionPing) {
@@ -390,6 +401,7 @@ func (ss *sessions) handlePing(ping *sessionPing) {
 			for i := range conn.nodeMask {
 				conn.nodeMask[i] = 0xFF
 			}
+			sinfo.setConn(ss.router, conn)
 			c := ss.listener.conn
 			go func() { c <- conn }()
 		}

From 9948e3d6591e905dd6095b10eae5f1232d37bd0f Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 00:44:02 -0500
Subject: [PATCH 16/62] add Conn.WriteFrom to allow actor-based sending

---
 src/yggdrasil/conn.go | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index 7121fc15..9e237436 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -217,7 +217,17 @@ func (c *Conn) _write(msg FlowKeyMessage) error {
 	return nil
 }
 
-// Used internally by Write, the caller must not reuse the argument bytes when no error occurs
+// WriteFrom should be called by a phony.IActor, and tells the Conn to send a message.
+// This is used internaly by WriteNoCopy and Write.
+// If the callback is called with a non-nil value, then it is safe to reuse the argument FlowKeyMessage.
+func (c *Conn) WriteFrom(from phony.IActor, msg FlowKeyMessage, callback func(error)) {
+	c.EnqueueFrom(from, func() {
+		callback(c._write(msg))
+	})
+}
+
+// WriteNoCopy is used internally by Write and makes use of WriteFrom under the hood.
+// The caller must not reuse the argument FlowKeyMessage when a nil error is returned.
 func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error {
 	var cancel util.Cancellation
 	var doCancel bool
@@ -231,12 +241,15 @@ func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error {
 			err = ConnError{errors.New("session closed"), false, false, true, 0}
 		}
 	default:
-		<-c.SyncExec(func() { err = c._write(msg) })
+		done := make(chan struct{})
+		callback := func(e error) { err = e; close(done) }
+		c.WriteFrom(nil, msg, callback)
+		<-done
 	}
 	return err
 }
 
-// Implements net.Conn.Write
+// Write implement the Write function of a net.Conn, and makes use of WriteNoCopy under the hood.
 func (c *Conn) Write(b []byte) (int, error) {
 	written := len(b)
 	msg := FlowKeyMessage{Message: append(util.GetBytes(), b...)}

From 1e346aaad052a23ada257f07e34355b622dafc90 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 01:52:21 -0500
Subject: [PATCH 17/62] have the conn actor receive messages from the session
 actor and either pass them to a callback or buffer them in a channel for Read
 to use if no callback was set

---
 src/yggdrasil/conn.go    | 55 ++++++++++++++++++++++++++++++++++++----
 src/yggdrasil/session.go | 20 ++-------------
 2 files changed, 52 insertions(+), 23 deletions(-)

diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index 9e237436..18c0b880 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -62,15 +62,18 @@ type Conn struct {
 	nodeMask      *crypto.NodeID
 	session       *sessionInfo
 	mtu           uint16
+	readCallback  func([]byte)
+	readBuffer    chan []byte
 }
 
 // TODO func NewConn() that initializes additional fields as needed
 func newConn(core *Core, nodeID *crypto.NodeID, nodeMask *crypto.NodeID, session *sessionInfo) *Conn {
 	conn := Conn{
-		core:     core,
-		nodeID:   nodeID,
-		nodeMask: nodeMask,
-		session:  session,
+		core:       core,
+		nodeID:     nodeID,
+		nodeMask:   nodeMask,
+		session:    session,
+		readBuffer: make(chan []byte, 1024),
 	}
 	return &conn
 }
@@ -154,6 +157,45 @@ func (c *Conn) _getDeadlineCancellation(t *time.Time) (util.Cancellation, bool)
 	}
 }
 
+// SetReadCallback sets a callback which will be called whenever a packet is received.
+// Note that calls to Read will fail if the callback has been set to a non-nil value.
+func (c *Conn) SetReadCallback(callback func([]byte)) {
+	c.EnqueueFrom(nil, func() {
+		c._setReadCallback(callback)
+	})
+}
+
+func (c *Conn) _setReadCallback(callback func([]byte)) {
+	c.readCallback = callback
+	c._drainReadBuffer()
+}
+
+func (c *Conn) _drainReadBuffer() {
+	if c.readCallback == nil {
+		return
+	}
+	select {
+	case bs := <-c.readBuffer:
+		c.readCallback(bs)
+		c.EnqueueFrom(nil, c._drainReadBuffer) // In case there's more
+	default:
+	}
+}
+
+// Called by the session to pass a new message to the Conn
+func (c *Conn) recvMsg(from phony.IActor, msg []byte) {
+	c.EnqueueFrom(from, func() {
+		if c.readCallback != nil {
+			c.readCallback(msg)
+		} else {
+			select {
+			case c.readBuffer <- msg:
+			default:
+			}
+		}
+	})
+}
+
 // Used internally by Read, the caller is responsible for util.PutBytes when they're done.
 func (c *Conn) ReadNoCopy() ([]byte, error) {
 	var cancel util.Cancellation
@@ -170,7 +212,7 @@ func (c *Conn) ReadNoCopy() ([]byte, error) {
 		} else {
 			return nil, ConnError{errors.New("session closed"), false, false, true, 0}
 		}
-	case bs := <-c.session.toConn:
+	case bs := <-c.readBuffer:
 		return bs, nil
 	}
 }
@@ -278,6 +320,7 @@ func (c *Conn) LocalAddr() crypto.NodeID {
 }
 
 func (c *Conn) RemoteAddr() crypto.NodeID {
+	// TODO warn that this can block while waiting for the Conn actor to run, so don't call it from other actors...
 	var n crypto.NodeID
 	<-c.SyncExec(func() { n = *c.nodeID })
 	return n
@@ -290,11 +333,13 @@ func (c *Conn) SetDeadline(t time.Time) error {
 }
 
 func (c *Conn) SetReadDeadline(t time.Time) error {
+	// TODO warn that this can block while waiting for the Conn actor to run, so don't call it from other actors...
 	<-c.SyncExec(func() { c.readDeadline = &t })
 	return nil
 }
 
 func (c *Conn) SetWriteDeadline(t time.Time) error {
+	// TODO warn that this can block while waiting for the Conn actor to run, so don't call it from other actors...
 	<-c.SyncExec(func() { c.writeDeadline = &t })
 	return nil
 }
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 854eb24d..a483a059 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -70,7 +70,6 @@ type sessionInfo struct {
 	bytesRecvd     uint64                        // Bytes of real traffic received in this session
 	init           chan struct{}                 // Closed when the first session pong arrives, used to signal that the session is ready for initial use
 	cancel         util.Cancellation             // Used to terminate workers
-	toConn         chan []byte                   // Decrypted packets go here, picked up by the associated Conn
 	conn           *Conn                         // The associated Conn object
 	callbacks      []chan func()                 // Finished work from crypto workers
 }
@@ -254,7 +253,6 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 	sinfo.myHandle = *crypto.NewHandle()
 	sinfo.theirAddr = *address.AddrForNodeID(crypto.GetNodeID(&sinfo.theirPermPub))
 	sinfo.theirSubnet = *address.SubnetForNodeID(crypto.GetNodeID(&sinfo.theirPermPub))
-	sinfo.toConn = make(chan []byte, 32)
 	ss.sinfos[sinfo.myHandle] = &sinfo
 	ss.byTheirPerm[sinfo.theirPermPub] = &sinfo.myHandle
 	go func() {
@@ -505,15 +503,10 @@ func (sinfo *sessionInfo) _recvPacket(p *wire_trafficPacket) {
 		util.PutBytes(p.Payload)
 		return
 	}
-	switch {
-	case sinfo._nonceIsOK(&p.Nonce):
-	case len(sinfo.toConn) < cap(sinfo.toConn):
-	default:
-		// We're either full or don't like this nonce
+	if !sinfo._nonceIsOK(&p.Nonce) {
 		util.PutBytes(p.Payload)
 		return
 	}
-
 	k := sinfo.sharedSesKey
 	var isOK bool
 	var bs []byte
@@ -530,16 +523,7 @@ func (sinfo *sessionInfo) _recvPacket(p *wire_trafficPacket) {
 			sinfo._updateNonce(&p.Nonce)
 			sinfo.time = time.Now()
 			sinfo.bytesRecvd += uint64(len(bs))
-			select {
-			case sinfo.toConn <- bs:
-			case <-sinfo.cancel.Finished():
-				util.PutBytes(bs)
-			default:
-				// We seem to have filled up the buffer in the mean time
-				// Since we need to not block, but the conn isn't an actor, we need to drop this packet
-				// TODO find some nicer way to interact with the Conn...
-				util.PutBytes(bs)
-			}
+			sinfo.conn.recvMsg(sinfo, bs)
 		}
 		ch <- callback
 		sinfo.checkCallbacks()

From b582c444f876cf50866fefdd70cf88357fd87f3a Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 01:57:08 -0500
Subject: [PATCH 18/62] minor cleanup

---
 src/yggdrasil/conn.go | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index 18c0b880..2f69139e 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -158,18 +158,13 @@ func (c *Conn) _getDeadlineCancellation(t *time.Time) (util.Cancellation, bool)
 }
 
 // SetReadCallback sets a callback which will be called whenever a packet is received.
-// Note that calls to Read will fail if the callback has been set to a non-nil value.
 func (c *Conn) SetReadCallback(callback func([]byte)) {
 	c.EnqueueFrom(nil, func() {
-		c._setReadCallback(callback)
+		c.readCallback = callback
+		c._drainReadBuffer()
 	})
 }
 
-func (c *Conn) _setReadCallback(callback func([]byte)) {
-	c.readCallback = callback
-	c._drainReadBuffer()
-}
-
 func (c *Conn) _drainReadBuffer() {
 	if c.readCallback == nil {
 		return

From 4893a07696c25aaed69f71cb4304f7f3312764b6 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 11:38:47 -0500
Subject: [PATCH 19/62] start migrating tunConn to the actor model

---
 src/tuntap/conn.go  | 183 ++++++++++++++++++++++++++++++++++++++++++++
 src/tuntap/iface.go |   4 +-
 src/tuntap/tun.go   |   9 ++-
 3 files changed, 192 insertions(+), 4 deletions(-)

diff --git a/src/tuntap/conn.go b/src/tuntap/conn.go
index 3fb7a544..32998e99 100644
--- a/src/tuntap/conn.go
+++ b/src/tuntap/conn.go
@@ -5,6 +5,7 @@ import (
 	"errors"
 	"time"
 
+	"github.com/Arceliar/phony"
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
@@ -16,6 +17,7 @@ import (
 const tunConnTimeout = 2 * time.Minute
 
 type tunConn struct {
+	phony.Actor
 	tun   *TunAdapter
 	conn  *yggdrasil.Conn
 	addr  address.Address
@@ -45,6 +47,83 @@ func (s *tunConn) _close_nomutex() {
 	}()
 }
 
+func (s *tunConn) _read(bs []byte) (err error) {
+	select {
+	case <-s.stop:
+		err = errors.New("session was already closed")
+		util.PutBytes(bs)
+		return
+	default:
+	}
+	if len(bs) == 0 {
+		err = errors.New("read packet with 0 size")
+		util.PutBytes(bs)
+		return
+	}
+	ipv4 := len(bs) > 20 && bs[0]&0xf0 == 0x40
+	ipv6 := len(bs) > 40 && bs[0]&0xf0 == 0x60
+	isCGA := true
+	// Check source addresses
+	switch {
+	case ipv6 && bs[8] == 0x02 && bytes.Equal(s.addr[:16], bs[8:24]): // source
+	case ipv6 && bs[8] == 0x03 && bytes.Equal(s.snet[:8], bs[8:16]): // source
+	default:
+		isCGA = false
+	}
+	// Check destiantion addresses
+	switch {
+	case ipv6 && bs[24] == 0x02 && bytes.Equal(s.tun.addr[:16], bs[24:40]): // destination
+	case ipv6 && bs[24] == 0x03 && bytes.Equal(s.tun.subnet[:8], bs[24:32]): // destination
+	default:
+		isCGA = false
+	}
+	// Decide how to handle the packet
+	var skip bool
+	switch {
+	case isCGA: // Allowed
+	case s.tun.ckr.isEnabled() && (ipv4 || ipv6):
+		var srcAddr address.Address
+		var dstAddr address.Address
+		var addrlen int
+		if ipv4 {
+			copy(srcAddr[:], bs[12:16])
+			copy(dstAddr[:], bs[16:20])
+			addrlen = 4
+		}
+		if ipv6 {
+			copy(srcAddr[:], bs[8:24])
+			copy(dstAddr[:], bs[24:40])
+			addrlen = 16
+		}
+		if !s.tun.ckr.isValidLocalAddress(dstAddr, addrlen) {
+			// The destination address isn't in our CKR allowed range
+			skip = true
+		} else if key, err := s.tun.ckr.getPublicKeyForAddress(srcAddr, addrlen); err == nil {
+			srcNodeID := crypto.GetNodeID(&key)
+			if s.conn.RemoteAddr() == *srcNodeID {
+				// This is the one allowed CKR case, where source and destination addresses are both good
+			} else {
+				// The CKR key associated with this address doesn't match the sender's NodeID
+				skip = true
+			}
+		} else {
+			// We have no CKR route for this source address
+			skip = true
+		}
+	default:
+		skip = true
+	}
+	if skip {
+		err = errors.New("address not allowed")
+		util.PutBytes(bs)
+		return
+	}
+	// FIXME this send can block if the tuntap isn't running, which isn't really safe...
+	s.tun.send <- bs
+	s.stillAlive()
+	return
+}
+
 func (s *tunConn) reader() (err error) {
 	select {
 	case _, ok := <-s.stop:
@@ -137,6 +216,110 @@ func (s *tunConn) reader() (err error) {
 	}
 }
 
+func (s *tunConn) _write(bs []byte) (err error) {
+	select {
+	case <-s.stop:
+		err = errors.New("session was already closed")
+		util.PutBytes(bs)
+		return
+	default:
+	}
+	v4 := len(bs) > 20 && bs[0]&0xf0 == 0x40
+	v6 := len(bs) > 40 && bs[0]&0xf0 == 0x60
+	isCGA := true
+	// Check source addresses
+	switch {
+	case v6 && bs[8] == 0x02 && bytes.Equal(s.tun.addr[:16], bs[8:24]): // source
+	case v6 && bs[8] == 0x03 && bytes.Equal(s.tun.subnet[:8], bs[8:16]): // source
+	default:
+		isCGA = false
+	}
+	// Check destiantion addresses
+	switch {
+	case v6 && bs[24] == 0x02 && bytes.Equal(s.addr[:16], bs[24:40]): // destination
+	case v6 && bs[24] == 0x03 && bytes.Equal(s.snet[:8], bs[24:32]): // destination
+	default:
+		isCGA = false
+	}
+	// Decide how to handle the packet
+	var skip bool
+	switch {
+	case isCGA: // Allowed
+	case s.tun.ckr.isEnabled() && (v4 || v6):
+		var srcAddr address.Address
+		var dstAddr address.Address
+		var addrlen int
+		if v4 {
+			copy(srcAddr[:], bs[12:16])
+			copy(dstAddr[:], bs[16:20])
+			addrlen = 4
+		}
+		if v6 {
+			copy(srcAddr[:], bs[8:24])
+			copy(dstAddr[:], bs[24:40])
+			addrlen = 16
+		}
+		if !s.tun.ckr.isValidLocalAddress(srcAddr, addrlen) {
+			// The source address isn't in our CKR allowed range
+			skip = true
+		} else if key, err := s.tun.ckr.getPublicKeyForAddress(dstAddr, addrlen); err == nil {
+			dstNodeID := crypto.GetNodeID(&key)
+			if s.conn.RemoteAddr() == *dstNodeID {
+				// This is the one allowed CKR case, where source and destination addresses are both good
+			} else {
+				// The CKR key associated with this address doesn't match the sender's NodeID
+				skip = true
+			}
+		} else {
+			// We have no CKR route for this destination address... why do we have the packet in the first place?
+			skip = true
+		}
+	default:
+		skip = true
+	}
+	if skip {
+		err = errors.New("address not allowed")
+		util.PutBytes(bs)
+		return
+	}
+	msg := yggdrasil.FlowKeyMessage{
+		FlowKey: util.GetFlowKey(bs),
+		Message: bs,
+	}
+	s.conn.WriteFrom(s, msg, func(err error) {
+		if err == nil {
+			// No point in wasting resources to send back an error if there was none
+			return
+		}
+		s.EnqueueFrom(s.conn, func() {
+			if e, eok := err.(yggdrasil.ConnError); !eok {
+				if e.Closed() {
+					s.tun.log.Debugln(s.conn.String(), "TUN/TAP generic write debug:", err)
+				} else {
+					s.tun.log.Errorln(s.conn.String(), "TUN/TAP generic write error:", err)
+				}
+			} else if e.PacketTooBig() {
+				// TODO: This currently isn't aware of IPv4 for CKR
+				ptb := &icmp.PacketTooBig{
+					MTU:  int(e.PacketMaximumSize()),
+					Data: bs[:900],
+				}
+				if packet, err := CreateICMPv6(bs[8:24], bs[24:40], ipv6.ICMPTypePacketTooBig, 0, ptb); err == nil {
+					s.tun.send <- packet
+				}
+			} else {
+				if e.Closed() {
+					s.tun.log.Debugln(s.conn.String(), "TUN/TAP conn write debug:", err)
+				} else {
+					s.tun.log.Errorln(s.conn.String(), "TUN/TAP conn write error:", err)
+				}
+			}
+		})
+	})
+	s.stillAlive()
+	return
+}
+
 func (s *tunConn) writer() error {
 	select {
 	case _, ok := <-s.stop:
diff --git a/src/tuntap/iface.go b/src/tuntap/iface.go
index 35657a9f..e0693e56 100644
--- a/src/tuntap/iface.go
+++ b/src/tuntap/iface.go
@@ -232,7 +232,7 @@ func (tun *TunAdapter) readerPacketHandler(ch chan []byte) {
 				if tc != nil {
 					for _, packet := range packets {
 						p := packet // Possibly required because of how range
-						tc.send <- p
+						<-tc.SyncExec(func() { tc._write(p) })
 					}
 				}
 			}()
@@ -242,7 +242,7 @@ func (tun *TunAdapter) readerPacketHandler(ch chan []byte) {
 		}
 		// If we have a connection now, try writing to it
 		if isIn && session != nil {
-			session.send <- bs
+			<-session.SyncExec(func() { session._write(bs) })
 		}
 	}
 }
diff --git a/src/tuntap/tun.go b/src/tuntap/tun.go
index eef05b87..dfc9ac3a 100644
--- a/src/tuntap/tun.go
+++ b/src/tuntap/tun.go
@@ -257,8 +257,13 @@ func (tun *TunAdapter) wrap(conn *yggdrasil.Conn) (c *tunConn, err error) {
 	tun.addrToConn[s.addr] = &s
 	tun.subnetToConn[s.snet] = &s
 	// Start the connection goroutines
-	go s.reader()
-	go s.writer()
+	conn.SetReadCallback(func(bs []byte) {
+		s.EnqueueFrom(conn, func() {
+			s._read(bs)
+		})
+	})
+	//go s.reader()
+	//go s.writer()
 	go s.checkForTimeouts()
 	// Return
 	return c, err

From ef15a6bd794145e49abdab53ca7247004046af3b Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 11:44:21 -0500
Subject: [PATCH 20/62] tunConn cleanup

---
 src/tuntap/conn.go | 202 ---------------------------------------------
 src/tuntap/tun.go  |   5 +-
 2 files changed, 1 insertion(+), 206 deletions(-)

diff --git a/src/tuntap/conn.go b/src/tuntap/conn.go
index 32998e99..744ca9fc 100644
--- a/src/tuntap/conn.go
+++ b/src/tuntap/conn.go
@@ -22,7 +22,6 @@ type tunConn struct {
 	conn  *yggdrasil.Conn
 	addr  address.Address
 	snet  address.Subnet
-	send  chan []byte
 	stop  chan struct{}
 	alive chan struct{}
 }
@@ -124,98 +123,6 @@ func (s *tunConn) _read(bs []byte) (err error) {
 	return
 }
 
-func (s *tunConn) reader() (err error) {
-	select {
-	case _, ok := <-s.stop:
-		if !ok {
-			return errors.New("session was already closed")
-		}
-	default:
-	}
-	s.tun.log.Debugln("Starting conn reader for", s.conn.String())
-	defer s.tun.log.Debugln("Stopping conn reader for", s.conn.String())
-	for {
-		select {
-		case <-s.stop:
-			return nil
-		default:
-		}
-		var bs []byte
-		if bs, err = s.conn.ReadNoCopy(); err != nil {
-			if e, eok := err.(yggdrasil.ConnError); eok && !e.Temporary() {
-				if e.Closed() {
-					s.tun.log.Debugln(s.conn.String(), "TUN/TAP conn read debug:", err)
-				} else {
-					s.tun.log.Errorln(s.conn.String(), "TUN/TAP conn read error:", err)
-				}
-				return e
-			}
-		} else if len(bs) > 0 {
-			ipv4 := len(bs) > 20 && bs[0]&0xf0 == 0x40
-			ipv6 := len(bs) > 40 && bs[0]&0xf0 == 0x60
-			isCGA := true
-			// Check source addresses
-			switch {
-			case ipv6 && bs[8] == 0x02 && bytes.Equal(s.addr[:16], bs[8:24]): // source
-			case ipv6 && bs[8] == 0x03 && bytes.Equal(s.snet[:8], bs[8:16]): // source
-			default:
-				isCGA = false
-			}
-			// Check destiantion addresses
-			switch {
-			case ipv6 && bs[24] == 0x02 && bytes.Equal(s.tun.addr[:16], bs[24:40]): // destination
-			case ipv6 && bs[24] == 0x03 && bytes.Equal(s.tun.subnet[:8], bs[24:32]): // destination
-			default:
-				isCGA = false
-			}
-			// Decide how to handle the packet
-			var skip bool
-			switch {
-			case isCGA: // Allowed
-			case s.tun.ckr.isEnabled() && (ipv4 || ipv6):
-				var srcAddr address.Address
-				var dstAddr address.Address
-				var addrlen int
-				if ipv4 {
-					copy(srcAddr[:], bs[12:16])
-					copy(dstAddr[:], bs[16:20])
-					addrlen = 4
-				}
-				if ipv6 {
-					copy(srcAddr[:], bs[8:24])
-					copy(dstAddr[:], bs[24:40])
-					addrlen = 16
-				}
-				if !s.tun.ckr.isValidLocalAddress(dstAddr, addrlen) {
-					// The destination address isn't in our CKR allowed range
-					skip = true
-				} else if key, err := s.tun.ckr.getPublicKeyForAddress(srcAddr, addrlen); err == nil {
-					srcNodeID := crypto.GetNodeID(&key)
-					if s.conn.RemoteAddr() == *srcNodeID {
-						// This is the one allowed CKR case, where source and destination addresses are both good
-					} else {
-						// The CKR key associated with this address doesn't match the sender's NodeID
-						skip = true
-					}
-				} else {
-					// We have no CKR route for this source address
-					skip = true
-				}
-			default:
-				skip = true
-			}
-			if skip {
-				util.PutBytes(bs)
-				continue
-			}
-			s.tun.send <- bs
-			s.stillAlive()
-		} else {
-			util.PutBytes(bs)
-		}
-	}
-}
-
 func (s *tunConn) _write(bs []byte) (err error) {
 	select {
 	case <-s.stop:
@@ -320,115 +227,6 @@ func (s *tunConn) _write(bs []byte) (err error) {
 	return
 }
 
-func (s *tunConn) writer() error {
-	select {
-	case _, ok := <-s.stop:
-		if !ok {
-			return errors.New("session was already closed")
-		}
-	default:
-	}
-	s.tun.log.Debugln("Starting conn writer for", s.conn.String())
-	defer s.tun.log.Debugln("Stopping conn writer for", s.conn.String())
-	for {
-		select {
-		case <-s.stop:
-			return nil
-		case bs, ok := <-s.send:
-			if !ok {
-				return errors.New("send closed")
-			}
-			v4 := len(bs) > 20 && bs[0]&0xf0 == 0x40
-			v6 := len(bs) > 40 && bs[0]&0xf0 == 0x60
-			isCGA := true
-			// Check source addresses
-			switch {
-			case v6 && bs[8] == 0x02 && bytes.Equal(s.tun.addr[:16], bs[8:24]): // source
-			case v6 && bs[8] == 0x03 && bytes.Equal(s.tun.subnet[:8], bs[8:16]): // source
-			default:
-				isCGA = false
-			}
-			// Check destiantion addresses
-			switch {
-			case v6 && bs[24] == 0x02 && bytes.Equal(s.addr[:16], bs[24:40]): // destination
-			case v6 && bs[24] == 0x03 && bytes.Equal(s.snet[:8], bs[24:32]): // destination
-			default:
-				isCGA = false
-			}
-			// Decide how to handle the packet
-			var skip bool
-			switch {
-			case isCGA: // Allowed
-			case s.tun.ckr.isEnabled() && (v4 || v6):
-				var srcAddr address.Address
-				var dstAddr address.Address
-				var addrlen int
-				if v4 {
-					copy(srcAddr[:], bs[12:16])
-					copy(dstAddr[:], bs[16:20])
-					addrlen = 4
-				}
-				if v6 {
-					copy(srcAddr[:], bs[8:24])
-					copy(dstAddr[:], bs[24:40])
-					addrlen = 16
-				}
-				if !s.tun.ckr.isValidLocalAddress(srcAddr, addrlen) {
-					// The source address isn't in our CKR allowed range
-					skip = true
-				} else if key, err := s.tun.ckr.getPublicKeyForAddress(dstAddr, addrlen); err == nil {
-					dstNodeID := crypto.GetNodeID(&key)
-					if s.conn.RemoteAddr() == *dstNodeID {
-						// This is the one allowed CKR case, where source and destination addresses are both good
-					} else {
-						// The CKR key associated with this address doesn't match the sender's NodeID
-						skip = true
-					}
-				} else {
-					// We have no CKR route for this destination address... why do we have the packet in the first place?
-					skip = true
-				}
-			default:
-				skip = true
-			}
-			if skip {
-				util.PutBytes(bs)
-				continue
-			}
-			msg := yggdrasil.FlowKeyMessage{
-				FlowKey: util.GetFlowKey(bs),
-				Message: bs,
-			}
-			if err := s.conn.WriteNoCopy(msg); err != nil {
-				if e, eok := err.(yggdrasil.ConnError); !eok {
-					if e.Closed() {
-						s.tun.log.Debugln(s.conn.String(), "TUN/TAP generic write debug:", err)
-					} else {
-						s.tun.log.Errorln(s.conn.String(), "TUN/TAP generic write error:", err)
-					}
-				} else if e.PacketTooBig() {
-					// TODO: This currently isn't aware of IPv4 for CKR
-					ptb := &icmp.PacketTooBig{
-						MTU:  int(e.PacketMaximumSize()),
-						Data: bs[:900],
-					}
-					if packet, err := CreateICMPv6(bs[8:24], bs[24:40], ipv6.ICMPTypePacketTooBig, 0, ptb); err == nil {
-						s.tun.send <- packet
-					}
-				} else {
-					if e.Closed() {
-						s.tun.log.Debugln(s.conn.String(), "TUN/TAP conn write debug:", err)
-					} else {
-						s.tun.log.Errorln(s.conn.String(), "TUN/TAP conn write error:", err)
-					}
-				}
-			} else {
-				s.stillAlive()
-			}
-		}
-	}
-}
-
 func (s *tunConn) stillAlive() {
 	defer func() { recover() }()
 	select {
diff --git a/src/tuntap/tun.go b/src/tuntap/tun.go
index dfc9ac3a..895f893b 100644
--- a/src/tuntap/tun.go
+++ b/src/tuntap/tun.go
@@ -229,7 +229,6 @@ func (tun *TunAdapter) wrap(conn *yggdrasil.Conn) (c *tunConn, err error) {
 	s := tunConn{
 		tun:   tun,
 		conn:  conn,
-		send:  make(chan []byte, 32), // TODO: is this a sensible value?
 		stop:  make(chan struct{}),
 		alive: make(chan struct{}, 1),
 	}
@@ -256,14 +255,12 @@ func (tun *TunAdapter) wrap(conn *yggdrasil.Conn) (c *tunConn, err error) {
 	// we receive a packet through the interface for this address
 	tun.addrToConn[s.addr] = &s
 	tun.subnetToConn[s.snet] = &s
-	// Start the connection goroutines
+	// Set the read callback and start the timeout goroutine
 	conn.SetReadCallback(func(bs []byte) {
 		s.EnqueueFrom(conn, func() {
 			s._read(bs)
 		})
 	})
-	//go s.reader()
-	//go s.writer()
 	go s.checkForTimeouts()
 	// Return
 	return c, err

From 775fb535dce6ad3d1639dff10fa50d518a9f041b Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 12:46:24 -0500
Subject: [PATCH 21/62] start converting the peer struct into an actor

---
 src/yggdrasil/peer.go   | 34 +++++++++++++++++++++-------------
 src/yggdrasil/switch.go |  8 +++++---
 2 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 8869bd2a..851a376a 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -12,6 +12,8 @@ import (
 
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
+
+	"github.com/Arceliar/phony"
 )
 
 // The peers struct represents peers with an active connection.
@@ -97,6 +99,7 @@ type peer struct {
 	bytesSent  uint64 // To track bandwidth usage for getPeers
 	bytesRecvd uint64 // To track bandwidth usage for getPeers
 	// BUG: sync/atomic, 32 bit platforms need the above to be the first element
+	phony.Actor
 	core       *Core
 	intf       *linkInterface
 	port       switchPort
@@ -206,7 +209,7 @@ func (p *peer) linkLoop() {
 			if !ok {
 				return
 			}
-			p.sendSwitchMsg()
+			<-p.SyncExec(p._sendSwitchMsg)
 		case dinfo = <-p.dinfo:
 		case _ = <-tick.C:
 			if dinfo != nil {
@@ -227,20 +230,19 @@ func (p *peer) handlePacket(packet []byte) {
 	}
 	switch pType {
 	case wire_Traffic:
-		p.handleTraffic(packet, pTypeLen)
+		p._handleTraffic(packet, pTypeLen)
 	case wire_ProtocolTraffic:
-		p.handleTraffic(packet, pTypeLen)
+		p._handleTraffic(packet, pTypeLen)
 	case wire_LinkProtocolTraffic:
-		p.handleLinkTraffic(packet)
+		p._handleLinkTraffic(packet)
 	default:
 		util.PutBytes(packet)
 	}
-	return
 }
 
 // Called to handle traffic or protocolTraffic packets.
 // In either case, this reads from the coords of the packet header, does a switch lookup, and forwards to the next node.
-func (p *peer) handleTraffic(packet []byte, pTypeLen int) {
+func (p *peer) _handleTraffic(packet []byte, pTypeLen int) {
 	table := p.core.switchTable.getTable()
 	if _, isIn := table.elems[p.port]; !isIn && p.port != 0 {
 		// Drop traffic if the peer isn't in the switch
@@ -249,8 +251,14 @@ func (p *peer) handleTraffic(packet []byte, pTypeLen int) {
 	p.core.switchTable.packetIn <- packet
 }
 
+func (p *peer) sendPacketsFrom(from phony.IActor, packets [][]byte) {
+	p.EnqueueFrom(from, func() {
+		p._sendPackets(packets)
+	})
+}
+
 // This just calls p.out(packet) for now.
-func (p *peer) sendPackets(packets [][]byte) {
+func (p *peer) _sendPackets(packets [][]byte) {
 	// Is there ever a case where something more complicated is needed?
 	// What if p.out blocks?
 	var size int
@@ -263,7 +271,7 @@ func (p *peer) sendPackets(packets [][]byte) {
 
 // This wraps the packet in the inner (ephemeral) and outer (permanent) crypto layers.
 // It sends it to p.linkOut, which bypasses the usual packet queues.
-func (p *peer) sendLinkPacket(packet []byte) {
+func (p *peer) _sendLinkPacket(packet []byte) {
 	innerPayload, innerNonce := crypto.BoxSeal(&p.linkShared, packet, nil)
 	innerLinkPacket := wire_linkProtoTrafficPacket{
 		Nonce:   *innerNonce,
@@ -281,7 +289,7 @@ func (p *peer) sendLinkPacket(packet []byte) {
 
 // Decrypts the outer (permanent) and inner (ephemeral) crypto layers on link traffic.
 // Identifies the link traffic type and calls the appropriate handler.
-func (p *peer) handleLinkTraffic(bs []byte) {
+func (p *peer) _handleLinkTraffic(bs []byte) {
 	packet := wire_linkProtoTrafficPacket{}
 	if !packet.decode(bs) {
 		return
@@ -304,14 +312,14 @@ func (p *peer) handleLinkTraffic(bs []byte) {
 	}
 	switch pType {
 	case wire_SwitchMsg:
-		p.handleSwitchMsg(payload)
+		p._handleSwitchMsg(payload)
 	default:
 		util.PutBytes(bs)
 	}
 }
 
 // Gets a switchMsg from the switch, adds signed next-hop info for this peer, and sends it to them.
-func (p *peer) sendSwitchMsg() {
+func (p *peer) _sendSwitchMsg() {
 	msg := p.core.switchTable.getMsg()
 	if msg == nil {
 		return
@@ -323,12 +331,12 @@ func (p *peer) sendSwitchMsg() {
 		Sig:  *crypto.Sign(&p.core.sigPriv, bs),
 	})
 	packet := msg.encode()
-	p.sendLinkPacket(packet)
+	p._sendLinkPacket(packet)
 }
 
 // Handles a switchMsg from the peer, checking signatures and passing good messages to the switch.
 // Also creates a dhtInfo struct and arranges for it to be added to the dht (this is how dht bootstrapping begins).
-func (p *peer) handleSwitchMsg(packet []byte) {
+func (p *peer) _handleSwitchMsg(packet []byte) {
 	var msg switchMsg
 	if !msg.decode(packet) {
 		return
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 1fa75a6c..d87b7880 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -703,7 +703,7 @@ func (t *switchTable) handleIn(packet []byte, idle map[switchPort]time.Time) boo
 	if best != nil {
 		// Send to the best idle next hop
 		delete(idle, best.port)
-		best.sendPackets([][]byte{packet})
+		best.sendPacketsFrom(nil, [][]byte{packet})
 		return true
 	}
 	// Didn't find anyone idle to send it to
@@ -817,7 +817,8 @@ func (t *switchTable) handleIdle(port switchPort) bool {
 		}
 	}
 	if len(packets) > 0 {
-		to.sendPackets(packets)
+		// TODO rewrite if/when the switch becomes an actor
+		to.sendPacketsFrom(nil, packets)
 		return true
 	}
 	return false
@@ -830,7 +831,8 @@ func (t *switchTable) doWorker() {
 		// Keep sending packets to the router
 		self := t.core.peers.getPorts()[0]
 		for bs := range sendingToRouter {
-			self.sendPackets([][]byte{bs})
+			// TODO remove this ugly mess of goroutines if/when the switch becomes an actor
+			<-self.SyncExec(func() { self._sendPackets([][]byte{bs}) })
 		}
 	}()
 	go func() {

From 88161009e94740511fb25b73dbb5b19c68f00af1 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 12:55:49 -0500
Subject: [PATCH 22/62] more peer migration

---
 src/yggdrasil/link.go   | 3 ++-
 src/yggdrasil/peer.go   | 8 +++++++-
 src/yggdrasil/router.go | 7 ++-----
 3 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index 4ce374b4..824afd34 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -387,7 +387,8 @@ func (intf *linkInterface) handler() error {
 	for {
 		msg, err := intf.msgIO.readMsg()
 		if len(msg) > 0 {
-			intf.peer.handlePacket(msg)
+			// TODO rewrite this if the link becomes an actor
+			<-intf.peer.SyncExec(func() { intf.peer._handlePacket(msg) })
 		}
 		if err != nil {
 			if err != io.EOF {
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 851a376a..22db88d1 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -219,9 +219,15 @@ func (p *peer) linkLoop() {
 	}
 }
 
+func (p *peer) handlePacketFrom(from phony.IActor, packet []byte) {
+	p.EnqueueFrom(from, func() {
+		p._handlePacket(packet)
+	})
+}
+
 // Called to handle incoming packets.
 // Passes the packet to a handler for that packet type.
-func (p *peer) handlePacket(packet []byte) {
+func (p *peer) _handlePacket(packet []byte) {
 	// FIXME this is off by stream padding and msg length overhead, should be done in tcp.go
 	atomic.AddUint64(&p.bytesRecvd, uint64(len(packet)))
 	pType, pTypeLen := wire_decode_uint64(packet)
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index 464a4778..adf1b1d4 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -64,11 +64,8 @@ func (r *router) init(core *Core) {
 		},
 	}
 	p := r.core.peers.newPeer(&r.core.boxPub, &r.core.sigPub, &crypto.BoxSharedKey{}, &self, nil)
-	p.out = func(packets [][]byte) {
-		// TODO make peers and/or the switch into actors, have them pass themselves as the from field
-		r.handlePackets(nil, packets)
-	}
-	r.out = p.handlePacket // TODO if the peer becomes its own actor, then send a message here
+	p.out = func(packets [][]byte) { r.handlePackets(p, packets) }
+	r.out = func(bs []byte) { p.handlePacketFrom(r, bs) }
 	r.nodeinfo.init(r.core)
 	r.core.config.Mutex.RLock()
 	r.nodeinfo.setNodeInfo(r.core.config.Current.NodeInfo, r.core.config.Current.NodeInfoPrivacy)

From ecd23ce9fc1e46c016697c4ac945162b9fb159e5 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 12:59:20 -0500
Subject: [PATCH 23/62] safer linkloop

---
 src/yggdrasil/peer.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 22db88d1..8c38c06a 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -213,7 +213,7 @@ func (p *peer) linkLoop() {
 		case dinfo = <-p.dinfo:
 		case _ = <-tick.C:
 			if dinfo != nil {
-				p.core.router.insertPeer(nil, dinfo)
+				<-p.SyncExec(func() { p.core.router.insertPeer(p, dinfo) })
 			}
 		}
 	}

From 034fece33f4fbabbf2bdf8c4fb643b91791438a7 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 13:15:29 -0500
Subject: [PATCH 24/62] more peer migration

---
 src/yggdrasil/peer.go   | 50 ++++++++++++++++-------------------------
 src/yggdrasil/switch.go |  4 ++--
 2 files changed, 21 insertions(+), 33 deletions(-)

diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 8c38c06a..c94cf266 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -110,9 +110,9 @@ type peer struct {
 	endpoint   string
 	firstSeen  time.Time       // To track uptime for getPeers
 	linkOut    (chan []byte)   // used for protocol traffic (to bypass queues)
-	doSend     (chan struct{}) // tell the linkLoop to send a switchMsg
-	dinfo      (chan *dhtInfo) // used to keep the DHT working
+	dinfo      *dhtInfo        // used to keep the DHT working
 	out        func([][]byte)  // Set up by whatever created the peers struct, used to send packets to other nodes
+	done       (chan struct{}) // closed to exit the linkLoop
 	close      func()          // Called when a peer is removed, to close the underlying connection, or via admin api
 }
 
@@ -124,8 +124,7 @@ func (ps *peers) newPeer(box *crypto.BoxPubKey, sig *crypto.SigPubKey, linkShare
 		shared:     *crypto.GetSharedKey(&ps.core.boxPriv, box),
 		linkShared: *linkShared,
 		firstSeen:  now,
-		doSend:     make(chan struct{}, 1),
-		dinfo:      make(chan *dhtInfo, 1),
+		done:       make(chan struct{}),
 		close:      closer,
 		core:       ps.core,
 		intf:       intf,
@@ -170,29 +169,19 @@ func (ps *peers) removePeer(port switchPort) {
 		if p.close != nil {
 			p.close()
 		}
-		close(p.doSend)
+		close(p.done)
 	}
 }
 
 // If called, sends a notification to each peer that they should send a new switch message.
 // Mainly called by the switch after an update.
-func (ps *peers) sendSwitchMsgs() {
+func (ps *peers) sendSwitchMsgs(from phony.IActor) {
 	ports := ps.getPorts()
 	for _, p := range ports {
 		if p.port == 0 {
 			continue
 		}
-		p.doSendSwitchMsgs()
-	}
-}
-
-// If called, sends a notification to the peer's linkLoop to trigger a switchMsg send.
-// Mainly called by sendSwitchMsgs or during linkLoop startup.
-func (p *peer) doSendSwitchMsgs() {
-	defer func() { recover() }() // In case there's a race with close(p.doSend)
-	select {
-	case p.doSend <- struct{}{}:
-	default:
+		p.EnqueueFrom(from, p._sendSwitchMsg)
 	}
 }
 
@@ -201,24 +190,23 @@ func (p *peer) doSendSwitchMsgs() {
 func (p *peer) linkLoop() {
 	tick := time.NewTicker(time.Second)
 	defer tick.Stop()
-	p.doSendSwitchMsgs()
-	var dinfo *dhtInfo
+	<-p.SyncExec(p._sendSwitchMsg) // Startup message
 	for {
 		select {
-		case _, ok := <-p.doSend:
-			if !ok {
-				return
-			}
-			<-p.SyncExec(p._sendSwitchMsg)
-		case dinfo = <-p.dinfo:
+		case <-p.done:
+			return
 		case _ = <-tick.C:
-			if dinfo != nil {
-				<-p.SyncExec(func() { p.core.router.insertPeer(p, dinfo) })
-			}
+			<-p.SyncExec(p._updateDHT)
 		}
 	}
 }
 
+func (p *peer) _updateDHT() {
+	if p.dinfo != nil {
+		p.core.router.insertPeer(p, p.dinfo)
+	}
+}
+
 func (p *peer) handlePacketFrom(from phony.IActor, packet []byte) {
 	p.EnqueueFrom(from, func() {
 		p._handlePacket(packet)
@@ -366,16 +354,16 @@ func (p *peer) _handleSwitchMsg(packet []byte) {
 	p.core.switchTable.handleMsg(&msg, p.port)
 	if !p.core.switchTable.checkRoot(&msg) {
 		// Bad switch message
-		p.dinfo <- nil
+		p.dinfo = nil
 		return
 	}
 	// Pass a mesage to the dht informing it that this peer (still) exists
 	loc.coords = loc.coords[:len(loc.coords)-1]
-	dinfo := dhtInfo{
+	p.dinfo = &dhtInfo{
 		key:    p.box,
 		coords: loc.getCoords(),
 	}
-	p.dinfo <- &dinfo
+	p._updateDHT()
 }
 
 // This generates the bytes that we sign or check the signature of for a switchMsg.
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index d87b7880..0be01240 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -248,7 +248,7 @@ func (t *switchTable) cleanRoot() {
 			t.core.router.reset(nil)
 		}
 		t.data.locator = switchLocator{root: t.key, tstamp: now.Unix()}
-		t.core.peers.sendSwitchMsgs()
+		t.core.peers.sendSwitchMsgs(nil) // TODO update if/when the switch becomes an actor
 	}
 }
 
@@ -515,7 +515,7 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort, rep
 		}
 		t.data.locator = sender.locator
 		t.parent = sender.port
-		t.core.peers.sendSwitchMsgs()
+		t.core.peers.sendSwitchMsgs(nil) // TODO update if/when the switch becomes an actor
 	}
 	if doUpdate {
 		t.updater.Store(&sync.Once{})

From 0539dee900342a9fad784e83bea51f32459cba9f Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 13:25:38 -0500
Subject: [PATCH 25/62] warning about possible deadlock in legacy channel send,
 need to migrate the link code to fix it

---
 src/yggdrasil/peer.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index c94cf266..f3af140f 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -278,6 +278,8 @@ func (p *peer) _sendLinkPacket(packet []byte) {
 		Payload: bs,
 	}
 	packet = linkPacket.encode()
+	// TODO replace this with a message send if/when the link becomes an actor
+	// FIXME not 100% sure the channel send version is deadlock-free...
 	p.linkOut <- packet
 }
 

From b337228aa45436736640492b54cd64059f21a199 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 14:24:42 -0500
Subject: [PATCH 26/62] minor fixes to peer stuff

---
 src/yggdrasil/api.go   | 43 +++++++++++++++++++++++-------------------
 src/yggdrasil/debug.go |  2 +-
 src/yggdrasil/peer.go  | 20 +++++++++++++-------
 3 files changed, 38 insertions(+), 27 deletions(-)

diff --git a/src/yggdrasil/api.go b/src/yggdrasil/api.go
index f50c8ceb..cbf232aa 100644
--- a/src/yggdrasil/api.go
+++ b/src/yggdrasil/api.go
@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"net"
 	"sort"
-	"sync/atomic"
 	"time"
 
 	"github.com/gologme/log"
@@ -106,15 +105,18 @@ func (c *Core) GetPeers() []Peer {
 	sort.Slice(ps, func(i, j int) bool { return ps[i] < ps[j] })
 	for _, port := range ps {
 		p := ports[port]
-		info := Peer{
-			Endpoint:   p.intf.name,
-			BytesSent:  atomic.LoadUint64(&p.bytesSent),
-			BytesRecvd: atomic.LoadUint64(&p.bytesRecvd),
-			Protocol:   p.intf.info.linkType,
-			Port:       uint64(port),
-			Uptime:     time.Since(p.firstSeen),
-		}
-		copy(info.PublicKey[:], p.box[:])
+		var info Peer
+		<-p.SyncExec(func() {
+			info = Peer{
+				Endpoint:   p.intf.name,
+				BytesSent:  p.bytesSent,
+				BytesRecvd: p.bytesRecvd,
+				Protocol:   p.intf.info.linkType,
+				Port:       uint64(port),
+				Uptime:     time.Since(p.firstSeen),
+			}
+			copy(info.PublicKey[:], p.box[:])
+		})
 		peers = append(peers, info)
 	}
 	return peers
@@ -135,15 +137,18 @@ func (c *Core) GetSwitchPeers() []SwitchPeer {
 			continue
 		}
 		coords := elem.locator.getCoords()
-		info := SwitchPeer{
-			Coords:     append([]uint64{}, wire_coordsBytestoUint64s(coords)...),
-			BytesSent:  atomic.LoadUint64(&peer.bytesSent),
-			BytesRecvd: atomic.LoadUint64(&peer.bytesRecvd),
-			Port:       uint64(elem.port),
-			Protocol:   peer.intf.info.linkType,
-			Endpoint:   peer.intf.info.remote,
-		}
-		copy(info.PublicKey[:], peer.box[:])
+		var info SwitchPeer
+		<-peer.SyncExec(func() {
+			info = SwitchPeer{
+				Coords:     append([]uint64{}, wire_coordsBytestoUint64s(coords)...),
+				BytesSent:  peer.bytesSent,
+				BytesRecvd: peer.bytesRecvd,
+				Port:       uint64(elem.port),
+				Protocol:   peer.intf.info.linkType,
+				Endpoint:   peer.intf.info.remote,
+			}
+			copy(info.PublicKey[:], peer.box[:])
+		})
 		switchpeers = append(switchpeers, info)
 	}
 	return switchpeers
diff --git a/src/yggdrasil/debug.go b/src/yggdrasil/debug.go
index d90bf1f4..30eb874d 100644
--- a/src/yggdrasil/debug.go
+++ b/src/yggdrasil/debug.go
@@ -576,7 +576,7 @@ func DEBUG_simLinkPeers(p, q *peer) {
 				default:
 				}
 				if len(packets) > 0 {
-					dest.handlePacket(packets[0])
+					<-dest.SyncExec(func() { dest._handlePacket(packets[0]) })
 					packets = packets[1:]
 					continue
 				}
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index f3af140f..4bebc17e 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -96,9 +96,6 @@ func (ps *peers) putPorts(ports map[switchPort]*peer) {
 
 // Information known about a peer, including thier box/sig keys, precomputed shared keys (static and ephemeral) and a handler for their outgoing traffic
 type peer struct {
-	bytesSent  uint64 // To track bandwidth usage for getPeers
-	bytesRecvd uint64 // To track bandwidth usage for getPeers
-	// BUG: sync/atomic, 32 bit platforms need the above to be the first element
 	phony.Actor
 	core       *Core
 	intf       *linkInterface
@@ -114,6 +111,9 @@ type peer struct {
 	out        func([][]byte)  // Set up by whatever created the peers struct, used to send packets to other nodes
 	done       (chan struct{}) // closed to exit the linkLoop
 	close      func()          // Called when a peer is removed, to close the underlying connection, or via admin api
+	// The below aren't actually useful internally, they're just gathered for getPeers statistics
+	bytesSent  uint64
+	bytesRecvd uint64
 }
 
 // Creates a new peer with the specified box, sig, and linkShared keys, using the lowest unoccupied port number.
@@ -217,7 +217,7 @@ func (p *peer) handlePacketFrom(from phony.IActor, packet []byte) {
 // Passes the packet to a handler for that packet type.
 func (p *peer) _handlePacket(packet []byte) {
 	// FIXME this is off by stream padding and msg length overhead, should be done in tcp.go
-	atomic.AddUint64(&p.bytesRecvd, uint64(len(packet)))
+	p.bytesRecvd += uint64(len(packet))
 	pType, pTypeLen := wire_decode_uint64(packet)
 	if pTypeLen == 0 {
 		return
@@ -259,10 +259,12 @@ func (p *peer) _sendPackets(packets [][]byte) {
 	for _, packet := range packets {
 		size += len(packet)
 	}
-	atomic.AddUint64(&p.bytesSent, uint64(size))
+	p.bytesSent += uint64(size)
 	p.out(packets)
 }
 
+var peerLinkOutHelper phony.Actor
+
 // This wraps the packet in the inner (ephemeral) and outer (permanent) crypto layers.
 // It sends it to p.linkOut, which bypasses the usual packet queues.
 func (p *peer) _sendLinkPacket(packet []byte) {
@@ -279,8 +281,12 @@ func (p *peer) _sendLinkPacket(packet []byte) {
 	}
 	packet = linkPacket.encode()
 	// TODO replace this with a message send if/when the link becomes an actor
-	// FIXME not 100% sure the channel send version is deadlock-free...
-	p.linkOut <- packet
+	peerLinkOutHelper.EnqueueFrom(nil, func() {
+		select {
+		case p.linkOut <- packet:
+		case <-p.done:
+		}
+	})
 }
 
 // Decrypts the outer (permanent) and inner (ephemeral) crypto layers on link traffic.

From 498bc395e2ed8adc55813f8aeda24f2827c3d4d3 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 14:56:33 -0500
Subject: [PATCH 27/62] start migrating switch to the actor model

---
 src/yggdrasil/switch.go | 117 ++++++++++++++++------------------------
 1 file changed, 47 insertions(+), 70 deletions(-)

diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 0be01240..7f3c7a78 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -19,6 +19,8 @@ import (
 
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
+
+	"github.com/Arceliar/phony"
 )
 
 const (
@@ -172,12 +174,13 @@ type switchTable struct {
 	data              switchData                 //
 	updater           atomic.Value               // *sync.Once
 	table             atomic.Value               // lookupTable
+	phony.Actor                                  // Owns the below
 	packetIn          chan []byte                // Incoming packets for the worker to handle
 	idleIn            chan switchPort            // Incoming idle notifications from peer links
 	admin             chan func()                // Pass a lambda for the admin socket to query stuff
 	queues            switch_buffers             // Queues - not atomic so ONLY use through admin chan
 	queueTotalMaxSize uint64                     // Maximum combined size of queues
-	toRouter          chan []byte                // Packets to be sent to the router
+	idle              map[switchPort]time.Time   // idle peers
 }
 
 // Minimum allowed total size of switch queues.
@@ -199,7 +202,7 @@ func (t *switchTable) init(core *Core) {
 	t.idleIn = make(chan switchPort, 1024)
 	t.admin = make(chan func())
 	t.queueTotalMaxSize = SwitchQueueTotalMinSize
-	t.toRouter = make(chan []byte, 1)
+	t.idle = make(map[switchPort]time.Time)
 }
 
 // Safely gets a copy of this node's locator.
@@ -653,12 +656,13 @@ func (t *switchTable) bestPortForCoords(coords []byte) switchPort {
 // Handle an incoming packet
 // Either send it to ourself, or to the first idle peer that's free
 // Returns true if the packet has been handled somehow, false if it should be queued
-func (t *switchTable) handleIn(packet []byte, idle map[switchPort]time.Time) bool {
+func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]time.Time) bool {
 	coords := switch_getPacketCoords(packet)
 	closer := t.getCloser(coords)
 	if len(closer) == 0 {
 		// TODO? call the router directly, and remove the whole concept of a self peer?
-		t.toRouter <- packet
+		self := t.core.peers.getPorts()[0]
+		self.sendPacketsFrom(t, [][]byte{packet})
 		return true
 	}
 	var best *peer
@@ -731,7 +735,7 @@ type switch_buffers struct {
 	closer      []closerInfo // Scratch space
 }
 
-func (b *switch_buffers) cleanup(t *switchTable) {
+func (b *switch_buffers) _cleanup(t *switchTable) {
 	for streamID, buf := range b.bufs {
 		// Remove queues for which we have no next hop
 		packet := buf.packets[0]
@@ -773,14 +777,14 @@ func (b *switch_buffers) cleanup(t *switchTable) {
 // Handles incoming idle notifications
 // Loops over packets and sends the newest one that's OK for this peer to send
 // Returns true if the peer is no longer idle, false if it should be added to the idle list
-func (t *switchTable) handleIdle(port switchPort) bool {
+func (t *switchTable) _handleIdle(port switchPort) bool {
 	to := t.core.peers.getPorts()[port]
 	if to == nil {
 		return true
 	}
 	var packets [][]byte
 	var psize int
-	t.queues.cleanup(t)
+	t.queues._cleanup(t)
 	now := time.Now()
 	for psize < 65535 {
 		var best string
@@ -824,79 +828,52 @@ func (t *switchTable) handleIdle(port switchPort) bool {
 	return false
 }
 
-// The switch worker does routing lookups and sends packets to where they need to be
-func (t *switchTable) doWorker() {
-	sendingToRouter := make(chan []byte, 1)
-	go func() {
-		// Keep sending packets to the router
-		self := t.core.peers.getPorts()[0]
-		for bs := range sendingToRouter {
-			// TODO remove this ugly mess of goroutines if/when the switch becomes an actor
-			<-self.SyncExec(func() { self._sendPackets([][]byte{bs}) })
+func (t *switchTable) _packetIn(bytes []byte) {
+	// Try to send it somewhere (or drop it if it's corrupt or at a dead end)
+	if !t._handleIn(bytes, t.idle) {
+		// There's nobody free to take it right now, so queue it for later
+		packet := switch_packetInfo{bytes, time.Now()}
+		streamID := switch_getPacketStreamID(packet.bytes)
+		buf, bufExists := t.queues.bufs[streamID]
+		buf.packets = append(buf.packets, packet)
+		buf.size += uint64(len(packet.bytes))
+		t.queues.size += uint64(len(packet.bytes))
+		// Keep a track of the max total queue size
+		if t.queues.size > t.queues.maxsize {
+			t.queues.maxsize = t.queues.size
 		}
-	}()
-	go func() {
-		// Keep taking packets from the idle worker and sending them to the above whenever it's idle, keeping anything extra in a (fifo, head-drop) buffer
-		var buf [][]byte
-		var size int
-		for {
-			bs := <-t.toRouter
-			size += len(bs)
-			buf = append(buf, bs)
-			for len(buf) > 0 {
-				select {
-				case bs := <-t.toRouter:
-					size += len(bs)
-					buf = append(buf, bs)
-					for size > int(t.queueTotalMaxSize) {
-						size -= len(buf[0])
-						util.PutBytes(buf[0])
-						buf = buf[1:]
-					}
-				case sendingToRouter <- buf[0]:
-					size -= len(buf[0])
-					buf = buf[1:]
-				}
+		t.queues.bufs[streamID] = buf
+		if !bufExists {
+			// Keep a track of the max total queue count. Only recalculate this
+			// when the queue is new because otherwise repeating len(dict) might
+			// cause unnecessary processing overhead
+			if len(t.queues.bufs) > t.queues.maxbufs {
+				t.queues.maxbufs = len(t.queues.bufs)
 			}
 		}
-	}()
+		t.queues._cleanup(t)
+	}
+}
+
+func (t *switchTable) _idleIn(port switchPort) {
+	// Try to find something to send to this peer
+	if !t._handleIdle(port) {
+		// Didn't find anything ready to send yet, so stay idle
+		t.idle[port] = time.Now()
+	}
+}
+
+// The switch worker does routing lookups and sends packets to where they need to be
+func (t *switchTable) doWorker() {
 	t.queues.switchTable = t
 	t.queues.bufs = make(map[string]switch_buffer) // Packets per PacketStreamID (string)
-	idle := make(map[switchPort]time.Time)         // this is to deduplicate things
 	for {
 		//t.core.log.Debugf("Switch state: idle = %d, buffers = %d", len(idle), len(t.queues.bufs))
 		select {
 		case bytes := <-t.packetIn:
-			// Try to send it somewhere (or drop it if it's corrupt or at a dead end)
-			if !t.handleIn(bytes, idle) {
-				// There's nobody free to take it right now, so queue it for later
-				packet := switch_packetInfo{bytes, time.Now()}
-				streamID := switch_getPacketStreamID(packet.bytes)
-				buf, bufExists := t.queues.bufs[streamID]
-				buf.packets = append(buf.packets, packet)
-				buf.size += uint64(len(packet.bytes))
-				t.queues.size += uint64(len(packet.bytes))
-				// Keep a track of the max total queue size
-				if t.queues.size > t.queues.maxsize {
-					t.queues.maxsize = t.queues.size
-				}
-				t.queues.bufs[streamID] = buf
-				if !bufExists {
-					// Keep a track of the max total queue count. Only recalculate this
-					// when the queue is new because otherwise repeating len(dict) might
-					// cause unnecessary processing overhead
-					if len(t.queues.bufs) > t.queues.maxbufs {
-						t.queues.maxbufs = len(t.queues.bufs)
-					}
-				}
-				t.queues.cleanup(t)
-			}
+			<-t.SyncExec(func() { t._packetIn(bytes) })
 		case port := <-t.idleIn:
-			// Try to find something to send to this peer
-			if !t.handleIdle(port) {
-				// Didn't find anything ready to send yet, so stay idle
-				idle[port] = time.Now()
-			}
+			<-t.SyncExec(func() { t._idleIn(port) })
 		case f := <-t.admin:
 			f()
 		case e := <-t.reconfigure:

From 555b4c18d4938dd2ad70941e8ffaf95490057059 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 15:05:18 -0500
Subject: [PATCH 28/62] a little switch cleanup

---
 src/yggdrasil/switch.go | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)

diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 7f3c7a78..3f4c41fe 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -177,7 +177,6 @@ type switchTable struct {
 	phony.Actor                                  // Owns the below
 	packetIn          chan []byte                // Incoming packets for the worker to handle
 	idleIn            chan switchPort            // Incoming idle notifications from peer links
-	admin             chan func()                // Pass a lambda for the admin socket to query stuff
 	queues            switch_buffers             // Queues - not atomic so ONLY use through admin chan
 	queueTotalMaxSize uint64                     // Maximum combined size of queues
 	idle              map[switchPort]time.Time   // idle peers
@@ -200,9 +199,10 @@ func (t *switchTable) init(core *Core) {
 	t.drop = make(map[crypto.SigPubKey]int64)
 	t.packetIn = make(chan []byte, 1024)
 	t.idleIn = make(chan switchPort, 1024)
-	t.admin = make(chan func())
 	t.queueTotalMaxSize = SwitchQueueTotalMinSize
 	t.idle = make(map[switchPort]time.Time)
+	t.queues.switchTable = t
+	t.queues.bufs = make(map[string]switch_buffer) // Packets per PacketStreamID (string)
 }
 
 // Safely gets a copy of this node's locator.
@@ -865,8 +865,6 @@ func (t *switchTable) _idleIn(port switchPort) {
 
 // The switch worker does routing lookups and sends packets to where they need to be
 func (t *switchTable) doWorker() {
-	t.queues.switchTable = t
-	t.queues.bufs = make(map[string]switch_buffer) // Packets per PacketStreamID (string)
 	for {
 		//t.core.log.Debugf("Switch state: idle = %d, buffers = %d", len(idle), len(t.queues.bufs))
 		select {
@@ -874,8 +872,6 @@ func (t *switchTable) doWorker() {
 			<-t.SyncExec(func() { t._packetIn(bytes) })
 		case port := <-t.idleIn:
 			<-t.SyncExec(func() { t._idleIn(port) })
-		case f := <-t.admin:
-			f()
 		case e := <-t.reconfigure:
 			e <- nil
 		}
@@ -885,13 +881,5 @@ func (t *switchTable) doWorker() {
 // Passed a function to call.
 // This will send the function to t.admin and block until it finishes.
 func (t *switchTable) doAdmin(f func()) {
-	// Pass this a function that needs to be run by the router's main goroutine
-	// It will pass the function to the router and wait for the router to finish
-	done := make(chan struct{})
-	newF := func() {
-		f()
-		close(done)
-	}
-	t.admin <- newF
-	<-done
+	<-t.SyncExec(f)
 }

From 998c76fd8cd3c97131a4ba31a24b53e0475b8f10 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 15:22:46 -0500
Subject: [PATCH 29/62] more switch migration

---
 src/yggdrasil/api.go    |  2 +-
 src/yggdrasil/core.go   |  4 ++-
 src/yggdrasil/link.go   |  8 ++++--
 src/yggdrasil/peer.go   |  2 +-
 src/yggdrasil/switch.go | 64 ++++++++++++++++++++---------------------
 5 files changed, 42 insertions(+), 38 deletions(-)

diff --git a/src/yggdrasil/api.go b/src/yggdrasil/api.go
index cbf232aa..2bc5c816 100644
--- a/src/yggdrasil/api.go
+++ b/src/yggdrasil/api.go
@@ -191,7 +191,7 @@ func (c *Core) GetSwitchQueues() SwitchQueues {
 			Size:         switchTable.queues.size,
 			HighestCount: uint64(switchTable.queues.maxbufs),
 			HighestSize:  switchTable.queues.maxsize,
-			MaximumSize:  switchTable.queueTotalMaxSize,
+			MaximumSize:  switchTable.queues.totalMaxSize,
 		}
 		for k, v := range switchTable.queues.bufs {
 			nexthop := switchTable.bestPortForCoords([]byte(k))
diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index fb2142c0..40982cdd 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -174,7 +174,9 @@ func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState,
 
 	c.config.Mutex.RLock()
 	if c.config.Current.SwitchOptions.MaxTotalQueueSize >= SwitchQueueTotalMinSize {
-		c.switchTable.queueTotalMaxSize = c.config.Current.SwitchOptions.MaxTotalQueueSize
+		c.switchTable.doAdmin(func() {
+			c.switchTable.queues.totalMaxSize = c.config.Current.SwitchOptions.MaxTotalQueueSize
+		})
 	}
 	c.config.Mutex.RUnlock()
 
diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index 824afd34..1389f41e 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -318,7 +318,9 @@ func (intf *linkInterface) handler() error {
 				isAlive = true
 				if !isReady {
 					// (Re-)enable in the switch
-					intf.link.core.switchTable.idleIn <- intf.peer.port
+					intf.link.core.switchTable.EnqueueFrom(nil, func() {
+						intf.link.core.switchTable._idleIn(intf.peer.port)
+					})
 					isReady = true
 				}
 				if gotMsg && !sendTimerRunning {
@@ -355,7 +357,9 @@ func (intf *linkInterface) handler() error {
 					isReady = false
 				} else {
 					// Keep enabled in the switch
-					intf.link.core.switchTable.idleIn <- intf.peer.port
+					intf.link.core.switchTable.EnqueueFrom(nil, func() {
+						intf.link.core.switchTable._idleIn(intf.peer.port)
+					})
 					isReady = true
 				}
 			case <-sendBlocked.C:
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 4bebc17e..9a6f75f1 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -242,7 +242,7 @@ func (p *peer) _handleTraffic(packet []byte, pTypeLen int) {
 		// Drop traffic if the peer isn't in the switch
 		return
 	}
-	p.core.switchTable.packetIn <- packet
+	p.core.switchTable.packetInFrom(p, packet)
 }
 
 func (p *peer) sendPacketsFrom(from phony.IActor, packets [][]byte) {
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 3f4c41fe..20552ae0 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -164,22 +164,19 @@ type switchData struct {
 
 // All the information stored by the switch.
 type switchTable struct {
-	core              *Core
-	reconfigure       chan chan error
-	key               crypto.SigPubKey           // Our own key
-	time              time.Time                  // Time when locator.tstamp was last updated
-	drop              map[crypto.SigPubKey]int64 // Tstamp associated with a dropped root
-	mutex             sync.RWMutex               // Lock for reads/writes of switchData
-	parent            switchPort                 // Port of whatever peer is our parent, or self if we're root
-	data              switchData                 //
-	updater           atomic.Value               // *sync.Once
-	table             atomic.Value               // lookupTable
-	phony.Actor                                  // Owns the below
-	packetIn          chan []byte                // Incoming packets for the worker to handle
-	idleIn            chan switchPort            // Incoming idle notifications from peer links
-	queues            switch_buffers             // Queues - not atomic so ONLY use through admin chan
-	queueTotalMaxSize uint64                     // Maximum combined size of queues
-	idle              map[switchPort]time.Time   // idle peers
+	core        *Core
+	reconfigure chan chan error
+	key         crypto.SigPubKey           // Our own key
+	time        time.Time                  // Time when locator.tstamp was last updated
+	drop        map[crypto.SigPubKey]int64 // Tstamp associated with a dropped root
+	mutex       sync.RWMutex               // Lock for reads/writes of switchData
+	parent      switchPort                 // Port of whatever peer is our parent, or self if we're root
+	data        switchData                 //
+	updater     atomic.Value               // *sync.Once
+	table       atomic.Value               // lookupTable
+	phony.Actor                            // Owns the below
+	queues      switch_buffers             // Queues - not atomic so ONLY use through the actor
+	idle        map[switchPort]time.Time   // idle peers - not atomic so ONLY use through the actor
 }
 
 // Minimum allowed total size of switch queues.
@@ -197,12 +194,11 @@ func (t *switchTable) init(core *Core) {
 	t.updater.Store(&sync.Once{})
 	t.table.Store(lookupTable{})
 	t.drop = make(map[crypto.SigPubKey]int64)
-	t.packetIn = make(chan []byte, 1024)
-	t.idleIn = make(chan switchPort, 1024)
-	t.queueTotalMaxSize = SwitchQueueTotalMinSize
-	t.idle = make(map[switchPort]time.Time)
-	t.queues.switchTable = t
-	t.queues.bufs = make(map[string]switch_buffer) // Packets per PacketStreamID (string)
+	<-t.SyncExec(func() {
+		t.queues.totalMaxSize = SwitchQueueTotalMinSize
+		t.queues.bufs = make(map[string]switch_buffer)
+		t.idle = make(map[switchPort]time.Time)
+	})
 }
 
 // Safely gets a copy of this node's locator.
@@ -727,12 +723,12 @@ type switch_buffer struct {
 }
 
 type switch_buffers struct {
-	switchTable *switchTable
-	bufs        map[string]switch_buffer // Buffers indexed by StreamID
-	size        uint64                   // Total size of all buffers, in bytes
-	maxbufs     int
-	maxsize     uint64
-	closer      []closerInfo // Scratch space
+	totalMaxSize uint64
+	bufs         map[string]switch_buffer // Buffers indexed by StreamID
+	size         uint64                   // Total size of all buffers, in bytes
+	maxbufs      int
+	maxsize      uint64
+	closer       []closerInfo // Scratch space
 }
 
 func (b *switch_buffers) _cleanup(t *switchTable) {
@@ -749,7 +745,7 @@ func (b *switch_buffers) _cleanup(t *switchTable) {
 		}
 	}
 
-	for b.size > b.switchTable.queueTotalMaxSize {
+	for b.size > b.totalMaxSize {
 		// Drop a random queue
 		target := rand.Uint64() % b.size
 		var size uint64 // running total
@@ -828,6 +824,12 @@ func (t *switchTable) _handleIdle(port switchPort) bool {
 	return false
 }
 
+func (t *switchTable) packetInFrom(from phony.IActor, bytes []byte) {
+	t.EnqueueFrom(from, func() {
+		t._packetIn(bytes)
+	})
+}
+
 func (t *switchTable) _packetIn(bytes []byte) {
 	// Try to send it somewhere (or drop it if it's corrupt or at a dead end)
 	if !t._handleIn(bytes, t.idle) {
@@ -868,10 +870,6 @@ func (t *switchTable) doWorker() {
 	for {
 		//t.core.log.Debugf("Switch state: idle = %d, buffers = %d", len(idle), len(t.queues.bufs))
 		select {
-		case bytes := <-t.packetIn:
-			<-t.SyncExec(func() { t._packetIn(bytes) })
-		case port := <-t.idleIn:
-			<-t.SyncExec(func() { t._idleIn(port) })
 		case e := <-t.reconfigure:
 			e <- nil
 		}

From c573170886a85e9a88fe3d8e770949ccfd78c4ea Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 15:27:56 -0500
Subject: [PATCH 30/62] remove switch doworker loop, start a dummy loop to
 respond to (unused) reconfiguration instead

---
 src/yggdrasil/switch.go | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 20552ae0..905f2a79 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -566,7 +566,12 @@ func (t *switchTable) getTable() lookupTable {
 // Starts the switch worker
 func (t *switchTable) start() error {
 	t.core.log.Infoln("Starting switch")
-	go t.doWorker()
+	go func() {
+		// TODO find a better way to handle reconfiguration... and have the switch do something with the new configuration
+		for ch := range t.reconfigure {
+			ch <- nil
+		}
+	}()
 	return nil
 }
 
@@ -865,17 +870,6 @@ func (t *switchTable) _idleIn(port switchPort) {
 	}
 }
 
-// The switch worker does routing lookups and sends packets to where they need to be
-func (t *switchTable) doWorker() {
-	for {
-		//t.core.log.Debugf("Switch state: idle = %d, buffers = %d", len(idle), len(t.queues.bufs))
-		select {
-		case e := <-t.reconfigure:
-			e <- nil
-		}
-	}
-}
-
 // Passed a function to call.
 // This will send the function to t.admin and block until it finishes.
 func (t *switchTable) doAdmin(f func()) {

From 8c7e9ec7c0266fce7ca074a45857233e5b583d4a Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 15:32:19 -0500
Subject: [PATCH 31/62] fix debug builds

---
 src/yggdrasil/debug.go | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/yggdrasil/debug.go b/src/yggdrasil/debug.go
index 30eb874d..f6ec8716 100644
--- a/src/yggdrasil/debug.go
+++ b/src/yggdrasil/debug.go
@@ -571,7 +571,9 @@ func DEBUG_simLinkPeers(p, q *peer) {
 					continue
 				case packet := <-send:
 					packets = append(packets, packet)
-					source.core.switchTable.idleIn <- source.port
+					<-source.core.switchTable.SyncExec(func() {
+						source.core.switchTable._idleIn(source.port)
+					})
 					continue
 				default:
 				}
@@ -585,15 +587,21 @@ func DEBUG_simLinkPeers(p, q *peer) {
 					packets = append(packets, packet)
 				case packet := <-send:
 					packets = append(packets, packet)
-					source.core.switchTable.idleIn <- source.port
+					<-source.core.switchTable.SyncExec(func() {
+						source.core.switchTable._idleIn(source.port)
+					})
 				}
 			}
 		}()
 	}
 	goWorkers(p, q)
 	goWorkers(q, p)
-	p.core.switchTable.idleIn <- p.port
-	q.core.switchTable.idleIn <- q.port
+	<-p.core.switchTable.SyncExec(func() {
+		p.core.switchTable._idleIn(p.port)
+	})
+	<-q.core.switchTable.SyncExec(func() {
+		q.core.switchTable._idleIn(q.port)
+	})
 }
 
 /*

From 209d2ffea55a6ce3f1f3d22a2f06bda4ec25aae3 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 16:04:05 -0500
Subject: [PATCH 32/62] correctly call peer.sendPacketsFrom in the switch

---
 src/yggdrasil/peer.go   | 6 +++---
 src/yggdrasil/switch.go | 5 ++---
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 9a6f75f1..7fe33ecd 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -224,9 +224,9 @@ func (p *peer) _handlePacket(packet []byte) {
 	}
 	switch pType {
 	case wire_Traffic:
-		p._handleTraffic(packet, pTypeLen)
+		p._handleTraffic(packet)
 	case wire_ProtocolTraffic:
-		p._handleTraffic(packet, pTypeLen)
+		p._handleTraffic(packet)
 	case wire_LinkProtocolTraffic:
 		p._handleLinkTraffic(packet)
 	default:
@@ -236,7 +236,7 @@ func (p *peer) _handlePacket(packet []byte) {
 
 // Called to handle traffic or protocolTraffic packets.
 // In either case, this reads from the coords of the packet header, does a switch lookup, and forwards to the next node.
-func (p *peer) _handleTraffic(packet []byte, pTypeLen int) {
+func (p *peer) _handleTraffic(packet []byte) {
 	table := p.core.switchTable.getTable()
 	if _, isIn := table.elems[p.port]; !isIn && p.port != 0 {
 		// Drop traffic if the peer isn't in the switch
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 905f2a79..b6fc26f8 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -708,7 +708,7 @@ func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]time.Time) bo
 	if best != nil {
 		// Send to the best idle next hop
 		delete(idle, best.port)
-		best.sendPacketsFrom(nil, [][]byte{packet})
+		best.sendPacketsFrom(t, [][]byte{packet})
 		return true
 	}
 	// Didn't find anyone idle to send it to
@@ -822,8 +822,7 @@ func (t *switchTable) _handleIdle(port switchPort) bool {
 		}
 	}
 	if len(packets) > 0 {
-		// TODO rewrite if/when the switch becomes an actor
-		to.sendPacketsFrom(nil, packets)
+		to.sendPacketsFrom(t, packets)
 		return true
 	}
 	return false

From 99be6b037d5cccccf5f7103eaf582bec002d4eee Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 16:13:34 -0500
Subject: [PATCH 33/62] stop synchronizing message reads for now, not 100% safe
 but I don't have any better ideas

---
 src/yggdrasil/link.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index 1389f41e..4b7f766d 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -392,7 +392,10 @@ func (intf *linkInterface) handler() error {
 		msg, err := intf.msgIO.readMsg()
 		if len(msg) > 0 {
 			// TODO rewrite this if the link becomes an actor
-			<-intf.peer.SyncExec(func() { intf.peer._handlePacket(msg) })
+			// FIXME this could theoretically send traffic faster than the peer can handle
+			//  The alternative is to SyncExec, but that causes traffic to block while *other* links work
+			//  Need to figure out why and find a workaround
+			intf.peer.handlePacketFrom(nil, msg)
 		}
 		if err != nil {
 			if err != io.EOF {

From 48bbdac9b3596c513e2530a5bc20dad6bfc4faea Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 16:27:12 -0500
Subject: [PATCH 34/62] add a helper actor to the link reader to make it play
 nicer with backpressure

---
 src/yggdrasil/link.go | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index 4b7f766d..98f44bc7 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -16,6 +16,8 @@ import (
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
+
+	"github.com/Arceliar/phony"
 )
 
 type link struct {
@@ -388,14 +390,17 @@ func (intf *linkInterface) handler() error {
 		}
 	}()
 	// Run reader loop
-	for {
+	var helper phony.Actor
+	done := make(chan struct{})
+	var helperFunc func()
+	helperFunc = func() {
+		// The helper reads in a loop and sends to the peer
+		// It loops by sending itself a message, which can be delayed by backpressure
+		// So if the peer is busy, backpressure will pause reading until the peer catches up
 		msg, err := intf.msgIO.readMsg()
 		if len(msg) > 0 {
 			// TODO rewrite this if the link becomes an actor
-			// FIXME this could theoretically send traffic faster than the peer can handle
-			//  The alternative is to SyncExec, but that causes traffic to block while *other* links work
-			//  Need to figure out why and find a workaround
-			intf.peer.handlePacketFrom(nil, msg)
+			intf.peer.handlePacketFrom(&helper, msg)
 		}
 		if err != nil {
 			if err != io.EOF {
@@ -404,13 +409,19 @@ func (intf *linkInterface) handler() error {
 				default:
 				}
 			}
-			break
+			close(done)
+			return
 		}
 		select {
 		case signalAlive <- len(msg) > 0:
 		default:
 		}
+		// Now try to read again
+		helper.EnqueueFrom(nil, helperFunc)
 	}
+	// Start the read loop
+	helper.EnqueueFrom(nil, helperFunc)
+	<-done // Wait for the helper to exit
 	////////////////////////////////////////////////////////////////////////////////
 	// Remember to set `err` to something useful before returning
 	select {

From 68c380ff452c4bc5fcbe70786f39b8db05b041a6 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 17:03:19 -0500
Subject: [PATCH 35/62] update phony dependency

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index fff4ae6f..db307d33 100644
--- a/go.mod
+++ b/go.mod
@@ -1,7 +1,7 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
 require (
-	github.com/Arceliar/phony v0.0.0-20190824031448-b53e115f69b5
+	github.com/Arceliar/phony v0.0.0-20190824220211-2c0698579651
 	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
 	github.com/hashicorp/go-syslog v1.0.0
 	github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222
diff --git a/go.sum b/go.sum
index 29854cbf..3839b51a 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-github.com/Arceliar/phony v0.0.0-20190824031448-b53e115f69b5 h1:D2Djqo/q7mftrtHLCpW4Rpplm8jj+Edc9jNz8Ll6E0A=
-github.com/Arceliar/phony v0.0.0-20190824031448-b53e115f69b5/go.mod h1:2Q9yJvg2PlMrnOEa3RTEy9hElWAICo/D8HTUDqAHUAo=
+github.com/Arceliar/phony v0.0.0-20190824220211-2c0698579651 h1:FDO8ctwDOOs8mI7EVE/hR3lpGjWoY2y21tAlblIYvs8=
+github.com/Arceliar/phony v0.0.0-20190824220211-2c0698579651/go.mod h1:+/sVcxsqK1Sjm3Vd+yCfMAohJOfTRyNh24apkxhqU3Q=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=

From cff1366146456e6421ab6eb4199879794938b784 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 24 Aug 2019 22:28:20 -0500
Subject: [PATCH 36/62] update phony dependency

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index db307d33..a71eb4bc 100644
--- a/go.mod
+++ b/go.mod
@@ -1,7 +1,7 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
 require (
-	github.com/Arceliar/phony v0.0.0-20190824220211-2c0698579651
+	github.com/Arceliar/phony v0.0.0-20190825032731-f8ba56f9093e
 	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
 	github.com/hashicorp/go-syslog v1.0.0
 	github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222
diff --git a/go.sum b/go.sum
index 3839b51a..0aba11ef 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-github.com/Arceliar/phony v0.0.0-20190824220211-2c0698579651 h1:FDO8ctwDOOs8mI7EVE/hR3lpGjWoY2y21tAlblIYvs8=
-github.com/Arceliar/phony v0.0.0-20190824220211-2c0698579651/go.mod h1:+/sVcxsqK1Sjm3Vd+yCfMAohJOfTRyNh24apkxhqU3Q=
+github.com/Arceliar/phony v0.0.0-20190825032731-f8ba56f9093e h1:qsNZzfxSvlSE4JZ3OpmDmAeqCRpOO3RI9eJ7U6z23Gk=
+github.com/Arceliar/phony v0.0.0-20190825032731-f8ba56f9093e/go.mod h1:+/sVcxsqK1Sjm3Vd+yCfMAohJOfTRyNh24apkxhqU3Q=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=

From aa30c6cc98071cb4c953049baf3ff8e19e213044 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 25 Aug 2019 10:36:09 -0500
Subject: [PATCH 37/62] upgrade phony dependency and switch to its new
 interface

---
 go.mod                   |  2 +-
 go.sum                   |  4 ++--
 src/tuntap/conn.go       |  4 ++--
 src/tuntap/tun.go        |  2 +-
 src/yggdrasil/conn.go    | 24 ++++++++++++------------
 src/yggdrasil/link.go    | 10 +++++-----
 src/yggdrasil/peer.go    | 18 +++++++++---------
 src/yggdrasil/router.go  | 14 +++++++-------
 src/yggdrasil/session.go | 24 ++++++++++++------------
 src/yggdrasil/switch.go  |  6 +++---
 10 files changed, 54 insertions(+), 54 deletions(-)

diff --git a/go.mod b/go.mod
index a71eb4bc..f3d8417f 100644
--- a/go.mod
+++ b/go.mod
@@ -1,7 +1,7 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
 require (
-	github.com/Arceliar/phony v0.0.0-20190825032731-f8ba56f9093e
+	github.com/Arceliar/phony v0.0.0-20190825152505-180ac75690fe
 	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
 	github.com/hashicorp/go-syslog v1.0.0
 	github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222
diff --git a/go.sum b/go.sum
index 0aba11ef..29e0dfec 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-github.com/Arceliar/phony v0.0.0-20190825032731-f8ba56f9093e h1:qsNZzfxSvlSE4JZ3OpmDmAeqCRpOO3RI9eJ7U6z23Gk=
-github.com/Arceliar/phony v0.0.0-20190825032731-f8ba56f9093e/go.mod h1:+/sVcxsqK1Sjm3Vd+yCfMAohJOfTRyNh24apkxhqU3Q=
+github.com/Arceliar/phony v0.0.0-20190825152505-180ac75690fe h1:U5bediuXjZ1y6bByIXXraoE319yFp9kx1z8K6el7Ftc=
+github.com/Arceliar/phony v0.0.0-20190825152505-180ac75690fe/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=
diff --git a/src/tuntap/conn.go b/src/tuntap/conn.go
index 744ca9fc..0e0dd461 100644
--- a/src/tuntap/conn.go
+++ b/src/tuntap/conn.go
@@ -17,7 +17,7 @@ import (
 const tunConnTimeout = 2 * time.Minute
 
 type tunConn struct {
-	phony.Actor
+	phony.Inbox
 	tun   *TunAdapter
 	conn  *yggdrasil.Conn
 	addr  address.Address
@@ -198,7 +198,7 @@ func (s *tunConn) _write(bs []byte) (err error) {
 			// No point in wasting resources to send back an error if there was none
 			return
 		}
-		s.EnqueueFrom(s.conn, func() {
+		s.RecvFrom(s.conn, func() {
 			if e, eok := err.(yggdrasil.ConnError); !eok {
 				if e.Closed() {
 					s.tun.log.Debugln(s.conn.String(), "TUN/TAP generic write debug:", err)
diff --git a/src/tuntap/tun.go b/src/tuntap/tun.go
index 895f893b..dfc343df 100644
--- a/src/tuntap/tun.go
+++ b/src/tuntap/tun.go
@@ -257,7 +257,7 @@ func (tun *TunAdapter) wrap(conn *yggdrasil.Conn) (c *tunConn, err error) {
 	tun.subnetToConn[s.snet] = &s
 	// Set the read callback and start the timeout goroutine
 	conn.SetReadCallback(func(bs []byte) {
-		s.EnqueueFrom(conn, func() {
+		s.RecvFrom(conn, func() {
 			s._read(bs)
 		})
 	})
diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index 2f69139e..b0c26867 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -54,7 +54,7 @@ func (e *ConnError) Closed() bool {
 }
 
 type Conn struct {
-	phony.Actor
+	phony.Inbox
 	core          *Core
 	readDeadline  *time.Time
 	writeDeadline *time.Time
@@ -84,8 +84,8 @@ func (c *Conn) String() string {
 	return s
 }
 
-func (c *Conn) setMTU(from phony.IActor, mtu uint16) {
-	c.EnqueueFrom(from, func() { c.mtu = mtu })
+func (c *Conn) setMTU(from phony.Actor, mtu uint16) {
+	c.RecvFrom(from, func() { c.mtu = mtu })
 }
 
 // This should never be called from the router goroutine, used in the dial functions
@@ -143,7 +143,7 @@ func (c *Conn) doSearch() {
 			sinfo.continueSearch()
 		}
 	}
-	c.core.router.EnqueueFrom(c.session, routerWork)
+	c.core.router.RecvFrom(c.session, routerWork)
 }
 
 func (c *Conn) _getDeadlineCancellation(t *time.Time) (util.Cancellation, bool) {
@@ -159,7 +159,7 @@ func (c *Conn) _getDeadlineCancellation(t *time.Time) (util.Cancellation, bool)
 
 // SetReadCallback sets a callback which will be called whenever a packet is received.
 func (c *Conn) SetReadCallback(callback func([]byte)) {
-	c.EnqueueFrom(nil, func() {
+	c.RecvFrom(nil, func() {
 		c.readCallback = callback
 		c._drainReadBuffer()
 	})
@@ -172,14 +172,14 @@ func (c *Conn) _drainReadBuffer() {
 	select {
 	case bs := <-c.readBuffer:
 		c.readCallback(bs)
-		c.EnqueueFrom(nil, c._drainReadBuffer) // In case there's more
+		c.RecvFrom(nil, c._drainReadBuffer) // In case there's more
 	default:
 	}
 }
 
 // Called by the session to pass a new message to the Conn
-func (c *Conn) recvMsg(from phony.IActor, msg []byte) {
-	c.EnqueueFrom(from, func() {
+func (c *Conn) recvMsg(from phony.Actor, msg []byte) {
+	c.RecvFrom(from, func() {
 		if c.readCallback != nil {
 			c.readCallback(msg)
 		} else {
@@ -234,7 +234,7 @@ func (c *Conn) _write(msg FlowKeyMessage) error {
 	if len(msg.Message) > int(c.mtu) {
 		return ConnError{errors.New("packet too big"), true, false, false, int(c.mtu)}
 	}
-	c.session.EnqueueFrom(c, func() {
+	c.session.RecvFrom(c, func() {
 		// Send the packet
 		c.session._send(msg)
 		// Session keep-alive, while we wait for the crypto workers from send
@@ -254,11 +254,11 @@ func (c *Conn) _write(msg FlowKeyMessage) error {
 	return nil
 }
 
-// WriteFrom should be called by a phony.IActor, and tells the Conn to send a message.
+// WriteFrom should be called by a phony.Actor, and tells the Conn to send a message.
 // This is used internaly by WriteNoCopy and Write.
 // If the callback is called with a non-nil value, then it is safe to reuse the argument FlowKeyMessage.
-func (c *Conn) WriteFrom(from phony.IActor, msg FlowKeyMessage, callback func(error)) {
-	c.EnqueueFrom(from, func() {
+func (c *Conn) WriteFrom(from phony.Actor, msg FlowKeyMessage, callback func(error)) {
+	c.RecvFrom(from, func() {
 		callback(c._write(msg))
 	})
 }
diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index 98f44bc7..d4779ead 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -320,7 +320,7 @@ func (intf *linkInterface) handler() error {
 				isAlive = true
 				if !isReady {
 					// (Re-)enable in the switch
-					intf.link.core.switchTable.EnqueueFrom(nil, func() {
+					intf.link.core.switchTable.RecvFrom(nil, func() {
 						intf.link.core.switchTable._idleIn(intf.peer.port)
 					})
 					isReady = true
@@ -359,7 +359,7 @@ func (intf *linkInterface) handler() error {
 					isReady = false
 				} else {
 					// Keep enabled in the switch
-					intf.link.core.switchTable.EnqueueFrom(nil, func() {
+					intf.link.core.switchTable.RecvFrom(nil, func() {
 						intf.link.core.switchTable._idleIn(intf.peer.port)
 					})
 					isReady = true
@@ -390,7 +390,7 @@ func (intf *linkInterface) handler() error {
 		}
 	}()
 	// Run reader loop
-	var helper phony.Actor
+	var helper phony.Inbox
 	done := make(chan struct{})
 	var helperFunc func()
 	helperFunc = func() {
@@ -417,10 +417,10 @@ func (intf *linkInterface) handler() error {
 		default:
 		}
 		// Now try to read again
-		helper.EnqueueFrom(nil, helperFunc)
+		helper.RecvFrom(nil, helperFunc)
 	}
 	// Start the read loop
-	helper.EnqueueFrom(nil, helperFunc)
+	helper.RecvFrom(nil, helperFunc)
 	<-done // Wait for the helper to exit
 	////////////////////////////////////////////////////////////////////////////////
 	// Remember to set `err` to something useful before returning
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 7fe33ecd..aa31bb1b 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -96,7 +96,7 @@ func (ps *peers) putPorts(ports map[switchPort]*peer) {
 
 // Information known about a peer, including thier box/sig keys, precomputed shared keys (static and ephemeral) and a handler for their outgoing traffic
 type peer struct {
-	phony.Actor
+	phony.Inbox
 	core       *Core
 	intf       *linkInterface
 	port       switchPort
@@ -175,13 +175,13 @@ func (ps *peers) removePeer(port switchPort) {
 
 // If called, sends a notification to each peer that they should send a new switch message.
 // Mainly called by the switch after an update.
-func (ps *peers) sendSwitchMsgs(from phony.IActor) {
+func (ps *peers) sendSwitchMsgs(from phony.Actor) {
 	ports := ps.getPorts()
 	for _, p := range ports {
 		if p.port == 0 {
 			continue
 		}
-		p.EnqueueFrom(from, p._sendSwitchMsg)
+		p.RecvFrom(from, p._sendSwitchMsg)
 	}
 }
 
@@ -207,8 +207,8 @@ func (p *peer) _updateDHT() {
 	}
 }
 
-func (p *peer) handlePacketFrom(from phony.IActor, packet []byte) {
-	p.EnqueueFrom(from, func() {
+func (p *peer) handlePacketFrom(from phony.Actor, packet []byte) {
+	p.RecvFrom(from, func() {
 		p._handlePacket(packet)
 	})
 }
@@ -245,8 +245,8 @@ func (p *peer) _handleTraffic(packet []byte) {
 	p.core.switchTable.packetInFrom(p, packet)
 }
 
-func (p *peer) sendPacketsFrom(from phony.IActor, packets [][]byte) {
-	p.EnqueueFrom(from, func() {
+func (p *peer) sendPacketsFrom(from phony.Actor, packets [][]byte) {
+	p.RecvFrom(from, func() {
 		p._sendPackets(packets)
 	})
 }
@@ -263,7 +263,7 @@ func (p *peer) _sendPackets(packets [][]byte) {
 	p.out(packets)
 }
 
-var peerLinkOutHelper phony.Actor
+var peerLinkOutHelper phony.Inbox
 
 // This wraps the packet in the inner (ephemeral) and outer (permanent) crypto layers.
 // It sends it to p.linkOut, which bypasses the usual packet queues.
@@ -281,7 +281,7 @@ func (p *peer) _sendLinkPacket(packet []byte) {
 	}
 	packet = linkPacket.encode()
 	// TODO replace this with a message send if/when the link becomes an actor
-	peerLinkOutHelper.EnqueueFrom(nil, func() {
+	peerLinkOutHelper.RecvFrom(nil, func() {
 		select {
 		case p.linkOut <- packet:
 		case <-p.done:
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index adf1b1d4..7b6a9b80 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -37,7 +37,7 @@ import (
 // The router struct has channels to/from the adapter device and a self peer (0), which is how messages are passed between this node and the peers/switch layer.
 // The router's mainLoop goroutine is responsible for managing all information related to the dht, searches, and crypto sessions.
 type router struct {
-	phony.Actor
+	phony.Inbox
 	core        *Core
 	reconfigure chan chan error
 	addr        address.Address
@@ -83,8 +83,8 @@ func (r *router) start() error {
 }
 
 // In practice, the switch will call this with 1 packet
-func (r *router) handlePackets(from phony.IActor, packets [][]byte) {
-	r.EnqueueFrom(from, func() {
+func (r *router) handlePackets(from phony.Actor, packets [][]byte) {
+	r.RecvFrom(from, func() {
 		for _, packet := range packets {
 			r._handlePacket(packet)
 		}
@@ -92,15 +92,15 @@ func (r *router) handlePackets(from phony.IActor, packets [][]byte) {
 }
 
 // Insert a peer info into the dht, TODO? make the dht a separate actor
-func (r *router) insertPeer(from phony.IActor, info *dhtInfo) {
-	r.EnqueueFrom(from, func() {
+func (r *router) insertPeer(from phony.Actor, info *dhtInfo) {
+	r.RecvFrom(from, func() {
 		r.dht.insertPeer(info)
 	})
 }
 
 // Reset sessions and DHT after the switch sees our coords change
-func (r *router) reset(from phony.IActor) {
-	r.EnqueueFrom(from, func() {
+func (r *router) reset(from phony.Actor) {
+	r.RecvFrom(from, func() {
 		r.sessions.reset()
 		r.dht.reset()
 	})
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 5243fb0f..f1263379 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -38,7 +38,7 @@ func (h nonceHeap) peek() *crypto.BoxNonce { return &h[0] }
 // All the information we know about an active session.
 // This includes coords, permanent and ephemeral keys, handles and nonces, various sorts of timing information for timeout and maintenance, and some metadata for the admin API.
 type sessionInfo struct {
-	phony.Actor                                  // Protects all of the below, use it any time you read/change the contents of a session
+	phony.Inbox                                  // Protects all of the below, use it any time you read/change the contents of a session
 	sessions       *sessions                     //
 	reconfigure    chan chan error               //
 	theirAddr      address.Address               //
@@ -342,8 +342,8 @@ func (ss *sessions) getSharedKey(myPriv *crypto.BoxPrivKey,
 }
 
 // Sends a session ping by calling sendPingPong in ping mode.
-func (sinfo *sessionInfo) ping(from phony.IActor) {
-	sinfo.EnqueueFrom(from, func() {
+func (sinfo *sessionInfo) ping(from phony.Actor) {
+	sinfo.RecvFrom(from, func() {
 		sinfo._sendPingPong(false)
 	})
 }
@@ -364,14 +364,14 @@ func (sinfo *sessionInfo) _sendPingPong(isPong bool) {
 	}
 	packet := p.encode()
 	// TODO rewrite the below if/when the peer struct becomes an actor, to not go through the router first
-	sinfo.sessions.router.EnqueueFrom(sinfo, func() { sinfo.sessions.router.out(packet) })
+	sinfo.sessions.router.RecvFrom(sinfo, func() { sinfo.sessions.router.out(packet) })
 	if sinfo.pingTime.Before(sinfo.time) {
 		sinfo.pingTime = time.Now()
 	}
 }
 
-func (sinfo *sessionInfo) setConn(from phony.IActor, conn *Conn) {
-	sinfo.EnqueueFrom(from, func() {
+func (sinfo *sessionInfo) setConn(from phony.Actor, conn *Conn) {
+	sinfo.RecvFrom(from, func() {
 		sinfo.conn = conn
 		sinfo.conn.setMTU(sinfo, sinfo._getMTU())
 	})
@@ -406,7 +406,7 @@ func (ss *sessions) handlePing(ping *sessionPing) {
 		ss.listenerMutex.Unlock()
 	}
 	if sinfo != nil {
-		sinfo.EnqueueFrom(ss.router, func() {
+		sinfo.RecvFrom(ss.router, func() {
 			// Update the session
 			if !sinfo._update(ping) { /*panic("Should not happen in testing")*/
 				return
@@ -474,7 +474,7 @@ func (sinfo *sessionInfo) _updateNonce(theirNonce *crypto.BoxNonce) {
 // Called after coord changes, so attemtps to use a session will trigger a new ping and notify the remote end of the coord change.
 func (ss *sessions) reset() {
 	for _, sinfo := range ss.sinfos {
-		sinfo.EnqueueFrom(ss.router, func() {
+		sinfo.RecvFrom(ss.router, func() {
 			sinfo.reset = true
 		})
 	}
@@ -489,8 +489,8 @@ type FlowKeyMessage struct {
 	Message []byte
 }
 
-func (sinfo *sessionInfo) recv(from phony.IActor, packet *wire_trafficPacket) {
-	sinfo.EnqueueFrom(from, func() {
+func (sinfo *sessionInfo) recv(from phony.Actor, packet *wire_trafficPacket) {
+	sinfo.RecvFrom(from, func() {
 		sinfo._recvPacket(packet)
 	})
 }
@@ -564,7 +564,7 @@ func (sinfo *sessionInfo) _send(msg FlowKeyMessage) {
 			util.PutBytes(p.Payload)
 			// Send the packet
 			// TODO replace this with a send to the peer struct if that becomes an actor
-			sinfo.sessions.router.EnqueueFrom(sinfo, func() {
+			sinfo.sessions.router.RecvFrom(sinfo, func() {
 				sinfo.sessions.router.out(packet)
 			})
 		}
@@ -576,7 +576,7 @@ func (sinfo *sessionInfo) _send(msg FlowKeyMessage) {
 }
 
 func (sinfo *sessionInfo) checkCallbacks() {
-	sinfo.EnqueueFrom(nil, func() {
+	sinfo.RecvFrom(nil, func() {
 		if len(sinfo.callbacks) > 0 {
 			select {
 			case callback := <-sinfo.callbacks[0]:
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index b6fc26f8..db63a845 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -174,7 +174,7 @@ type switchTable struct {
 	data        switchData                 //
 	updater     atomic.Value               // *sync.Once
 	table       atomic.Value               // lookupTable
-	phony.Actor                            // Owns the below
+	phony.Inbox                            // Owns the below
 	queues      switch_buffers             // Queues - not atomic so ONLY use through the actor
 	idle        map[switchPort]time.Time   // idle peers - not atomic so ONLY use through the actor
 }
@@ -828,8 +828,8 @@ func (t *switchTable) _handleIdle(port switchPort) bool {
 	return false
 }
 
-func (t *switchTable) packetInFrom(from phony.IActor, bytes []byte) {
-	t.EnqueueFrom(from, func() {
+func (t *switchTable) packetInFrom(from phony.Actor, bytes []byte) {
+	t.RecvFrom(from, func() {
 		t._packetIn(bytes)
 	})
 }

From a3d4d8125b4acb438779d4d5263ea0faa48d0125 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 25 Aug 2019 12:10:59 -0500
Subject: [PATCH 38/62] make the main library reconfiguration more
 actor-friendly

---
 src/yggdrasil/core.go    | 15 ++++----
 src/yggdrasil/dht.go     | 21 +++++------
 src/yggdrasil/link.go    | 32 +++++++----------
 src/yggdrasil/peer.go    | 19 +++++-----
 src/yggdrasil/router.go  | 63 +++++++++++++++++---------------
 src/yggdrasil/search.go  | 17 ++++-----
 src/yggdrasil/session.go | 42 +++++++++++-----------
 src/yggdrasil/switch.go  | 16 ++++-----
 src/yggdrasil/tcp.go     | 78 ++++++++++++++++++----------------------
 9 files changed, 144 insertions(+), 159 deletions(-)

diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index 40982cdd..ec530745 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -117,20 +117,21 @@ func (c *Core) UpdateConfig(config *config.NodeConfig) {
 
 	errors := 0
 
-	components := []chan chan error{
-		c.router.searches.reconfigure,
-		c.router.dht.reconfigure,
-		c.router.sessions.reconfigure,
+	// Each reconfigure function should pass any errors to the channel, then close it
+	components := []func(chan error){
+		c.link.reconfigure,
 		c.peers.reconfigure,
 		c.router.reconfigure,
+		c.router.dht.reconfigure,
+		c.router.searches.reconfigure,
+		c.router.sessions.reconfigure,
 		c.switchTable.reconfigure,
-		c.link.reconfigure,
 	}
 
 	for _, component := range components {
 		response := make(chan error)
-		component <- response
-		if err := <-response; err != nil {
+		go component(response)
+		for err := range response {
 			c.log.Errorln(err)
 			errors++
 		}
diff --git a/src/yggdrasil/dht.go b/src/yggdrasil/dht.go
index adfc40e7..4f380363 100644
--- a/src/yggdrasil/dht.go
+++ b/src/yggdrasil/dht.go
@@ -65,11 +65,10 @@ type dhtReqKey struct {
 
 // The main DHT struct.
 type dht struct {
-	router      *router
-	reconfigure chan chan error
-	nodeID      crypto.NodeID
-	reqs        map[dhtReqKey]time.Time          // Keeps track of recent outstanding requests
-	callbacks   map[dhtReqKey][]dht_callbackInfo // Search and admin lookup callbacks
+	router    *router
+	nodeID    crypto.NodeID
+	reqs      map[dhtReqKey]time.Time          // Keeps track of recent outstanding requests
+	callbacks map[dhtReqKey][]dht_callbackInfo // Search and admin lookup callbacks
 	// These next two could be replaced by a single linked list or similar...
 	table map[crypto.NodeID]*dhtInfo
 	imp   []*dhtInfo
@@ -78,18 +77,16 @@ type dht struct {
 // Initializes the DHT.
 func (t *dht) init(r *router) {
 	t.router = r
-	t.reconfigure = make(chan chan error, 1)
-	go func() {
-		for {
-			e := <-t.reconfigure
-			e <- nil
-		}
-	}()
 	t.nodeID = *t.router.core.NodeID()
 	t.callbacks = make(map[dhtReqKey][]dht_callbackInfo)
 	t.reset()
 }
 
+func (t *dht) reconfigure(e chan error) {
+	defer close(e)
+	// This is where reconfiguration would go, if we had anything to do
+}
+
 // Resets the DHT in response to coord changes.
 // This empties all info from the DHT and drops outstanding requests.
 func (t *dht) reset() {
diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index d4779ead..bfbcc99b 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -21,11 +21,10 @@ import (
 )
 
 type link struct {
-	core        *Core
-	reconfigure chan chan error
-	mutex       sync.RWMutex // protects interfaces below
-	interfaces  map[linkInfo]*linkInterface
-	tcp         tcp // TCP interface support
+	core       *Core
+	mutex      sync.RWMutex // protects interfaces below
+	interfaces map[linkInfo]*linkInterface
+	tcp        tcp // TCP interface support
 	// TODO timeout (to remove from switch), read from config.ReadTimeout
 }
 
@@ -61,7 +60,6 @@ func (l *link) init(c *Core) error {
 	l.core = c
 	l.mutex.Lock()
 	l.interfaces = make(map[linkInfo]*linkInterface)
-	l.reconfigure = make(chan chan error)
 	l.mutex.Unlock()
 
 	if err := l.tcp.init(l); err != nil {
@@ -69,22 +67,18 @@ func (l *link) init(c *Core) error {
 		return err
 	}
 
-	go func() {
-		for {
-			e := <-l.reconfigure
-			tcpresponse := make(chan error)
-			l.tcp.reconfigure <- tcpresponse
-			if err := <-tcpresponse; err != nil {
-				e <- err
-				continue
-			}
-			e <- nil
-		}
-	}()
-
 	return nil
 }
 
+func (l *link) reconfigure(e chan error) {
+	defer close(e)
+	tcpResponse := make(chan error)
+	go l.tcp.reconfigure(tcpResponse)
+	for err := range tcpResponse {
+		e <- err
+	}
+}
+
 func (l *link) call(uri string, sintf string) error {
 	u, err := url.Parse(uri)
 	if err != nil {
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index aa31bb1b..989d9ee1 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -21,10 +21,9 @@ import (
 // In most cases, this involves passing the packet to the handler for outgoing traffic to another peer.
 // In other cases, it's link protocol traffic used to build the spanning tree, in which case this checks signatures and passes the message along to the switch.
 type peers struct {
-	core        *Core
-	reconfigure chan chan error
-	mutex       sync.Mutex   // Synchronize writes to atomic
-	ports       atomic.Value //map[switchPort]*peer, use CoW semantics
+	core  *Core
+	mutex sync.Mutex   // Synchronize writes to atomic
+	ports atomic.Value //map[switchPort]*peer, use CoW semantics
 }
 
 // Initializes the peers struct.
@@ -33,13 +32,11 @@ func (ps *peers) init(c *Core) {
 	defer ps.mutex.Unlock()
 	ps.putPorts(make(map[switchPort]*peer))
 	ps.core = c
-	ps.reconfigure = make(chan chan error, 1)
-	go func() {
-		for {
-			e := <-ps.reconfigure
-			e <- nil
-		}
-	}()
+}
+
+func (ps *peers) reconfigure(e chan error) {
+	defer close(e)
+	// This is where reconfiguration would go, if we had anything to do
 }
 
 // Returns true if an incoming peer connection to a key is allowed, either
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index 7b6a9b80..002905bc 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -35,24 +35,22 @@ import (
 )
 
 // The router struct has channels to/from the adapter device and a self peer (0), which is how messages are passed between this node and the peers/switch layer.
-// The router's mainLoop goroutine is responsible for managing all information related to the dht, searches, and crypto sessions.
+// The router's phony.Inbox goroutine is responsible for managing all information related to the dht, searches, and crypto sessions.
 type router struct {
 	phony.Inbox
-	core        *Core
-	reconfigure chan chan error
-	addr        address.Address
-	subnet      address.Subnet
-	out         func([]byte) // packets we're sending to the network, link to peer's "in"
-	dht         dht
-	nodeinfo    nodeinfo
-	searches    searches
-	sessions    sessions
+	core     *Core
+	addr     address.Address
+	subnet   address.Subnet
+	out      func([]byte) // packets we're sending to the network, link to peer's "in"
+	dht      dht
+	nodeinfo nodeinfo
+	searches searches
+	sessions sessions
 }
 
 // Initializes the router struct, which includes setting up channels to/from the adapter.
 func (r *router) init(core *Core) {
 	r.core = core
-	r.reconfigure = make(chan chan error, 1)
 	r.addr = *address.AddrForNodeID(&r.dht.nodeID)
 	r.subnet = *address.SubnetForNodeID(&r.dht.nodeID)
 	self := linkInterface{
@@ -75,10 +73,26 @@ func (r *router) init(core *Core) {
 	r.sessions.init(r)
 }
 
-// Starts the mainLoop goroutine.
+func (r *router) reconfigure(e chan error) {
+	defer close(e)
+	var errs []error
+	// Reconfigure the router
+	<-r.SyncExec(func() {
+		current := r.core.config.GetCurrent()
+		err := r.nodeinfo.setNodeInfo(current.NodeInfo, current.NodeInfoPrivacy)
+		if err != nil {
+			errs = append(errs, err)
+		}
+	})
+	for _, err := range errs {
+		e <- err
+	}
+}
+
+// Starts the tickerLoop goroutine.
 func (r *router) start() error {
 	r.core.log.Infoln("Starting router")
-	go r._mainLoop()
+	go r.tickerLoop()
 	return nil
 }
 
@@ -108,24 +122,17 @@ func (r *router) reset(from phony.Actor) {
 
 // TODO remove reconfigure so this is just a ticker loop
 // and then find something better than a ticker loop to schedule things...
-func (r *router) _mainLoop() {
+func (r *router) tickerLoop() {
 	ticker := time.NewTicker(time.Second)
 	defer ticker.Stop()
 	for {
-		select {
-		case <-ticker.C:
-			<-r.SyncExec(func() {
-				// Any periodic maintenance stuff goes here
-				r.core.switchTable.doMaintenance()
-				r.dht.doMaintenance()
-				r.sessions.cleanup()
-			})
-		case e := <-r.reconfigure:
-			<-r.SyncExec(func() {
-				current := r.core.config.GetCurrent()
-				e <- r.nodeinfo.setNodeInfo(current.NodeInfo, current.NodeInfoPrivacy)
-			})
-		}
+		<-ticker.C
+		<-r.SyncExec(func() {
+			// Any periodic maintenance stuff goes here
+			r.core.switchTable.doMaintenance()
+			r.dht.doMaintenance()
+			r.sessions.cleanup()
+		})
 	}
 }
 
diff --git a/src/yggdrasil/search.go b/src/yggdrasil/search.go
index 397c28a9..5fb36584 100644
--- a/src/yggdrasil/search.go
+++ b/src/yggdrasil/search.go
@@ -45,24 +45,21 @@ type searchInfo struct {
 
 // This stores a map of active searches.
 type searches struct {
-	router      *router
-	reconfigure chan chan error
-	searches    map[crypto.NodeID]*searchInfo
+	router   *router
+	searches map[crypto.NodeID]*searchInfo
 }
 
 // Initializes the searches struct.
 func (s *searches) init(r *router) {
 	s.router = r
-	s.reconfigure = make(chan chan error, 1)
-	go func() {
-		for {
-			e := <-s.reconfigure
-			e <- nil
-		}
-	}()
 	s.searches = make(map[crypto.NodeID]*searchInfo)
 }
 
+func (s *searches) reconfigure(e chan error) {
+	defer close(e)
+	// This is where reconfiguration would go, if we had anything to do
+}
+
 // Creates a new search info, adds it to the searches struct, and returns a pointer to the info.
 func (s *searches) createSearch(dest *crypto.NodeID, mask *crypto.NodeID, callback func(*sessionInfo, error)) *searchInfo {
 	info := searchInfo{
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index f1263379..e448cf25 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -40,7 +40,6 @@ func (h nonceHeap) peek() *crypto.BoxNonce { return &h[0] }
 type sessionInfo struct {
 	phony.Inbox                                  // Protects all of the below, use it any time you read/change the contents of a session
 	sessions       *sessions                     //
-	reconfigure    chan chan error               //
 	theirAddr      address.Address               //
 	theirSubnet    address.Subnet                //
 	theirPermPub   crypto.BoxPubKey              //
@@ -74,6 +73,11 @@ type sessionInfo struct {
 	callbacks      []chan func()                 // Finished work from crypto workers
 }
 
+func (sinfo *sessionInfo) reconfigure(e chan error) {
+	defer close(e)
+	// This is where reconfiguration would go, if we had anything to do
+}
+
 // TODO remove this, call SyncExec directly
 func (sinfo *sessionInfo) doFunc(f func()) {
 	<-sinfo.SyncExec(f)
@@ -140,7 +144,6 @@ type sessions struct {
 	router           *router
 	listener         *Listener
 	listenerMutex    sync.Mutex
-	reconfigure      chan chan error
 	lastCleanup      time.Time
 	isAllowedHandler func(pubkey *crypto.BoxPubKey, initiator bool) bool // Returns true or false if session setup is allowed
 	isAllowedMutex   sync.RWMutex                                        // Protects the above
@@ -152,30 +155,28 @@ type sessions struct {
 // Initializes the session struct.
 func (ss *sessions) init(r *router) {
 	ss.router = r
-	ss.reconfigure = make(chan chan error, 1)
-	go func() {
-		for {
-			e := <-ss.reconfigure
-			responses := make(map[crypto.Handle]chan error)
-			for index, session := range ss.sinfos {
-				responses[index] = make(chan error)
-				session.reconfigure <- responses[index]
-			}
-			for _, response := range responses {
-				if err := <-response; err != nil {
-					e <- err
-					continue
-				}
-			}
-			e <- nil
-		}
-	}()
 	ss.permShared = make(map[crypto.BoxPubKey]*crypto.BoxSharedKey)
 	ss.sinfos = make(map[crypto.Handle]*sessionInfo)
 	ss.byTheirPerm = make(map[crypto.BoxPubKey]*crypto.Handle)
 	ss.lastCleanup = time.Now()
 }
 
+func (ss *sessions) reconfigure(e chan error) {
+	defer close(e)
+	responses := make(map[crypto.Handle]chan error)
+	<-ss.router.SyncExec(func() {
+		for index, session := range ss.sinfos {
+			responses[index] = make(chan error)
+			go session.reconfigure(responses[index])
+		}
+	})
+	for _, response := range responses {
+		for err := range response {
+			e <- err
+		}
+	}
+}
+
 // Determines whether the session with a given publickey is allowed based on
 // session firewall rules.
 func (ss *sessions) isSessionAllowed(pubkey *crypto.BoxPubKey, initiator bool) bool {
@@ -215,7 +216,6 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 	}
 	sinfo := sessionInfo{}
 	sinfo.sessions = ss
-	sinfo.reconfigure = make(chan chan error, 1)
 	sinfo.theirPermPub = *theirPermKey
 	sinfo.sharedPermKey = *ss.getSharedKey(&ss.router.core.boxPriv, &sinfo.theirPermPub)
 	pub, priv := crypto.NewBoxKeys()
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index db63a845..cb5cf1eb 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -165,7 +165,6 @@ type switchData struct {
 // All the information stored by the switch.
 type switchTable struct {
 	core        *Core
-	reconfigure chan chan error
 	key         crypto.SigPubKey           // Our own key
 	time        time.Time                  // Time when locator.tstamp was last updated
 	drop        map[crypto.SigPubKey]int64 // Tstamp associated with a dropped root
@@ -186,7 +185,6 @@ const SwitchQueueTotalMinSize = 4 * 1024 * 1024
 func (t *switchTable) init(core *Core) {
 	now := time.Now()
 	t.core = core
-	t.reconfigure = make(chan chan error, 1)
 	t.key = t.core.sigPub
 	locator := switchLocator{root: t.key, tstamp: now.Unix()}
 	peers := make(map[switchPort]peerInfo)
@@ -201,6 +199,13 @@ func (t *switchTable) init(core *Core) {
 	})
 }
 
+func (t *switchTable) reconfigure(e chan error) {
+	go func() {
+		defer close(e)
+		// This is where reconfiguration would go, if we had anything useful to do.
+	}()
+}
+
 // Safely gets a copy of this node's locator.
 func (t *switchTable) getLocator() switchLocator {
 	t.mutex.RLock()
@@ -566,12 +571,7 @@ func (t *switchTable) getTable() lookupTable {
 // Starts the switch worker
 func (t *switchTable) start() error {
 	t.core.log.Infoln("Starting switch")
-	go func() {
-		// TODO find a better way to handle reconfiguration... and have the switch do something with the new configuration
-		for ch := range t.reconfigure {
-			ch <- nil
-		}
-	}()
+	// There's actually nothing to do to start it...
 	return nil
 }
 
diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go
index dfb41510..ccb488f9 100644
--- a/src/yggdrasil/tcp.go
+++ b/src/yggdrasil/tcp.go
@@ -33,12 +33,11 @@ const tcp_ping_interval = (default_timeout * 2 / 3)
 
 // The TCP listener and information about active TCP connections, to avoid duplication.
 type tcp struct {
-	link        *link
-	reconfigure chan chan error
-	mutex       sync.Mutex // Protecting the below
-	listeners   map[string]*TcpListener
-	calls       map[string]struct{}
-	conns       map[linkInfo](chan struct{})
+	link      *link
+	mutex     sync.Mutex // Protecting the below
+	listeners map[string]*TcpListener
+	calls     map[string]struct{}
+	conns     map[linkInfo](chan struct{})
 }
 
 // TcpListener is a stoppable TCP listener interface. These are typically
@@ -76,49 +75,12 @@ func (t *tcp) getAddr() *net.TCPAddr {
 // Initializes the struct.
 func (t *tcp) init(l *link) error {
 	t.link = l
-	t.reconfigure = make(chan chan error, 1)
 	t.mutex.Lock()
 	t.calls = make(map[string]struct{})
 	t.conns = make(map[linkInfo](chan struct{}))
 	t.listeners = make(map[string]*TcpListener)
 	t.mutex.Unlock()
 
-	go func() {
-		for {
-			e := <-t.reconfigure
-			t.link.core.config.Mutex.RLock()
-			added := util.Difference(t.link.core.config.Current.Listen, t.link.core.config.Previous.Listen)
-			deleted := util.Difference(t.link.core.config.Previous.Listen, t.link.core.config.Current.Listen)
-			t.link.core.config.Mutex.RUnlock()
-			if len(added) > 0 || len(deleted) > 0 {
-				for _, a := range added {
-					if a[:6] != "tcp://" {
-						continue
-					}
-					if _, err := t.listen(a[6:]); err != nil {
-						e <- err
-						continue
-					}
-				}
-				for _, d := range deleted {
-					if d[:6] != "tcp://" {
-						continue
-					}
-					t.mutex.Lock()
-					if listener, ok := t.listeners[d[6:]]; ok {
-						t.mutex.Unlock()
-						listener.Stop <- true
-					} else {
-						t.mutex.Unlock()
-					}
-				}
-				e <- nil
-			} else {
-				e <- nil
-			}
-		}
-	}()
-
 	t.link.core.config.Mutex.RLock()
 	defer t.link.core.config.Mutex.RUnlock()
 	for _, listenaddr := range t.link.core.config.Current.Listen {
@@ -133,6 +95,36 @@ func (t *tcp) init(l *link) error {
 	return nil
 }
 
+func (t *tcp) reconfigure(e chan error) {
+	defer close(e)
+	t.link.core.config.Mutex.RLock()
+	added := util.Difference(t.link.core.config.Current.Listen, t.link.core.config.Previous.Listen)
+	deleted := util.Difference(t.link.core.config.Previous.Listen, t.link.core.config.Current.Listen)
+	t.link.core.config.Mutex.RUnlock()
+	if len(added) > 0 || len(deleted) > 0 {
+		for _, a := range added {
+			if a[:6] != "tcp://" {
+				continue
+			}
+			if _, err := t.listen(a[6:]); err != nil {
+				e <- err
+			}
+		}
+		for _, d := range deleted {
+			if d[:6] != "tcp://" {
+				continue
+			}
+			t.mutex.Lock()
+			if listener, ok := t.listeners[d[6:]]; ok {
+				t.mutex.Unlock()
+				listener.Stop <- true
+			} else {
+				t.mutex.Unlock()
+			}
+		}
+	}
+}
+
 func (t *tcp) listen(listenaddr string) (*TcpListener, error) {
 	var err error
 

From 502f2937a9419fabc7985da9a55d5913a40499db Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 25 Aug 2019 17:00:02 -0500
Subject: [PATCH 39/62] a couple race fixes and use timer.AfterFunc instead of
 sleeping goroutines or ticker in a few places

---
 src/tuntap/conn.go        | 31 ++++---------------------------
 src/tuntap/tun.go         | 11 +++++------
 src/yggdrasil/api.go      |  6 ++----
 src/yggdrasil/debug.go    |  4 ++--
 src/yggdrasil/link.go     |  2 +-
 src/yggdrasil/nodeinfo.go | 34 +++++++++++++++++-----------------
 src/yggdrasil/peer.go     | 25 ++++++++++++++-----------
 src/yggdrasil/router.go   | 22 +++++++++-------------
 src/yggdrasil/search.go   | 22 ++++++++++------------
 src/yggdrasil/session.go  |  1 -
 src/yggdrasil/switch.go   |  6 ++----
 11 files changed, 66 insertions(+), 98 deletions(-)

diff --git a/src/tuntap/conn.go b/src/tuntap/conn.go
index 0e0dd461..31490916 100644
--- a/src/tuntap/conn.go
+++ b/src/tuntap/conn.go
@@ -23,7 +23,7 @@ type tunConn struct {
 	addr  address.Address
 	snet  address.Subnet
 	stop  chan struct{}
-	alive chan struct{}
+	alive *time.Timer // From calling time.AfterFunc
 }
 
 func (s *tunConn) close() {
@@ -40,10 +40,6 @@ func (s *tunConn) _close_nomutex() {
 		defer func() { recover() }()
 		close(s.stop) // Closes reader/writer goroutines
 	}()
-	func() {
-		defer func() { recover() }()
-		close(s.alive) // Closes timeout goroutine
-	}()
 }
 
 func (s *tunConn) _read(bs []byte) (err error) {
@@ -228,27 +224,8 @@ func (s *tunConn) _write(bs []byte) (err error) {
 }
 
 func (s *tunConn) stillAlive() {
-	defer func() { recover() }()
-	select {
-	case s.alive <- struct{}{}:
-	default:
-	}
-}
-
-func (s *tunConn) checkForTimeouts() error {
-	timer := time.NewTimer(tunConnTimeout)
-	defer util.TimerStop(timer)
-	defer s.close()
-	for {
-		select {
-		case _, ok := <-s.alive:
-			if !ok {
-				return errors.New("connection closed")
-			}
-			util.TimerStop(timer)
-			timer.Reset(tunConnTimeout)
-		case <-timer.C:
-			return errors.New("timed out")
-		}
+	if s.alive != nil {
+		s.alive.Stop()
 	}
+	s.alive = time.AfterFunc(tunConnTimeout, s.close)
 }
diff --git a/src/tuntap/tun.go b/src/tuntap/tun.go
index dfc343df..d73384d4 100644
--- a/src/tuntap/tun.go
+++ b/src/tuntap/tun.go
@@ -227,10 +227,9 @@ func (tun *TunAdapter) handler() error {
 func (tun *TunAdapter) wrap(conn *yggdrasil.Conn) (c *tunConn, err error) {
 	// Prepare a session wrapper for the given connection
 	s := tunConn{
-		tun:   tun,
-		conn:  conn,
-		stop:  make(chan struct{}),
-		alive: make(chan struct{}, 1),
+		tun:  tun,
+		conn: conn,
+		stop: make(chan struct{}),
 	}
 	c = &s
 	// Get the remote address and subnet of the other side
@@ -255,13 +254,13 @@ func (tun *TunAdapter) wrap(conn *yggdrasil.Conn) (c *tunConn, err error) {
 	// we receive a packet through the interface for this address
 	tun.addrToConn[s.addr] = &s
 	tun.subnetToConn[s.snet] = &s
-	// Set the read callback and start the timeout goroutine
+	// Set the read callback and start the timeout
 	conn.SetReadCallback(func(bs []byte) {
 		s.RecvFrom(conn, func() {
 			s._read(bs)
 		})
 	})
-	go s.checkForTimeouts()
+	s.RecvFrom(nil, s.stillAlive)
 	// Return
 	return c, err
 }
diff --git a/src/yggdrasil/api.go b/src/yggdrasil/api.go
index 2bc5c816..c6c15e24 100644
--- a/src/yggdrasil/api.go
+++ b/src/yggdrasil/api.go
@@ -344,10 +344,8 @@ func (c *Core) GetNodeInfo(key crypto.BoxPubKey, coords []uint64, nocache bool)
 		c.router.nodeinfo.sendNodeInfo(key, wire_coordsUint64stoBytes(coords), false)
 	}
 	c.router.doAdmin(sendNodeInfoRequest)
-	go func() {
-		time.Sleep(6 * time.Second)
-		close(response)
-	}()
+	timer := time.AfterFunc(6*time.Second, func() { close(response) })
+	defer timer.Stop()
 	for res := range response {
 		return *res, nil
 	}
diff --git a/src/yggdrasil/debug.go b/src/yggdrasil/debug.go
index f6ec8716..c6f98ca0 100644
--- a/src/yggdrasil/debug.go
+++ b/src/yggdrasil/debug.go
@@ -552,7 +552,7 @@ func (c *Core) DEBUG_addAllowedEncryptionPublicKey(boxStr string) {
 ////////////////////////////////////////////////////////////////////////////////
 
 func DEBUG_simLinkPeers(p, q *peer) {
-	// Sets q.out() to point to p and starts p.linkLoop()
+	// Sets q.out() to point to p and starts p.start()
 	goWorkers := func(source, dest *peer) {
 		source.linkOut = make(chan []byte, 1)
 		send := make(chan []byte, 1)
@@ -561,7 +561,7 @@ func DEBUG_simLinkPeers(p, q *peer) {
 				send <- bs
 			}
 		}
-		go source.linkLoop()
+		go source.start()
 		go func() {
 			var packets [][]byte
 			for {
diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index bfbcc99b..1b48f391 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -216,7 +216,7 @@ func (intf *linkInterface) handler() error {
 	intf.link.core.log.Infof("Connected %s: %s, source %s",
 		strings.ToUpper(intf.info.linkType), themString, intf.info.local)
 	// Start the link loop
-	go intf.peer.linkLoop()
+	go intf.peer.start()
 	// Start the writer
 	signalReady := make(chan struct{}, 1)
 	signalSent := make(chan bool, 1)
diff --git a/src/yggdrasil/nodeinfo.go b/src/yggdrasil/nodeinfo.go
index 50f5bf9c..8a5d7872 100644
--- a/src/yggdrasil/nodeinfo.go
+++ b/src/yggdrasil/nodeinfo.go
@@ -47,25 +47,25 @@ func (m *nodeinfo) init(core *Core) {
 	m.callbacks = make(map[crypto.BoxPubKey]nodeinfoCallback)
 	m.cache = make(map[crypto.BoxPubKey]nodeinfoCached)
 
-	go func() {
-		for {
-			m.callbacksMutex.Lock()
-			for boxPubKey, callback := range m.callbacks {
-				if time.Since(callback.created) > time.Minute {
-					delete(m.callbacks, boxPubKey)
-				}
+	var f func()
+	f = func() {
+		m.callbacksMutex.Lock()
+		for boxPubKey, callback := range m.callbacks {
+			if time.Since(callback.created) > time.Minute {
+				delete(m.callbacks, boxPubKey)
 			}
-			m.callbacksMutex.Unlock()
-			m.cacheMutex.Lock()
-			for boxPubKey, cache := range m.cache {
-				if time.Since(cache.created) > time.Hour {
-					delete(m.cache, boxPubKey)
-				}
-			}
-			m.cacheMutex.Unlock()
-			time.Sleep(time.Second * 30)
 		}
-	}()
+		m.callbacksMutex.Unlock()
+		m.cacheMutex.Lock()
+		for boxPubKey, cache := range m.cache {
+			if time.Since(cache.created) > time.Hour {
+				delete(m.cache, boxPubKey)
+			}
+		}
+		m.cacheMutex.Unlock()
+		time.AfterFunc(time.Second*30, f)
+	}
+	go f()
 }
 
 // Add a callback for a nodeinfo lookup
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 989d9ee1..50fb03f8 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -184,18 +184,21 @@ func (ps *peers) sendSwitchMsgs(from phony.Actor) {
 
 // This must be launched in a separate goroutine by whatever sets up the peer struct.
 // It handles link protocol traffic.
-func (p *peer) linkLoop() {
-	tick := time.NewTicker(time.Second)
-	defer tick.Stop()
-	<-p.SyncExec(p._sendSwitchMsg) // Startup message
-	for {
-		select {
-		case <-p.done:
-			return
-		case _ = <-tick.C:
-			<-p.SyncExec(p._updateDHT)
-		}
+func (p *peer) start() {
+	var updateDHT func()
+	updateDHT = func() {
+		<-p.SyncExec(func() {
+			select {
+			case <-p.done:
+			default:
+				p._updateDHT()
+				time.AfterFunc(time.Second, updateDHT)
+			}
+		})
 	}
+	updateDHT()
+	// Just for good measure, immediately send a switch message to this peer when we start
+	<-p.SyncExec(p._sendSwitchMsg)
 }
 
 func (p *peer) _updateDHT() {
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index 002905bc..fec55b7c 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -92,7 +92,7 @@ func (r *router) reconfigure(e chan error) {
 // Starts the tickerLoop goroutine.
 func (r *router) start() error {
 	r.core.log.Infoln("Starting router")
-	go r.tickerLoop()
+	go r.doMaintenance()
 	return nil
 }
 
@@ -122,18 +122,14 @@ func (r *router) reset(from phony.Actor) {
 
 // TODO remove reconfigure so this is just a ticker loop
 // and then find something better than a ticker loop to schedule things...
-func (r *router) tickerLoop() {
-	ticker := time.NewTicker(time.Second)
-	defer ticker.Stop()
-	for {
-		<-ticker.C
-		<-r.SyncExec(func() {
-			// Any periodic maintenance stuff goes here
-			r.core.switchTable.doMaintenance()
-			r.dht.doMaintenance()
-			r.sessions.cleanup()
-		})
-	}
+func (r *router) doMaintenance() {
+	<-r.SyncExec(func() {
+		// Any periodic maintenance stuff goes here
+		r.core.switchTable.doMaintenance()
+		r.dht.doMaintenance()
+		r.sessions.cleanup()
+	})
+	time.AfterFunc(time.Second, r.doMaintenance)
 }
 
 // Checks incoming traffic type and passes it to the appropriate handler.
diff --git a/src/yggdrasil/search.go b/src/yggdrasil/search.go
index 5fb36584..9cf5f06e 100644
--- a/src/yggdrasil/search.go
+++ b/src/yggdrasil/search.go
@@ -152,18 +152,16 @@ func (sinfo *searchInfo) continueSearch() {
 	// In case the search dies, try to spawn another thread later
 	// Note that this will spawn multiple parallel searches as time passes
 	// Any that die aren't restarted, but a new one will start later
-	retryLater := func() {
-		// FIXME this keeps the search alive forever if not for the searches map, fix that
-		newSearchInfo := sinfo.searches.searches[sinfo.dest]
-		if newSearchInfo != sinfo {
-			return
-		}
-		sinfo.continueSearch()
-	}
-	go func() {
-		time.Sleep(search_RETRY_TIME)
-		sinfo.searches.router.doAdmin(retryLater)
-	}()
+	time.AfterFunc(search_RETRY_TIME, func() {
+		sinfo.searches.router.RecvFrom(nil, func() {
+			// FIXME this keeps the search alive forever if not for the searches map, fix that
+			newSearchInfo := sinfo.searches.searches[sinfo.dest]
+			if newSearchInfo != sinfo {
+				return
+			}
+			sinfo.continueSearch()
+		})
+	})
 }
 
 // Calls create search, and initializes the iterative search parts of the struct before returning it.
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index e448cf25..94ee41e6 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -262,7 +262,6 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 			sinfo.sessions.removeSession(&sinfo)
 		})
 	}()
-	//go sinfo.startWorkers()
 	return &sinfo
 }
 
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index cb5cf1eb..6d882252 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -200,10 +200,8 @@ func (t *switchTable) init(core *Core) {
 }
 
 func (t *switchTable) reconfigure(e chan error) {
-	go func() {
-		defer close(e)
-		// This is where reconfiguration would go, if we had anything useful to do.
-	}()
+	defer close(e)
+	// This is where reconfiguration would go, if we had anything useful to do.
 }
 
 // Safely gets a copy of this node's locator.

From aaf34c63049ed8314fd222367cfbc1fa51813ca7 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 25 Aug 2019 18:08:43 -0500
Subject: [PATCH 40/62] start migrating the TunAdapter to the actor model

---
 src/tuntap/conn.go  |   5 +-
 src/tuntap/iface.go | 501 +++++++++++++++++++++++---------------------
 src/tuntap/tun.go   |  11 +-
 3 files changed, 267 insertions(+), 250 deletions(-)

diff --git a/src/tuntap/conn.go b/src/tuntap/conn.go
index 31490916..cdb8c32e 100644
--- a/src/tuntap/conn.go
+++ b/src/tuntap/conn.go
@@ -113,8 +113,7 @@ func (s *tunConn) _read(bs []byte) (err error) {
 		util.PutBytes(bs)
 		return
 	}
-	// FIXME this send can block if the tuntap isn't running, which isn't really safe...
-	s.tun.send <- bs
+	s.tun.writer.writeFrom(s, bs)
 	s.stillAlive()
 	return
 }
@@ -208,7 +207,7 @@ func (s *tunConn) _write(bs []byte) (err error) {
 					Data: bs[:900],
 				}
 				if packet, err := CreateICMPv6(bs[8:24], bs[24:40], ipv6.ICMPTypePacketTooBig, 0, ptb); err == nil {
-					s.tun.send <- packet
+					s.tun.writer.writeFrom(s, packet)
 				}
 			} else {
 				if e.Closed() {
diff --git a/src/tuntap/iface.go b/src/tuntap/iface.go
index e0693e56..3f923513 100644
--- a/src/tuntap/iface.go
+++ b/src/tuntap/iface.go
@@ -9,264 +9,279 @@ import (
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
+
+	"github.com/Arceliar/phony"
 )
 
-func (tun *TunAdapter) writer() error {
-	var w int
+type tunWriter struct {
+	phony.Inbox
+	tun *TunAdapter
+}
+
+func (w *tunWriter) writeFrom(from phony.Actor, b []byte) {
+	w.RecvFrom(from, func() {
+		w._write(b)
+	})
+}
+
+// write is pretty loose with the memory safety rules, e.g. it assumes it can read w.tun.iface.IsTap() safely
+func (w *tunWriter) _write(b []byte) {
+	var written int
 	var err error
-	for {
-		b := <-tun.send
-		n := len(b)
-		if n == 0 {
-			continue
-		}
-		if tun.iface.IsTAP() {
-			sendndp := func(dstAddr address.Address) {
-				neigh, known := tun.icmpv6.getNeighbor(dstAddr)
-				known = known && (time.Since(neigh.lastsolicitation).Seconds() < 30)
-				if !known {
-					tun.icmpv6.Solicit(dstAddr)
-				}
-			}
-			peermac := net.HardwareAddr{0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
-			var dstAddr address.Address
-			var peerknown bool
-			if b[0]&0xf0 == 0x40 {
-				dstAddr = tun.addr
-			} else if b[0]&0xf0 == 0x60 {
-				if !bytes.Equal(tun.addr[:16], dstAddr[:16]) && !bytes.Equal(tun.subnet[:8], dstAddr[:8]) {
-					dstAddr = tun.addr
-				}
-			}
-			if neighbor, ok := tun.icmpv6.getNeighbor(dstAddr); ok && neighbor.learned {
-				// If we've learned the MAC of a 300::/7 address, for example, or a CKR
-				// address, use the MAC address of that
-				peermac = neighbor.mac
-				peerknown = true
-			} else if neighbor, ok := tun.icmpv6.getNeighbor(tun.addr); ok && neighbor.learned {
-				// Otherwise send directly to the MAC address of the host if that's
-				// known instead
-				peermac = neighbor.mac
-				peerknown = true
-			} else {
-				// Nothing has been discovered, try to discover the destination
-				sendndp(tun.addr)
-			}
-			if peerknown {
-				var proto ethernet.Ethertype
-				switch {
-				case b[0]&0xf0 == 0x60:
-					proto = ethernet.IPv6
-				case b[0]&0xf0 == 0x40:
-					proto = ethernet.IPv4
-				}
-				var frame ethernet.Frame
-				frame.Prepare(
-					peermac[:6],          // Destination MAC address
-					tun.icmpv6.mymac[:6], // Source MAC address
-					ethernet.NotTagged,   // VLAN tagging
-					proto,                // Ethertype
-					len(b))               // Payload length
-				copy(frame[tun_ETHER_HEADER_LENGTH:], b[:n])
-				n += tun_ETHER_HEADER_LENGTH
-				w, err = tun.iface.Write(frame[:n])
-			} else {
-				tun.log.Errorln("TUN/TAP iface write error: no peer MAC known for", net.IP(dstAddr[:]).String(), "- dropping packet")
-			}
-		} else {
-			w, err = tun.iface.Write(b[:n])
-			util.PutBytes(b)
-		}
-		if err != nil {
-			if !tun.isOpen {
-				return err
-			}
-			tun.log.Errorln("TUN/TAP iface write error:", err)
-			continue
-		}
-		if w != n {
-			tun.log.Errorln("TUN/TAP iface write mismatch:", w, "bytes written vs", n, "bytes given")
-			continue
-		}
+	n := len(b)
+	if n == 0 {
+		return
 	}
-}
-
-// Run in a separate goroutine by the reader
-// Does all of the per-packet ICMP checks, passes packets to the right Conn worker
-func (tun *TunAdapter) readerPacketHandler(ch chan []byte) {
-	for recvd := range ch {
-		// If it's a TAP adapter, update the buffer slice so that we no longer
-		// include the ethernet headers
-		offset := 0
-		if tun.iface.IsTAP() {
-			// Set our offset to beyond the ethernet headers
-			offset = tun_ETHER_HEADER_LENGTH
-			// Check first of all that we can go beyond the ethernet headers
-			if len(recvd) <= offset {
-				continue
+	if w.tun.iface.IsTAP() {
+		sendndp := func(dstAddr address.Address) {
+			neigh, known := w.tun.icmpv6.getNeighbor(dstAddr)
+			known = known && (time.Since(neigh.lastsolicitation).Seconds() < 30)
+			if !known {
+				w.tun.icmpv6.Solicit(dstAddr)
 			}
 		}
-		// Offset the buffer from now on so that we can ignore ethernet frames if
-		// they are present
-		bs := recvd[offset:]
-		// If we detect an ICMP packet then hand it to the ICMPv6 module
-		if bs[6] == 58 {
-			// Found an ICMPv6 packet - we need to make sure to give ICMPv6 the full
-			// Ethernet frame rather than just the IPv6 packet as this is needed for
-			// NDP to work correctly
-			if err := tun.icmpv6.ParsePacket(recvd); err == nil {
-				// We acted on the packet in the ICMPv6 module so don't forward or do
-				// anything else with it
-				continue
-			}
-		}
-		if offset != 0 {
-			// Shift forward to avoid leaking bytes off the front of the slice when we eventually store it
-			bs = append(recvd[:0], bs...)
-		}
-		// From the IP header, work out what our source and destination addresses
-		// and node IDs are. We will need these in order to work out where to send
-		// the packet
+		peermac := net.HardwareAddr{0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
 		var dstAddr address.Address
-		var dstSnet address.Subnet
-		var addrlen int
-		n := len(bs)
-		// Check the IP protocol - if it doesn't match then we drop the packet and
-		// do nothing with it
-		if bs[0]&0xf0 == 0x60 {
-			// Check if we have a fully-sized IPv6 header
-			if len(bs) < 40 {
-				continue
+		var peerknown bool
+		if b[0]&0xf0 == 0x40 {
+			dstAddr = w.tun.addr
+		} else if b[0]&0xf0 == 0x60 {
+			if !bytes.Equal(w.tun.addr[:16], dstAddr[:16]) && !bytes.Equal(w.tun.subnet[:8], dstAddr[:8]) {
+				dstAddr = w.tun.addr
 			}
-			// Check the packet size
-			if n-tun_IPv6_HEADER_LENGTH != 256*int(bs[4])+int(bs[5]) {
-				continue
-			}
-			// IPv6 address
-			addrlen = 16
-			copy(dstAddr[:addrlen], bs[24:])
-			copy(dstSnet[:addrlen/2], bs[24:])
-		} else if bs[0]&0xf0 == 0x40 {
-			// Check if we have a fully-sized IPv4 header
-			if len(bs) < 20 {
-				continue
-			}
-			// Check the packet size
-			if n != 256*int(bs[2])+int(bs[3]) {
-				continue
-			}
-			// IPv4 address
-			addrlen = 4
-			copy(dstAddr[:addrlen], bs[16:])
+		}
+		if neighbor, ok := w.tun.icmpv6.getNeighbor(dstAddr); ok && neighbor.learned {
+			// If we've learned the MAC of a 300::/7 address, for example, or a CKR
+			// address, use the MAC address of that
+			peermac = neighbor.mac
+			peerknown = true
+		} else if neighbor, ok := w.tun.icmpv6.getNeighbor(w.tun.addr); ok && neighbor.learned {
+			// Otherwise send directly to the MAC address of the host if that's
+			// known instead
+			peermac = neighbor.mac
+			peerknown = true
 		} else {
-			// Unknown address length or protocol, so drop the packet and ignore it
-			tun.log.Traceln("Unknown packet type, dropping")
-			continue
+			// Nothing has been discovered, try to discover the destination
+			sendndp(w.tun.addr)
 		}
-		if tun.ckr.isEnabled() {
-			if addrlen != 16 || (!dstAddr.IsValid() && !dstSnet.IsValid()) {
-				if key, err := tun.ckr.getPublicKeyForAddress(dstAddr, addrlen); err == nil {
-					// A public key was found, get the node ID for the search
-					dstNodeID := crypto.GetNodeID(&key)
-					dstAddr = *address.AddrForNodeID(dstNodeID)
-					dstSnet = *address.SubnetForNodeID(dstNodeID)
-					addrlen = 16
-				}
+		if peerknown {
+			var proto ethernet.Ethertype
+			switch {
+			case b[0]&0xf0 == 0x60:
+				proto = ethernet.IPv6
+			case b[0]&0xf0 == 0x40:
+				proto = ethernet.IPv4
 			}
+			var frame ethernet.Frame
+			frame.Prepare(
+				peermac[:6],            // Destination MAC address
+				w.tun.icmpv6.mymac[:6], // Source MAC address
+				ethernet.NotTagged,     // VLAN tagging
+				proto,                  // Ethertype
+				len(b))                 // Payload length
+			copy(frame[tun_ETHER_HEADER_LENGTH:], b[:n])
+			n += tun_ETHER_HEADER_LENGTH
+			written, err = w.tun.iface.Write(frame[:n])
+		} else {
+			w.tun.log.Errorln("TUN/TAP iface write error: no peer MAC known for", net.IP(dstAddr[:]).String(), "- dropping packet")
 		}
-		if addrlen != 16 || (!dstAddr.IsValid() && !dstSnet.IsValid()) {
-			// Couldn't find this node's ygg IP
-			continue
-		}
-		// Do we have an active connection for this node address?
-		var dstNodeID, dstNodeIDMask *crypto.NodeID
-		tun.mutex.RLock()
-		session, isIn := tun.addrToConn[dstAddr]
-		if !isIn || session == nil {
-			session, isIn = tun.subnetToConn[dstSnet]
-			if !isIn || session == nil {
-				// Neither an address nor a subnet mapping matched, therefore populate
-				// the node ID and mask to commence a search
-				if dstAddr.IsValid() {
-					dstNodeID, dstNodeIDMask = dstAddr.GetNodeIDandMask()
-				} else {
-					dstNodeID, dstNodeIDMask = dstSnet.GetNodeIDandMask()
-				}
-			}
-		}
-		tun.mutex.RUnlock()
-		// If we don't have a connection then we should open one
-		if !isIn || session == nil {
-			// Check we haven't been given empty node ID, really this shouldn't ever
-			// happen but just to be sure...
-			if dstNodeID == nil || dstNodeIDMask == nil {
-				panic("Given empty dstNodeID and dstNodeIDMask - this shouldn't happen")
-			}
-			// Dial to the remote node
-			go func() {
-				// FIXME just spitting out a goroutine to do this is kind of ugly and means we drop packets until the dial finishes
-				tun.mutex.Lock()
-				_, known := tun.dials[*dstNodeID]
-				tun.dials[*dstNodeID] = append(tun.dials[*dstNodeID], bs)
-				for len(tun.dials[*dstNodeID]) > 32 {
-					util.PutBytes(tun.dials[*dstNodeID][0])
-					tun.dials[*dstNodeID] = tun.dials[*dstNodeID][1:]
-				}
-				tun.mutex.Unlock()
-				if known {
-					return
-				}
-				var tc *tunConn
-				if conn, err := tun.dialer.DialByNodeIDandMask(dstNodeID, dstNodeIDMask); err == nil {
-					// We've been given a connection so prepare the session wrapper
-					if tc, err = tun.wrap(conn); err != nil {
-						// Something went wrong when storing the connection, typically that
-						// something already exists for this address or subnet
-						tun.log.Debugln("TUN/TAP iface wrap:", err)
-					}
-				}
-				tun.mutex.Lock()
-				packets := tun.dials[*dstNodeID]
-				delete(tun.dials, *dstNodeID)
-				tun.mutex.Unlock()
-				if tc != nil {
-					for _, packet := range packets {
-						p := packet // Possibly required because of how range
-						<-tc.SyncExec(func() { tc._write(p) })
-					}
-				}
-			}()
-			// While the dial is going on we can't do much else
-			// continuing this iteration - skip to the next one
-			continue
-		}
-		// If we have a connection now, try writing to it
-		if isIn && session != nil {
-			<-session.SyncExec(func() { session._write(bs) })
+	} else {
+		written, err = w.tun.iface.Write(b[:n])
+		util.PutBytes(b)
+	}
+	if err != nil {
+		w.tun.mutex.Lock()
+		open := w.tun.isOpen
+		w.tun.mutex.Unlock()
+		if !open {
+			return
 		}
+		w.tun.log.Errorln("TUN/TAP iface write error:", err)
+	}
+	if written != n {
+		w.tun.log.Errorln("TUN/TAP iface write mismatch:", written, "bytes written vs", n, "bytes given")
 	}
 }
 
-func (tun *TunAdapter) reader() error {
-	toWorker := make(chan []byte, 32)
-	defer close(toWorker)
-	go tun.readerPacketHandler(toWorker)
-	for {
-		// Get a slice to store the packet in
-		recvd := util.ResizeBytes(util.GetBytes(), 65535+tun_ETHER_HEADER_LENGTH)
-		// Wait for a packet to be delivered to us through the TUN/TAP adapter
-		n, err := tun.iface.Read(recvd)
-		if err != nil {
-			if !tun.isOpen {
-				return err
-			}
-			panic(err)
-		}
-		if n == 0 {
-			util.PutBytes(recvd)
-			continue
-		}
-		// Send the packet to the worker
-		toWorker <- recvd[:n]
+type tunReader struct {
+	phony.Inbox
+	tun *TunAdapter
+}
+
+func (r *tunReader) _read() {
+	// Get a slice to store the packet in
+	recvd := util.ResizeBytes(util.GetBytes(), 65535+tun_ETHER_HEADER_LENGTH)
+	// Wait for a packet to be delivered to us through the TUN/TAP adapter
+	n, err := r.tun.iface.Read(recvd)
+	if n == 0 {
+		util.PutBytes(recvd)
+	} else {
+		r.tun.handlePacketFrom(r, recvd[:n], err)
+	}
+	if err == nil {
+		// Now read again
+		r.RecvFrom(nil, r._read)
+	}
+}
+
+func (tun *TunAdapter) handlePacketFrom(from phony.Actor, packet []byte, err error) {
+	tun.RecvFrom(from, func() {
+		tun._handlePacket(packet, err)
+	})
+}
+
+// does the work of reading a packet and sending it to the correct tunConn
+func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
+	if err != nil {
+		tun.log.Errorln("TUN/TAP iface read error:", err)
+		return
+	}
+	// If it's a TAP adapter, update the buffer slice so that we no longer
+	// include the ethernet headers
+	offset := 0
+	if tun.iface.IsTAP() {
+		// Set our offset to beyond the ethernet headers
+		offset = tun_ETHER_HEADER_LENGTH
+		// Check first of all that we can go beyond the ethernet headers
+		if len(recvd) <= offset {
+			return
+		}
+	}
+	// Offset the buffer from now on so that we can ignore ethernet frames if
+	// they are present
+	bs := recvd[offset:]
+	// If we detect an ICMP packet then hand it to the ICMPv6 module
+	if bs[6] == 58 {
+		// Found an ICMPv6 packet - we need to make sure to give ICMPv6 the full
+		// Ethernet frame rather than just the IPv6 packet as this is needed for
+		// NDP to work correctly
+		if err := tun.icmpv6.ParsePacket(recvd); err == nil {
+			// We acted on the packet in the ICMPv6 module so don't forward or do
+			// anything else with it
+			return
+		}
+	}
+	if offset != 0 {
+		// Shift forward to avoid leaking bytes off the front of the slice when we eventually store it
+		bs = append(recvd[:0], bs...)
+	}
+	// From the IP header, work out what our source and destination addresses
+	// and node IDs are. We will need these in order to work out where to send
+	// the packet
+	var dstAddr address.Address
+	var dstSnet address.Subnet
+	var addrlen int
+	n := len(bs)
+	// Check the IP protocol - if it doesn't match then we drop the packet and
+	// do nothing with it
+	if bs[0]&0xf0 == 0x60 {
+		// Check if we have a fully-sized IPv6 header
+		if len(bs) < 40 {
+			return
+		}
+		// Check the packet size
+		if n-tun_IPv6_HEADER_LENGTH != 256*int(bs[4])+int(bs[5]) {
+			return
+		}
+		// IPv6 address
+		addrlen = 16
+		copy(dstAddr[:addrlen], bs[24:])
+		copy(dstSnet[:addrlen/2], bs[24:])
+	} else if bs[0]&0xf0 == 0x40 {
+		// Check if we have a fully-sized IPv4 header
+		if len(bs) < 20 {
+			return
+		}
+		// Check the packet size
+		if n != 256*int(bs[2])+int(bs[3]) {
+			return
+		}
+		// IPv4 address
+		addrlen = 4
+		copy(dstAddr[:addrlen], bs[16:])
+	} else {
+		// Unknown address length or protocol, so drop the packet and ignore it
+		tun.log.Traceln("Unknown packet type, dropping")
+		return
+	}
+	if tun.ckr.isEnabled() {
+		if addrlen != 16 || (!dstAddr.IsValid() && !dstSnet.IsValid()) {
+			if key, err := tun.ckr.getPublicKeyForAddress(dstAddr, addrlen); err == nil {
+				// A public key was found, get the node ID for the search
+				dstNodeID := crypto.GetNodeID(&key)
+				dstAddr = *address.AddrForNodeID(dstNodeID)
+				dstSnet = *address.SubnetForNodeID(dstNodeID)
+				addrlen = 16
+			}
+		}
+	}
+	if addrlen != 16 || (!dstAddr.IsValid() && !dstSnet.IsValid()) {
+		// Couldn't find this node's ygg IP
+		return
+	}
+	// Do we have an active connection for this node address?
+	var dstNodeID, dstNodeIDMask *crypto.NodeID
+	tun.mutex.RLock()
+	session, isIn := tun.addrToConn[dstAddr]
+	if !isIn || session == nil {
+		session, isIn = tun.subnetToConn[dstSnet]
+		if !isIn || session == nil {
+			// Neither an address nor a subnet mapping matched, therefore populate
+			// the node ID and mask to commence a search
+			if dstAddr.IsValid() {
+				dstNodeID, dstNodeIDMask = dstAddr.GetNodeIDandMask()
+			} else {
+				dstNodeID, dstNodeIDMask = dstSnet.GetNodeIDandMask()
+			}
+		}
+	}
+	tun.mutex.RUnlock()
+	// If we don't have a connection then we should open one
+	if !isIn || session == nil {
+		// Check we haven't been given empty node ID, really this shouldn't ever
+		// happen but just to be sure...
+		if dstNodeID == nil || dstNodeIDMask == nil {
+			panic("Given empty dstNodeID and dstNodeIDMask - this shouldn't happen")
+		}
+		// Dial to the remote node
+		go func() {
+			// FIXME just spitting out a goroutine to do this is kind of ugly and means we drop packets until the dial finishes
+			tun.mutex.Lock()
+			_, known := tun.dials[*dstNodeID]
+			tun.dials[*dstNodeID] = append(tun.dials[*dstNodeID], bs)
+			for len(tun.dials[*dstNodeID]) > 32 {
+				util.PutBytes(tun.dials[*dstNodeID][0])
+				tun.dials[*dstNodeID] = tun.dials[*dstNodeID][1:]
+			}
+			tun.mutex.Unlock()
+			if known {
+				return
+			}
+			var tc *tunConn
+			if conn, err := tun.dialer.DialByNodeIDandMask(dstNodeID, dstNodeIDMask); err == nil {
+				// We've been given a connection so prepare the session wrapper
+				if tc, err = tun.wrap(conn); err != nil {
+					// Something went wrong when storing the connection, typically that
+					// something already exists for this address or subnet
+					tun.log.Debugln("TUN/TAP iface wrap:", err)
+				}
+			}
+			tun.mutex.Lock()
+			packets := tun.dials[*dstNodeID]
+			delete(tun.dials, *dstNodeID)
+			tun.mutex.Unlock()
+			if tc != nil {
+				for _, packet := range packets {
+					p := packet // Possibly required because of how range
+					<-tc.SyncExec(func() { tc._write(p) })
+				}
+			}
+		}()
+		// While the dial is going on we can't do much else
+		return
+	}
+	// If we have a connection now, try writing to it
+	if isIn && session != nil {
+		session.RecvFrom(tun, func() { session._write(bs) })
 	}
 }
diff --git a/src/tuntap/tun.go b/src/tuntap/tun.go
index d73384d4..7bb3a59f 100644
--- a/src/tuntap/tun.go
+++ b/src/tuntap/tun.go
@@ -15,6 +15,7 @@ import (
 	"net"
 	"sync"
 
+	"github.com/Arceliar/phony"
 	"github.com/gologme/log"
 	"github.com/yggdrasil-network/water"
 
@@ -33,6 +34,8 @@ const tun_ETHER_HEADER_LENGTH = 14
 // you should pass this object to the yggdrasil.SetRouterAdapter() function
 // before calling yggdrasil.Start().
 type TunAdapter struct {
+	writer       tunWriter
+	reader       tunReader
 	config       *config.NodeState
 	log          *log.Logger
 	reconfigure  chan chan error
@@ -44,7 +47,7 @@ type TunAdapter struct {
 	icmpv6       ICMPv6
 	mtu          int
 	iface        *water.Interface
-	send         chan []byte
+	phony.Inbox               // Currently only used for _handlePacket from the reader, TODO: all the stuff that currently needs a mutex below
 	mutex        sync.RWMutex // Protects the below
 	addrToConn   map[address.Address]*tunConn
 	subnetToConn map[address.Subnet]*tunConn
@@ -114,6 +117,8 @@ func (tun *TunAdapter) Init(config *config.NodeState, log *log.Logger, listener
 	tun.addrToConn = make(map[address.Address]*tunConn)
 	tun.subnetToConn = make(map[address.Subnet]*tunConn)
 	tun.dials = make(map[crypto.NodeID][][]byte)
+	tun.writer.tun = tun
+	tun.reader.tun = tun
 }
 
 // Start the setup process for the TUN/TAP adapter. If successful, starts the
@@ -147,7 +152,6 @@ func (tun *TunAdapter) Start() error {
 	}
 	tun.mutex.Lock()
 	tun.isOpen = true
-	tun.send = make(chan []byte, 32) // TODO: is this a sensible value?
 	tun.reconfigure = make(chan chan error)
 	tun.mutex.Unlock()
 	go func() {
@@ -157,8 +161,7 @@ func (tun *TunAdapter) Start() error {
 		}
 	}()
 	go tun.handler()
-	go tun.reader()
-	go tun.writer()
+	tun.reader.RecvFrom(nil, tun.reader._read) // Start the reader
 	tun.icmpv6.Init(tun)
 	if iftapmode {
 		go tun.icmpv6.Solicit(tun.addr)

From b2a2e251ad173686c752463f38efdf77e6814487 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 25 Aug 2019 18:53:11 -0500
Subject: [PATCH 41/62] more TunAdapter migration

---
 src/tuntap/conn.go  | 12 ++++---
 src/tuntap/iface.go | 78 +++++++++++++++++++--------------------------
 src/tuntap/tun.go   | 72 ++++++++++++++++++++++++-----------------
 3 files changed, 84 insertions(+), 78 deletions(-)

diff --git a/src/tuntap/conn.go b/src/tuntap/conn.go
index cdb8c32e..afb0f09e 100644
--- a/src/tuntap/conn.go
+++ b/src/tuntap/conn.go
@@ -27,12 +27,10 @@ type tunConn struct {
 }
 
 func (s *tunConn) close() {
-	s.tun.mutex.Lock()
-	defer s.tun.mutex.Unlock()
-	s._close_nomutex()
+	s.tun.RecvFrom(s, s._close_from_tun)
 }
 
-func (s *tunConn) _close_nomutex() {
+func (s *tunConn) _close_from_tun() {
 	s.conn.Close()
 	delete(s.tun.addrToConn, s.addr)
 	delete(s.tun.subnetToConn, s.snet)
@@ -118,6 +116,12 @@ func (s *tunConn) _read(bs []byte) (err error) {
 	return
 }
 
+func (s *tunConn) writeFrom(from phony.Actor, bs []byte) {
+	s.RecvFrom(from, func() {
+		s._write(bs)
+	})
+}
+
 func (s *tunConn) _write(bs []byte) (err error) {
 	select {
 	case <-s.stop:
diff --git a/src/tuntap/iface.go b/src/tuntap/iface.go
index 3f923513..16e8d25d 100644
--- a/src/tuntap/iface.go
+++ b/src/tuntap/iface.go
@@ -90,13 +90,11 @@ func (w *tunWriter) _write(b []byte) {
 		util.PutBytes(b)
 	}
 	if err != nil {
-		w.tun.mutex.Lock()
-		open := w.tun.isOpen
-		w.tun.mutex.Unlock()
-		if !open {
-			return
-		}
-		w.tun.log.Errorln("TUN/TAP iface write error:", err)
+		w.tun.RecvFrom(w, func() {
+			if !w.tun.isOpen {
+				w.tun.log.Errorln("TUN/TAP iface write error:", err)
+			}
+		})
 	}
 	if written != n {
 		w.tun.log.Errorln("TUN/TAP iface write mismatch:", written, "bytes written vs", n, "bytes given")
@@ -221,7 +219,6 @@ func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
 	}
 	// Do we have an active connection for this node address?
 	var dstNodeID, dstNodeIDMask *crypto.NodeID
-	tun.mutex.RLock()
 	session, isIn := tun.addrToConn[dstAddr]
 	if !isIn || session == nil {
 		session, isIn = tun.subnetToConn[dstSnet]
@@ -235,7 +232,6 @@ func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
 			}
 		}
 	}
-	tun.mutex.RUnlock()
 	// If we don't have a connection then we should open one
 	if !isIn || session == nil {
 		// Check we haven't been given empty node ID, really this shouldn't ever
@@ -243,45 +239,37 @@ func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
 		if dstNodeID == nil || dstNodeIDMask == nil {
 			panic("Given empty dstNodeID and dstNodeIDMask - this shouldn't happen")
 		}
-		// Dial to the remote node
-		go func() {
-			// FIXME just spitting out a goroutine to do this is kind of ugly and means we drop packets until the dial finishes
-			tun.mutex.Lock()
-			_, known := tun.dials[*dstNodeID]
-			tun.dials[*dstNodeID] = append(tun.dials[*dstNodeID], bs)
-			for len(tun.dials[*dstNodeID]) > 32 {
-				util.PutBytes(tun.dials[*dstNodeID][0])
-				tun.dials[*dstNodeID] = tun.dials[*dstNodeID][1:]
-			}
-			tun.mutex.Unlock()
-			if known {
-				return
-			}
-			var tc *tunConn
-			if conn, err := tun.dialer.DialByNodeIDandMask(dstNodeID, dstNodeIDMask); err == nil {
-				// We've been given a connection so prepare the session wrapper
-				if tc, err = tun.wrap(conn); err != nil {
-					// Something went wrong when storing the connection, typically that
-					// something already exists for this address or subnet
-					tun.log.Debugln("TUN/TAP iface wrap:", err)
+		_, known := tun.dials[*dstNodeID]
+		tun.dials[*dstNodeID] = append(tun.dials[*dstNodeID], bs)
+		for len(tun.dials[*dstNodeID]) > 32 {
+			util.PutBytes(tun.dials[*dstNodeID][0])
+			tun.dials[*dstNodeID] = tun.dials[*dstNodeID][1:]
+		}
+		if !known {
+			go func() {
+				if conn, err := tun.dialer.DialByNodeIDandMask(dstNodeID, dstNodeIDMask); err == nil {
+					tun.RecvFrom(nil, func() {
+						// We've been given a connection so prepare the session wrapper
+						packets := tun.dials[*dstNodeID]
+						delete(tun.dials, *dstNodeID)
+						var tc *tunConn
+						var err error
+						if tc, err = tun._wrap(conn); err != nil {
+							// Something went wrong when storing the connection, typically that
+							// something already exists for this address or subnet
+							tun.log.Debugln("TUN/TAP iface wrap:", err)
+							return
+						}
+						for _, packet := range packets {
+							tc.writeFrom(nil, packet)
+						}
+					})
 				}
-			}
-			tun.mutex.Lock()
-			packets := tun.dials[*dstNodeID]
-			delete(tun.dials, *dstNodeID)
-			tun.mutex.Unlock()
-			if tc != nil {
-				for _, packet := range packets {
-					p := packet // Possibly required because of how range
-					<-tc.SyncExec(func() { tc._write(p) })
-				}
-			}
-		}()
-		// While the dial is going on we can't do much else
-		return
+			}()
+		}
 	}
 	// If we have a connection now, try writing to it
 	if isIn && session != nil {
-		session.RecvFrom(tun, func() { session._write(bs) })
+		session.writeFrom(tun, bs)
 	}
 }
diff --git a/src/tuntap/tun.go b/src/tuntap/tun.go
index 7bb3a59f..2b163e3b 100644
--- a/src/tuntap/tun.go
+++ b/src/tuntap/tun.go
@@ -13,7 +13,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
-	"sync"
+	//"sync"
 
 	"github.com/Arceliar/phony"
 	"github.com/gologme/log"
@@ -34,21 +34,21 @@ const tun_ETHER_HEADER_LENGTH = 14
 // you should pass this object to the yggdrasil.SetRouterAdapter() function
 // before calling yggdrasil.Start().
 type TunAdapter struct {
-	writer       tunWriter
-	reader       tunReader
-	config       *config.NodeState
-	log          *log.Logger
-	reconfigure  chan chan error
-	listener     *yggdrasil.Listener
-	dialer       *yggdrasil.Dialer
-	addr         address.Address
-	subnet       address.Subnet
-	ckr          cryptokey
-	icmpv6       ICMPv6
-	mtu          int
-	iface        *water.Interface
-	phony.Inbox               // Currently only used for _handlePacket from the reader, TODO: all the stuff that currently needs a mutex below
-	mutex        sync.RWMutex // Protects the below
+	writer      tunWriter
+	reader      tunReader
+	config      *config.NodeState
+	log         *log.Logger
+	reconfigure chan chan error
+	listener    *yggdrasil.Listener
+	dialer      *yggdrasil.Dialer
+	addr        address.Address
+	subnet      address.Subnet
+	ckr         cryptokey
+	icmpv6      ICMPv6
+	mtu         int
+	iface       *water.Interface
+	phony.Inbox // Currently only used for _handlePacket from the reader, TODO: all the stuff that currently needs a mutex below
+	//mutex        sync.RWMutex // Protects the below
 	addrToConn   map[address.Address]*tunConn
 	subnetToConn map[address.Subnet]*tunConn
 	dials        map[crypto.NodeID][][]byte // Buffer of packets to send after dialing finishes
@@ -122,8 +122,16 @@ func (tun *TunAdapter) Init(config *config.NodeState, log *log.Logger, listener
 }
 
 // Start the setup process for the TUN/TAP adapter. If successful, starts the
-// read/write goroutines to handle packets on that interface.
+// reader actor to handle packets on that interface.
 func (tun *TunAdapter) Start() error {
+	var err error
+	<-tun.SyncExec(func() {
+		err = tun._start()
+	})
+	return err
+}
+
+func (tun *TunAdapter) _start() error {
 	current := tun.config.GetCurrent()
 	if tun.config == nil || tun.listener == nil || tun.dialer == nil {
 		return errors.New("No configuration available to TUN/TAP")
@@ -150,10 +158,8 @@ func (tun *TunAdapter) Start() error {
 		tun.log.Debugln("Not starting TUN/TAP as ifname is none or dummy")
 		return nil
 	}
-	tun.mutex.Lock()
 	tun.isOpen = true
 	tun.reconfigure = make(chan chan error)
-	tun.mutex.Unlock()
 	go func() {
 		for {
 			e := <-tun.reconfigure
@@ -173,6 +179,14 @@ func (tun *TunAdapter) Start() error {
 // Start the setup process for the TUN/TAP adapter. If successful, starts the
 // read/write goroutines to handle packets on that interface.
 func (tun *TunAdapter) Stop() error {
+	var err error
+	<-tun.SyncExec(func() {
+		err = tun._stop()
+	})
+	return err
+}
+
+func (tun *TunAdapter) _stop() error {
 	tun.isOpen = false
 	// TODO: we have nothing that cleanly stops all the various goroutines opened
 	// by TUN/TAP, e.g. readers/writers, sessions
@@ -219,15 +233,17 @@ func (tun *TunAdapter) handler() error {
 			tun.log.Errorln("TUN/TAP connection accept error:", err)
 			return err
 		}
-		if _, err := tun.wrap(conn); err != nil {
-			// Something went wrong when storing the connection, typically that
-			// something already exists for this address or subnet
-			tun.log.Debugln("TUN/TAP handler wrap:", err)
-		}
+		<-tun.SyncExec(func() {
+			if _, err := tun._wrap(conn); err != nil {
+				// Something went wrong when storing the connection, typically that
+				// something already exists for this address or subnet
+				tun.log.Debugln("TUN/TAP handler wrap:", err)
+			}
+		})
 	}
 }
 
-func (tun *TunAdapter) wrap(conn *yggdrasil.Conn) (c *tunConn, err error) {
+func (tun *TunAdapter) _wrap(conn *yggdrasil.Conn) (c *tunConn, err error) {
 	// Prepare a session wrapper for the given connection
 	s := tunConn{
 		tun:  tun,
@@ -240,17 +256,15 @@ func (tun *TunAdapter) wrap(conn *yggdrasil.Conn) (c *tunConn, err error) {
 	s.addr = *address.AddrForNodeID(&remoteNodeID)
 	s.snet = *address.SubnetForNodeID(&remoteNodeID)
 	// Work out if this is already a destination we already know about
-	tun.mutex.Lock()
-	defer tun.mutex.Unlock()
 	atc, aok := tun.addrToConn[s.addr]
 	stc, sok := tun.subnetToConn[s.snet]
 	// If we know about a connection for this destination already then assume it
 	// is no longer valid and close it
 	if aok {
-		atc._close_nomutex()
+		atc._close_from_tun()
 		err = errors.New("replaced connection for address")
 	} else if sok {
-		stc._close_nomutex()
+		stc._close_from_tun()
 		err = errors.New("replaced connection for subnet")
 	}
 	// Save the session wrapper so that we can look it up quickly next time

From dffd70119da10ef92665a14edc224d9db690a036 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 25 Aug 2019 19:13:47 -0500
Subject: [PATCH 42/62] remove session shutdown goroutine, just send a message
 instead

---
 src/yggdrasil/conn.go    |  2 ++
 src/yggdrasil/session.go | 13 ++++++-------
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index b0c26867..efc0c81e 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -304,6 +304,8 @@ func (c *Conn) Close() (err error) {
 			// Close the session, if it hasn't been closed already
 			if e := c.session.cancel.Cancel(errors.New("connection closed")); e != nil {
 				err = ConnError{errors.New("close failed, session already closed"), false, false, true, 0}
+			} else {
+				c.session.doRemove()
 			}
 		}
 	})
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 94ee41e6..0fc7ec80 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -255,13 +255,6 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 	sinfo.theirSubnet = *address.SubnetForNodeID(crypto.GetNodeID(&sinfo.theirPermPub))
 	ss.sinfos[sinfo.myHandle] = &sinfo
 	ss.byTheirPerm[sinfo.theirPermPub] = &sinfo.myHandle
-	go func() {
-		// Run cleanup when the session is canceled
-		<-sinfo.cancel.Finished()
-		sinfo.sessions.router.doAdmin(func() {
-			sinfo.sessions.removeSession(&sinfo)
-		})
-	}()
 	return &sinfo
 }
 
@@ -293,6 +286,12 @@ func (ss *sessions) cleanup() {
 	ss.lastCleanup = time.Now()
 }
 
+func (sinfo *sessionInfo) doRemove() {
+	sinfo.sessions.router.RecvFrom(nil, func() {
+		sinfo.sessions.removeSession(sinfo)
+	})
+}
+
 // Closes a session, removing it from sessions maps.
 func (ss *sessions) removeSession(sinfo *sessionInfo) {
 	if s := sinfo.sessions.sinfos[sinfo.myHandle]; s == sinfo {

From b5b179904b89a668c03e735a430e9a534f5140b2 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 25 Aug 2019 22:19:20 -0500
Subject: [PATCH 43/62] ugly work-in-progress to migrate link to the actor
 model

---
 src/yggdrasil/link.go | 211 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 210 insertions(+), 1 deletion(-)

diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index 1b48f391..5489c015 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -54,6 +54,15 @@ type linkInterface struct {
 	incoming bool
 	force    bool
 	closed   chan struct{}
+  reader linkReader // Reads packets, notifies this linkInterface, passes packets to switch
+  writer linkWriter // Writes packets, notifies this linkInterface
+  phony.Inbox // Protects the below
+  sendTimer *time.Timer // Fires to signal that sending is blocked
+  stallTimer *time.Timer // Fires to signal that no incoming traffic (including keep-alive) has been seen
+  recvTimer *time.Timer // Fires to send keep-alive traffic
+  closeTimer *time.Timer // Fires when the link has been idle so long we need to close it
+  inSwitch bool // True if the switch is tracking this link
+  stalled bool // True if we haven't been receiving any response traffic
 }
 
 func (l *link) init(c *Core) error {
@@ -124,6 +133,9 @@ func (l *link) create(msgIO linkInterfaceMsgIO, name, linkType, local, remote st
 		incoming: incoming,
 		force:    force,
 	}
+  intf.writer.intf = &intf
+  intf.reader.intf = &intf
+  intf.reader.err = make(chan error)
 	return &intf, nil
 }
 
@@ -187,7 +199,7 @@ func (intf *linkInterface) handler() error {
 			intf.link.mutex.Lock()
 			delete(intf.link.interfaces, intf.info)
 			intf.link.mutex.Unlock()
-			close(intf.closed)
+			//close(intf.closed)
 		}()
 		intf.link.core.log.Debugln("DEBUG: registered interface for", intf.name)
 	}
@@ -203,13 +215,24 @@ func (intf *linkInterface) handler() error {
 		intf.link.core.peers.removePeer(intf.peer.port)
 	}()
 	// Finish setting up the peer struct
+  /*
 	out := make(chan [][]byte, 1)
 	defer close(out)
 	intf.peer.out = func(msgs [][]byte) {
 		defer func() { recover() }()
 		out <- msgs
 	}
+  */
+  intf.peer.out = func(msgs [][]byte) {
+    intf.writer.sendFrom(intf.peer, msgs, false)
+  }
 	intf.peer.linkOut = make(chan []byte, 1)
+  go func() {
+    // TODO fix this
+    for bs := range intf.peer.linkOut {
+      intf.writer.sendFrom(intf.peer, [][]byte{bs}, true)
+    }
+  }()
 	themAddr := address.AddrForNodeID(crypto.GetNodeID(&intf.info.box))
 	themAddrString := net.IP(themAddr[:]).String()
 	themString := fmt.Sprintf("%s@%s", themAddrString, intf.info.remote)
@@ -218,6 +241,7 @@ func (intf *linkInterface) handler() error {
 	// Start the link loop
 	go intf.peer.start()
 	// Start the writer
+  /*
 	signalReady := make(chan struct{}, 1)
 	signalSent := make(chan bool, 1)
 	sendAck := make(chan struct{}, 1)
@@ -413,6 +437,7 @@ func (intf *linkInterface) handler() error {
 		// Now try to read again
 		helper.RecvFrom(nil, helperFunc)
 	}
+  
 	// Start the read loop
 	helper.RecvFrom(nil, helperFunc)
 	<-done // Wait for the helper to exit
@@ -427,5 +452,189 @@ func (intf *linkInterface) handler() error {
 		intf.link.core.log.Infof("Disconnected %s: %s, source %s",
 			strings.ToUpper(intf.info.linkType), themString, intf.info.local)
 	}
+  */
+  // Start the reader
+  intf.reader.RecvFrom(nil, intf.reader._read)
+  // Wait for the reader to finish
+  err = <- intf.reader.err
+  if err != nil {
+		intf.link.core.log.Infof("Disconnected %s: %s, source %s; error: %s",
+			strings.ToUpper(intf.info.linkType), themString, intf.info.local, err)
+  } else {
+		intf.link.core.log.Infof("Disconnected %s: %s, source %s",
+			strings.ToUpper(intf.info.linkType), themString, intf.info.local)
+  }
 	return err
 }
+
+////////////////////////////////////////////////////////////////////////////////
+
+
+/*
+  phony.Inbox // Protects the below
+  sendTimer *time.Timer // Fires to signal that sending is blocked
+  stallTimer *time.Time // Fires to signal that no incoming traffic (including keep-alive) has been seen
+  recvTimer *time.Timer // Fires to send keep-alive traffic
+  closeTimer *time.Timer // Fires when the link has been idle so long we need to close it
+  inSwitch bool // True if the switch is tracking this link
+  stalled bool // True if we haven't been receiving any response traffic
+*/
+
+const (
+  sendBlockedTime = time.Second // How long to wait before deciding a send is blocked
+  keepAliveTime = 2*time.Second // How long to wait before sending a keep-alive response if we have no real traffic to send
+  stallTime = 6*time.Second // How long to wait for response traffic before deciding the connection has stalled
+  closeTime = 2*switch_timeout // How long to wait before closing the link
+)
+
+// notify the intf that we're currently sending
+func (intf *linkInterface) notifySending(size int, isLinkTraffic bool) {
+  intf.RecvFrom(nil, func() {
+    if !isLinkTraffic && size > 0 {
+      intf.inSwitch = false
+    }
+    intf.sendTimer = time.AfterFunc(sendBlockedTime, intf.notifyBlockedSend)
+    intf._cancelRecvTimer()
+  })
+}
+
+// we just sent something, so cancel any pending timer to send keep-alive traffic
+func (intf *linkInterface) _cancelRecvTimer() {
+  intf.RecvFrom(nil, func() {
+    if intf.recvTimer != nil {
+      intf.recvTimer.Stop()
+      intf.recvTimer = nil
+    }
+  })
+}
+
+// called by an AfterFunc if we appear to have timed out
+func (intf *linkInterface) notifyBlockedSend() {
+  intf.RecvFrom(nil, func() {
+    if intf.sendTimer != nil {
+      //As far as we know, we're still trying to send, and the timer fired.
+			intf.link.core.switchTable.blockPeer(intf.peer.port)
+    }
+  })
+}
+
+// notify the intf that we've finished sending, returning the peer to the switch
+func (intf *linkInterface) notifySent(size int, isLinkTraffic bool) {
+  intf.RecvFrom(nil, func() {
+    intf.sendTimer.Stop()
+    intf.sendTimer = nil
+    if !isLinkTraffic {
+      intf._notifySwitch()
+    }
+    if size > 0 && intf.stallTimer == nil {
+      intf.stallTimer = time.AfterFunc(stallTime, intf.notifyStalled)
+    }
+  })
+}
+
+// Notify the switch that we're ready for more traffic, assuming we're not in a stalled state
+func (intf *linkInterface) _notifySwitch() {
+  if !intf.inSwitch && !intf.stalled {
+    intf.inSwitch = true
+	  intf.link.core.switchTable.RecvFrom(intf, func() {
+		  intf.link.core.switchTable._idleIn(intf.peer.port)
+	  })
+  }
+}
+
+// Set the peer as stalled, to prevent them from returning to the switch until a read succeeds
+func (intf *linkInterface) notifyStalled() {
+  intf.RecvFrom(nil, func() {
+    if intf.stallTimer != nil {
+      intf.stallTimer = nil
+      intf.stalled = true
+			intf.link.core.switchTable.blockPeer(intf.peer.port)
+    }
+  })
+}
+
+
+// reset the close timer
+func (intf *linkInterface) notifyReading(from phony.Actor) {
+  intf.RecvFrom(from, func() {
+    if intf.closeTimer != nil {
+      intf.closeTimer.Stop()
+    }
+    intf.closeTimer = time.AfterFunc(closeTime, func() { intf.msgIO.close() })
+  })
+}
+
+// wake up the link if it was stalled, and (if size > 0) prepare to send keep-alive traffic
+func (intf *linkInterface) notifyReadFrom(from phony.Actor, size int) {
+  intf.RecvFrom(from, func() {
+    intf.link.core.log.Printf("DEBUG notifyReadFrom: inSwitch %v, stalled %v\n", intf.inSwitch, intf.stalled)
+    if intf.stallTimer != nil {
+      intf.stallTimer.Stop()
+      intf.stallTimer = nil
+    }
+    intf.stalled = false
+    intf._notifySwitch()
+    if size > 0 && intf.recvTimer == nil {
+      intf.recvTimer = time.AfterFunc(keepAliveTime, intf.notifyDoKeepAlive)
+    }
+  })
+}
+
+// We need to send keep-alive traffic now
+func (intf *linkInterface) notifyDoKeepAlive() {
+  intf.RecvFrom(nil, func() {
+    if intf.recvTimer != nil {
+      intf.recvTimer.Stop()
+      intf.recvTimer = nil
+      intf.writer.sendFrom(nil, [][]byte{nil}, true) // Empty keep-alive traffic
+    }
+  })
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+type linkWriter struct {
+  phony.Inbox
+  intf *linkInterface
+}
+
+func (w *linkWriter) sendFrom(from phony.Actor, bss [][]byte, isLinkTraffic bool) {
+  w.RecvFrom(from, func() {
+    var size int
+    for _, bs := range bss {
+      size += len(bs)
+    }
+    w.intf.notifySending(size, isLinkTraffic)
+	  w.intf.msgIO.writeMsgs(bss)
+    w.intf.notifySent(size, isLinkTraffic)
+    w.intf.link.core.log.Println("DEBUG: wrote something, size:", size, "isLinkTraffic:", isLinkTraffic)
+  })
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+type linkReader struct {
+  phony.Inbox
+  intf *linkInterface
+  err chan error
+}
+
+func (r *linkReader) _read() {
+    r.intf.notifyReading(r)
+		msg, err := r.intf.msgIO.readMsg()
+    r.intf.link.core.log.Println("DEBUG read something")
+    r.intf.notifyReadFrom(r, len(msg))
+		if len(msg) > 0 {
+			r.intf.peer.handlePacketFrom(r, msg)
+		}
+		if err != nil {
+			if err != io.EOF {
+        r.err<-err
+			}
+      close(r.err)
+			return
+		}
+		// Now try to read again
+		r.RecvFrom(nil, r._read)
+}
+

From bd3eaefb72675473ca46a0c0b943ffc9ac2ffdee Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 25 Aug 2019 22:55:17 -0500
Subject: [PATCH 44/62] more link migration

---
 src/yggdrasil/debug.go  |   7 +-
 src/yggdrasil/link.go   | 497 ++++++++++------------------------------
 src/yggdrasil/peer.go   |  12 +-
 src/yggdrasil/switch.go |   4 +-
 4 files changed, 135 insertions(+), 385 deletions(-)

diff --git a/src/yggdrasil/debug.go b/src/yggdrasil/debug.go
index c6f98ca0..1cc7eae4 100644
--- a/src/yggdrasil/debug.go
+++ b/src/yggdrasil/debug.go
@@ -554,7 +554,8 @@ func (c *Core) DEBUG_addAllowedEncryptionPublicKey(boxStr string) {
 func DEBUG_simLinkPeers(p, q *peer) {
 	// Sets q.out() to point to p and starts p.start()
 	goWorkers := func(source, dest *peer) {
-		source.linkOut = make(chan []byte, 1)
+		linkOut := make(chan []byte, 1)
+		source.linkOut = func(bs []byte) { linkOut <- bs }
 		send := make(chan []byte, 1)
 		source.out = func(bss [][]byte) {
 			for _, bs := range bss {
@@ -566,7 +567,7 @@ func DEBUG_simLinkPeers(p, q *peer) {
 			var packets [][]byte
 			for {
 				select {
-				case packet := <-source.linkOut:
+				case packet := <-linkOut:
 					packets = append(packets, packet)
 					continue
 				case packet := <-send:
@@ -583,7 +584,7 @@ func DEBUG_simLinkPeers(p, q *peer) {
 					continue
 				}
 				select {
-				case packet := <-source.linkOut:
+				case packet := <-linkOut:
 					packets = append(packets, packet)
 				case packet := <-send:
 					packets = append(packets, packet)
diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index 5489c015..b4ca38c4 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -46,23 +46,23 @@ type linkInterfaceMsgIO interface {
 }
 
 type linkInterface struct {
-	name     string
-	link     *link
-	peer     *peer
-	msgIO    linkInterfaceMsgIO
-	info     linkInfo
-	incoming bool
-	force    bool
-	closed   chan struct{}
-  reader linkReader // Reads packets, notifies this linkInterface, passes packets to switch
-  writer linkWriter // Writes packets, notifies this linkInterface
-  phony.Inbox // Protects the below
-  sendTimer *time.Timer // Fires to signal that sending is blocked
-  stallTimer *time.Timer // Fires to signal that no incoming traffic (including keep-alive) has been seen
-  recvTimer *time.Timer // Fires to send keep-alive traffic
-  closeTimer *time.Timer // Fires when the link has been idle so long we need to close it
-  inSwitch bool // True if the switch is tracking this link
-  stalled bool // True if we haven't been receiving any response traffic
+	name        string
+	link        *link
+	peer        *peer
+	msgIO       linkInterfaceMsgIO
+	info        linkInfo
+	incoming    bool
+	force       bool
+	closed      chan struct{}
+	reader      linkReader  // Reads packets, notifies this linkInterface, passes packets to switch
+	writer      linkWriter  // Writes packets, notifies this linkInterface
+	phony.Inbox             // Protects the below
+	sendTimer   *time.Timer // Fires to signal that sending is blocked
+	stallTimer  *time.Timer // Fires to signal that no incoming traffic (including keep-alive) has been seen
+	recvTimer   *time.Timer // Fires to send keep-alive traffic
+	closeTimer  *time.Timer // Fires when the link has been idle so long we need to close it
+	inSwitch    bool        // True if the switch is tracking this link
+	stalled     bool        // True if we haven't been receiving any response traffic
 }
 
 func (l *link) init(c *Core) error {
@@ -133,9 +133,9 @@ func (l *link) create(msgIO linkInterfaceMsgIO, name, linkType, local, remote st
 		incoming: incoming,
 		force:    force,
 	}
-  intf.writer.intf = &intf
-  intf.reader.intf = &intf
-  intf.reader.err = make(chan error)
+	intf.writer.intf = &intf
+	intf.reader.intf = &intf
+	intf.reader.err = make(chan error)
 	return &intf, nil
 }
 
@@ -199,7 +199,7 @@ func (intf *linkInterface) handler() error {
 			intf.link.mutex.Lock()
 			delete(intf.link.interfaces, intf.info)
 			intf.link.mutex.Unlock()
-			//close(intf.closed)
+			close(intf.closed)
 		}()
 		intf.link.core.log.Debugln("DEBUG: registered interface for", intf.name)
 	}
@@ -214,427 +214,184 @@ func (intf *linkInterface) handler() error {
 		// More cleanup can go here
 		intf.link.core.peers.removePeer(intf.peer.port)
 	}()
-	// Finish setting up the peer struct
-  /*
-	out := make(chan [][]byte, 1)
-	defer close(out)
 	intf.peer.out = func(msgs [][]byte) {
-		defer func() { recover() }()
-		out <- msgs
+		intf.writer.sendFrom(intf.peer, msgs, false)
+	}
+	intf.peer.linkOut = func(bs []byte) {
+		intf.writer.sendFrom(intf.peer, [][]byte{bs}, true)
 	}
-  */
-  intf.peer.out = func(msgs [][]byte) {
-    intf.writer.sendFrom(intf.peer, msgs, false)
-  }
-	intf.peer.linkOut = make(chan []byte, 1)
-  go func() {
-    // TODO fix this
-    for bs := range intf.peer.linkOut {
-      intf.writer.sendFrom(intf.peer, [][]byte{bs}, true)
-    }
-  }()
 	themAddr := address.AddrForNodeID(crypto.GetNodeID(&intf.info.box))
 	themAddrString := net.IP(themAddr[:]).String()
 	themString := fmt.Sprintf("%s@%s", themAddrString, intf.info.remote)
 	intf.link.core.log.Infof("Connected %s: %s, source %s",
 		strings.ToUpper(intf.info.linkType), themString, intf.info.local)
-	// Start the link loop
+	// Start things
 	go intf.peer.start()
-	// Start the writer
-  /*
-	signalReady := make(chan struct{}, 1)
-	signalSent := make(chan bool, 1)
-	sendAck := make(chan struct{}, 1)
-	sendBlocked := time.NewTimer(time.Second)
-	defer util.TimerStop(sendBlocked)
-	util.TimerStop(sendBlocked)
-	go func() {
-		defer close(signalReady)
-		defer close(signalSent)
-		interval := 4 * time.Second
-		tcpTimer := time.NewTimer(interval) // used for backwards compat with old tcp
-		defer util.TimerStop(tcpTimer)
-		send := func(bss [][]byte) {
-			sendBlocked.Reset(time.Second)
-			size, _ := intf.msgIO.writeMsgs(bss)
-			util.TimerStop(sendBlocked)
-			select {
-			case signalSent <- size > 0:
-			default:
-			}
-		}
-		for {
-			// First try to send any link protocol traffic
-			select {
-			case msg := <-intf.peer.linkOut:
-				send([][]byte{msg})
-				continue
-			default:
-			}
-			// No protocol traffic to send, so reset the timer
-			util.TimerStop(tcpTimer)
-			tcpTimer.Reset(interval)
-			// Now block until something is ready or the timer triggers keepalive traffic
-			select {
-			case <-tcpTimer.C:
-				intf.link.core.log.Tracef("Sending (legacy) keep-alive to %s: %s, source %s",
-					strings.ToUpper(intf.info.linkType), themString, intf.info.local)
-				send([][]byte{nil})
-			case <-sendAck:
-				intf.link.core.log.Tracef("Sending ack to %s: %s, source %s",
-					strings.ToUpper(intf.info.linkType), themString, intf.info.local)
-				send([][]byte{nil})
-			case msg := <-intf.peer.linkOut:
-				send([][]byte{msg})
-			case msgs, ok := <-out:
-				if !ok {
-					return
-				}
-				send(msgs)
-				for _, msg := range msgs {
-					util.PutBytes(msg)
-				}
-				select {
-				case signalReady <- struct{}{}:
-				default:
-				}
-				//intf.link.core.log.Tracef("Sending packet to %s: %s, source %s",
-				//	strings.ToUpper(intf.info.linkType), themString, intf.info.local)
-			}
-		}
-	}()
-	//intf.link.core.switchTable.idleIn <- intf.peer.port // notify switch that we're idle
-	// Used to enable/disable activity in the switch
-	signalAlive := make(chan bool, 1) // True = real packet, false = keep-alive
-	defer close(signalAlive)
-	ret := make(chan error, 1) // How we signal the return value when multiple goroutines are involved
-	go func() {
-		var isAlive bool
-		var isReady bool
-		var sendTimerRunning bool
-		var recvTimerRunning bool
-		recvTime := 6 * time.Second     // TODO set to ReadTimeout from the config, reset if it gets changed
-		closeTime := 2 * switch_timeout // TODO or maybe this makes more sense for ReadTimeout?...
-		sendTime := time.Second
-		sendTimer := time.NewTimer(sendTime)
-		defer util.TimerStop(sendTimer)
-		recvTimer := time.NewTimer(recvTime)
-		defer util.TimerStop(recvTimer)
-		closeTimer := time.NewTimer(closeTime)
-		defer util.TimerStop(closeTimer)
-		for {
-			//intf.link.core.log.Debugf("State of %s: %s, source %s :: isAlive %t isReady %t sendTimerRunning %t recvTimerRunning %t",
-			//	strings.ToUpper(intf.info.linkType), themString, intf.info.local,
-			//	isAlive, isReady, sendTimerRunning, recvTimerRunning)
-			select {
-			case gotMsg, ok := <-signalAlive:
-				if !ok {
-					return
-				}
-				util.TimerStop(closeTimer)
-				closeTimer.Reset(closeTime)
-				util.TimerStop(recvTimer)
-				recvTimerRunning = false
-				isAlive = true
-				if !isReady {
-					// (Re-)enable in the switch
-					intf.link.core.switchTable.RecvFrom(nil, func() {
-						intf.link.core.switchTable._idleIn(intf.peer.port)
-					})
-					isReady = true
-				}
-				if gotMsg && !sendTimerRunning {
-					// We got a message
-					// Start a timer, if it expires then send a 0-sized ack to let them know we're alive
-					util.TimerStop(sendTimer)
-					sendTimer.Reset(sendTime)
-					sendTimerRunning = true
-				}
-				if !gotMsg {
-					intf.link.core.log.Tracef("Received ack from %s: %s, source %s",
-						strings.ToUpper(intf.info.linkType), themString, intf.info.local)
-				}
-			case sentMsg, ok := <-signalSent:
-				// Stop any running ack timer
-				if !ok {
-					return
-				}
-				util.TimerStop(sendTimer)
-				sendTimerRunning = false
-				if sentMsg && !recvTimerRunning {
-					// We sent a message
-					// Start a timer, if it expires and we haven't gotten any return traffic (including a 0-sized ack), then assume there's a problem
-					util.TimerStop(recvTimer)
-					recvTimer.Reset(recvTime)
-					recvTimerRunning = true
-				}
-			case _, ok := <-signalReady:
-				if !ok {
-					return
-				}
-				if !isAlive {
-					// Disable in the switch
-					isReady = false
-				} else {
-					// Keep enabled in the switch
-					intf.link.core.switchTable.RecvFrom(nil, func() {
-						intf.link.core.switchTable._idleIn(intf.peer.port)
-					})
-					isReady = true
-				}
-			case <-sendBlocked.C:
-				// We blocked while trying to send something
-				isReady = false
-				intf.link.core.switchTable.blockPeer(intf.peer.port)
-			case <-sendTimer.C:
-				// We haven't sent anything, so signal a send of a 0 packet to let them know we're alive
-				select {
-				case sendAck <- struct{}{}:
-				default:
-				}
-			case <-recvTimer.C:
-				// We haven't received anything, so assume there's a problem and don't return this node to the switch until they start responding
-				isAlive = false
-				intf.link.core.switchTable.blockPeer(intf.peer.port)
-			case <-closeTimer.C:
-				// We haven't received anything in a really long time, so things have died at the switch level and then some...
-				// Just close the connection at this point...
-				select {
-				case ret <- errors.New("timeout"):
-				default:
-				}
-				intf.msgIO.close()
-			}
-		}
-	}()
-	// Run reader loop
-	var helper phony.Inbox
-	done := make(chan struct{})
-	var helperFunc func()
-	helperFunc = func() {
-		// The helper reads in a loop and sends to the peer
-		// It loops by sending itself a message, which can be delayed by backpressure
-		// So if the peer is busy, backpressure will pause reading until the peer catches up
-		msg, err := intf.msgIO.readMsg()
-		if len(msg) > 0 {
-			// TODO rewrite this if the link becomes an actor
-			intf.peer.handlePacketFrom(&helper, msg)
-		}
-		if err != nil {
-			if err != io.EOF {
-				select {
-				case ret <- err:
-				default:
-				}
-			}
-			close(done)
-			return
-		}
-		select {
-		case signalAlive <- len(msg) > 0:
-		default:
-		}
-		// Now try to read again
-		helper.RecvFrom(nil, helperFunc)
-	}
-  
-	// Start the read loop
-	helper.RecvFrom(nil, helperFunc)
-	<-done // Wait for the helper to exit
-	////////////////////////////////////////////////////////////////////////////////
-	// Remember to set `err` to something useful before returning
-	select {
-	case err = <-ret:
+	intf.reader.RecvFrom(nil, intf.reader._read)
+	// Wait for the reader to finish
+	err = <-intf.reader.err
+	if err != nil {
 		intf.link.core.log.Infof("Disconnected %s: %s, source %s; error: %s",
 			strings.ToUpper(intf.info.linkType), themString, intf.info.local, err)
-	default:
-		err = nil
+	} else {
 		intf.link.core.log.Infof("Disconnected %s: %s, source %s",
 			strings.ToUpper(intf.info.linkType), themString, intf.info.local)
 	}
-  */
-  // Start the reader
-  intf.reader.RecvFrom(nil, intf.reader._read)
-  // Wait for the reader to finish
-  err = <- intf.reader.err
-  if err != nil {
-		intf.link.core.log.Infof("Disconnected %s: %s, source %s; error: %s",
-			strings.ToUpper(intf.info.linkType), themString, intf.info.local, err)
-  } else {
-		intf.link.core.log.Infof("Disconnected %s: %s, source %s",
-			strings.ToUpper(intf.info.linkType), themString, intf.info.local)
-  }
 	return err
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 
-
-/*
-  phony.Inbox // Protects the below
-  sendTimer *time.Timer // Fires to signal that sending is blocked
-  stallTimer *time.Time // Fires to signal that no incoming traffic (including keep-alive) has been seen
-  recvTimer *time.Timer // Fires to send keep-alive traffic
-  closeTimer *time.Timer // Fires when the link has been idle so long we need to close it
-  inSwitch bool // True if the switch is tracking this link
-  stalled bool // True if we haven't been receiving any response traffic
-*/
-
 const (
-  sendBlockedTime = time.Second // How long to wait before deciding a send is blocked
-  keepAliveTime = 2*time.Second // How long to wait before sending a keep-alive response if we have no real traffic to send
-  stallTime = 6*time.Second // How long to wait for response traffic before deciding the connection has stalled
-  closeTime = 2*switch_timeout // How long to wait before closing the link
+	sendBlockedTime = time.Second        // How long to wait before deciding a send is blocked
+	keepAliveTime   = 2 * time.Second    // How long to wait before sending a keep-alive response if we have no real traffic to send
+	stallTime       = 6 * time.Second    // How long to wait for response traffic before deciding the connection has stalled
+	closeTime       = 2 * switch_timeout // How long to wait before closing the link
 )
 
 // notify the intf that we're currently sending
 func (intf *linkInterface) notifySending(size int, isLinkTraffic bool) {
-  intf.RecvFrom(nil, func() {
-    if !isLinkTraffic && size > 0 {
-      intf.inSwitch = false
-    }
-    intf.sendTimer = time.AfterFunc(sendBlockedTime, intf.notifyBlockedSend)
-    intf._cancelRecvTimer()
-  })
+	intf.RecvFrom(nil, func() {
+		if !isLinkTraffic && size > 0 {
+			intf.inSwitch = false
+		}
+		intf.sendTimer = time.AfterFunc(sendBlockedTime, intf.notifyBlockedSend)
+		intf._cancelRecvTimer()
+	})
 }
 
 // we just sent something, so cancel any pending timer to send keep-alive traffic
 func (intf *linkInterface) _cancelRecvTimer() {
-  intf.RecvFrom(nil, func() {
-    if intf.recvTimer != nil {
-      intf.recvTimer.Stop()
-      intf.recvTimer = nil
-    }
-  })
+	intf.RecvFrom(nil, func() {
+		if intf.recvTimer != nil {
+			intf.recvTimer.Stop()
+			intf.recvTimer = nil
+		}
+	})
 }
 
 // called by an AfterFunc if we appear to have timed out
 func (intf *linkInterface) notifyBlockedSend() {
-  intf.RecvFrom(nil, func() {
-    if intf.sendTimer != nil {
-      //As far as we know, we're still trying to send, and the timer fired.
+	intf.RecvFrom(nil, func() {
+		if intf.sendTimer != nil {
+			//As far as we know, we're still trying to send, and the timer fired.
 			intf.link.core.switchTable.blockPeer(intf.peer.port)
-    }
-  })
+		}
+	})
 }
 
 // notify the intf that we've finished sending, returning the peer to the switch
 func (intf *linkInterface) notifySent(size int, isLinkTraffic bool) {
-  intf.RecvFrom(nil, func() {
-    intf.sendTimer.Stop()
-    intf.sendTimer = nil
-    if !isLinkTraffic {
-      intf._notifySwitch()
-    }
-    if size > 0 && intf.stallTimer == nil {
-      intf.stallTimer = time.AfterFunc(stallTime, intf.notifyStalled)
-    }
-  })
+	intf.RecvFrom(nil, func() {
+		intf.sendTimer.Stop()
+		intf.sendTimer = nil
+		if !isLinkTraffic {
+			intf._notifySwitch()
+		}
+		if size > 0 && intf.stallTimer == nil {
+			intf.stallTimer = time.AfterFunc(stallTime, intf.notifyStalled)
+		}
+	})
 }
 
 // Notify the switch that we're ready for more traffic, assuming we're not in a stalled state
 func (intf *linkInterface) _notifySwitch() {
-  if !intf.inSwitch && !intf.stalled {
-    intf.inSwitch = true
-	  intf.link.core.switchTable.RecvFrom(intf, func() {
-		  intf.link.core.switchTable._idleIn(intf.peer.port)
-	  })
-  }
+	if !intf.inSwitch && !intf.stalled {
+		intf.inSwitch = true
+		intf.link.core.switchTable.RecvFrom(intf, func() {
+			intf.link.core.switchTable._idleIn(intf.peer.port)
+		})
+	}
 }
 
 // Set the peer as stalled, to prevent them from returning to the switch until a read succeeds
 func (intf *linkInterface) notifyStalled() {
-  intf.RecvFrom(nil, func() {
-    if intf.stallTimer != nil {
-      intf.stallTimer = nil
-      intf.stalled = true
+	intf.RecvFrom(nil, func() {
+		if intf.stallTimer != nil {
+			intf.stallTimer = nil
+			intf.stalled = true
 			intf.link.core.switchTable.blockPeer(intf.peer.port)
-    }
-  })
+		}
+	})
 }
 
-
 // reset the close timer
 func (intf *linkInterface) notifyReading(from phony.Actor) {
-  intf.RecvFrom(from, func() {
-    if intf.closeTimer != nil {
-      intf.closeTimer.Stop()
-    }
-    intf.closeTimer = time.AfterFunc(closeTime, func() { intf.msgIO.close() })
-  })
+	intf.RecvFrom(from, func() {
+		if intf.closeTimer != nil {
+			intf.closeTimer.Stop()
+		}
+		intf.closeTimer = time.AfterFunc(closeTime, func() { intf.msgIO.close() })
+	})
 }
 
 // wake up the link if it was stalled, and (if size > 0) prepare to send keep-alive traffic
 func (intf *linkInterface) notifyReadFrom(from phony.Actor, size int) {
-  intf.RecvFrom(from, func() {
-    intf.link.core.log.Printf("DEBUG notifyReadFrom: inSwitch %v, stalled %v\n", intf.inSwitch, intf.stalled)
-    if intf.stallTimer != nil {
-      intf.stallTimer.Stop()
-      intf.stallTimer = nil
-    }
-    intf.stalled = false
-    intf._notifySwitch()
-    if size > 0 && intf.recvTimer == nil {
-      intf.recvTimer = time.AfterFunc(keepAliveTime, intf.notifyDoKeepAlive)
-    }
-  })
+	intf.RecvFrom(from, func() {
+		if intf.stallTimer != nil {
+			intf.stallTimer.Stop()
+			intf.stallTimer = nil
+		}
+		intf.stalled = false
+		intf._notifySwitch()
+		if size > 0 && intf.recvTimer == nil {
+			intf.recvTimer = time.AfterFunc(keepAliveTime, intf.notifyDoKeepAlive)
+		}
+	})
 }
 
 // We need to send keep-alive traffic now
 func (intf *linkInterface) notifyDoKeepAlive() {
-  intf.RecvFrom(nil, func() {
-    if intf.recvTimer != nil {
-      intf.recvTimer.Stop()
-      intf.recvTimer = nil
-      intf.writer.sendFrom(nil, [][]byte{nil}, true) // Empty keep-alive traffic
-    }
-  })
+	intf.RecvFrom(nil, func() {
+		if intf.recvTimer != nil {
+			intf.recvTimer.Stop()
+			intf.recvTimer = nil
+			intf.writer.sendFrom(nil, [][]byte{nil}, true) // Empty keep-alive traffic
+		}
+	})
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 
 type linkWriter struct {
-  phony.Inbox
-  intf *linkInterface
+	phony.Inbox
+	intf *linkInterface
 }
 
 func (w *linkWriter) sendFrom(from phony.Actor, bss [][]byte, isLinkTraffic bool) {
-  w.RecvFrom(from, func() {
-    var size int
-    for _, bs := range bss {
-      size += len(bs)
-    }
-    w.intf.notifySending(size, isLinkTraffic)
-	  w.intf.msgIO.writeMsgs(bss)
-    w.intf.notifySent(size, isLinkTraffic)
-    w.intf.link.core.log.Println("DEBUG: wrote something, size:", size, "isLinkTraffic:", isLinkTraffic)
-  })
+	w.RecvFrom(from, func() {
+		var size int
+		for _, bs := range bss {
+			size += len(bs)
+		}
+		w.intf.notifySending(size, isLinkTraffic)
+		w.intf.msgIO.writeMsgs(bss)
+		w.intf.notifySent(size, isLinkTraffic)
+	})
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 
 type linkReader struct {
-  phony.Inbox
-  intf *linkInterface
-  err chan error
+	phony.Inbox
+	intf *linkInterface
+	err  chan error
 }
 
 func (r *linkReader) _read() {
-    r.intf.notifyReading(r)
-		msg, err := r.intf.msgIO.readMsg()
-    r.intf.link.core.log.Println("DEBUG read something")
-    r.intf.notifyReadFrom(r, len(msg))
-		if len(msg) > 0 {
-			r.intf.peer.handlePacketFrom(r, msg)
+	r.intf.notifyReading(r)
+	msg, err := r.intf.msgIO.readMsg()
+	r.intf.notifyReadFrom(r, len(msg))
+	if len(msg) > 0 {
+		r.intf.peer.handlePacketFrom(r, msg)
+	}
+	if err != nil {
+		if err != io.EOF {
+			r.err <- err
 		}
-		if err != nil {
-			if err != io.EOF {
-        r.err<-err
-			}
-      close(r.err)
-			return
-		}
-		// Now try to read again
-		r.RecvFrom(nil, r._read)
+		close(r.err)
+		return
+	}
+	// Now try to read again
+	r.RecvFrom(nil, r._read)
 }
-
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 50fb03f8..4cf0068b 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -103,7 +103,7 @@ type peer struct {
 	linkShared crypto.BoxSharedKey
 	endpoint   string
 	firstSeen  time.Time       // To track uptime for getPeers
-	linkOut    (chan []byte)   // used for protocol traffic (to bypass queues)
+	linkOut    func([]byte)    // used for protocol traffic (bypasses the switch)
 	dinfo      *dhtInfo        // used to keep the DHT working
 	out        func([][]byte)  // Set up by whatever created the peers struct, used to send packets to other nodes
 	done       (chan struct{}) // closed to exit the linkLoop
@@ -263,8 +263,6 @@ func (p *peer) _sendPackets(packets [][]byte) {
 	p.out(packets)
 }
 
-var peerLinkOutHelper phony.Inbox
-
 // This wraps the packet in the inner (ephemeral) and outer (permanent) crypto layers.
 // It sends it to p.linkOut, which bypasses the usual packet queues.
 func (p *peer) _sendLinkPacket(packet []byte) {
@@ -280,13 +278,7 @@ func (p *peer) _sendLinkPacket(packet []byte) {
 		Payload: bs,
 	}
 	packet = linkPacket.encode()
-	// TODO replace this with a message send if/when the link becomes an actor
-	peerLinkOutHelper.RecvFrom(nil, func() {
-		select {
-		case p.linkOut <- packet:
-		case <-p.done:
-		}
-	})
+	p.linkOut(packet)
 }
 
 // Decrypts the outer (permanent) and inner (ephemeral) crypto layers on link traffic.
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 6d882252..b30e59d5 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -250,7 +250,7 @@ func (t *switchTable) cleanRoot() {
 			t.core.router.reset(nil)
 		}
 		t.data.locator = switchLocator{root: t.key, tstamp: now.Unix()}
-		t.core.peers.sendSwitchMsgs(nil) // TODO update if/when the switch becomes an actor
+		t.core.peers.sendSwitchMsgs(t)
 	}
 }
 
@@ -517,7 +517,7 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort, rep
 		}
 		t.data.locator = sender.locator
 		t.parent = sender.port
-		t.core.peers.sendSwitchMsgs(nil) // TODO update if/when the switch becomes an actor
+		t.core.peers.sendSwitchMsgs(t)
 	}
 	if doUpdate {
 		t.updater.Store(&sync.Once{})

From e5b88c0da3998cd7582d07b2385054f6005f402f Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 25 Aug 2019 23:07:56 -0500
Subject: [PATCH 45/62] update switch

---
 src/yggdrasil/switch.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 6d882252..b30e59d5 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -250,7 +250,7 @@ func (t *switchTable) cleanRoot() {
 			t.core.router.reset(nil)
 		}
 		t.data.locator = switchLocator{root: t.key, tstamp: now.Unix()}
-		t.core.peers.sendSwitchMsgs(nil) // TODO update if/when the switch becomes an actor
+		t.core.peers.sendSwitchMsgs(t)
 	}
 }
 
@@ -517,7 +517,7 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort, rep
 		}
 		t.data.locator = sender.locator
 		t.parent = sender.port
-		t.core.peers.sendSwitchMsgs(nil) // TODO update if/when the switch becomes an actor
+		t.core.peers.sendSwitchMsgs(t)
 	}
 	if doUpdate {
 		t.updater.Store(&sync.Once{})

From ab591295578e66aae15452b843194562c56ab490 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sun, 25 Aug 2019 23:24:18 -0500
Subject: [PATCH 46/62] have the writer clean things up. note that their still
 seem to be bugs in the main linkInterface actor's state machine--links
 sometimes just die, presumably because they're dropped from the switch and
 never replaced

---
 src/yggdrasil/link.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index b4ca38c4..548f5341 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -367,6 +367,10 @@ func (w *linkWriter) sendFrom(from phony.Actor, bss [][]byte, isLinkTraffic bool
 		w.intf.notifySending(size, isLinkTraffic)
 		w.intf.msgIO.writeMsgs(bss)
 		w.intf.notifySent(size, isLinkTraffic)
+		// Cleanup
+		for _, bs := range bss {
+			util.PutBytes(bs)
+		}
 	})
 }
 

From c97dd4ad28c167b09e0b3ea9d26d41128085273e Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Mon, 26 Aug 2019 00:38:14 -0500
Subject: [PATCH 47/62] fix dial bug

---
 src/tuntap/iface.go   | 38 ++++++++++++++++++++------------------
 src/yggdrasil/link.go |  2 +-
 2 files changed, 21 insertions(+), 19 deletions(-)

diff --git a/src/tuntap/iface.go b/src/tuntap/iface.go
index 16e8d25d..da419369 100644
--- a/src/tuntap/iface.go
+++ b/src/tuntap/iface.go
@@ -247,24 +247,26 @@ func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
 		}
 		if !known {
 			go func() {
-				if conn, err := tun.dialer.DialByNodeIDandMask(dstNodeID, dstNodeIDMask); err == nil {
-					tun.RecvFrom(nil, func() {
-						// We've been given a connection so prepare the session wrapper
-						packets := tun.dials[*dstNodeID]
-						delete(tun.dials, *dstNodeID)
-						var tc *tunConn
-						var err error
-						if tc, err = tun._wrap(conn); err != nil {
-							// Something went wrong when storing the connection, typically that
-							// something already exists for this address or subnet
-							tun.log.Debugln("TUN/TAP iface wrap:", err)
-							return
-						}
-						for _, packet := range packets {
-							tc.writeFrom(nil, packet)
-						}
-					})
-				}
+				conn, err := tun.dialer.DialByNodeIDandMask(dstNodeID, dstNodeIDMask)
+				tun.RecvFrom(nil, func() {
+					packets := tun.dials[*dstNodeID]
+					delete(tun.dials, *dstNodeID)
+					if err != nil {
+						return
+					}
+					// We've been given a connection so prepare the session wrapper
+					var tc *tunConn
+					if tc, err = tun._wrap(conn); err != nil {
+						// Something went wrong when storing the connection, typically that
+						// something already exists for this address or subnet
+						tun.log.Debugln("TUN/TAP iface wrap:", err)
+						return
+					}
+					for _, packet := range packets {
+						tc.writeFrom(nil, packet)
+					}
+				})
+				return
 			}()
 		}
 	}
diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index 548f5341..8fbd0317 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -252,7 +252,7 @@ const (
 // notify the intf that we're currently sending
 func (intf *linkInterface) notifySending(size int, isLinkTraffic bool) {
 	intf.RecvFrom(nil, func() {
-		if !isLinkTraffic && size > 0 {
+		if !isLinkTraffic {
 			intf.inSwitch = false
 		}
 		intf.sendTimer = time.AfterFunc(sendBlockedTime, intf.notifyBlockedSend)

From 4d9c6342a7383a4481851be11c9357af770b6f18 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Mon, 26 Aug 2019 18:37:38 -0500
Subject: [PATCH 48/62] more link updates

---
 src/yggdrasil/link.go | 91 +++++++++++++++++++++----------------------
 1 file changed, 45 insertions(+), 46 deletions(-)

diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index 8fbd0317..6b7f446f 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -46,23 +46,23 @@ type linkInterfaceMsgIO interface {
 }
 
 type linkInterface struct {
-	name        string
-	link        *link
-	peer        *peer
-	msgIO       linkInterfaceMsgIO
-	info        linkInfo
-	incoming    bool
-	force       bool
-	closed      chan struct{}
-	reader      linkReader  // Reads packets, notifies this linkInterface, passes packets to switch
-	writer      linkWriter  // Writes packets, notifies this linkInterface
-	phony.Inbox             // Protects the below
-	sendTimer   *time.Timer // Fires to signal that sending is blocked
-	stallTimer  *time.Timer // Fires to signal that no incoming traffic (including keep-alive) has been seen
-	recvTimer   *time.Timer // Fires to send keep-alive traffic
-	closeTimer  *time.Timer // Fires when the link has been idle so long we need to close it
-	inSwitch    bool        // True if the switch is tracking this link
-	stalled     bool        // True if we haven't been receiving any response traffic
+	name           string
+	link           *link
+	peer           *peer
+	msgIO          linkInterfaceMsgIO
+	info           linkInfo
+	incoming       bool
+	force          bool
+	closed         chan struct{}
+	reader         linkReader  // Reads packets, notifies this linkInterface, passes packets to switch
+	writer         linkWriter  // Writes packets, notifies this linkInterface
+	phony.Inbox                // Protects the below
+	sendTimer      *time.Timer // Fires to signal that sending is blocked
+	keepAliveTimer *time.Timer // Fires to send keep-alive traffic
+	stallTimer     *time.Timer // Fires to signal that no incoming traffic (including keep-alive) has been seen
+	closeTimer     *time.Timer // Fires when the link has been idle so long we need to close it
+	inSwitch       bool        // True if the switch is tracking this link
+	stalled        bool        // True if we haven't been receiving any response traffic
 }
 
 func (l *link) init(c *Core) error {
@@ -243,36 +243,34 @@ func (intf *linkInterface) handler() error {
 ////////////////////////////////////////////////////////////////////////////////
 
 const (
-	sendBlockedTime = time.Second        // How long to wait before deciding a send is blocked
-	keepAliveTime   = 2 * time.Second    // How long to wait before sending a keep-alive response if we have no real traffic to send
-	stallTime       = 6 * time.Second    // How long to wait for response traffic before deciding the connection has stalled
-	closeTime       = 2 * switch_timeout // How long to wait before closing the link
+	sendTime      = 1 * time.Second    // How long to wait before deciding a send is blocked
+	keepAliveTime = 2 * time.Second    // How long to wait before sending a keep-alive response if we have no real traffic to send
+	stallTime     = 6 * time.Second    // How long to wait for response traffic before deciding the connection has stalled
+	closeTime     = 2 * switch_timeout // How long to wait before closing the link
 )
 
 // notify the intf that we're currently sending
 func (intf *linkInterface) notifySending(size int, isLinkTraffic bool) {
-	intf.RecvFrom(nil, func() {
+	intf.RecvFrom(&intf.writer, func() {
 		if !isLinkTraffic {
 			intf.inSwitch = false
 		}
-		intf.sendTimer = time.AfterFunc(sendBlockedTime, intf.notifyBlockedSend)
-		intf._cancelRecvTimer()
+		intf.sendTimer = time.AfterFunc(sendTime, intf.notifyBlockedSend)
+		intf._cancelStallTimer()
 	})
 }
 
 // we just sent something, so cancel any pending timer to send keep-alive traffic
-func (intf *linkInterface) _cancelRecvTimer() {
-	intf.RecvFrom(nil, func() {
-		if intf.recvTimer != nil {
-			intf.recvTimer.Stop()
-			intf.recvTimer = nil
-		}
-	})
+func (intf *linkInterface) _cancelStallTimer() {
+	if intf.stallTimer != nil {
+		intf.stallTimer.Stop()
+		intf.stallTimer = nil
+	}
 }
 
 // called by an AfterFunc if we appear to have timed out
 func (intf *linkInterface) notifyBlockedSend() {
-	intf.RecvFrom(nil, func() {
+	intf.RecvFrom(nil, func() { // Sent from a time.AfterFunc
 		if intf.sendTimer != nil {
 			//As far as we know, we're still trying to send, and the timer fired.
 			intf.link.core.switchTable.blockPeer(intf.peer.port)
@@ -282,7 +280,7 @@ func (intf *linkInterface) notifyBlockedSend() {
 
 // notify the intf that we've finished sending, returning the peer to the switch
 func (intf *linkInterface) notifySent(size int, isLinkTraffic bool) {
-	intf.RecvFrom(nil, func() {
+	intf.RecvFrom(&intf.writer, func() {
 		intf.sendTimer.Stop()
 		intf.sendTimer = nil
 		if !isLinkTraffic {
@@ -306,8 +304,9 @@ func (intf *linkInterface) _notifySwitch() {
 
 // Set the peer as stalled, to prevent them from returning to the switch until a read succeeds
 func (intf *linkInterface) notifyStalled() {
-	intf.RecvFrom(nil, func() {
+	intf.RecvFrom(nil, func() { // Sent from a time.AfterFunc
 		if intf.stallTimer != nil {
+			intf.stallTimer.Stop()
 			intf.stallTimer = nil
 			intf.stalled = true
 			intf.link.core.switchTable.blockPeer(intf.peer.port)
@@ -316,8 +315,8 @@ func (intf *linkInterface) notifyStalled() {
 }
 
 // reset the close timer
-func (intf *linkInterface) notifyReading(from phony.Actor) {
-	intf.RecvFrom(from, func() {
+func (intf *linkInterface) notifyReading() {
+	intf.RecvFrom(&intf.reader, func() {
 		if intf.closeTimer != nil {
 			intf.closeTimer.Stop()
 		}
@@ -326,26 +325,26 @@ func (intf *linkInterface) notifyReading(from phony.Actor) {
 }
 
 // wake up the link if it was stalled, and (if size > 0) prepare to send keep-alive traffic
-func (intf *linkInterface) notifyReadFrom(from phony.Actor, size int) {
-	intf.RecvFrom(from, func() {
+func (intf *linkInterface) notifyRead(size int) {
+	intf.RecvFrom(&intf.reader, func() {
 		if intf.stallTimer != nil {
 			intf.stallTimer.Stop()
 			intf.stallTimer = nil
 		}
 		intf.stalled = false
 		intf._notifySwitch()
-		if size > 0 && intf.recvTimer == nil {
-			intf.recvTimer = time.AfterFunc(keepAliveTime, intf.notifyDoKeepAlive)
+		if size > 0 && intf.stallTimer == nil {
+			intf.stallTimer = time.AfterFunc(keepAliveTime, intf.notifyDoKeepAlive)
 		}
 	})
 }
 
 // We need to send keep-alive traffic now
 func (intf *linkInterface) notifyDoKeepAlive() {
-	intf.RecvFrom(nil, func() {
-		if intf.recvTimer != nil {
-			intf.recvTimer.Stop()
-			intf.recvTimer = nil
+	intf.RecvFrom(nil, func() { // Sent from a time.AfterFunc
+		if intf.stallTimer != nil {
+			intf.stallTimer.Stop()
+			intf.stallTimer = nil
 			intf.writer.sendFrom(nil, [][]byte{nil}, true) // Empty keep-alive traffic
 		}
 	})
@@ -383,9 +382,9 @@ type linkReader struct {
 }
 
 func (r *linkReader) _read() {
-	r.intf.notifyReading(r)
+	r.intf.notifyReading()
 	msg, err := r.intf.msgIO.readMsg()
-	r.intf.notifyReadFrom(r, len(msg))
+	r.intf.notifyRead(len(msg))
 	if len(msg) > 0 {
 		r.intf.peer.handlePacketFrom(r, msg)
 	}

From 3845f81357dae0f29062e7caa60b1a723efddf05 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Tue, 27 Aug 2019 19:43:54 -0500
Subject: [PATCH 49/62] update to latest phony, adjust interface use
 accordingly

---
 go.mod                   |  2 +-
 go.sum                   |  4 ++--
 src/tuntap/conn.go       |  6 +++---
 src/tuntap/iface.go      | 10 +++++-----
 src/tuntap/tun.go        | 12 ++++++------
 src/yggdrasil/api.go     |  6 ++++--
 src/yggdrasil/conn.go    | 28 ++++++++++++++--------------
 src/yggdrasil/debug.go   | 12 +++++++-----
 src/yggdrasil/link.go    | 22 +++++++++++-----------
 src/yggdrasil/peer.go    | 10 +++++-----
 src/yggdrasil/router.go  | 14 +++++++-------
 src/yggdrasil/search.go  |  2 +-
 src/yggdrasil/session.go | 22 +++++++++++-----------
 src/yggdrasil/switch.go  |  6 +++---
 14 files changed, 80 insertions(+), 76 deletions(-)

diff --git a/go.mod b/go.mod
index f3d8417f..9ff6031a 100644
--- a/go.mod
+++ b/go.mod
@@ -1,7 +1,7 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
 require (
-	github.com/Arceliar/phony v0.0.0-20190825152505-180ac75690fe
+	github.com/Arceliar/phony v0.0.0-20190828002416-0337564e2c44
 	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
 	github.com/hashicorp/go-syslog v1.0.0
 	github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222
diff --git a/go.sum b/go.sum
index 29e0dfec..60388964 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-github.com/Arceliar/phony v0.0.0-20190825152505-180ac75690fe h1:U5bediuXjZ1y6bByIXXraoE319yFp9kx1z8K6el7Ftc=
-github.com/Arceliar/phony v0.0.0-20190825152505-180ac75690fe/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
+github.com/Arceliar/phony v0.0.0-20190828002416-0337564e2c44 h1:8EiuIp65v8aLkLc4LWxtn4NTH+P2LwDmrKKWdBzn9cI=
+github.com/Arceliar/phony v0.0.0-20190828002416-0337564e2c44/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=
diff --git a/src/tuntap/conn.go b/src/tuntap/conn.go
index afb0f09e..31875d94 100644
--- a/src/tuntap/conn.go
+++ b/src/tuntap/conn.go
@@ -27,7 +27,7 @@ type tunConn struct {
 }
 
 func (s *tunConn) close() {
-	s.tun.RecvFrom(s, s._close_from_tun)
+	s.tun.Act(s, s._close_from_tun)
 }
 
 func (s *tunConn) _close_from_tun() {
@@ -117,7 +117,7 @@ func (s *tunConn) _read(bs []byte) (err error) {
 }
 
 func (s *tunConn) writeFrom(from phony.Actor, bs []byte) {
-	s.RecvFrom(from, func() {
+	s.Act(from, func() {
 		s._write(bs)
 	})
 }
@@ -197,7 +197,7 @@ func (s *tunConn) _write(bs []byte) (err error) {
 			// No point in wasting resources to send back an error if there was none
 			return
 		}
-		s.RecvFrom(s.conn, func() {
+		s.Act(s.conn, func() {
 			if e, eok := err.(yggdrasil.ConnError); !eok {
 				if e.Closed() {
 					s.tun.log.Debugln(s.conn.String(), "TUN/TAP generic write debug:", err)
diff --git a/src/tuntap/iface.go b/src/tuntap/iface.go
index da419369..92ba36ab 100644
--- a/src/tuntap/iface.go
+++ b/src/tuntap/iface.go
@@ -19,7 +19,7 @@ type tunWriter struct {
 }
 
 func (w *tunWriter) writeFrom(from phony.Actor, b []byte) {
-	w.RecvFrom(from, func() {
+	w.Act(from, func() {
 		w._write(b)
 	})
 }
@@ -90,7 +90,7 @@ func (w *tunWriter) _write(b []byte) {
 		util.PutBytes(b)
 	}
 	if err != nil {
-		w.tun.RecvFrom(w, func() {
+		w.tun.Act(w, func() {
 			if !w.tun.isOpen {
 				w.tun.log.Errorln("TUN/TAP iface write error:", err)
 			}
@@ -118,12 +118,12 @@ func (r *tunReader) _read() {
 	}
 	if err == nil {
 		// Now read again
-		r.RecvFrom(nil, r._read)
+		r.Act(nil, r._read)
 	}
 }
 
 func (tun *TunAdapter) handlePacketFrom(from phony.Actor, packet []byte, err error) {
-	tun.RecvFrom(from, func() {
+	tun.Act(from, func() {
 		tun._handlePacket(packet, err)
 	})
 }
@@ -248,7 +248,7 @@ func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
 		if !known {
 			go func() {
 				conn, err := tun.dialer.DialByNodeIDandMask(dstNodeID, dstNodeIDMask)
-				tun.RecvFrom(nil, func() {
+				tun.Act(nil, func() {
 					packets := tun.dials[*dstNodeID]
 					delete(tun.dials, *dstNodeID)
 					if err != nil {
diff --git a/src/tuntap/tun.go b/src/tuntap/tun.go
index 2b163e3b..6317459c 100644
--- a/src/tuntap/tun.go
+++ b/src/tuntap/tun.go
@@ -125,7 +125,7 @@ func (tun *TunAdapter) Init(config *config.NodeState, log *log.Logger, listener
 // reader actor to handle packets on that interface.
 func (tun *TunAdapter) Start() error {
 	var err error
-	<-tun.SyncExec(func() {
+	phony.Block(tun, func() {
 		err = tun._start()
 	})
 	return err
@@ -167,7 +167,7 @@ func (tun *TunAdapter) _start() error {
 		}
 	}()
 	go tun.handler()
-	tun.reader.RecvFrom(nil, tun.reader._read) // Start the reader
+	tun.reader.Act(nil, tun.reader._read) // Start the reader
 	tun.icmpv6.Init(tun)
 	if iftapmode {
 		go tun.icmpv6.Solicit(tun.addr)
@@ -180,7 +180,7 @@ func (tun *TunAdapter) _start() error {
 // read/write goroutines to handle packets on that interface.
 func (tun *TunAdapter) Stop() error {
 	var err error
-	<-tun.SyncExec(func() {
+	phony.Block(tun, func() {
 		err = tun._stop()
 	})
 	return err
@@ -233,7 +233,7 @@ func (tun *TunAdapter) handler() error {
 			tun.log.Errorln("TUN/TAP connection accept error:", err)
 			return err
 		}
-		<-tun.SyncExec(func() {
+		phony.Block(tun, func() {
 			if _, err := tun._wrap(conn); err != nil {
 				// Something went wrong when storing the connection, typically that
 				// something already exists for this address or subnet
@@ -273,11 +273,11 @@ func (tun *TunAdapter) _wrap(conn *yggdrasil.Conn) (c *tunConn, err error) {
 	tun.subnetToConn[s.snet] = &s
 	// Set the read callback and start the timeout
 	conn.SetReadCallback(func(bs []byte) {
-		s.RecvFrom(conn, func() {
+		s.Act(conn, func() {
 			s._read(bs)
 		})
 	})
-	s.RecvFrom(nil, s.stillAlive)
+	s.Act(nil, s.stillAlive)
 	// Return
 	return c, err
 }
diff --git a/src/yggdrasil/api.go b/src/yggdrasil/api.go
index c6c15e24..44cbad1d 100644
--- a/src/yggdrasil/api.go
+++ b/src/yggdrasil/api.go
@@ -11,6 +11,8 @@ import (
 	"github.com/gologme/log"
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
+
+	"github.com/Arceliar/phony"
 )
 
 // Peer represents a single peer object. This contains information from the
@@ -106,7 +108,7 @@ func (c *Core) GetPeers() []Peer {
 	for _, port := range ps {
 		p := ports[port]
 		var info Peer
-		<-p.SyncExec(func() {
+		phony.Block(p, func() {
 			info = Peer{
 				Endpoint:   p.intf.name,
 				BytesSent:  p.bytesSent,
@@ -138,7 +140,7 @@ func (c *Core) GetSwitchPeers() []SwitchPeer {
 		}
 		coords := elem.locator.getCoords()
 		var info SwitchPeer
-		<-peer.SyncExec(func() {
+		phony.Block(peer, func() {
 			info = SwitchPeer{
 				Coords:     append([]uint64{}, wire_coordsBytestoUint64s(coords)...),
 				BytesSent:  peer.bytesSent,
diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index efc0c81e..c56c2a90 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -80,12 +80,12 @@ func newConn(core *Core, nodeID *crypto.NodeID, nodeMask *crypto.NodeID, session
 
 func (c *Conn) String() string {
 	var s string
-	<-c.SyncExec(func() { s = fmt.Sprintf("conn=%p", c) })
+	phony.Block(c, func() { s = fmt.Sprintf("conn=%p", c) })
 	return s
 }
 
 func (c *Conn) setMTU(from phony.Actor, mtu uint16) {
-	c.RecvFrom(from, func() { c.mtu = mtu })
+	c.Act(from, func() { c.mtu = mtu })
 }
 
 // This should never be called from the router goroutine, used in the dial functions
@@ -143,7 +143,7 @@ func (c *Conn) doSearch() {
 			sinfo.continueSearch()
 		}
 	}
-	c.core.router.RecvFrom(c.session, routerWork)
+	c.core.router.Act(c.session, routerWork)
 }
 
 func (c *Conn) _getDeadlineCancellation(t *time.Time) (util.Cancellation, bool) {
@@ -159,7 +159,7 @@ func (c *Conn) _getDeadlineCancellation(t *time.Time) (util.Cancellation, bool)
 
 // SetReadCallback sets a callback which will be called whenever a packet is received.
 func (c *Conn) SetReadCallback(callback func([]byte)) {
-	c.RecvFrom(nil, func() {
+	c.Act(nil, func() {
 		c.readCallback = callback
 		c._drainReadBuffer()
 	})
@@ -172,14 +172,14 @@ func (c *Conn) _drainReadBuffer() {
 	select {
 	case bs := <-c.readBuffer:
 		c.readCallback(bs)
-		c.RecvFrom(nil, c._drainReadBuffer) // In case there's more
+		c.Act(nil, c._drainReadBuffer) // In case there's more
 	default:
 	}
 }
 
 // Called by the session to pass a new message to the Conn
 func (c *Conn) recvMsg(from phony.Actor, msg []byte) {
-	c.RecvFrom(from, func() {
+	c.Act(from, func() {
 		if c.readCallback != nil {
 			c.readCallback(msg)
 		} else {
@@ -195,7 +195,7 @@ func (c *Conn) recvMsg(from phony.Actor, msg []byte) {
 func (c *Conn) ReadNoCopy() ([]byte, error) {
 	var cancel util.Cancellation
 	var doCancel bool
-	<-c.SyncExec(func() { cancel, doCancel = c._getDeadlineCancellation(c.readDeadline) })
+	phony.Block(c, func() { cancel, doCancel = c._getDeadlineCancellation(c.readDeadline) })
 	if doCancel {
 		defer cancel.Cancel(nil)
 	}
@@ -234,7 +234,7 @@ func (c *Conn) _write(msg FlowKeyMessage) error {
 	if len(msg.Message) > int(c.mtu) {
 		return ConnError{errors.New("packet too big"), true, false, false, int(c.mtu)}
 	}
-	c.session.RecvFrom(c, func() {
+	c.session.Act(c, func() {
 		// Send the packet
 		c.session._send(msg)
 		// Session keep-alive, while we wait for the crypto workers from send
@@ -258,7 +258,7 @@ func (c *Conn) _write(msg FlowKeyMessage) error {
 // This is used internaly by WriteNoCopy and Write.
 // If the callback is called with a non-nil value, then it is safe to reuse the argument FlowKeyMessage.
 func (c *Conn) WriteFrom(from phony.Actor, msg FlowKeyMessage, callback func(error)) {
-	c.RecvFrom(from, func() {
+	c.Act(from, func() {
 		callback(c._write(msg))
 	})
 }
@@ -268,7 +268,7 @@ func (c *Conn) WriteFrom(from phony.Actor, msg FlowKeyMessage, callback func(err
 func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error {
 	var cancel util.Cancellation
 	var doCancel bool
-	<-c.SyncExec(func() { cancel, doCancel = c._getDeadlineCancellation(c.writeDeadline) })
+	phony.Block(c, func() { cancel, doCancel = c._getDeadlineCancellation(c.writeDeadline) })
 	var err error
 	select {
 	case <-cancel.Finished():
@@ -299,7 +299,7 @@ func (c *Conn) Write(b []byte) (int, error) {
 }
 
 func (c *Conn) Close() (err error) {
-	<-c.SyncExec(func() {
+	phony.Block(c, func() {
 		if c.session != nil {
 			// Close the session, if it hasn't been closed already
 			if e := c.session.cancel.Cancel(errors.New("connection closed")); e != nil {
@@ -319,7 +319,7 @@ func (c *Conn) LocalAddr() crypto.NodeID {
 func (c *Conn) RemoteAddr() crypto.NodeID {
 	// TODO warn that this can block while waiting for the Conn actor to run, so don't call it from other actors...
 	var n crypto.NodeID
-	<-c.SyncExec(func() { n = *c.nodeID })
+	phony.Block(c, func() { n = *c.nodeID })
 	return n
 }
 
@@ -331,12 +331,12 @@ func (c *Conn) SetDeadline(t time.Time) error {
 
 func (c *Conn) SetReadDeadline(t time.Time) error {
 	// TODO warn that this can block while waiting for the Conn actor to run, so don't call it from other actors...
-	<-c.SyncExec(func() { c.readDeadline = &t })
+	phony.Block(c, func() { c.readDeadline = &t })
 	return nil
 }
 
 func (c *Conn) SetWriteDeadline(t time.Time) error {
 	// TODO warn that this can block while waiting for the Conn actor to run, so don't call it from other actors...
-	<-c.SyncExec(func() { c.writeDeadline = &t })
+	phony.Block(c, func() { c.writeDeadline = &t })
 	return nil
 }
diff --git a/src/yggdrasil/debug.go b/src/yggdrasil/debug.go
index 1cc7eae4..9d2199d0 100644
--- a/src/yggdrasil/debug.go
+++ b/src/yggdrasil/debug.go
@@ -29,6 +29,8 @@ import "github.com/yggdrasil-network/yggdrasil-go/src/config"
 import "github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 import "github.com/yggdrasil-network/yggdrasil-go/src/defaults"
 
+import "github.com/Arceliar/phony"
+
 // Start the profiler in debug builds, if the required environment variable is set.
 func init() {
 	envVarName := "PPROFLISTEN"
@@ -572,14 +574,14 @@ func DEBUG_simLinkPeers(p, q *peer) {
 					continue
 				case packet := <-send:
 					packets = append(packets, packet)
-					<-source.core.switchTable.SyncExec(func() {
+					phony.Block(&source.core.switchTable, func() {
 						source.core.switchTable._idleIn(source.port)
 					})
 					continue
 				default:
 				}
 				if len(packets) > 0 {
-					<-dest.SyncExec(func() { dest._handlePacket(packets[0]) })
+					phony.Block(dest, func() { dest._handlePacket(packets[0]) })
 					packets = packets[1:]
 					continue
 				}
@@ -588,7 +590,7 @@ func DEBUG_simLinkPeers(p, q *peer) {
 					packets = append(packets, packet)
 				case packet := <-send:
 					packets = append(packets, packet)
-					<-source.core.switchTable.SyncExec(func() {
+					phony.Block(&source.core.switchTable, func() {
 						source.core.switchTable._idleIn(source.port)
 					})
 				}
@@ -597,10 +599,10 @@ func DEBUG_simLinkPeers(p, q *peer) {
 	}
 	goWorkers(p, q)
 	goWorkers(q, p)
-	<-p.core.switchTable.SyncExec(func() {
+	phony.Block(&p.core.switchTable, func() {
 		p.core.switchTable._idleIn(p.port)
 	})
-	<-q.core.switchTable.SyncExec(func() {
+	phony.Block(&q.core.switchTable, func() {
 		q.core.switchTable._idleIn(q.port)
 	})
 }
diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index 6b7f446f..e65055bc 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -227,7 +227,7 @@ func (intf *linkInterface) handler() error {
 		strings.ToUpper(intf.info.linkType), themString, intf.info.local)
 	// Start things
 	go intf.peer.start()
-	intf.reader.RecvFrom(nil, intf.reader._read)
+	intf.reader.Act(nil, intf.reader._read)
 	// Wait for the reader to finish
 	err = <-intf.reader.err
 	if err != nil {
@@ -251,7 +251,7 @@ const (
 
 // notify the intf that we're currently sending
 func (intf *linkInterface) notifySending(size int, isLinkTraffic bool) {
-	intf.RecvFrom(&intf.writer, func() {
+	intf.Act(&intf.writer, func() {
 		if !isLinkTraffic {
 			intf.inSwitch = false
 		}
@@ -270,7 +270,7 @@ func (intf *linkInterface) _cancelStallTimer() {
 
 // called by an AfterFunc if we appear to have timed out
 func (intf *linkInterface) notifyBlockedSend() {
-	intf.RecvFrom(nil, func() { // Sent from a time.AfterFunc
+	intf.Act(nil, func() { // Sent from a time.AfterFunc
 		if intf.sendTimer != nil {
 			//As far as we know, we're still trying to send, and the timer fired.
 			intf.link.core.switchTable.blockPeer(intf.peer.port)
@@ -280,7 +280,7 @@ func (intf *linkInterface) notifyBlockedSend() {
 
 // notify the intf that we've finished sending, returning the peer to the switch
 func (intf *linkInterface) notifySent(size int, isLinkTraffic bool) {
-	intf.RecvFrom(&intf.writer, func() {
+	intf.Act(&intf.writer, func() {
 		intf.sendTimer.Stop()
 		intf.sendTimer = nil
 		if !isLinkTraffic {
@@ -296,7 +296,7 @@ func (intf *linkInterface) notifySent(size int, isLinkTraffic bool) {
 func (intf *linkInterface) _notifySwitch() {
 	if !intf.inSwitch && !intf.stalled {
 		intf.inSwitch = true
-		intf.link.core.switchTable.RecvFrom(intf, func() {
+		intf.link.core.switchTable.Act(intf, func() {
 			intf.link.core.switchTable._idleIn(intf.peer.port)
 		})
 	}
@@ -304,7 +304,7 @@ func (intf *linkInterface) _notifySwitch() {
 
 // Set the peer as stalled, to prevent them from returning to the switch until a read succeeds
 func (intf *linkInterface) notifyStalled() {
-	intf.RecvFrom(nil, func() { // Sent from a time.AfterFunc
+	intf.Act(nil, func() { // Sent from a time.AfterFunc
 		if intf.stallTimer != nil {
 			intf.stallTimer.Stop()
 			intf.stallTimer = nil
@@ -316,7 +316,7 @@ func (intf *linkInterface) notifyStalled() {
 
 // reset the close timer
 func (intf *linkInterface) notifyReading() {
-	intf.RecvFrom(&intf.reader, func() {
+	intf.Act(&intf.reader, func() {
 		if intf.closeTimer != nil {
 			intf.closeTimer.Stop()
 		}
@@ -326,7 +326,7 @@ func (intf *linkInterface) notifyReading() {
 
 // wake up the link if it was stalled, and (if size > 0) prepare to send keep-alive traffic
 func (intf *linkInterface) notifyRead(size int) {
-	intf.RecvFrom(&intf.reader, func() {
+	intf.Act(&intf.reader, func() {
 		if intf.stallTimer != nil {
 			intf.stallTimer.Stop()
 			intf.stallTimer = nil
@@ -341,7 +341,7 @@ func (intf *linkInterface) notifyRead(size int) {
 
 // We need to send keep-alive traffic now
 func (intf *linkInterface) notifyDoKeepAlive() {
-	intf.RecvFrom(nil, func() { // Sent from a time.AfterFunc
+	intf.Act(nil, func() { // Sent from a time.AfterFunc
 		if intf.stallTimer != nil {
 			intf.stallTimer.Stop()
 			intf.stallTimer = nil
@@ -358,7 +358,7 @@ type linkWriter struct {
 }
 
 func (w *linkWriter) sendFrom(from phony.Actor, bss [][]byte, isLinkTraffic bool) {
-	w.RecvFrom(from, func() {
+	w.Act(from, func() {
 		var size int
 		for _, bs := range bss {
 			size += len(bs)
@@ -396,5 +396,5 @@ func (r *linkReader) _read() {
 		return
 	}
 	// Now try to read again
-	r.RecvFrom(nil, r._read)
+	r.Act(nil, r._read)
 }
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 4cf0068b..d33c413a 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -178,7 +178,7 @@ func (ps *peers) sendSwitchMsgs(from phony.Actor) {
 		if p.port == 0 {
 			continue
 		}
-		p.RecvFrom(from, p._sendSwitchMsg)
+		p.Act(from, p._sendSwitchMsg)
 	}
 }
 
@@ -187,7 +187,7 @@ func (ps *peers) sendSwitchMsgs(from phony.Actor) {
 func (p *peer) start() {
 	var updateDHT func()
 	updateDHT = func() {
-		<-p.SyncExec(func() {
+		phony.Block(p, func() {
 			select {
 			case <-p.done:
 			default:
@@ -198,7 +198,7 @@ func (p *peer) start() {
 	}
 	updateDHT()
 	// Just for good measure, immediately send a switch message to this peer when we start
-	<-p.SyncExec(p._sendSwitchMsg)
+	p.Act(nil, p._sendSwitchMsg)
 }
 
 func (p *peer) _updateDHT() {
@@ -208,7 +208,7 @@ func (p *peer) _updateDHT() {
 }
 
 func (p *peer) handlePacketFrom(from phony.Actor, packet []byte) {
-	p.RecvFrom(from, func() {
+	p.Act(from, func() {
 		p._handlePacket(packet)
 	})
 }
@@ -246,7 +246,7 @@ func (p *peer) _handleTraffic(packet []byte) {
 }
 
 func (p *peer) sendPacketsFrom(from phony.Actor, packets [][]byte) {
-	p.RecvFrom(from, func() {
+	p.Act(from, func() {
 		p._sendPackets(packets)
 	})
 }
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index fec55b7c..bbc87c51 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -77,7 +77,7 @@ func (r *router) reconfigure(e chan error) {
 	defer close(e)
 	var errs []error
 	// Reconfigure the router
-	<-r.SyncExec(func() {
+	phony.Block(r, func() {
 		current := r.core.config.GetCurrent()
 		err := r.nodeinfo.setNodeInfo(current.NodeInfo, current.NodeInfoPrivacy)
 		if err != nil {
@@ -98,7 +98,7 @@ func (r *router) start() error {
 
 // In practice, the switch will call this with 1 packet
 func (r *router) handlePackets(from phony.Actor, packets [][]byte) {
-	r.RecvFrom(from, func() {
+	r.Act(from, func() {
 		for _, packet := range packets {
 			r._handlePacket(packet)
 		}
@@ -107,14 +107,14 @@ func (r *router) handlePackets(from phony.Actor, packets [][]byte) {
 
 // Insert a peer info into the dht, TODO? make the dht a separate actor
 func (r *router) insertPeer(from phony.Actor, info *dhtInfo) {
-	r.RecvFrom(from, func() {
+	r.Act(from, func() {
 		r.dht.insertPeer(info)
 	})
 }
 
 // Reset sessions and DHT after the switch sees our coords change
 func (r *router) reset(from phony.Actor) {
-	r.RecvFrom(from, func() {
+	r.Act(from, func() {
 		r.sessions.reset()
 		r.dht.reset()
 	})
@@ -123,7 +123,7 @@ func (r *router) reset(from phony.Actor) {
 // TODO remove reconfigure so this is just a ticker loop
 // and then find something better than a ticker loop to schedule things...
 func (r *router) doMaintenance() {
-	<-r.SyncExec(func() {
+	phony.Block(r, func() {
 		// Any periodic maintenance stuff goes here
 		r.core.switchTable.doMaintenance()
 		r.dht.doMaintenance()
@@ -252,7 +252,7 @@ func (r *router) _handleNodeInfo(bs []byte, fromKey *crypto.BoxPubKey) {
 	r.nodeinfo.handleNodeInfo(&req)
 }
 
-// TODO remove this, have things either be actors that send message or else call SyncExec directly
+// TODO remove this, have things either be actors that send message or else call Block directly
 func (r *router) doAdmin(f func()) {
-	<-r.SyncExec(f)
+	phony.Block(r, f)
 }
diff --git a/src/yggdrasil/search.go b/src/yggdrasil/search.go
index 9cf5f06e..ca357cc2 100644
--- a/src/yggdrasil/search.go
+++ b/src/yggdrasil/search.go
@@ -153,7 +153,7 @@ func (sinfo *searchInfo) continueSearch() {
 	// Note that this will spawn multiple parallel searches as time passes
 	// Any that die aren't restarted, but a new one will start later
 	time.AfterFunc(search_RETRY_TIME, func() {
-		sinfo.searches.router.RecvFrom(nil, func() {
+		sinfo.searches.router.Act(nil, func() {
 			// FIXME this keeps the search alive forever if not for the searches map, fix that
 			newSearchInfo := sinfo.searches.searches[sinfo.dest]
 			if newSearchInfo != sinfo {
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 0fc7ec80..06780e65 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -80,7 +80,7 @@ func (sinfo *sessionInfo) reconfigure(e chan error) {
 
 // TODO remove this, call SyncExec directly
 func (sinfo *sessionInfo) doFunc(f func()) {
-	<-sinfo.SyncExec(f)
+	phony.Block(sinfo, f)
 }
 
 // Represents a session ping/pong packet, andincludes information like public keys, a session handle, coords, a timestamp to prevent replays, and the tun/tap MTU.
@@ -164,7 +164,7 @@ func (ss *sessions) init(r *router) {
 func (ss *sessions) reconfigure(e chan error) {
 	defer close(e)
 	responses := make(map[crypto.Handle]chan error)
-	<-ss.router.SyncExec(func() {
+	phony.Block(ss.router, func() {
 		for index, session := range ss.sinfos {
 			responses[index] = make(chan error)
 			go session.reconfigure(responses[index])
@@ -287,7 +287,7 @@ func (ss *sessions) cleanup() {
 }
 
 func (sinfo *sessionInfo) doRemove() {
-	sinfo.sessions.router.RecvFrom(nil, func() {
+	sinfo.sessions.router.Act(nil, func() {
 		sinfo.sessions.removeSession(sinfo)
 	})
 }
@@ -341,7 +341,7 @@ func (ss *sessions) getSharedKey(myPriv *crypto.BoxPrivKey,
 
 // Sends a session ping by calling sendPingPong in ping mode.
 func (sinfo *sessionInfo) ping(from phony.Actor) {
-	sinfo.RecvFrom(from, func() {
+	sinfo.Act(from, func() {
 		sinfo._sendPingPong(false)
 	})
 }
@@ -362,14 +362,14 @@ func (sinfo *sessionInfo) _sendPingPong(isPong bool) {
 	}
 	packet := p.encode()
 	// TODO rewrite the below if/when the peer struct becomes an actor, to not go through the router first
-	sinfo.sessions.router.RecvFrom(sinfo, func() { sinfo.sessions.router.out(packet) })
+	sinfo.sessions.router.Act(sinfo, func() { sinfo.sessions.router.out(packet) })
 	if sinfo.pingTime.Before(sinfo.time) {
 		sinfo.pingTime = time.Now()
 	}
 }
 
 func (sinfo *sessionInfo) setConn(from phony.Actor, conn *Conn) {
-	sinfo.RecvFrom(from, func() {
+	sinfo.Act(from, func() {
 		sinfo.conn = conn
 		sinfo.conn.setMTU(sinfo, sinfo._getMTU())
 	})
@@ -404,7 +404,7 @@ func (ss *sessions) handlePing(ping *sessionPing) {
 		ss.listenerMutex.Unlock()
 	}
 	if sinfo != nil {
-		sinfo.RecvFrom(ss.router, func() {
+		sinfo.Act(ss.router, func() {
 			// Update the session
 			if !sinfo._update(ping) { /*panic("Should not happen in testing")*/
 				return
@@ -472,7 +472,7 @@ func (sinfo *sessionInfo) _updateNonce(theirNonce *crypto.BoxNonce) {
 // Called after coord changes, so attemtps to use a session will trigger a new ping and notify the remote end of the coord change.
 func (ss *sessions) reset() {
 	for _, sinfo := range ss.sinfos {
-		sinfo.RecvFrom(ss.router, func() {
+		sinfo.Act(ss.router, func() {
 			sinfo.reset = true
 		})
 	}
@@ -488,7 +488,7 @@ type FlowKeyMessage struct {
 }
 
 func (sinfo *sessionInfo) recv(from phony.Actor, packet *wire_trafficPacket) {
-	sinfo.RecvFrom(from, func() {
+	sinfo.Act(from, func() {
 		sinfo._recvPacket(packet)
 	})
 }
@@ -562,7 +562,7 @@ func (sinfo *sessionInfo) _send(msg FlowKeyMessage) {
 			util.PutBytes(p.Payload)
 			// Send the packet
 			// TODO replace this with a send to the peer struct if that becomes an actor
-			sinfo.sessions.router.RecvFrom(sinfo, func() {
+			sinfo.sessions.router.Act(sinfo, func() {
 				sinfo.sessions.router.out(packet)
 			})
 		}
@@ -574,7 +574,7 @@ func (sinfo *sessionInfo) _send(msg FlowKeyMessage) {
 }
 
 func (sinfo *sessionInfo) checkCallbacks() {
-	sinfo.RecvFrom(nil, func() {
+	sinfo.Act(nil, func() {
 		if len(sinfo.callbacks) > 0 {
 			select {
 			case callback := <-sinfo.callbacks[0]:
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index b30e59d5..93d79d29 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -192,7 +192,7 @@ func (t *switchTable) init(core *Core) {
 	t.updater.Store(&sync.Once{})
 	t.table.Store(lookupTable{})
 	t.drop = make(map[crypto.SigPubKey]int64)
-	<-t.SyncExec(func() {
+	phony.Block(t, func() {
 		t.queues.totalMaxSize = SwitchQueueTotalMinSize
 		t.queues.bufs = make(map[string]switch_buffer)
 		t.idle = make(map[switchPort]time.Time)
@@ -827,7 +827,7 @@ func (t *switchTable) _handleIdle(port switchPort) bool {
 }
 
 func (t *switchTable) packetInFrom(from phony.Actor, bytes []byte) {
-	t.RecvFrom(from, func() {
+	t.Act(from, func() {
 		t._packetIn(bytes)
 	})
 }
@@ -870,5 +870,5 @@ func (t *switchTable) _idleIn(port switchPort) {
 // Passed a function to call.
 // This will send the function to t.admin and block until it finishes.
 func (t *switchTable) doAdmin(f func()) {
-	<-t.SyncExec(f)
+	phony.Block(t, f)
 }

From a8b323acdd2236fb3f984cee6829395829196d14 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Tue, 27 Aug 2019 20:01:37 -0500
Subject: [PATCH 50/62] have an actor manage the crypto worker pool instead of
 each session trying to use it directly, this should result in a fairer
 round-robin behavior in cases where crypto congestion is the bottleneck

---
 src/yggdrasil/session.go | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 06780e65..9abfeca6 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -482,6 +482,18 @@ func (ss *sessions) reset() {
 //////////////////////////// Worker Functions Below ////////////////////////////
 ////////////////////////////////////////////////////////////////////////////////
 
+type sessionCryptoManager struct {
+	phony.Inbox
+}
+
+func (m *sessionCryptoManager) workerGo(from phony.Actor, f func()) {
+	m.Act(from, func() {
+		util.WorkerGo(f)
+	})
+}
+
+var manager = sessionCryptoManager{}
+
 type FlowKeyMessage struct {
 	FlowKey uint64
 	Message []byte
@@ -527,7 +539,7 @@ func (sinfo *sessionInfo) _recvPacket(p *wire_trafficPacket) {
 		sinfo.checkCallbacks()
 	}
 	sinfo.callbacks = append(sinfo.callbacks, ch)
-	util.WorkerGo(poolFunc)
+	manager.workerGo(sinfo, poolFunc)
 }
 
 func (sinfo *sessionInfo) _send(msg FlowKeyMessage) {
@@ -570,7 +582,7 @@ func (sinfo *sessionInfo) _send(msg FlowKeyMessage) {
 		sinfo.checkCallbacks()
 	}
 	sinfo.callbacks = append(sinfo.callbacks, ch)
-	util.WorkerGo(poolFunc)
+	manager.workerGo(sinfo, poolFunc)
 }
 
 func (sinfo *sessionInfo) checkCallbacks() {

From 5d7d84f82716c3689dca19d294ca239fa3c2f9f6 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 28 Aug 2019 12:17:19 +0100
Subject: [PATCH 51/62] Remove router.doAdmin and switchTable.doAdmin

---
 src/tuntap/ckr.go       |  3 +--
 src/yggdrasil/api.go    | 11 +++++------
 src/yggdrasil/conn.go   |  6 ++++--
 src/yggdrasil/core.go   |  3 ++-
 src/yggdrasil/debug.go  |  2 +-
 src/yggdrasil/peer.go   |  2 +-
 src/yggdrasil/router.go |  5 -----
 src/yggdrasil/switch.go |  8 +-------
 8 files changed, 15 insertions(+), 25 deletions(-)

diff --git a/src/tuntap/ckr.go b/src/tuntap/ckr.go
index ad8c89d4..229d2607 100644
--- a/src/tuntap/ckr.go
+++ b/src/tuntap/ckr.go
@@ -56,8 +56,7 @@ func (c *cryptokey) init(tun *TunAdapter) {
 	}
 }
 
-// Configure the CKR routes - this must only ever be called from the router
-// goroutine, e.g. through router.doAdmin
+// Configure the CKR routes.
 func (c *cryptokey) configure() error {
 	current := c.tun.config.GetCurrent()
 
diff --git a/src/yggdrasil/api.go b/src/yggdrasil/api.go
index 44cbad1d..2f684646 100644
--- a/src/yggdrasil/api.go
+++ b/src/yggdrasil/api.go
@@ -178,7 +178,7 @@ func (c *Core) GetDHT() []DHTEntry {
 			dhtentries = append(dhtentries, info)
 		}
 	}
-	c.router.doAdmin(getDHT)
+	phony.Block(c.router, getDHT)
 	return dhtentries
 }
 
@@ -205,9 +205,8 @@ func (c *Core) GetSwitchQueues() SwitchQueues {
 			}
 			switchqueues.Queues = append(switchqueues.Queues, queue)
 		}
-
 	}
-	c.switchTable.doAdmin(getSwitchQueues)
+	phony.Block(c.switchTable, getSwitchQueues)
 	return switchqueues
 }
 
@@ -244,7 +243,7 @@ func (c *Core) GetSessions() []Session {
 			sessions = append(sessions, session)
 		}
 	}
-	c.router.doAdmin(getSessions)
+	phony.Block(c.router, getSessions)
 	return sessions
 }
 
@@ -345,7 +344,7 @@ func (c *Core) GetNodeInfo(key crypto.BoxPubKey, coords []uint64, nocache bool)
 		})
 		c.router.nodeinfo.sendNodeInfo(key, wire_coordsUint64stoBytes(coords), false)
 	}
-	c.router.doAdmin(sendNodeInfoRequest)
+	phony.Block(c.router, sendNodeInfoRequest)
 	timer := time.AfterFunc(6*time.Second, func() { close(response) })
 	defer timer.Stop()
 	for res := range response {
@@ -455,7 +454,7 @@ func (c *Core) DHTPing(key crypto.BoxPubKey, coords []uint64, target *crypto.Nod
 		})
 		c.router.dht.ping(&info, &rq.dest)
 	}
-	c.router.doAdmin(sendPing)
+	phony.Block(c.router, sendPing)
 	// TODO: do something better than the below...
 	res := <-resCh
 	if res != nil {
diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index c56c2a90..4c222266 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -92,7 +92,9 @@ func (c *Conn) setMTU(from phony.Actor, mtu uint16) {
 func (c *Conn) search() error {
 	var sinfo *searchInfo
 	var isIn bool
-	c.core.router.doAdmin(func() { sinfo, isIn = c.core.router.searches.searches[*c.nodeID] })
+	phony.Block(c.core.router, func() {
+		sinfo, isIn = c.core.router.searches.searches[*c.nodeID]
+	})
 	if !isIn {
 		done := make(chan struct{}, 1)
 		var sess *sessionInfo
@@ -106,7 +108,7 @@ func (c *Conn) search() error {
 			default:
 			}
 		}
-		c.core.router.doAdmin(func() {
+		phony.Block(c.core.router, func() {
 			sinfo = c.core.router.searches.newIterSearch(c.nodeID, c.nodeMask, searchCompleted)
 			sinfo.continueSearch()
 		})
diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index ec530745..3719818b 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"time"
 
+	"github.com/Arceliar/phony"
 	"github.com/gologme/log"
 
 	"github.com/yggdrasil-network/yggdrasil-go/src/config"
@@ -175,7 +176,7 @@ func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState,
 
 	c.config.Mutex.RLock()
 	if c.config.Current.SwitchOptions.MaxTotalQueueSize >= SwitchQueueTotalMinSize {
-		c.switchTable.doAdmin(func() {
+		phony.Block(c.switchTable, func() {
 			c.switchTable.queues.totalMaxSize = c.config.Current.SwitchOptions.MaxTotalQueueSize
 		})
 	}
diff --git a/src/yggdrasil/debug.go b/src/yggdrasil/debug.go
index 9d2199d0..4502c11e 100644
--- a/src/yggdrasil/debug.go
+++ b/src/yggdrasil/debug.go
@@ -259,7 +259,7 @@ func DEBUG_wire_encode_coords(coords []byte) []byte {
 
 func (c *Core) DEBUG_getDHTSize() int {
 	var total int
-	c.router.doAdmin(func() {
+	phony.Block(c.router, func() {
 		total = len(c.router.dht.table)
 	})
 	return total
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index d33c413a..72ff2847 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -149,7 +149,7 @@ func (ps *peers) removePeer(port switchPort) {
 	if port == 0 {
 		return
 	} // Can't remove self peer
-	ps.core.router.doAdmin(func() {
+	phony.Block(ps.core.router, func() {
 		ps.core.switchTable.forgetPeer(port)
 	})
 	ps.mutex.Lock()
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index bbc87c51..d93ba433 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -251,8 +251,3 @@ func (r *router) _handleNodeInfo(bs []byte, fromKey *crypto.BoxPubKey) {
 	req.SendPermPub = *fromKey
 	r.nodeinfo.handleNodeInfo(&req)
 }
-
-// TODO remove this, have things either be actors that send message or else call Block directly
-func (r *router) doAdmin(f func()) {
-	phony.Block(r, f)
-}
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 93d79d29..5c613d8d 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -278,7 +278,7 @@ func (t *switchTable) blockPeer(port switchPort) {
 }
 
 // Removes a peer.
-// Must be called by the router mainLoop goroutine, e.g. call router.doAdmin with a lambda that calls this.
+// Must be called by the router actor with a lambda that calls this.
 // If the removed peer was this node's parent, it immediately tries to find a new parent.
 func (t *switchTable) forgetPeer(port switchPort) {
 	t.mutex.Lock()
@@ -866,9 +866,3 @@ func (t *switchTable) _idleIn(port switchPort) {
 		t.idle[port] = time.Now()
 	}
 }
-
-// Passed a function to call.
-// This will send the function to t.admin and block until it finishes.
-func (t *switchTable) doAdmin(f func()) {
-	phony.Block(t, f)
-}

From 607c9068201d10b49861a06b0d1fbd1cf2400582 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 28 Aug 2019 12:26:44 +0100
Subject: [PATCH 52/62] Pointer receivers for phony.Block

---
 src/yggdrasil/api.go  | 10 +++++-----
 src/yggdrasil/conn.go |  4 ++--
 src/yggdrasil/core.go |  2 +-
 src/yggdrasil/peer.go |  2 +-
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/yggdrasil/api.go b/src/yggdrasil/api.go
index 2f684646..9e82ece1 100644
--- a/src/yggdrasil/api.go
+++ b/src/yggdrasil/api.go
@@ -178,7 +178,7 @@ func (c *Core) GetDHT() []DHTEntry {
 			dhtentries = append(dhtentries, info)
 		}
 	}
-	phony.Block(c.router, getDHT)
+	phony.Block(&c.router, getDHT)
 	return dhtentries
 }
 
@@ -206,7 +206,7 @@ func (c *Core) GetSwitchQueues() SwitchQueues {
 			switchqueues.Queues = append(switchqueues.Queues, queue)
 		}
 	}
-	phony.Block(c.switchTable, getSwitchQueues)
+	phony.Block(&c.switchTable, getSwitchQueues)
 	return switchqueues
 }
 
@@ -243,7 +243,7 @@ func (c *Core) GetSessions() []Session {
 			sessions = append(sessions, session)
 		}
 	}
-	phony.Block(c.router, getSessions)
+	phony.Block(&c.router, getSessions)
 	return sessions
 }
 
@@ -344,7 +344,7 @@ func (c *Core) GetNodeInfo(key crypto.BoxPubKey, coords []uint64, nocache bool)
 		})
 		c.router.nodeinfo.sendNodeInfo(key, wire_coordsUint64stoBytes(coords), false)
 	}
-	phony.Block(c.router, sendNodeInfoRequest)
+	phony.Block(&c.router, sendNodeInfoRequest)
 	timer := time.AfterFunc(6*time.Second, func() { close(response) })
 	defer timer.Stop()
 	for res := range response {
@@ -454,7 +454,7 @@ func (c *Core) DHTPing(key crypto.BoxPubKey, coords []uint64, target *crypto.Nod
 		})
 		c.router.dht.ping(&info, &rq.dest)
 	}
-	phony.Block(c.router, sendPing)
+	phony.Block(&c.router, sendPing)
 	// TODO: do something better than the below...
 	res := <-resCh
 	if res != nil {
diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go
index 4c222266..0a4b84aa 100644
--- a/src/yggdrasil/conn.go
+++ b/src/yggdrasil/conn.go
@@ -92,7 +92,7 @@ func (c *Conn) setMTU(from phony.Actor, mtu uint16) {
 func (c *Conn) search() error {
 	var sinfo *searchInfo
 	var isIn bool
-	phony.Block(c.core.router, func() {
+	phony.Block(&c.core.router, func() {
 		sinfo, isIn = c.core.router.searches.searches[*c.nodeID]
 	})
 	if !isIn {
@@ -108,7 +108,7 @@ func (c *Conn) search() error {
 			default:
 			}
 		}
-		phony.Block(c.core.router, func() {
+		phony.Block(&c.core.router, func() {
 			sinfo = c.core.router.searches.newIterSearch(c.nodeID, c.nodeMask, searchCompleted)
 			sinfo.continueSearch()
 		})
diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index 3719818b..b466e3e8 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -176,7 +176,7 @@ func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState,
 
 	c.config.Mutex.RLock()
 	if c.config.Current.SwitchOptions.MaxTotalQueueSize >= SwitchQueueTotalMinSize {
-		phony.Block(c.switchTable, func() {
+		phony.Block(&c.switchTable, func() {
 			c.switchTable.queues.totalMaxSize = c.config.Current.SwitchOptions.MaxTotalQueueSize
 		})
 	}
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 72ff2847..8bd638c4 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -149,7 +149,7 @@ func (ps *peers) removePeer(port switchPort) {
 	if port == 0 {
 		return
 	} // Can't remove self peer
-	phony.Block(ps.core.router, func() {
+	phony.Block(&ps.core.router, func() {
 		ps.core.switchTable.forgetPeer(port)
 	})
 	ps.mutex.Lock()

From e553f3e013ecb5146f1fe9512a135446d8ea30ef Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 28 Aug 2019 12:46:12 +0100
Subject: [PATCH 53/62] Reconfigure functions now ran by actors

---
 src/yggdrasil/core.go    | 42 +++++++++++++++++++++++++---------------
 src/yggdrasil/link.go    |  2 +-
 src/yggdrasil/router.go  | 12 +++++-------
 src/yggdrasil/session.go | 10 ++++------
 4 files changed, 36 insertions(+), 30 deletions(-)

diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index b466e3e8..d48b6479 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -112,29 +112,39 @@ func (c *Core) addPeerLoop() {
 // config.NodeConfig and then signals the various module goroutines to
 // reconfigure themselves if needed.
 func (c *Core) UpdateConfig(config *config.NodeConfig) {
-	c.log.Debugln("Reloading node configuration...")
+	c.log.Infoln("Reloading node configuration...")
 
 	c.config.Replace(*config)
-
 	errors := 0
 
 	// Each reconfigure function should pass any errors to the channel, then close it
-	components := []func(chan error){
-		c.link.reconfigure,
-		c.peers.reconfigure,
-		c.router.reconfigure,
-		c.router.dht.reconfigure,
-		c.router.searches.reconfigure,
-		c.router.sessions.reconfigure,
-		c.switchTable.reconfigure,
+	components := map[phony.Actor][]func(chan error){
+		&c.router: []func(chan error){
+			c.router.reconfigure,
+			c.router.dht.reconfigure,
+			c.router.searches.reconfigure,
+			c.router.sessions.reconfigure,
+		},
+		&c.switchTable: []func(chan error){
+			c.switchTable.reconfigure,
+			c.link.reconfigure,
+			c.peers.reconfigure,
+		},
 	}
 
-	for _, component := range components {
-		response := make(chan error)
-		go component(response)
-		for err := range response {
-			c.log.Errorln(err)
-			errors++
+	// TODO: We count errors here but honestly that provides us with absolutely no
+	// benefit over components reporting errors themselves, so maybe we can use
+	// actor.Act() here instead and stop counting errors
+	for actor, functions := range components {
+		for _, function := range functions {
+			response := make(chan error)
+			phony.Block(actor, func() {
+				function(response)
+			})
+			for err := range response {
+				c.log.Errorln(err)
+				errors++
+			}
 		}
 	}
 
diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index e65055bc..ee4b9815 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -82,7 +82,7 @@ func (l *link) init(c *Core) error {
 func (l *link) reconfigure(e chan error) {
 	defer close(e)
 	tcpResponse := make(chan error)
-	go l.tcp.reconfigure(tcpResponse)
+	l.tcp.reconfigure(tcpResponse)
 	for err := range tcpResponse {
 		e <- err
 	}
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index d93ba433..5f894d9b 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -77,13 +77,11 @@ func (r *router) reconfigure(e chan error) {
 	defer close(e)
 	var errs []error
 	// Reconfigure the router
-	phony.Block(r, func() {
-		current := r.core.config.GetCurrent()
-		err := r.nodeinfo.setNodeInfo(current.NodeInfo, current.NodeInfoPrivacy)
-		if err != nil {
-			errs = append(errs, err)
-		}
-	})
+	current := r.core.config.GetCurrent()
+	err := r.nodeinfo.setNodeInfo(current.NodeInfo, current.NodeInfoPrivacy)
+	if err != nil {
+		errs = append(errs, err)
+	}
 	for _, err := range errs {
 		e <- err
 	}
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 9abfeca6..f6855b4d 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -164,12 +164,10 @@ func (ss *sessions) init(r *router) {
 func (ss *sessions) reconfigure(e chan error) {
 	defer close(e)
 	responses := make(map[crypto.Handle]chan error)
-	phony.Block(ss.router, func() {
-		for index, session := range ss.sinfos {
-			responses[index] = make(chan error)
-			go session.reconfigure(responses[index])
-		}
-	})
+	for index, session := range ss.sinfos {
+		responses[index] = make(chan error)
+		session.reconfigure(responses[index])
+	}
 	for _, response := range responses {
 		for err := range response {
 			e <- err

From 881d0a1ada34fa6fc57cf1c182995a7ca1686954 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 28 Aug 2019 12:46:49 +0100
Subject: [PATCH 54/62] Fix DEBUG_getDHTSize

---
 src/yggdrasil/debug.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/yggdrasil/debug.go b/src/yggdrasil/debug.go
index 4502c11e..8fdb2786 100644
--- a/src/yggdrasil/debug.go
+++ b/src/yggdrasil/debug.go
@@ -259,7 +259,7 @@ func DEBUG_wire_encode_coords(coords []byte) []byte {
 
 func (c *Core) DEBUG_getDHTSize() int {
 	var total int
-	phony.Block(c.router, func() {
+	phony.Block(&c.router, func() {
 		total = len(c.router.dht.table)
 	})
 	return total

From 764f9c8e117f8ee6a3db99fbbc5b8e5ba7f94ba7 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 28 Aug 2019 17:24:41 +0100
Subject: [PATCH 55/62] Remove legacy debug functions

---
 src/yggdrasil/debug.go | 606 -----------------------------------------
 1 file changed, 606 deletions(-)

diff --git a/src/yggdrasil/debug.go b/src/yggdrasil/debug.go
index 8fdb2786..b9bd5cfb 100644
--- a/src/yggdrasil/debug.go
+++ b/src/yggdrasil/debug.go
@@ -2,20 +2,7 @@
 
 package yggdrasil
 
-// These are functions that should not exist
-// They are (or were) used during development, to work around missing features
-// They're also used to configure things from the outside
-//  It would be better to define and export a few config functions elsewhere
-//  Or define some remote API and call it to send/request configuration info
-
-import _ "golang.org/x/net/ipv6" // TODO put this somewhere better
-
-//import "golang.org/x/net/proxy"
-
 import "fmt"
-import "net"
-import "regexp"
-import "encoding/hex"
 
 import _ "net/http/pprof"
 import "net/http"
@@ -24,13 +11,6 @@ import "os"
 
 import "github.com/gologme/log"
 
-import "github.com/yggdrasil-network/yggdrasil-go/src/address"
-import "github.com/yggdrasil-network/yggdrasil-go/src/config"
-import "github.com/yggdrasil-network/yggdrasil-go/src/crypto"
-import "github.com/yggdrasil-network/yggdrasil-go/src/defaults"
-
-import "github.com/Arceliar/phony"
-
 // Start the profiler in debug builds, if the required environment variable is set.
 func init() {
 	envVarName := "PPROFLISTEN"
@@ -51,589 +31,3 @@ func StartProfiler(log *log.Logger) error {
 	go func() { log.Println(http.ListenAndServe("localhost:6060", nil)) }()
 	return nil
 }
-
-// This function is only called by the simulator to set up a node with random
-// keys. It should not be used and may be removed in the future.
-func (c *Core) Init() {
-	bpub, bpriv := crypto.NewBoxKeys()
-	spub, spriv := crypto.NewSigKeys()
-	hbpub := hex.EncodeToString(bpub[:])
-	hbpriv := hex.EncodeToString(bpriv[:])
-	hspub := hex.EncodeToString(spub[:])
-	hspriv := hex.EncodeToString(spriv[:])
-	cfg := config.NodeConfig{
-		EncryptionPublicKey:  hbpub,
-		EncryptionPrivateKey: hbpriv,
-		SigningPublicKey:     hspub,
-		SigningPrivateKey:    hspriv,
-	}
-	c.config = config.NodeState{
-		Current:  cfg,
-		Previous: cfg,
-	}
-	c.init()
-	c.switchTable.start()
-	c.router.start()
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
-// Core
-
-func (c *Core) DEBUG_getSigningPublicKey() crypto.SigPubKey {
-	return (crypto.SigPubKey)(c.sigPub)
-}
-
-func (c *Core) DEBUG_getEncryptionPublicKey() crypto.BoxPubKey {
-	return (crypto.BoxPubKey)(c.boxPub)
-}
-
-/*
-func (c *Core) DEBUG_getSend() chan<- []byte {
-	return c.router.tun.send
-}
-
-func (c *Core) DEBUG_getRecv() <-chan []byte {
-	return c.router.tun.recv
-}
-*/
-
-// Peer
-
-func (c *Core) DEBUG_getPeers() *peers {
-	return &c.peers
-}
-
-func (ps *peers) DEBUG_newPeer(box crypto.BoxPubKey, sig crypto.SigPubKey, link crypto.BoxSharedKey) *peer {
-	sim := linkInterface{
-		name: "(simulator)",
-		info: linkInfo{
-			local:    "(simulator)",
-			remote:   "(simulator)",
-			linkType: "sim",
-		},
-	}
-	return ps.newPeer(&box, &sig, &link, &sim, nil)
-}
-
-/*
-func (ps *peers) DEBUG_startPeers() {
-  ps.mutex.RLock()
-  defer ps.mutex.RUnlock()
-  for _, p := range ps.ports {
-    if p == nil { continue }
-    go p.MainLoop()
-  }
-}
-*/
-
-func (ps *peers) DEBUG_hasPeer(key crypto.SigPubKey) bool {
-	ports := ps.ports.Load().(map[switchPort]*peer)
-	for _, p := range ports {
-		if p == nil {
-			continue
-		}
-		if p.sig == key {
-			return true
-		}
-	}
-	return false
-}
-
-func (ps *peers) DEBUG_getPorts() map[switchPort]*peer {
-	ports := ps.ports.Load().(map[switchPort]*peer)
-	newPeers := make(map[switchPort]*peer)
-	for port, p := range ports {
-		newPeers[port] = p
-	}
-	return newPeers
-}
-
-func (p *peer) DEBUG_getSigKey() crypto.SigPubKey {
-	return p.sig
-}
-
-func (p *peer) DEEBUG_getPort() switchPort {
-	return p.port
-}
-
-// Router
-
-func (c *Core) DEBUG_getSwitchTable() *switchTable {
-	return &c.switchTable
-}
-
-func (c *Core) DEBUG_getLocator() switchLocator {
-	return c.switchTable.getLocator()
-}
-
-func (l *switchLocator) DEBUG_getCoords() []byte {
-	return l.getCoords()
-}
-
-func (c *Core) DEBUG_switchLookup(dest []byte) switchPort {
-	return c.switchTable.DEBUG_lookup(dest)
-}
-
-// This does the switch layer lookups that decide how to route traffic.
-// Traffic uses greedy routing in a metric space, where the metric distance between nodes is equal to the distance between them on the tree.
-// Traffic must be routed to a node that is closer to the destination via the metric space distance.
-// In the event that two nodes are equally close, it gets routed to the one with the longest uptime (due to the order that things are iterated over).
-// The size of the outgoing packet queue is added to a node's tree distance when the cost of forwarding to a node, subject to the constraint that the real tree distance puts them closer to the destination than ourself.
-// Doing so adds a limited form of backpressure routing, based on local information, which allows us to forward traffic around *local* bottlenecks, provided that another greedy path exists.
-func (t *switchTable) DEBUG_lookup(dest []byte) switchPort {
-	table := t.getTable()
-	myDist := table.self.dist(dest)
-	if myDist == 0 {
-		return 0
-	}
-	// cost is in units of (expected distance) + (expected queue size), where expected distance is used as an approximation of the minimum backpressure gradient needed for packets to flow
-	ports := t.core.peers.getPorts()
-	var best switchPort
-	bestCost := int64(^uint64(0) >> 1)
-	for _, info := range table.elems {
-		dist := info.locator.dist(dest)
-		if !(dist < myDist) {
-			continue
-		}
-		//p, isIn := ports[info.port]
-		_, isIn := ports[info.port]
-		if !isIn {
-			continue
-		}
-		cost := int64(dist) // + p.getQueueSize()
-		if cost < bestCost {
-			best = info.port
-			bestCost = cost
-		}
-	}
-	return best
-}
-
-/*
-func (t *switchTable) DEBUG_isDirty() bool {
-  //data := t.data.Load().(*tabledata)
-  t.mutex.RLock()
-  defer t.mutex.RUnlock()
-  data := t.data
-  return data.dirty
-}
-*/
-
-func (t *switchTable) DEBUG_dumpTable() {
-	//data := t.data.Load().(*tabledata)
-	t.mutex.RLock()
-	defer t.mutex.RUnlock()
-	data := t.data
-	for _, peer := range data.peers {
-		//fmt.Println("DUMPTABLE:", t.treeID, peer.treeID, peer.port,
-		//            peer.locator.Root, peer.coords,
-		//            peer.reverse.Root, peer.reverse.Coords, peer.forward)
-		fmt.Println("DUMPTABLE:", t.key, peer.key, peer.locator.coords, peer.port /*, peer.forward*/)
-	}
-}
-
-func (t *switchTable) DEBUG_getReversePort(port switchPort) switchPort {
-	// Returns Port(0) if it cannot get the reverse peer for any reason
-	//data := t.data.Load().(*tabledata)
-	t.mutex.RLock()
-	defer t.mutex.RUnlock()
-	data := t.data
-	if port >= switchPort(len(data.peers)) {
-		return switchPort(0)
-	}
-	pinfo := data.peers[port]
-	if len(pinfo.locator.coords) < 1 {
-		return switchPort(0)
-	}
-	return pinfo.locator.coords[len(pinfo.locator.coords)-1]
-}
-
-// Wire
-
-func DEBUG_wire_encode_coords(coords []byte) []byte {
-	return wire_encode_coords(coords)
-}
-
-// DHT, via core
-
-func (c *Core) DEBUG_getDHTSize() int {
-	var total int
-	phony.Block(&c.router, func() {
-		total = len(c.router.dht.table)
-	})
-	return total
-}
-
-// TUN defaults
-
-func (c *Core) DEBUG_GetTUNDefaultIfName() string {
-	return defaults.GetDefaults().DefaultIfName
-}
-
-func (c *Core) DEBUG_GetTUNDefaultIfMTU() int {
-	return defaults.GetDefaults().DefaultIfMTU
-}
-
-func (c *Core) DEBUG_GetTUNDefaultIfTAPMode() bool {
-	return defaults.GetDefaults().DefaultIfTAPMode
-}
-
-// udpInterface
-//  FIXME udpInterface isn't exported
-//  So debug functions need to work differently...
-
-/*
-func (c *Core) DEBUG_setupLoopbackUDPInterface() {
-  iface := udpInterface{}
-  iface.init(c, "[::1]:0")
-  c.ifaces = append(c.ifaces[:0], &iface)
-}
-*/
-
-/*
-func (c *Core) DEBUG_getLoopbackAddr() net.Addr {
-  iface := c.ifaces[0]
-  return iface.sock.LocalAddr()
-}
-*/
-
-/*
-func (c *Core) DEBUG_addLoopbackPeer(addr *net.UDPAddr,
-                                     in (chan<- []byte),
-                                     out (<-chan []byte)) {
-  iface := c.ifaces[0]
-  iface.addPeer(addr, in, out)
-}
-*/
-
-/*
-func (c *Core) DEBUG_startLoopbackUDPInterface() {
-  iface := c.ifaces[0]
-  go iface.reader()
-  for addr, chs := range iface.peers {
-    udpAddr, err := net.ResolveUDPAddr("udp6", addr)
-    if err != nil { panic(err) }
-    go iface.writer(udpAddr, chs.out)
-  }
-}
-*/
-
-////////////////////////////////////////////////////////////////////////////////
-
-func (c *Core) DEBUG_getAddr() *address.Address {
-	return address.AddrForNodeID(&c.router.dht.nodeID)
-}
-
-/*
-func (c *Core) DEBUG_startTun(ifname string, iftapmode bool) {
-	c.DEBUG_startTunWithMTU(ifname, iftapmode, 1280)
-}
-
-func (c *Core) DEBUG_startTunWithMTU(ifname string, iftapmode bool, mtu int) {
-	addr := c.DEBUG_getAddr()
-	straddr := fmt.Sprintf("%s/%v", net.IP(addr[:]).String(), 8*len(address.GetPrefix()))
-	if ifname != "none" {
-		err := c.router.tun.setup(ifname, iftapmode, straddr, mtu)
-		if err != nil {
-			panic(err)
-		}
-		c.log.Println("Setup TUN/TAP:", c.router.tun.iface.Name(), straddr)
-		go func() { panic(c.router.tun.read()) }()
-	}
-	go func() { panic(c.router.tun.write()) }()
-}
-
-func (c *Core) DEBUG_stopTun() {
-	c.router.tun.close()
-}
-*/
-
-////////////////////////////////////////////////////////////////////////////////
-
-func (c *Core) DEBUG_newBoxKeys() (*crypto.BoxPubKey, *crypto.BoxPrivKey) {
-	return crypto.NewBoxKeys()
-}
-
-func (c *Core) DEBUG_getSharedKey(myPrivKey *crypto.BoxPrivKey, othersPubKey *crypto.BoxPubKey) *crypto.BoxSharedKey {
-	return crypto.GetSharedKey(myPrivKey, othersPubKey)
-}
-
-func (c *Core) DEBUG_newSigKeys() (*crypto.SigPubKey, *crypto.SigPrivKey) {
-	return crypto.NewSigKeys()
-}
-
-func (c *Core) DEBUG_getNodeID(pub *crypto.BoxPubKey) *crypto.NodeID {
-	return crypto.GetNodeID(pub)
-}
-
-func (c *Core) DEBUG_getTreeID(pub *crypto.SigPubKey) *crypto.TreeID {
-	return crypto.GetTreeID(pub)
-}
-
-func (c *Core) DEBUG_addrForNodeID(nodeID *crypto.NodeID) string {
-	return net.IP(address.AddrForNodeID(nodeID)[:]).String()
-}
-
-func (c *Core) DEBUG_init(bpub []byte,
-	bpriv []byte,
-	spub []byte,
-	spriv []byte) {
-	/*var boxPub crypto.BoxPubKey
-	var boxPriv crypto.BoxPrivKey
-	var sigPub crypto.SigPubKey
-	var sigPriv crypto.SigPrivKey
-	copy(boxPub[:], bpub)
-	copy(boxPriv[:], bpriv)
-	copy(sigPub[:], spub)
-	copy(sigPriv[:], spriv)
-	c.init(&boxPub, &boxPriv, &sigPub, &sigPriv)*/
-	hbpub := hex.EncodeToString(bpub[:])
-	hbpriv := hex.EncodeToString(bpriv[:])
-	hspub := hex.EncodeToString(spub[:])
-	hspriv := hex.EncodeToString(spriv[:])
-	cfg := config.NodeConfig{
-		EncryptionPublicKey:  hbpub,
-		EncryptionPrivateKey: hbpriv,
-		SigningPublicKey:     hspub,
-		SigningPrivateKey:    hspriv,
-	}
-	c.config = config.NodeState{
-		Current:  cfg,
-		Previous: cfg,
-	}
-	c.init()
-
-	if err := c.router.start(); err != nil {
-		panic(err)
-	}
-
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
-/*
-func (c *Core) DEBUG_setupAndStartGlobalUDPInterface(addrport string) {
-	if err := c.udp.init(c, addrport); err != nil {
-		c.log.Println("Failed to start UDP interface:", err)
-		panic(err)
-	}
-}
-
-func (c *Core) DEBUG_getGlobalUDPAddr() *net.UDPAddr {
-	return c.udp.sock.LocalAddr().(*net.UDPAddr)
-}
-
-func (c *Core) DEBUG_maybeSendUDPKeys(saddr string) {
-	udpAddr, err := net.ResolveUDPAddr("udp", saddr)
-	if err != nil {
-		panic(err)
-	}
-	var addr connAddr
-	addr.fromUDPAddr(udpAddr)
-	c.udp.mutex.RLock()
-	_, isIn := c.udp.conns[addr]
-	c.udp.mutex.RUnlock()
-	if !isIn {
-		c.udp.sendKeys(addr)
-	}
-}
-*/
-
-////////////////////////////////////////////////////////////////////////////////
-/*
-func (c *Core) DEBUG_addPeer(addr string) {
-	err := c.admin.addPeer(addr, "")
-	if err != nil {
-		panic(err)
-	}
-}
-*/
-/*
-func (c *Core) DEBUG_addSOCKSConn(socksaddr, peeraddr string) {
-	go func() {
-		dialer, err := proxy.SOCKS5("tcp", socksaddr, nil, proxy.Direct)
-		if err == nil {
-			conn, err := dialer.Dial("tcp", peeraddr)
-			if err == nil {
-				c.tcp.callWithConn(&wrappedConn{
-					c: conn,
-					raddr: &wrappedAddr{
-						network: "tcp",
-						addr:    peeraddr,
-					},
-				})
-			}
-		}
-	}()
-}
-*/
-
-/*
-func (c *Core) DEBUG_setupAndStartGlobalTCPInterface(addrport string) {
-	c.config.Listen = []string{addrport}
-	if err := c.link.init(c); err != nil {
-		c.log.Println("Failed to start interfaces:", err)
-		panic(err)
-	}
-}
-
-func (c *Core) DEBUG_getGlobalTCPAddr() *net.TCPAddr {
-	return c.link.tcp.getAddr()
-}
-
-func (c *Core) DEBUG_addTCPConn(saddr string) {
-	c.link.tcp.call(saddr, nil, "")
-}
-
-//*/
-
-/*
-func (c *Core) DEBUG_startSelfPeer() {
-  c.Peers.mutex.RLock()
-  defer c.Peers.mutex.RUnlock()
-  p := c.Peers.ports[0]
-  go p.MainLoop()
-}
-*/
-
-////////////////////////////////////////////////////////////////////////////////
-
-/*
-func (c *Core) DEBUG_setupAndStartGlobalKCPInterface(addrport string) {
-  iface := kcpInterface{}
-  iface.init(c, addrport)
-  c.kcp = &iface
-}
-
-func (c *Core) DEBUG_getGlobalKCPAddr() net.Addr {
-  return c.kcp.serv.Addr()
-}
-
-func (c *Core) DEBUG_addKCPConn(saddr string) {
-  c.kcp.call(saddr)
-}
-*/
-
-////////////////////////////////////////////////////////////////////////////////
-
-/*
-func (c *Core) DEBUG_setupAndStartAdminInterface(addrport string) {
-	a := admin{}
-	c.config.AdminListen = addrport
-	a.init()
-	c.admin = a
-}
-
-func (c *Core) DEBUG_setupAndStartMulticastInterface() {
-	m := multicast{}
-	m.init(c)
-	c.multicast = m
-	m.start()
-}
-*/
-
-////////////////////////////////////////////////////////////////////////////////
-
-func (c *Core) DEBUG_setLogger(log *log.Logger) {
-	c.log = log
-}
-
-func (c *Core) DEBUG_setIfceExpr(expr *regexp.Regexp) {
-	c.log.Println("DEBUG_setIfceExpr no longer implemented")
-}
-
-/*
-func (c *Core) DEBUG_addAllowedEncryptionPublicKey(boxStr string) {
-	err := c.admin.addAllowedEncryptionPublicKey(boxStr)
-	if err != nil {
-		panic(err)
-	}
-}
-*/
-////////////////////////////////////////////////////////////////////////////////
-
-func DEBUG_simLinkPeers(p, q *peer) {
-	// Sets q.out() to point to p and starts p.start()
-	goWorkers := func(source, dest *peer) {
-		linkOut := make(chan []byte, 1)
-		source.linkOut = func(bs []byte) { linkOut <- bs }
-		send := make(chan []byte, 1)
-		source.out = func(bss [][]byte) {
-			for _, bs := range bss {
-				send <- bs
-			}
-		}
-		go source.start()
-		go func() {
-			var packets [][]byte
-			for {
-				select {
-				case packet := <-linkOut:
-					packets = append(packets, packet)
-					continue
-				case packet := <-send:
-					packets = append(packets, packet)
-					phony.Block(&source.core.switchTable, func() {
-						source.core.switchTable._idleIn(source.port)
-					})
-					continue
-				default:
-				}
-				if len(packets) > 0 {
-					phony.Block(dest, func() { dest._handlePacket(packets[0]) })
-					packets = packets[1:]
-					continue
-				}
-				select {
-				case packet := <-linkOut:
-					packets = append(packets, packet)
-				case packet := <-send:
-					packets = append(packets, packet)
-					phony.Block(&source.core.switchTable, func() {
-						source.core.switchTable._idleIn(source.port)
-					})
-				}
-			}
-		}()
-	}
-	goWorkers(p, q)
-	goWorkers(q, p)
-	phony.Block(&p.core.switchTable, func() {
-		p.core.switchTable._idleIn(p.port)
-	})
-	phony.Block(&q.core.switchTable, func() {
-		q.core.switchTable._idleIn(q.port)
-	})
-}
-
-/*
-func (c *Core) DEBUG_simFixMTU() {
-	c.router.tun.mtu = 65535
-}
-*/
-
-////////////////////////////////////////////////////////////////////////////////
-
-func Util_testAddrIDMask() {
-	for idx := 0; idx < 16; idx++ {
-		var orig crypto.NodeID
-		orig[8] = 42
-		for bidx := 0; bidx < idx; bidx++ {
-			orig[bidx/8] |= (0x80 >> uint8(bidx%8))
-		}
-		addr := address.AddrForNodeID(&orig)
-		nid, mask := addr.GetNodeIDandMask()
-		for b := 0; b < len(mask); b++ {
-			nid[b] &= mask[b]
-			orig[b] &= mask[b]
-		}
-		if *nid != orig {
-			fmt.Println(orig)
-			fmt.Println(*addr)
-			fmt.Println(*nid)
-			fmt.Println(*mask)
-			panic(idx)
-		}
-	}
-}

From fc9a1c6c31b456fb55dd7b5c555975652573b44c Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 28 Aug 2019 19:31:04 +0100
Subject: [PATCH 56/62] Simplify reconfiguration

---
 cmd/yggdrasil/main.go      |  1 +
 src/multicast/multicast.go |  1 -
 src/tuntap/ckr.go          | 66 +++++++++++++++-----------------------
 src/tuntap/tun.go          | 25 +++------------
 src/yggdrasil/core.go      | 44 ++++---------------------
 src/yggdrasil/dht.go       |  3 +-
 src/yggdrasil/link.go      |  9 ++----
 src/yggdrasil/peer.go      |  3 +-
 src/yggdrasil/router.go    | 20 ++++++------
 src/yggdrasil/search.go    |  3 +-
 src/yggdrasil/session.go   | 17 +++-------
 src/yggdrasil/switch.go    |  5 +--
 src/yggdrasil/tcp.go       |  8 +++--
 13 files changed, 64 insertions(+), 141 deletions(-)

diff --git a/cmd/yggdrasil/main.go b/cmd/yggdrasil/main.go
index 2fa10720..e9a21c13 100644
--- a/cmd/yggdrasil/main.go
+++ b/cmd/yggdrasil/main.go
@@ -279,6 +279,7 @@ func main() {
 		case _ = <-r:
 			if *useconffile != "" {
 				cfg = readConfig(useconf, useconffile, normaliseconf)
+				logger.Infoln("Reloading configuration from", *useconffile)
 				n.core.UpdateConfig(cfg)
 				n.tuntap.UpdateConfig(cfg)
 				n.multicast.UpdateConfig(cfg)
diff --git a/src/multicast/multicast.go b/src/multicast/multicast.go
index 5ab9673e..22ac9356 100644
--- a/src/multicast/multicast.go
+++ b/src/multicast/multicast.go
@@ -83,7 +83,6 @@ func (m *Multicast) Stop() error {
 func (m *Multicast) UpdateConfig(config *config.NodeConfig) {
 	m.log.Debugln("Reloading multicast configuration...")
 	m.config.Replace(*config)
-	m.log.Infoln("Multicast configuration reloaded successfully")
 }
 
 // GetInterfaces returns the currently known/enabled multicast interfaces. It is
diff --git a/src/tuntap/ckr.go b/src/tuntap/ckr.go
index 229d2607..9af3564a 100644
--- a/src/tuntap/ckr.go
+++ b/src/tuntap/ckr.go
@@ -20,7 +20,6 @@ import (
 type cryptokey struct {
 	tun          *TunAdapter
 	enabled      atomic.Value // bool
-	reconfigure  chan chan error
 	ipv4remotes  []cryptokey_route
 	ipv6remotes  []cryptokey_route
 	ipv4cache    map[address.Address]cryptokey_route
@@ -40,24 +39,11 @@ type cryptokey_route struct {
 // Initialise crypto-key routing. This must be done before any other CKR calls.
 func (c *cryptokey) init(tun *TunAdapter) {
 	c.tun = tun
-	c.reconfigure = make(chan chan error, 1)
-	go func() {
-		for {
-			e := <-c.reconfigure
-			e <- nil
-		}
-	}()
-
-	c.tun.log.Debugln("Configuring CKR...")
-	if err := c.configure(); err != nil {
-		c.tun.log.Errorln("CKR configuration failed:", err)
-	} else {
-		c.tun.log.Debugln("CKR configured")
-	}
+	c.configure()
 }
 
-// Configure the CKR routes.
-func (c *cryptokey) configure() error {
+// Configure the CKR routes. This should only ever be ran by the TUN/TAP actor.
+func (c *cryptokey) configure() {
 	current := c.tun.config.GetCurrent()
 
 	// Set enabled/disabled state
@@ -72,14 +58,14 @@ func (c *cryptokey) configure() error {
 	// Add IPv6 routes
 	for ipv6, pubkey := range current.TunnelRouting.IPv6RemoteSubnets {
 		if err := c.addRemoteSubnet(ipv6, pubkey); err != nil {
-			return err
+			c.tun.log.Errorln("Error adding CKR IPv6 remote subnet:", err)
 		}
 	}
 
 	// Add IPv4 routes
 	for ipv4, pubkey := range current.TunnelRouting.IPv4RemoteSubnets {
 		if err := c.addRemoteSubnet(ipv4, pubkey); err != nil {
-			return err
+			c.tun.log.Errorln("Error adding CKR IPv4 remote subnet:", err)
 		}
 	}
 
@@ -93,7 +79,7 @@ func (c *cryptokey) configure() error {
 	c.ipv6locals = make([]net.IPNet, 0)
 	for _, source := range current.TunnelRouting.IPv6LocalSubnets {
 		if err := c.addLocalSubnet(source); err != nil {
-			return err
+			c.tun.log.Errorln("Error adding CKR IPv6 local subnet:", err)
 		}
 	}
 
@@ -101,7 +87,7 @@ func (c *cryptokey) configure() error {
 	c.ipv4locals = make([]net.IPNet, 0)
 	for _, source := range current.TunnelRouting.IPv4LocalSubnets {
 		if err := c.addLocalSubnet(source); err != nil {
-			return err
+			c.tun.log.Errorln("Error adding CKR IPv4 local subnet:", err)
 		}
 	}
 
@@ -110,8 +96,6 @@ func (c *cryptokey) configure() error {
 	c.ipv4cache = make(map[address.Address]cryptokey_route, 0)
 	c.ipv6cache = make(map[address.Address]cryptokey_route, 0)
 	c.mutexcaches.Unlock()
-
-	return nil
 }
 
 // Enable or disable crypto-key routing.
@@ -181,19 +165,19 @@ func (c *cryptokey) addLocalSubnet(cidr string) error {
 	} else if prefixsize == net.IPv4len*8 {
 		routingsources = &c.ipv4locals
 	} else {
-		return errors.New("Unexpected prefix size")
+		return errors.New("unexpected prefix size")
 	}
 
 	// Check if we already have this CIDR
 	for _, subnet := range *routingsources {
 		if subnet.String() == ipnet.String() {
-			return errors.New("Source subnet already configured")
+			return errors.New("local subnet already configured")
 		}
 	}
 
 	// Add the source subnet
 	*routingsources = append(*routingsources, *ipnet)
-	c.tun.log.Infoln("Added CKR source subnet", cidr)
+	c.tun.log.Infoln("Added CKR local subnet", cidr)
 	return nil
 }
 
@@ -226,7 +210,7 @@ func (c *cryptokey) addRemoteSubnet(cidr string, dest string) error {
 		routingtable = &c.ipv4remotes
 		routingcache = &c.ipv4cache
 	} else {
-		return errors.New("Unexpected prefix size")
+		return errors.New("unexpected prefix size")
 	}
 
 	// Is the route an Yggdrasil destination?
@@ -235,19 +219,19 @@ func (c *cryptokey) addRemoteSubnet(cidr string, dest string) error {
 	copy(addr[:], ipaddr)
 	copy(snet[:], ipnet.IP)
 	if addr.IsValid() || snet.IsValid() {
-		return errors.New("Can't specify Yggdrasil destination as crypto-key route")
+		return errors.New("can't specify Yggdrasil destination as crypto-key route")
 	}
 	// Do we already have a route for this subnet?
 	for _, route := range *routingtable {
 		if route.subnet.String() == ipnet.String() {
-			return errors.New(fmt.Sprintf("Route already exists for %s", cidr))
+			return fmt.Errorf("remote subnet already exists for %s", cidr)
 		}
 	}
 	// Decode the public key
 	if bpk, err := hex.DecodeString(dest); err != nil {
 		return err
 	} else if len(bpk) != crypto.BoxPubKeyLen {
-		return errors.New(fmt.Sprintf("Incorrect key length for %s", dest))
+		return fmt.Errorf("incorrect key length for %s", dest)
 	} else {
 		// Add the new crypto-key route
 		var key crypto.BoxPubKey
@@ -270,7 +254,7 @@ func (c *cryptokey) addRemoteSubnet(cidr string, dest string) error {
 			delete(*routingcache, k)
 		}
 
-		c.tun.log.Infoln("Added CKR destination subnet", cidr)
+		c.tun.log.Infoln("Added CKR remote subnet", cidr)
 		return nil
 	}
 }
@@ -284,7 +268,7 @@ func (c *cryptokey) getPublicKeyForAddress(addr address.Address, addrlen int) (c
 	// Check if the address is a valid Yggdrasil address - if so it
 	// is exempt from all CKR checking
 	if addr.IsValid() {
-		return crypto.BoxPubKey{}, errors.New("Cannot look up CKR for Yggdrasil addresses")
+		return crypto.BoxPubKey{}, errors.New("cannot look up CKR for Yggdrasil addresses")
 	}
 
 	// Build our references to the routing table and cache
@@ -297,7 +281,7 @@ func (c *cryptokey) getPublicKeyForAddress(addr address.Address, addrlen int) (c
 	} else if addrlen == net.IPv4len {
 		routingcache = &c.ipv4cache
 	} else {
-		return crypto.BoxPubKey{}, errors.New("Unexpected prefix size")
+		return crypto.BoxPubKey{}, errors.New("unexpected prefix size")
 	}
 
 	// Check if there's a cache entry for this addr
@@ -317,7 +301,7 @@ func (c *cryptokey) getPublicKeyForAddress(addr address.Address, addrlen int) (c
 	} else if addrlen == net.IPv4len {
 		routingtable = &c.ipv4remotes
 	} else {
-		return crypto.BoxPubKey{}, errors.New("Unexpected prefix size")
+		return crypto.BoxPubKey{}, errors.New("unexpected prefix size")
 	}
 
 	// No cache was found - start by converting the address into a net.IP
@@ -378,18 +362,18 @@ func (c *cryptokey) removeLocalSubnet(cidr string) error {
 	} else if prefixsize == net.IPv4len*8 {
 		routingsources = &c.ipv4locals
 	} else {
-		return errors.New("Unexpected prefix size")
+		return errors.New("unexpected prefix size")
 	}
 
 	// Check if we already have this CIDR
 	for idx, subnet := range *routingsources {
 		if subnet.String() == ipnet.String() {
 			*routingsources = append((*routingsources)[:idx], (*routingsources)[idx+1:]...)
-			c.tun.log.Infoln("Removed CKR source subnet", cidr)
+			c.tun.log.Infoln("Removed CKR local subnet", cidr)
 			return nil
 		}
 	}
-	return errors.New("Source subnet not found")
+	return errors.New("local subnet not found")
 }
 
 // Removes a destination route for the given CIDR to be tunnelled to the node
@@ -421,7 +405,7 @@ func (c *cryptokey) removeRemoteSubnet(cidr string, dest string) error {
 		routingtable = &c.ipv4remotes
 		routingcache = &c.ipv4cache
 	} else {
-		return errors.New("Unexpected prefix size")
+		return errors.New("unexpected prefix size")
 	}
 
 	// Decode the public key
@@ -429,7 +413,7 @@ func (c *cryptokey) removeRemoteSubnet(cidr string, dest string) error {
 	if err != nil {
 		return err
 	} else if len(bpk) != crypto.BoxPubKeyLen {
-		return errors.New(fmt.Sprintf("Incorrect key length for %s", dest))
+		return fmt.Errorf("incorrect key length for %s", dest)
 	}
 	netStr := ipnet.String()
 
@@ -439,9 +423,9 @@ func (c *cryptokey) removeRemoteSubnet(cidr string, dest string) error {
 			for k := range *routingcache {
 				delete(*routingcache, k)
 			}
-			c.tun.log.Infof("Removed CKR destination subnet %s via %s\n", cidr, dest)
+			c.tun.log.Infof("Removed CKR remote subnet %s via %s\n", cidr, dest)
 			return nil
 		}
 	}
-	return errors.New(fmt.Sprintf("Route does not exists for %s", cidr))
+	return fmt.Errorf("route does not exists for %s", cidr)
 }
diff --git a/src/tuntap/tun.go b/src/tuntap/tun.go
index 6317459c..8e1e5b0c 100644
--- a/src/tuntap/tun.go
+++ b/src/tuntap/tun.go
@@ -13,6 +13,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+
 	//"sync"
 
 	"github.com/Arceliar/phony"
@@ -200,29 +201,11 @@ func (tun *TunAdapter) _stop() error {
 func (tun *TunAdapter) UpdateConfig(config *config.NodeConfig) {
 	tun.log.Debugln("Reloading TUN/TAP configuration...")
 
+	// Replace the active configuration with the supplied one
 	tun.config.Replace(*config)
 
-	errors := 0
-
-	components := []chan chan error{
-		tun.reconfigure,
-		tun.ckr.reconfigure,
-	}
-
-	for _, component := range components {
-		response := make(chan error)
-		component <- response
-		if err := <-response; err != nil {
-			tun.log.Errorln(err)
-			errors++
-		}
-	}
-
-	if errors > 0 {
-		tun.log.Warnln(errors, "TUN/TAP module(s) reported errors during configuration reload")
-	} else {
-		tun.log.Infoln("TUN/TAP configuration reloaded successfully")
-	}
+	// Notify children about the configuration change
+	tun.Act(nil, tun.ckr.configure)
 }
 
 func (tun *TunAdapter) handler() error {
diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index d48b6479..831109dd 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -20,6 +20,7 @@ type Core struct {
 	// This is the main data structure that holds everything else for a node
 	// We're going to keep our own copy of the provided config - that way we can
 	// guarantee that it will be covered by the mutex
+	phony.Inbox
 	config      config.NodeState // Config
 	boxPub      crypto.BoxPubKey
 	boxPriv     crypto.BoxPrivKey
@@ -112,47 +113,14 @@ func (c *Core) addPeerLoop() {
 // config.NodeConfig and then signals the various module goroutines to
 // reconfigure themselves if needed.
 func (c *Core) UpdateConfig(config *config.NodeConfig) {
-	c.log.Infoln("Reloading node configuration...")
+	c.log.Debugln("Reloading node configuration...")
 
+	// Replace the active configuration with the supplied one
 	c.config.Replace(*config)
-	errors := 0
 
-	// Each reconfigure function should pass any errors to the channel, then close it
-	components := map[phony.Actor][]func(chan error){
-		&c.router: []func(chan error){
-			c.router.reconfigure,
-			c.router.dht.reconfigure,
-			c.router.searches.reconfigure,
-			c.router.sessions.reconfigure,
-		},
-		&c.switchTable: []func(chan error){
-			c.switchTable.reconfigure,
-			c.link.reconfigure,
-			c.peers.reconfigure,
-		},
-	}
-
-	// TODO: We count errors here but honestly that provides us with absolutely no
-	// benefit over components reporting errors themselves, so maybe we can use
-	// actor.Act() here instead and stop counting errors
-	for actor, functions := range components {
-		for _, function := range functions {
-			response := make(chan error)
-			phony.Block(actor, func() {
-				function(response)
-			})
-			for err := range response {
-				c.log.Errorln(err)
-				errors++
-			}
-		}
-	}
-
-	if errors > 0 {
-		c.log.Warnln(errors, "node module(s) reported errors during configuration reload")
-	} else {
-		c.log.Infoln("Node configuration reloaded successfully")
-	}
+	// Notify the router and switch about the new configuration
+	c.router.Act(c, c.router.reconfigure)
+	c.switchTable.Act(c, c.switchTable.reconfigure)
 }
 
 // Start starts up Yggdrasil using the provided config.NodeConfig, and outputs
diff --git a/src/yggdrasil/dht.go b/src/yggdrasil/dht.go
index 4f380363..575c8b1a 100644
--- a/src/yggdrasil/dht.go
+++ b/src/yggdrasil/dht.go
@@ -82,8 +82,7 @@ func (t *dht) init(r *router) {
 	t.reset()
 }
 
-func (t *dht) reconfigure(e chan error) {
-	defer close(e)
+func (t *dht) reconfigure() {
 	// This is where reconfiguration would go, if we had anything to do
 }
 
diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go
index ee4b9815..6e393514 100644
--- a/src/yggdrasil/link.go
+++ b/src/yggdrasil/link.go
@@ -79,13 +79,8 @@ func (l *link) init(c *Core) error {
 	return nil
 }
 
-func (l *link) reconfigure(e chan error) {
-	defer close(e)
-	tcpResponse := make(chan error)
-	l.tcp.reconfigure(tcpResponse)
-	for err := range tcpResponse {
-		e <- err
-	}
+func (l *link) reconfigure() {
+	l.tcp.reconfigure()
 }
 
 func (l *link) call(uri string, sintf string) error {
diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go
index 8bd638c4..381e6917 100644
--- a/src/yggdrasil/peer.go
+++ b/src/yggdrasil/peer.go
@@ -34,8 +34,7 @@ func (ps *peers) init(c *Core) {
 	ps.core = c
 }
 
-func (ps *peers) reconfigure(e chan error) {
-	defer close(e)
+func (ps *peers) reconfigure() {
 	// This is where reconfiguration would go, if we had anything to do
 }
 
diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go
index 5f894d9b..64c81701 100644
--- a/src/yggdrasil/router.go
+++ b/src/yggdrasil/router.go
@@ -73,18 +73,20 @@ func (r *router) init(core *Core) {
 	r.sessions.init(r)
 }
 
-func (r *router) reconfigure(e chan error) {
-	defer close(e)
-	var errs []error
+// Reconfigures the router and any child modules. This should only ever be run
+// by the router actor.
+func (r *router) reconfigure() {
 	// Reconfigure the router
 	current := r.core.config.GetCurrent()
-	err := r.nodeinfo.setNodeInfo(current.NodeInfo, current.NodeInfoPrivacy)
-	if err != nil {
-		errs = append(errs, err)
-	}
-	for _, err := range errs {
-		e <- err
+	if err := r.nodeinfo.setNodeInfo(current.NodeInfo, current.NodeInfoPrivacy); err != nil {
+		r.core.log.Errorln("Error reloading NodeInfo:", err)
+	} else {
+		r.core.log.Infoln("NodeInfo updated")
 	}
+	// Reconfigure children
+	r.dht.reconfigure()
+	r.searches.reconfigure()
+	r.sessions.reconfigure()
 }
 
 // Starts the tickerLoop goroutine.
diff --git a/src/yggdrasil/search.go b/src/yggdrasil/search.go
index ca357cc2..c128175b 100644
--- a/src/yggdrasil/search.go
+++ b/src/yggdrasil/search.go
@@ -55,8 +55,7 @@ func (s *searches) init(r *router) {
 	s.searches = make(map[crypto.NodeID]*searchInfo)
 }
 
-func (s *searches) reconfigure(e chan error) {
-	defer close(e)
+func (s *searches) reconfigure() {
 	// This is where reconfiguration would go, if we had anything to do
 }
 
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index f6855b4d..0b55aac8 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -73,8 +73,7 @@ type sessionInfo struct {
 	callbacks      []chan func()                 // Finished work from crypto workers
 }
 
-func (sinfo *sessionInfo) reconfigure(e chan error) {
-	defer close(e)
+func (sinfo *sessionInfo) reconfigure() {
 	// This is where reconfiguration would go, if we had anything to do
 }
 
@@ -161,17 +160,9 @@ func (ss *sessions) init(r *router) {
 	ss.lastCleanup = time.Now()
 }
 
-func (ss *sessions) reconfigure(e chan error) {
-	defer close(e)
-	responses := make(map[crypto.Handle]chan error)
-	for index, session := range ss.sinfos {
-		responses[index] = make(chan error)
-		session.reconfigure(responses[index])
-	}
-	for _, response := range responses {
-		for err := range response {
-			e <- err
-		}
+func (ss *sessions) reconfigure() {
+	for _, session := range ss.sinfos {
+		session.reconfigure()
 	}
 }
 
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index 5c613d8d..b6bd5b96 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -199,9 +199,10 @@ func (t *switchTable) init(core *Core) {
 	})
 }
 
-func (t *switchTable) reconfigure(e chan error) {
-	defer close(e)
+func (t *switchTable) reconfigure() {
 	// This is where reconfiguration would go, if we had anything useful to do.
+	t.core.link.reconfigure()
+	t.core.peers.reconfigure()
 }
 
 // Safely gets a copy of this node's locator.
diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go
index ccb488f9..cce352bd 100644
--- a/src/yggdrasil/tcp.go
+++ b/src/yggdrasil/tcp.go
@@ -95,8 +95,7 @@ func (t *tcp) init(l *link) error {
 	return nil
 }
 
-func (t *tcp) reconfigure(e chan error) {
-	defer close(e)
+func (t *tcp) reconfigure() {
 	t.link.core.config.Mutex.RLock()
 	added := util.Difference(t.link.core.config.Current.Listen, t.link.core.config.Previous.Listen)
 	deleted := util.Difference(t.link.core.config.Previous.Listen, t.link.core.config.Current.Listen)
@@ -107,7 +106,9 @@ func (t *tcp) reconfigure(e chan error) {
 				continue
 			}
 			if _, err := t.listen(a[6:]); err != nil {
-				e <- err
+				t.link.core.log.Errorln("Error adding TCP", a[6:], "listener:", err)
+			} else {
+				t.link.core.log.Infoln("Started TCP listener:", a[6:])
 			}
 		}
 		for _, d := range deleted {
@@ -118,6 +119,7 @@ func (t *tcp) reconfigure(e chan error) {
 			if listener, ok := t.listeners[d[6:]]; ok {
 				t.mutex.Unlock()
 				listener.Stop <- true
+				t.link.core.log.Infoln("Stopped TCP listener:", d[6:])
 			} else {
 				t.mutex.Unlock()
 			}

From aa0770546eae3b3931326d9df8fea9724715f95e Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 28 Aug 2019 19:39:23 +0100
Subject: [PATCH 57/62] Move responsibility for configuring max queue size into
 switch

---
 src/yggdrasil/core.go   | 8 --------
 src/yggdrasil/switch.go | 8 +++++++-
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index 831109dd..ba06899b 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -152,14 +152,6 @@ func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState,
 		return nil, err
 	}
 
-	c.config.Mutex.RLock()
-	if c.config.Current.SwitchOptions.MaxTotalQueueSize >= SwitchQueueTotalMinSize {
-		phony.Block(&c.switchTable, func() {
-			c.switchTable.queues.totalMaxSize = c.config.Current.SwitchOptions.MaxTotalQueueSize
-		})
-	}
-	c.config.Mutex.RUnlock()
-
 	if err := c.switchTable.start(); err != nil {
 		c.log.Errorln("Failed to start switch")
 		return nil, err
diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go
index b6bd5b96..163c85cf 100644
--- a/src/yggdrasil/switch.go
+++ b/src/yggdrasil/switch.go
@@ -193,7 +193,13 @@ func (t *switchTable) init(core *Core) {
 	t.table.Store(lookupTable{})
 	t.drop = make(map[crypto.SigPubKey]int64)
 	phony.Block(t, func() {
-		t.queues.totalMaxSize = SwitchQueueTotalMinSize
+		core.config.Mutex.RLock()
+		if core.config.Current.SwitchOptions.MaxTotalQueueSize > SwitchQueueTotalMinSize {
+			t.queues.totalMaxSize = core.config.Current.SwitchOptions.MaxTotalQueueSize
+		} else {
+			t.queues.totalMaxSize = SwitchQueueTotalMinSize
+		}
+		core.config.Mutex.RUnlock()
 		t.queues.bufs = make(map[string]switch_buffer)
 		t.idle = make(map[switchPort]time.Time)
 	})

From 1f658cce76c9180104dcbdf93d3b04cedc05f5b1 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 28 Aug 2019 19:53:52 +0100
Subject: [PATCH 58/62] Add Core actor

---
 src/yggdrasil/core.go | 72 ++++++++++++++++++++++++++-----------------
 1 file changed, 43 insertions(+), 29 deletions(-)

diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go
index ba06899b..754d7d64 100644
--- a/src/yggdrasil/core.go
+++ b/src/yggdrasil/core.go
@@ -33,7 +33,7 @@ type Core struct {
 	log         *log.Logger
 }
 
-func (c *Core) init() error {
+func (c *Core) _init() error {
 	// TODO separate init and start functions
 	//  Init sets up structs
 	//  Start launches goroutines that depend on structs being set up
@@ -85,42 +85,44 @@ func (c *Core) init() error {
 // If any static peers were provided in the configuration above then we should
 // configure them. The loop ensures that disconnected peers will eventually
 // be reconnected with.
-func (c *Core) addPeerLoop() {
-	for {
-		//  the peers from the config - these could change!
-		current := c.config.GetCurrent()
+func (c *Core) _addPeerLoop() {
+	// Get the peers from the config - these could change!
+	current := c.config.GetCurrent()
 
-		// Add peers from the Peers section
-		for _, peer := range current.Peers {
-			go c.AddPeer(peer, "")
+	// Add peers from the Peers section
+	for _, peer := range current.Peers {
+		go c.AddPeer(peer, "") // TODO: this should be acted and not in a goroutine?
+		time.Sleep(time.Second)
+	}
+
+	// Add peers from the InterfacePeers section
+	for intf, intfpeers := range current.InterfacePeers {
+		for _, peer := range intfpeers {
+			go c.AddPeer(peer, intf) // TODO: this should be acted and not in a goroutine?
 			time.Sleep(time.Second)
 		}
-
-		// Add peers from the InterfacePeers section
-		for intf, intfpeers := range current.InterfacePeers {
-			for _, peer := range intfpeers {
-				go c.AddPeer(peer, intf)
-				time.Sleep(time.Second)
-			}
-		}
-
-		// Sit for a while
-		time.Sleep(time.Minute)
 	}
+
+	// Sit for a while
+	time.AfterFunc(time.Minute, func() {
+		c.Act(c, c._addPeerLoop)
+	})
 }
 
 // UpdateConfig updates the configuration in Core with the provided
 // config.NodeConfig and then signals the various module goroutines to
 // reconfigure themselves if needed.
 func (c *Core) UpdateConfig(config *config.NodeConfig) {
-	c.log.Debugln("Reloading node configuration...")
+	c.Act(nil, func() {
+		c.log.Debugln("Reloading node configuration...")
 
-	// Replace the active configuration with the supplied one
-	c.config.Replace(*config)
+		// Replace the active configuration with the supplied one
+		c.config.Replace(*config)
 
-	// Notify the router and switch about the new configuration
-	c.router.Act(c, c.router.reconfigure)
-	c.switchTable.Act(c, c.switchTable.reconfigure)
+		// Notify the router and switch about the new configuration
+		c.router.Act(c, c.router.reconfigure)
+		c.switchTable.Act(c, c.switchTable.reconfigure)
+	})
 }
 
 // Start starts up Yggdrasil using the provided config.NodeConfig, and outputs
@@ -128,7 +130,15 @@ func (c *Core) UpdateConfig(config *config.NodeConfig) {
 // TCP and UDP sockets, a multicast discovery socket, an admin socket, router,
 // switch and DHT node. A config.NodeState is returned which contains both the
 // current and previous configurations (from reconfigures).
-func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState, error) {
+func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) (conf *config.NodeState, err error) {
+	phony.Block(c, func() {
+		conf, err = c._start(nc, log)
+	})
+	return
+}
+
+// This function is unsafe and should only be ran by the core actor.
+func (c *Core) _start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState, error) {
 	c.log = log
 
 	c.config = config.NodeState{
@@ -144,8 +154,7 @@ func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState,
 	}
 
 	c.log.Infoln("Starting up...")
-
-	c.init()
+	c._init()
 
 	if err := c.link.init(c); err != nil {
 		c.log.Errorln("Failed to start link interfaces")
@@ -162,7 +171,7 @@ func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState,
 		return nil, err
 	}
 
-	go c.addPeerLoop()
+	c.Act(c, c._addPeerLoop)
 
 	c.log.Infoln("Startup complete")
 	return &c.config, nil
@@ -170,5 +179,10 @@ func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState,
 
 // Stop shuts down the Yggdrasil node.
 func (c *Core) Stop() {
+	phony.Block(c, c._stop)
+}
+
+// This function is unsafe and should only be ran by the core actor.
+func (c *Core) _stop() {
 	c.log.Infoln("Stopping...")
 }

From 7649ea0f9fe25f191018cc862a3c9341dac234ed Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Thu, 29 Aug 2019 21:59:28 -0500
Subject: [PATCH 59/62] remove sessionInfo.doFunc, have the api just use
 phony.Block instead

---
 src/yggdrasil/api.go     | 13 +------------
 src/yggdrasil/session.go |  5 -----
 2 files changed, 1 insertion(+), 17 deletions(-)

diff --git a/src/yggdrasil/api.go b/src/yggdrasil/api.go
index 9e82ece1..d1753b69 100644
--- a/src/yggdrasil/api.go
+++ b/src/yggdrasil/api.go
@@ -227,18 +227,7 @@ func (c *Core) GetSessions() []Session {
 				}
 				copy(session.PublicKey[:], sinfo.theirPermPub[:])
 			}
-			var skip bool
-			func() {
-				defer func() {
-					if recover() != nil {
-						skip = true
-					}
-				}()
-				sinfo.doFunc(workerFunc)
-			}()
-			if skip {
-				continue
-			}
+			phony.Block(sinfo, workerFunc)
 			// TODO? skipped known but timed out sessions?
 			sessions = append(sessions, session)
 		}
diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go
index 0b55aac8..d209a0de 100644
--- a/src/yggdrasil/session.go
+++ b/src/yggdrasil/session.go
@@ -77,11 +77,6 @@ func (sinfo *sessionInfo) reconfigure() {
 	// This is where reconfiguration would go, if we had anything to do
 }
 
-// TODO remove this, call SyncExec directly
-func (sinfo *sessionInfo) doFunc(f func()) {
-	phony.Block(sinfo, f)
-}
-
 // Represents a session ping/pong packet, andincludes information like public keys, a session handle, coords, a timestamp to prevent replays, and the tun/tap MTU.
 type sessionPing struct {
 	SendPermPub crypto.BoxPubKey // Sender's permanent key

From 9e4d4f33bab6186a6415801ef80e361c49765037 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Thu, 29 Aug 2019 23:30:39 -0500
Subject: [PATCH 60/62] upgrade to latest phony

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 9ff6031a..fc9549ff 100644
--- a/go.mod
+++ b/go.mod
@@ -1,7 +1,7 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
 require (
-	github.com/Arceliar/phony v0.0.0-20190828002416-0337564e2c44
+	github.com/Arceliar/phony v0.0.0-20190830042734-c3fdbc251992
 	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
 	github.com/hashicorp/go-syslog v1.0.0
 	github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222
diff --git a/go.sum b/go.sum
index 60388964..7499e926 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-github.com/Arceliar/phony v0.0.0-20190828002416-0337564e2c44 h1:8EiuIp65v8aLkLc4LWxtn4NTH+P2LwDmrKKWdBzn9cI=
-github.com/Arceliar/phony v0.0.0-20190828002416-0337564e2c44/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
+github.com/Arceliar/phony v0.0.0-20190830042734-c3fdbc251992 h1:sT4p8AJps69R/8Tio29ywFQSKcKGvrp3LvnvezxUDJU=
+github.com/Arceliar/phony v0.0.0-20190830042734-c3fdbc251992/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=

From 32633011efa20399d83852e5b64dfd71ed9c4084 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Fri, 30 Aug 2019 22:10:34 -0500
Subject: [PATCH 61/62] upgrade phony dependency

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index fc9549ff..44227a4f 100644
--- a/go.mod
+++ b/go.mod
@@ -1,7 +1,7 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
 require (
-	github.com/Arceliar/phony v0.0.0-20190830042734-c3fdbc251992
+	github.com/Arceliar/phony v0.0.0-20190831030740-7cac84315954
 	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
 	github.com/hashicorp/go-syslog v1.0.0
 	github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222
diff --git a/go.sum b/go.sum
index 7499e926..40977e6a 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-github.com/Arceliar/phony v0.0.0-20190830042734-c3fdbc251992 h1:sT4p8AJps69R/8Tio29ywFQSKcKGvrp3LvnvezxUDJU=
-github.com/Arceliar/phony v0.0.0-20190830042734-c3fdbc251992/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
+github.com/Arceliar/phony v0.0.0-20190831030740-7cac84315954 h1:kwhCKZy6GHYxuqUvmBbicaLlWsYXwqph8tkuaMrl5T4=
+github.com/Arceliar/phony v0.0.0-20190831030740-7cac84315954/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=

From 08f69de1e266c30fb0bdb8d24f8b9b4054903c73 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 31 Aug 2019 00:04:35 -0500
Subject: [PATCH 62/62] another phony update

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 44227a4f..3b0af5a4 100644
--- a/go.mod
+++ b/go.mod
@@ -1,7 +1,7 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
 require (
-	github.com/Arceliar/phony v0.0.0-20190831030740-7cac84315954
+	github.com/Arceliar/phony v0.0.0-20190831050304-94a6d3da9ba4
 	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
 	github.com/hashicorp/go-syslog v1.0.0
 	github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222
diff --git a/go.sum b/go.sum
index 40977e6a..df7ff630 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-github.com/Arceliar/phony v0.0.0-20190831030740-7cac84315954 h1:kwhCKZy6GHYxuqUvmBbicaLlWsYXwqph8tkuaMrl5T4=
-github.com/Arceliar/phony v0.0.0-20190831030740-7cac84315954/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
+github.com/Arceliar/phony v0.0.0-20190831050304-94a6d3da9ba4 h1:OePImnPRBqS6JiHuVVq4Rfvt2yyhqMpWCB63PrwGrJE=
+github.com/Arceliar/phony v0.0.0-20190831050304-94a6d3da9ba4/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=