Merge pull request #770 from yggdrasil-network/develop

Version 0.3.16

.circleci/config.yml

@@ -3,9 +3,22 @@
 # Check https://circleci.com/docs/2.0/language-go/ for more details
 version: 2.1
 jobs:
+  lint:
+    docker:
+      - image: circleci/golang:1.16
+
+    steps:
+      - checkout
+
+      - run:
+          name: Run golangci-lint
+          command: |
+              go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.31.0
+              golangci-lint run
+
   build-linux:
     docker:
-      - image: circleci/golang:1.13.3
+      - image: circleci/golang:1.16
 
     steps:
       - checkout
@@ -106,11 +119,11 @@ jobs:
               echo -e "Host *\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
 
       - run:
-          name: Install Go 1.13.3
+          name: Install Go 1.16
           command: |
               cd /tmp
-              curl -LO https://dl.google.com/go/go1.13.3.darwin-amd64.pkg
-              sudo installer -pkg /tmp/go1.13.3.darwin-amd64.pkg -target /
+              curl -LO https://dl.google.com/go/go1.16.darwin-amd64.pkg
+              sudo installer -pkg /tmp/go1.16.darwin-amd64.pkg -target /
 
       #- run:
       #    name: Install Gomobile
@@ -144,9 +157,46 @@ jobs:
           paths:
             - upload
 
+  build-windows:
+    docker:
+      - image: circleci/golang:1.16
+
+    steps:
+      - checkout
+
+      - run:
+          name: Create artifact upload directory and set variables
+          command: |
+              mkdir /tmp/upload
+              echo 'export CINAME=$(sh contrib/semver/name.sh)' >> $BASH_ENV
+              echo 'export CIVERSION=$(sh contrib/semver/version.sh --bare)' >> $BASH_ENV
+              git config --global user.email "$(git log --format='%ae' HEAD -1)";
+              git config --global user.name "$(git log --format='%an' HEAD -1)";
+
+      - run:
+          name: Install tools
+          command: |
+              sudo apt-get update
+              sudo apt-get -y install msitools wixl
+
+      - run:
+          name: Build for Windows
+          command: |
+              rm -f {yggdrasil,yggdrasilctl}
+              GOOS=windows GOARCH=amd64 ./build && mv yggdrasil.exe /tmp/upload/$CINAME-$CIVERSION-windows-amd64.exe && mv yggdrasilctl.exe /tmp/upload/$CINAME-$CIVERSION-yggdrasilctl-windows-amd64.exe;
+              GOOS=windows GOARCH=386 ./build && mv yggdrasil.exe /tmp/upload/$CINAME-$CIVERSION-windows-i386.exe && mv yggdrasilctl.exe /tmp/upload/$CINAME-$CIVERSION-yggdrasilctl-windows-i386.exe;
+              bash contrib/msi/build-msi.sh x64
+              bash contrib/msi/build-msi.sh x86
+              mv *.msi /tmp/upload
+
+      - persist_to_workspace:
+          root: /tmp
+          paths:
+            - upload
+
   build-other:
     docker:
-      - image: circleci/golang:1.13.3
+      - image: circleci/golang:1.16
 
     steps:
       - checkout
@@ -174,13 +224,6 @@ jobs:
               GOOS=freebsd GOARCH=amd64 ./build && mv yggdrasil /tmp/upload/$CINAME-$CIVERSION-freebsd-amd64 && mv yggdrasilctl /tmp/upload/$CINAME-$CIVERSION-yggdrasilctl-freebsd-amd64;
               GOOS=freebsd GOARCH=386 ./build && mv yggdrasil /tmp/upload/$CINAME-$CIVERSION-freebsd-i386 && mv yggdrasilctl /tmp/upload/$CINAME-$CIVERSION-yggdrasilctl-freebsd-i386;
 
-      - run:
-          name: Build for Windows
-          command: |
-              rm -f {yggdrasil,yggdrasilctl}
-              GOOS=windows GOARCH=amd64 ./build && mv yggdrasil.exe /tmp/upload/$CINAME-$CIVERSION-windows-amd64.exe && mv yggdrasilctl.exe /tmp/upload/$CINAME-$CIVERSION-yggdrasilctl-windows-amd64.exe;
-              GOOS=windows GOARCH=386 ./build && mv yggdrasil.exe /tmp/upload/$CINAME-$CIVERSION-windows-i386.exe && mv yggdrasilctl.exe /tmp/upload/$CINAME-$CIVERSION-yggdrasilctl-windows-i386.exe;
-
       - persist_to_workspace:
           root: /tmp
           paths:
@@ -201,11 +244,14 @@ workflows:
   version: 2.1
   build:
     jobs:
+      - lint
       - build-linux
       - build-macos
+      - build-windows
       - build-other
       - upload:
           requires:
             - build-linux
             - build-macos
+            - build-windows
             - build-other

.golangci.yml

@@ -0,0 +1,10 @@
+run:
+  build-tags:
+    - lint
+  issues-exit-code: 0 # TODO: change this to 1 when we want it to fail builds
+  skip-dirs:
+    - contrib/
+    - misc/
+linters:
+  disable:
+    - gocyclo

CHANGELOG.md

@@ -25,6 +25,59 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - in case of vulnerabilities.
 -->
 
+## [0.3.16] - 2021-03-18
+### Added
+- New simulation code under `cmd/yggdrasilsim` (work-in-progress)
+
+### Changed
+- Multi-threading in the switch
+  - Swich lookups happen independently for each (incoming) peer connection, instead of being funneled to a single dedicated switch worker
+  - Packets are queued for each (outgoing) peer connection, instead of being handled by a single dedicated switch worker
+- Queue logic rewritten
+  - Heap structure per peer that traffic is routed to, with one FIFO queue per traffic flow
+  - The total size of each heap is configured automatically (we basically queue packets until we think we're blocked on a socket write)
+  - When adding to a full heap, the oldest packet from the largest queue is dropped
+  - Packets are popped from the queue in FIFO order (oldest packet from among all queues in the heap) to prevent packet reordering at the session level
+- Removed global `sync.Pool` of `[]byte`
+  - Local `sync.Pool`s are used in the hot loops, but not exported, to avoid memory corruption if libraries are reused by other projects
+  - This may increase allocations (and slightly reduce speed in CPU-bound benchmarks) when interacting with the tun/tap device, but traffic forwarded at the switch layer should be unaffected
+- Upgrade dependencies
+- Upgrade build to Go 1.16
+
+### Fixed
+- Fixed a bug where the connection listener could exit prematurely due to resoruce exhaustion (if e.g. too many connections were opened)
+- Fixed DefaultIfName for OpenBSD (`/dev/tun0` -> `tun0`)
+- Fixed an issue where a peer could sometimes never be added to the switch
+- Fixed a goroutine leak that could occur if a peer with an open connection continued to spam additional connection attempts
+
+## [0.3.15] - 2020-09-27
+### Added
+- Support for pinning remote public keys in peering strings has been added, e.g.
+  - By signing public key: `tcp://host:port?ed25519=key`
+  - By encryption public key: `tcp://host:port?curve25519=key`
+  - By both: `tcp://host:port?ed25519=key&curve25519=key`
+  - By multiple, in case of DNS round-robin or similar: `tcp://host:port?curve25519=key&curve25519=key&ed25519=key&ed25519=key`
+- Some checks to prevent Yggdrasil-over-Yggdrasil peerings have been added
+- Added support for SOCKS proxy authentication, e.g. `socks://user@password:host/...`
+
+### Fixed
+- Some bugs in the multicast code that could cause unnecessary CPU usage have been fixed
+- A possible multicast deadlock on macOS when enumerating interfaces has been fixed
+- A deadlock in the connection code has been fixed
+- Updated HJSON dependency that caused some build problems
+
+### Changed
+- `DisconnectPeer` and `RemovePeer` have been separated and implemented properly now
+- Less nodes are stored in the DHT now, reducing ambient network traffic and possible instability
+- Default config file for FreeBSD is now at `/usr/local/etc/yggdrasil.conf` instead of `/etc/yggdrasil.conf`
+
+## [0.3.14] - 2020-03-28
+### Fixed
+- Fixes a memory leak that may occur if packets are incorrectly never removed from a switch queue
+
+### Changed
+- Make DHT searches a bit more reliable by tracking the 16 most recently visited nodes
+
 ## [0.3.13] - 2020-02-21
 ### Added
 - Support for the Wireguard TUN driver, which now replaces Water and provides far better support and performance on Windows

README.md

@@ -26,7 +26,7 @@ some of the below:
 - Linux
   - `.deb` and `.rpm` packages are built by CI for Debian and Red Hat-based
     distributions
-  - Void and Arch packages also available within their respective repositories
+  - Arch, Nix, Void packages also available within their respective repositories
 - macOS
   - `.pkg` packages are built by CI
 - Ubiquiti EdgeOS
@@ -48,7 +48,7 @@ You may also find other platform-specific wrappers, scripts or tools in the
 If you want to build from source, as opposed to installing one of the pre-built
 packages:
 
-1. Install [Go](https://golang.org) (requires Go 1.13 or later)
+1. Install [Go](https://golang.org) (requires Go 1.16 or later)
 2. Clone this repository
 2. Run `./build`
 

appveyor.yml

@@ -1,20 +0,0 @@
-version: '{build}'
-pull_requests:
-  do_not_increment_build_number: true
-os: Visual Studio 2017
-shallow_clone: false
-
-environment:
-  MSYS2_PATH_TYPE: inherit
-  CHERE_INVOKING: enabled_from_arguments
-
-build_script:
-- cmd: >-
-    cd %APPVEYOR_BUILD_FOLDER%
-- c:\msys64\usr\bin\bash -lc "./contrib/msi/build-msi.sh x64"
-- c:\msys64\usr\bin\bash -lc "./contrib/msi/build-msi.sh x86"
-
-test: off
-
-artifacts:
-- path: '*.msi'

cmd/genkeys/main.go

@@ -51,7 +51,7 @@ func main() {
 
 	for {
 		newKey := <-newKeys
-		if isBetter(currentBest[:], newKey.id[:]) || len(currentBest) == 0 {
+		if isBetter(currentBest, newKey.id[:]) || len(currentBest) == 0 {
 			currentBest = newKey.id
 			for _, channel := range threadChannels {
 				select {
@@ -61,13 +61,13 @@ func main() {
 			fmt.Println("--------------------------------------------------------------------------------")
 			switch {
 			case *doSig:
-				fmt.Println("sigPriv:", hex.EncodeToString(newKey.priv[:]))
-				fmt.Println("sigPub:", hex.EncodeToString(newKey.pub[:]))
-				fmt.Println("TreeID:", hex.EncodeToString(newKey.id[:]))
+				fmt.Println("sigPriv:", hex.EncodeToString(newKey.priv))
+				fmt.Println("sigPub:", hex.EncodeToString(newKey.pub))
+				fmt.Println("TreeID:", hex.EncodeToString(newKey.id))
 			default:
-				fmt.Println("boxPriv:", hex.EncodeToString(newKey.priv[:]))
-				fmt.Println("boxPub:", hex.EncodeToString(newKey.pub[:]))
-				fmt.Println("NodeID:", hex.EncodeToString(newKey.id[:]))
+				fmt.Println("boxPriv:", hex.EncodeToString(newKey.priv))
+				fmt.Println("boxPub:", hex.EncodeToString(newKey.pub))
+				fmt.Println("NodeID:", hex.EncodeToString(newKey.id))
 				fmt.Println("IP:", newKey.ip)
 			}
 		}
@@ -76,11 +76,8 @@ func main() {
 
 func isBetter(oldID, newID []byte) bool {
 	for idx := range oldID {
-		if newID[idx] > oldID[idx] {
-			return true
-		}
-		if newID[idx] < oldID[idx] {
-			return false
+		if newID[idx] != oldID[idx] {
+			return newID[idx] > oldID[idx]
 		}
 	}
 	return false

cmd/yggdrasil/main.go

@@ -60,8 +60,8 @@ func readConfig(useconf *bool, useconffile *string, normaliseconf *bool) *config
 	// throwing everywhere when it's converting things into UTF-16 for the hell
 	// of it - remove it and decode back down into UTF-8. This is necessary
 	// because hjson doesn't know what to do with UTF-16 and will panic
-	if bytes.Compare(conf[0:2], []byte{0xFF, 0xFE}) == 0 ||
-		bytes.Compare(conf[0:2], []byte{0xFE, 0xFF}) == 0 {
+	if bytes.Equal(conf[0:2], []byte{0xFF, 0xFE}) ||
+		bytes.Equal(conf[0:2], []byte{0xFE, 0xFF}) {
 		utf := unicode.UTF16(unicode.BigEndian, unicode.UseBOM)
 		decoder := utf.NewDecoder()
 		conf, err = decoder.Bytes(conf)
@@ -222,7 +222,7 @@ func main() {
 	getNodeID := func() *crypto.NodeID {
 		if pubkey, err := hex.DecodeString(cfg.EncryptionPublicKey); err == nil {
 			var box crypto.BoxPubKey
-			copy(box[:], pubkey[:])
+			copy(box[:], pubkey)
 			return crypto.GetNodeID(&box)
 		}
 		return nil
@@ -328,9 +328,9 @@ func main() {
 	// deferred Stop function above will run which will shut down TUN/TAP.
 	for {
 		select {
-		case _ = <-c:
+		case <-c:
 			goto exit
-		case _ = <-r:
+		case <-r:
 			if *useconffile != "" {
 				cfg = readConfig(useconf, useconffile, normaliseconf)
 				logger.Infoln("Reloading configuration from", *useconffile)

cmd/yggdrasilctl/main.go

@@ -78,8 +78,8 @@ func run() int {
 
 	if *server == endpoint {
 		if config, err := ioutil.ReadFile(defaults.GetDefaults().DefaultConfigFile); err == nil {
-			if bytes.Compare(config[0:2], []byte{0xFF, 0xFE}) == 0 ||
-				bytes.Compare(config[0:2], []byte{0xFE, 0xFF}) == 0 {
+			if bytes.Equal(config[0:2], []byte{0xFF, 0xFE}) ||
+				bytes.Equal(config[0:2], []byte{0xFE, 0xFF}) {
 				utf := unicode.UTF16(unicode.BigEndian, unicode.UseBOM)
 				decoder := utf.NewDecoder()
 				config, err = decoder.Bytes(config)

cmd/yggdrasilsim/dial.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"fmt"
+	"sort"
+	"time"
+
+	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
+)
+
+func doListen(recvNode *simNode) {
+	// TODO be able to stop the listeners somehow so they don't leak across different tests
+	for {
+		c, err := recvNode.listener.Accept()
+		if err != nil {
+			panic(err)
+		}
+		c.Close()
+	}
+}
+
+func dialTest(sendNode, recvNode *simNode) {
+	if sendNode.id == recvNode.id {
+		fmt.Println("Skipping dial to self")
+		return
+	}
+	var mask crypto.NodeID
+	for idx := range mask {
+		mask[idx] = 0xff
+	}
+	for {
+		c, err := sendNode.dialer.DialByNodeIDandMask(nil, &recvNode.nodeID, &mask)
+		if c != nil {
+			c.Close()
+			return
+		}
+		if err != nil {
+			fmt.Println("Dial failed:", err)
+		}
+		time.Sleep(time.Second)
+	}
+}
+
+func dialStore(store nodeStore) {
+	var nodeIdxs []int
+	for idx, n := range store {
+		nodeIdxs = append(nodeIdxs, idx)
+		go doListen(n)
+	}
+	sort.Slice(nodeIdxs, func(i, j int) bool {
+		return nodeIdxs[i] < nodeIdxs[j]
+	})
+	for _, idx := range nodeIdxs {
+		sendNode := store[idx]
+		for _, jdx := range nodeIdxs {
+			recvNode := store[jdx]
+			fmt.Printf("Dialing from node %d to node %d / %d...\n", idx, jdx, len(store))
+			dialTest(sendNode, recvNode)
+		}
+	}
+}

cmd/yggdrasilsim/main.go

@@ -0,0 +1,6 @@
+package main
+
+func main() {
+	store := makeStoreSquareGrid(4)
+	dialStore(store)
+}

cmd/yggdrasilsim/node.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"io/ioutil"
+
+	"github.com/gologme/log"
+
+	"github.com/yggdrasil-network/yggdrasil-go/src/config"
+	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
+	"github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil"
+)
+
+type simNode struct {
+	core     yggdrasil.Core
+	id       int
+	nodeID   crypto.NodeID
+	dialer   *yggdrasil.Dialer
+	listener *yggdrasil.Listener
+}
+
+func newNode(id int) *simNode {
+	n := simNode{id: id}
+	n.core.Start(config.GenerateConfig(), log.New(ioutil.Discard, "", 0))
+	n.nodeID = *n.core.NodeID()
+	n.dialer, _ = n.core.ConnDialer()
+	n.listener, _ = n.core.ConnListen()
+	return &n
+}

cmd/yggdrasilsim/store.go

@@ -0,0 +1,41 @@
+package main
+
+type nodeStore map[int]*simNode
+
+func makeStoreSingle() nodeStore {
+	s := make(nodeStore)
+	s[0] = newNode(0)
+	return s
+}
+
+func linkNodes(a *simNode, b *simNode) {
+	la := a.core.NewSimlink()
+	lb := b.core.NewSimlink()
+	la.SetDestination(lb)
+	lb.SetDestination(la)
+	la.Start()
+	lb.Start()
+}
+
+func makeStoreSquareGrid(sideLength int) nodeStore {
+	store := make(nodeStore)
+	nNodes := sideLength * sideLength
+	idxs := make([]int, 0, nNodes)
+	// TODO shuffle nodeIDs
+	for idx := 1; idx <= nNodes; idx++ {
+		idxs = append(idxs, idx)
+	}
+	for _, idx := range idxs {
+		n := newNode(idx)
+		store[idx] = n
+	}
+	for idx := 0; idx < nNodes; idx++ {
+		if (idx % sideLength) != 0 {
+			linkNodes(store[idxs[idx]], store[idxs[idx-1]])
+		}
+		if idx >= sideLength {
+			linkNodes(store[idxs[idx]], store[idxs[idx-sideLength]])
+		}
+	}
+	return store
+}

contrib/ansible/genkeys.go

@@ -12,6 +12,7 @@ import (
 	"net"
 	"os"
 
+	"github.com/cheggaaa/pb/v3"
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 )
@@ -29,6 +30,8 @@ type keySet struct {
 func main() {
 	flag.Parse()
 
+	bar := pb.StartNew(*keyTries*2 + *numHosts)
+
 	if *numHosts > *keyTries {
 		println("Can't generate less keys than hosts.")
 		return
@@ -37,21 +40,25 @@ func main() {
 	var encryptionKeys []keySet
 	for i := 0; i < *numHosts+1; i++ {
 		encryptionKeys = append(encryptionKeys, newBoxKey())
+		bar.Increment()
 	}
 	encryptionKeys = sortKeySetArray(encryptionKeys)
 	for i := 0; i < *keyTries-*numHosts-1; i++ {
 		encryptionKeys[0] = newBoxKey()
 		encryptionKeys = bubbleUpTo(encryptionKeys, 0)
+		bar.Increment()
 	}
 
 	var signatureKeys []keySet
 	for i := 0; i < *numHosts+1; i++ {
 		signatureKeys = append(signatureKeys, newSigKey())
+		bar.Increment()
 	}
 	signatureKeys = sortKeySetArray(signatureKeys)
 	for i := 0; i < *keyTries-*numHosts-1; i++ {
 		signatureKeys[0] = newSigKey()
 		signatureKeys = bubbleUpTo(signatureKeys, 0)
+		bar.Increment()
 	}
 
 	os.MkdirAll("host_vars", 0755)
@@ -76,7 +83,9 @@ func main() {
 		defer file.Close()
 		file.WriteString(fmt.Sprintf("vault_yggdrasil_encryption_private_key: %v\n", hex.EncodeToString(encryptionKeys[i].priv)))
 		file.WriteString(fmt.Sprintf("vault_yggdrasil_signing_private_key: %v\n", hex.EncodeToString(signatureKeys[i].priv)))
+		bar.Increment()
 	}
+	bar.Finish()
 }
 
 func newBoxKey() keySet {

contrib/apparmor/usr.bin.yggdrasil

@@ -1,23 +1,17 @@
-# Last Modified: Sat Mar  9 06:08:02 2019
+# Last Modified: Fri Oct 30 11:33:31 2020
 #include <tunables/global>
 
 /usr/bin/yggdrasil {
   #include <abstractions/base>
+  #include <abstractions/nameservice>
 
   capability net_admin,
+  capability net_raw,
 
-  network inet stream,
-  network inet dgram,
-  network inet6 dgram,
-  network inet6 stream,
-  network netlink raw,
-
-  /lib/@{multiarch}/ld-*.so mr,
-  /proc/sys/net/core/somaxconn r,
   /dev/net/tun rw,
+  /proc/sys/net/core/somaxconn r,
+  /sys/kernel/mm/transparent_hugepage/hpage_pmd_size r,
 
-  /usr/bin/yggdrasil mr,
   /etc/yggdrasil.conf rw,
   /run/yggdrasil.sock rw,
-
 }

contrib/config/yggdrasilconf.go

@@ -1,97 +0,0 @@
-package main
-
-/*
-This is a small utility that is designed to accompany the vyatta-yggdrasil
-package. It takes a HJSON configuration file, makes changes to it based on
-the command line arguments, and then spits out an updated file.
-*/
-
-import (
-	"bytes"
-	"encoding/json"
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"strconv"
-
-	"github.com/hjson/hjson-go"
-	"golang.org/x/text/encoding/unicode"
-
-	"github.com/yggdrasil-network/yggdrasil-go/src/config"
-)
-
-type nodeConfig = config.NodeConfig
-
-func main() {
-	useconffile := flag.String("useconffile", "/etc/yggdrasil.conf", "update config at specified file path")
-	flag.Parse()
-	cfg := nodeConfig{}
-	var config []byte
-	var err error
-	config, err = ioutil.ReadFile(*useconffile)
-	if err != nil {
-		panic(err)
-	}
-	if bytes.Compare(config[0:2], []byte{0xFF, 0xFE}) == 0 ||
-		bytes.Compare(config[0:2], []byte{0xFE, 0xFF}) == 0 {
-		utf := unicode.UTF16(unicode.BigEndian, unicode.UseBOM)
-		decoder := utf.NewDecoder()
-		config, err = decoder.Bytes(config)
-		if err != nil {
-			panic(err)
-		}
-	}
-	var dat map[string]interface{}
-	if err := hjson.Unmarshal(config, &dat); err != nil {
-		panic(err)
-	}
-	confJson, err := json.Marshal(dat)
-	if err != nil {
-		panic(err)
-	}
-	json.Unmarshal(confJson, &cfg)
-	switch flag.Arg(0) {
-	case "setMTU":
-		cfg.IfMTU, err = strconv.Atoi(flag.Arg(1))
-		if err != nil {
-			cfg.IfMTU = 1280
-		}
-		if mtu, _ := strconv.Atoi(flag.Arg(1)); mtu < 1280 {
-			cfg.IfMTU = 1280
-		}
-	case "setIfName":
-		cfg.IfName = flag.Arg(1)
-	case "setListen":
-		cfg.Listen = flag.Arg(1)
-	case "setAdminListen":
-		cfg.AdminListen = flag.Arg(1)
-	case "setIfTapMode":
-		if flag.Arg(1) == "true" {
-			cfg.IfTAPMode = true
-		} else {
-			cfg.IfTAPMode = false
-		}
-	case "addPeer":
-		found := false
-		for _, v := range cfg.Peers {
-			if v == flag.Arg(1) {
-				found = true
-			}
-		}
-		if !found {
-			cfg.Peers = append(cfg.Peers, flag.Arg(1))
-		}
-	case "removePeer":
-		for k, v := range cfg.Peers {
-			if v == flag.Arg(1) {
-				cfg.Peers = append(cfg.Peers[:k], cfg.Peers[k+1:]...)
-			}
-		}
-	}
-	bs, err := hjson.Marshal(cfg)
-	if err != nil {
-		panic(err)
-	}
-	fmt.Println(string(bs))
-	return
-}

contrib/deb/generate.sh

@@ -83,7 +83,7 @@ then
   echo "Backing up configuration file to /var/backups/yggdrasil.conf.`date +%Y%m%d`"
   cp /etc/yggdrasil.conf /var/backups/yggdrasil.conf.`date +%Y%m%d`
   echo "Normalising and updating /etc/yggdrasil.conf"
-  /usr/bin/yggdrasil -useconffile /var/backups/yggdrasil.conf.`date +%Y%m%d` -normaliseconf > /etc/yggdrasil.conf
+  /usr/bin/yggdrasil -useconf -normaliseconf < /var/backups/yggdrasil.conf.`date +%Y%m%d` > /etc/yggdrasil.conf
   chgrp yggdrasil /etc/yggdrasil.conf
 
   if command -v systemctl >/dev/null; then
@@ -94,7 +94,7 @@ then
 else
   echo "Generating initial configuration file /etc/yggdrasil.conf"
   echo "Please familiarise yourself with this file before starting Yggdrasil"
-  /usr/bin/yggdrasil -genconf > /etc/yggdrasil.conf
+  sh -c 'umask 0027 && /usr/bin/yggdrasil -genconf > /etc/yggdrasil.conf'
   chgrp yggdrasil /etc/yggdrasil.conf
 fi
 EOF

contrib/msi/build-msi.sh

@@ -1,9 +1,7 @@
-#!/bin/sh
+#!/bin/bash
 
 # This script generates an MSI file for Yggdrasil for a given architecture. It
-# needs to run on Windows within MSYS2 and Go 1.13 or later must be installed on
-# the system and within the PATH. This is ran currently by Appveyor (see
-# appveyor.yml in the repository root) for both x86 and x64.
+# needs to run on Linux or macOS with Go 1.16, wixl and msitools installed.
 #
 # Author: Neil Alexander <neilalexander@users.noreply.github.com>
 
@@ -11,7 +9,7 @@
 PKGARCH=$1
 if [ "${PKGARCH}" == "" ];
 then
-  echo "tell me the architecture: x86 or x64"
+  echo "tell me the architecture: x86, x64 or arm"
   exit 1
 fi
 
@@ -28,28 +26,11 @@ then
   git checkout ${APPVEYOR_REPO_BRANCH}
 fi
 
-# Install prerequisites within MSYS2
-pacman -S --needed --noconfirm unzip git curl
-
-# Download the wix tools!
-if [ ! -d wixbin ];
-then
-  curl -LO https://github.com/wixtoolset/wix3/releases/download/wix3112rtm/wix311-binaries.zip
-  if [ `md5sum wix311-binaries.zip | cut -f 1 -d " "` != "47a506f8ab6666ee3cc502fb07d0ee2a" ];
-  then
-    echo "wix package didn't match expected checksum"
-    exit 1
-  fi
-  mkdir -p wixbin
-  unzip -o wix311-binaries.zip -d wixbin || (
-    echo "failed to unzip WiX"
-    exit 1
-  )
-fi
-
 # Build Yggdrasil!
-[ "${PKGARCH}" == "x64" ] && GOOS=windows GOARCH=amd64 CGO_ENABLED=0 ./build
-[ "${PKGARCH}" == "x86" ] && GOOS=windows GOARCH=386 CGO_ENABLED=0 ./build
+[ "${PKGARCH}" == "x64" ] && GOOS=windows GOARCH=amd64 CGO_ENABLED=0 ./build -p -l "-aslr"
+[ "${PKGARCH}" == "x86" ] && GOOS=windows GOARCH=386 CGO_ENABLED=0 ./build -p -l "-aslr"
+[ "${PKGARCH}" == "arm" ] && GOOS=windows GOARCH=arm CGO_ENABLED=0 ./build -p -l "-aslr"
+#[ "${PKGARCH}" == "arm64" ] && GOOS=windows GOARCH=arm64 CGO_ENABLED=0 ./build
 
 # Create the postinstall script
 cat > updateconfig.bat << EOF
@@ -58,7 +39,9 @@ if not exist %ALLUSERSPROFILE%\\Yggdrasil (
 )
 if not exist %ALLUSERSPROFILE%\\Yggdrasil\\yggdrasil.conf (
   if exist yggdrasil.exe (
-    yggdrasil.exe -genconf > %ALLUSERSPROFILE%\\Yggdrasil\\yggdrasil.conf
+    if not exist %ALLUSERSPROFILE%\\Yggdrasil\\yggdrasil.conf (
+      yggdrasil.exe -genconf > %ALLUSERSPROFILE%\\Yggdrasil\\yggdrasil.conf
+    )
   )
 )
 EOF
@@ -72,12 +55,16 @@ PKGVERSIONMS=$(echo $PKGVERSION | tr - .)
   PKGGUID="54a3294e-a441-4322-aefb-3bb40dd022bb" PKGINSTFOLDER="ProgramFilesFolder"
 
 # Download the Wintun driver
+curl -o wintun.zip https://www.wintun.net/builds/wintun-0.10.2.zip
+unzip wintun.zip
 if [ $PKGARCH = "x64" ]; then
-  PKGMSMNAME=wintun-x64.msm
-  curl -o ${PKGMSMNAME} https://www.wintun.net/builds/wintun-amd64-0.7.msm || (echo "couldn't get wintun"; exit 1)
+  PKGWINTUNDLL=wintun/bin/amd64/wintun.dll
 elif [ $PKGARCH = "x86" ]; then
-  PKGMSMNAME=wintun-x86.msm
-  curl -o ${PKGMSMNAME} https://www.wintun.net/builds/wintun-x86-0.7.msm || (echo "couldn't get wintun"; exit 1)
+  PKGWINTUNDLL=wintun/bin/x86/wintun.dll
+elif [ $PKGARCH = "arm" ]; then
+  PKGWINTUNDLL=wintun/bin/arm/wintun.dll
+#elif [ $PKGARCH = "arm64" ]; then
+#  PKGWINTUNDLL=wintun/bin/arm64/wintun.dll
 else
   echo "wasn't sure which architecture to get wintun for"
   exit 1
@@ -100,6 +87,7 @@ cat > wix.xml << EOF
     Language="1033"
     Codepage="1252"
     Version="${PKGVERSIONMS}"
+    Platform="${PKGARCH}"
     Manufacturer="github.com/yggdrasil-network">
 
     <Package
@@ -136,6 +124,12 @@ cat > wix.xml << EOF
               Source="yggdrasil.exe"
               KeyPath="yes" />
 
+            <File
+              Id="Wintun"
+              Name="wintun.dll"
+              DiskId="1"
+              Source="${PKGWINTUNDLL}" />
+
             <ServiceInstall
               Id="ServiceInstaller"
               Account="LocalSystem"
@@ -176,12 +170,6 @@ cat > wix.xml << EOF
           </Component>
         </Directory>
       </Directory>
-
-      <Merge
-        Id="Wintun"
-        Language="0"
-        DiskId="1"
-        SourceFile="${PKGMSMNAME}" />
     </Directory>
 
     <Feature Id="YggdrasilFeature" Title="Yggdrasil" Level="1">
@@ -190,13 +178,6 @@ cat > wix.xml << EOF
       <ComponentRef Id="ConfigScript" />
     </Feature>
 
-    <Feature Id="WintunFeature" Title="Wintun" Level="1">
-      <Condition Level="0">
-        UPGRADINGPRODUCTCODE
-      </Condition>
-      <MergeRef Id="Wintun" />
-    </Feature>
-
     <CustomAction
       Id="UpdateGenerateConfig"
       Directory="YggdrasilInstallFolder"
@@ -208,9 +189,7 @@ cat > wix.xml << EOF
     <InstallExecuteSequence>
       <Custom
         Action="UpdateGenerateConfig"
-        Before="StartServices">
-          NOT Installed AND NOT REMOVE
-      </Custom>
+        Before="StartServices" />
     </InstallExecuteSequence>
 
   </Product>
@@ -218,7 +197,4 @@ cat > wix.xml << EOF
 EOF
 
 # Generate the MSI
-CANDLEFLAGS="-nologo"
-LIGHTFLAGS="-nologo -spdb -sice:ICE71 -sice:ICE61"
-wixbin/candle $CANDLEFLAGS -out ${PKGNAME}-${PKGVERSION}-${PKGARCH}.wixobj -arch ${PKGARCH} wix.xml && \
-wixbin/light $LIGHTFLAGS -ext WixUtilExtension.dll -out ${PKGNAME}-${PKGVERSION}-${PKGARCH}.msi ${PKGNAME}-${PKGVERSION}-${PKGARCH}.wixobj
+wixl -v wix.xml -a ${PKGARCH} -o ${PKGNAME}-${PKGVERSION}-${PKGARCH}.msi

go.mod

@@ -1,20 +1,24 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
-go 1.13
+go 1.16
 
 require (
-	github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0
-	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
+	github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979
+	github.com/cheggaaa/pb/v3 v3.0.6
+	github.com/fatih/color v1.10.0 // indirect
+	github.com/gologme/log v1.2.0
 	github.com/hashicorp/go-syslog v1.0.0
-	github.com/hjson/hjson-go v3.0.1-0.20190209023717-9147687966d9+incompatible
-	github.com/kardianos/minwinsvc v0.0.0-20151122163309-cad6b2b879b0
-	github.com/mitchellh/mapstructure v1.1.2
-	github.com/vishvananda/netlink v1.0.0
-	github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f // indirect
-	golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876
-	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553
-	golang.org/x/sys v0.0.0-20200103143344-a1369afcdac7
-	golang.org/x/text v0.3.2
-	golang.zx2c4.com/wireguard v0.0.20200121
-	golang.zx2c4.com/wireguard/windows v0.0.35-0.20191123133119-cb4a03094c25
+	github.com/hjson/hjson-go v3.1.0+incompatible
+	github.com/kardianos/minwinsvc v1.0.0
+	github.com/mattn/go-runewidth v0.0.10 // indirect
+	github.com/mitchellh/mapstructure v1.4.1
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/vishvananda/netlink v1.1.0
+	github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
+	golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
+	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110
+	golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b
+	golang.org/x/text v0.3.6-0.20210220033129-8f690f22cf1c
+	golang.zx2c4.com/wireguard v0.0.0-20210306175010-7e3b8371a1bf
+	golang.zx2c4.com/wireguard/windows v0.3.8
 )

go.sum

@@ -1,46 +1,72 @@
-github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0 h1:p3puK8Sl2xK+2FnnIvY/C0N1aqJo2kbEsdAzU+Tnv48=
-github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
-github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
-github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
+github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 h1:WndgpSW13S32VLQ3ugUxx2EnnWmgba1kCqPkd4Gk1yQ=
+github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
+github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
+github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
+github.com/cheggaaa/pb/v3 v3.0.6 h1:ULPm1wpzvj60FvmCrX7bIaB80UgbhI+zSaQJKRfCbAs=
+github.com/cheggaaa/pb/v3 v3.0.6/go.mod h1:X1L61/+36nz9bjIsrDU52qHKOQukUQe2Ge+YvGuquCw=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
+github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
+github.com/gologme/log v1.2.0 h1:Ya5Ip/KD6FX7uH0S31QO87nCCSucKtF44TLbTtO7V4c=
+github.com/gologme/log v1.2.0/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
-github.com/hjson/hjson-go v3.0.1-0.20190209023717-9147687966d9+incompatible h1:bLQ2Ve+eW65id3b8xEMQiAwJT4qGZeywAEMLvXjznvw=
-github.com/hjson/hjson-go v3.0.1-0.20190209023717-9147687966d9+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
-github.com/kardianos/minwinsvc v0.0.0-20151122163309-cad6b2b879b0 h1:YnZmFjg0Nvk8851WTVWlqMC1ecJH07Ctz+Ezxx4u54g=
-github.com/kardianos/minwinsvc v0.0.0-20151122163309-cad6b2b879b0/go.mod h1:rUi0/YffDo1oXBOGn1KRq7Fr07LX48XEBecQnmwjsAo=
-github.com/lxn/walk v0.0.0-20191031081659-c0bb82ae46cb/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ=
-github.com/lxn/win v0.0.0-20191024121223-cc00c7492fe1 h1:h0wbuSK8xUNmMwDdCxZx2OLdkVck6Bb31zj4CxCN5I4=
-github.com/lxn/win v0.0.0-20191024121223-cc00c7492fe1/go.mod h1:ouWl4wViUNh8tPSIwxTVMuS014WakR1hqvBc2I0bMoA=
-github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
-github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/vishvananda/netlink v1.0.0 h1:bqNY2lgheFIu1meHUFSH3d7vG93AFyqg3oGbJCOJgSM=
-github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
-github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f h1:nBX3nTcmxEtHSERBJaIo1Qa26VwRaopnZmfDQUXsF4I=
-github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
+github.com/hjson/hjson-go v3.1.0+incompatible h1:DY/9yE8ey8Zv22bY+mHV1uk2yRy0h8tKhZ77hEdi0Aw=
+github.com/hjson/hjson-go v3.1.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
+github.com/kardianos/minwinsvc v1.0.0 h1:+JfAi8IBJna0jY2dJGZqi7o15z13JelFIklJCAENALA=
+github.com/kardianos/minwinsvc v1.0.0/go.mod h1:Bgd0oc+D0Qo3bBytmNtyRKVlp85dAloLKhfxanPFFRc=
+github.com/lxn/walk v0.0.0-20210112085537-c389da54e794/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ=
+github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg=
+github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
+github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
+github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
+github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
+github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191029031824-8986dd9e96cf/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876 h1:sKJQZMuxjOAR/Uo2LBfU90onWEf1dF4C+0hPJCc9Mpc=
-golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20191003171128-d98b1b443823/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8=
-golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191003212358-c178f38b412c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200103143344-a1369afcdac7 h1:/W9OPMnnpmFXHYkcp2rQsbFUbRlRzfECQjmAFiOyHE8=
-golang.org/x/sys v0.0.0-20200103143344-a1369afcdac7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210225014209-683adc9d29d7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210305215415-5cdee2b1b5a0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b h1:ggRgirZABFolTmi3sn6Ivd9SipZwLedQ5wR0aAKnFxU=
+golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6-0.20210220033129-8f690f22cf1c h1:SW/oilbeWd6f32u3ZvuYGqZ+wivcp//I3Dy/gByk7Wk=
+golang.org/x/text v0.3.6-0.20210220033129-8f690f22cf1c/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.zx2c4.com/wireguard v0.0.20191013-0.20191030132932-4cdf805b29b1 h1:KxtBKNgJUQG8vwZzJKkwBGOcqp95xLu6A6KIMde1kl0=
-golang.zx2c4.com/wireguard v0.0.20191013-0.20191030132932-4cdf805b29b1/go.mod h1:P2HsVp8SKwZEufsnezXZA4GRX/T49/HlU7DGuelXsU4=
-golang.zx2c4.com/wireguard v0.0.20200121 h1:vcswa5Q6f+sylDfjqyrVNNrjsFUUbPsgAQTBCAg/Qf8=
-golang.zx2c4.com/wireguard v0.0.20200121/go.mod h1:P2HsVp8SKwZEufsnezXZA4GRX/T49/HlU7DGuelXsU4=
-golang.zx2c4.com/wireguard/windows v0.0.35-0.20191123133119-cb4a03094c25 h1:TreP+furSwdqoSToFrwb1S5cwxb7jhOsnwj2MsDeT+4=
-golang.zx2c4.com/wireguard/windows v0.0.35-0.20191123133119-cb4a03094c25/go.mod h1:EO8KCpT944a9CnwHJLZ1sl84FfIrY42fP/fcXUuYhKM=
+golang.zx2c4.com/wireguard v0.0.0-20210225140808-70b7b7158fc9/go.mod h1:39ZQQ95hUxDxT7opsWy/rtfgvXXc8s30qfZ02df69Fo=
+golang.zx2c4.com/wireguard v0.0.0-20210306175010-7e3b8371a1bf h1:AtdIMfzvVNPXN4kVY/yWS8mvpQogSwtCRJk2y/LBPpg=
+golang.zx2c4.com/wireguard v0.0.0-20210306175010-7e3b8371a1bf/go.mod h1:ojGPy+9W6ZSM8anL+xC67fvh8zPQJwA6KpFOHyDWLX4=
+golang.zx2c4.com/wireguard/windows v0.3.8 h1:FvfBEhdZZTwthLuPHdyP6zpivYL3enopxd4XpggAufM=
+golang.zx2c4.com/wireguard/windows v0.3.8/go.mod h1:lm7dxHcBuzMNq706Ge1tZKZKw4+19vG9dLOhoDX05HQ=

misc/sim/treesim.go

@@ -1,20 +1,24 @@
+// +build !lint
+
 package main
 
-import "fmt"
-import "bufio"
-import "os"
-import "strings"
-import "strconv"
-import "time"
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"os"
+	"runtime"
+	"runtime/pprof"
+	"strconv"
+	"strings"
+	"time"
 
-import "runtime"
-import "runtime/pprof"
-import "flag"
+	"github.com/gologme/log"
 
-import "github.com/gologme/log"
+	. "github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil"
 
-import . "github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil"
-import . "github.com/yggdrasil-network/yggdrasil-go/src/crypto"
+	. "github.com/yggdrasil-network/yggdrasil-go/src/crypto"
+)
 
 ////////////////////////////////////////////////////////////////////////////////
 

src/admin/admin.go

@@ -53,7 +53,7 @@ func (a *AdminSocket) AddHandler(name string, args []string, handlerfunc func(In
 	return nil
 }
 
-// init runs the initial admin setup.
+// Init runs the initial admin setup.
 func (a *AdminSocket) Init(c *yggdrasil.Core, state *config.NodeState, log *log.Logger, options interface{}) error {
 	a.core = c
 	a.log = log
@@ -181,32 +181,57 @@ func (a *AdminSocket) SetupAdminHandlers(na *AdminSocket) {
 					in["uri"].(string),
 				},
 			}, nil
-		} else {
-			return Info{
-				"not_added": []string{
-					in["uri"].(string),
-				},
-			}, errors.New("Failed to add peer")
 		}
+		return Info{
+			"not_added": []string{
+				in["uri"].(string),
+			},
+		}, errors.New("Failed to add peer")
 	})
-	a.AddHandler("removePeer", []string{"port"}, func(in Info) (Info, error) {
+	a.AddHandler("disconnectPeer", []string{"port"}, func(in Info) (Info, error) {
 		port, err := strconv.ParseInt(fmt.Sprint(in["port"]), 10, 64)
 		if err != nil {
 			return Info{}, err
 		}
 		if a.core.DisconnectPeer(uint64(port)) == nil {
 			return Info{
-				"removed": []string{
+				"disconnected": []string{
 					fmt.Sprint(port),
 				},
 			}, nil
 		} else {
 			return Info{
-				"not_removed": []string{
+				"not_disconnected": []string{
 					fmt.Sprint(port),
 				},
+			}, errors.New("Failed to disconnect peer")
+		}
+	})
+	a.AddHandler("removePeer", []string{"uri", "[interface]"}, func(in Info) (Info, error) {
+		// Set sane defaults
+		intf := ""
+		// Has interface been specified?
+		if itf, ok := in["interface"]; ok {
+			intf = itf.(string)
+		}
+		if a.core.RemovePeer(in["uri"].(string), intf) == nil {
+			return Info{
+				"removed": []string{
+					in["uri"].(string),
+				},
+			}, nil
+		} else {
+			return Info{
+				"not_removed": []string{
+					in["uri"].(string),
+				},
 			}, errors.New("Failed to remove peer")
 		}
+		return Info{
+			"not_removed": []string{
+				in["uri"].(string),
+			},
+		}, errors.New("Failed to remove peer")
 	})
 	a.AddHandler("getAllowedEncryptionPublicKeys", []string{}, func(in Info) (Info, error) {
 		return Info{"allowed_box_pubs": a.core.GetAllowedEncryptionPublicKeys()}, nil
@@ -218,13 +243,12 @@ func (a *AdminSocket) SetupAdminHandlers(na *AdminSocket) {
 					in["box_pub_key"].(string),
 				},
 			}, nil
-		} else {
-			return Info{
-				"not_added": []string{
-					in["box_pub_key"].(string),
-				},
-			}, errors.New("Failed to add allowed key")
 		}
+		return Info{
+			"not_added": []string{
+				in["box_pub_key"].(string),
+			},
+		}, errors.New("Failed to add allowed key")
 	})
 	a.AddHandler("removeAllowedEncryptionPublicKey", []string{"box_pub_key"}, func(in Info) (Info, error) {
 		if a.core.RemoveAllowedEncryptionPublicKey(in["box_pub_key"].(string)) == nil {
@@ -233,13 +257,12 @@ func (a *AdminSocket) SetupAdminHandlers(na *AdminSocket) {
 					in["box_pub_key"].(string),
 				},
 			}, nil
-		} else {
-			return Info{
-				"not_removed": []string{
-					in["box_pub_key"].(string),
-				},
-			}, errors.New("Failed to remove allowed key")
 		}
+		return Info{
+			"not_removed": []string{
+				in["box_pub_key"].(string),
+			},
+		}, errors.New("Failed to remove allowed key")
 	})
 	a.AddHandler("dhtPing", []string{"box_pub_key", "coords", "[target]"}, func(in Info) (Info, error) {
 		var reserr error
@@ -250,10 +273,10 @@ func (a *AdminSocket) SetupAdminHandlers(na *AdminSocket) {
 		coords := util.DecodeCoordString(in["coords"].(string))
 		var boxPubKey crypto.BoxPubKey
 		if b, err := hex.DecodeString(in["box_pub_key"].(string)); err == nil {
-			copy(boxPubKey[:], b[:])
+			copy(boxPubKey[:], b)
 			if n, err := hex.DecodeString(in["target"].(string)); err == nil {
 				var targetNodeID crypto.NodeID
-				copy(targetNodeID[:], n[:])
+				copy(targetNodeID[:], n)
 				result, reserr = a.core.DHTPing(boxPubKey, coords, &targetNodeID)
 			} else {
 				result, reserr = a.core.DHTPing(boxPubKey, coords, nil)
@@ -287,14 +310,13 @@ func (a *AdminSocket) SetupAdminHandlers(na *AdminSocket) {
 			var jsoninfo interface{}
 			if err := json.Unmarshal(nodeinfo, &jsoninfo); err != nil {
 				return Info{}, err
-			} else {
-				return Info{"nodeinfo": jsoninfo}, nil
 			}
+			return Info{"nodeinfo": jsoninfo}, nil
 		} else if in["box_pub_key"] == nil || in["coords"] == nil {
 			return Info{}, errors.New("Expecting both box_pub_key and coords")
 		} else {
 			if b, err := hex.DecodeString(in["box_pub_key"].(string)); err == nil {
-				copy(boxPubKey[:], b[:])
+				copy(boxPubKey[:], b)
 			} else {
 				return Info{}, err
 			}
@@ -305,12 +327,10 @@ func (a *AdminSocket) SetupAdminHandlers(na *AdminSocket) {
 			var m map[string]interface{}
 			if err = json.Unmarshal(result, &m); err == nil {
 				return Info{"nodeinfo": m}, nil
-			} else {
-				return Info{}, err
 			}
-		} else {
 			return Info{}, err
 		}
+		return Info{}, err
 	})
 }
 
@@ -333,9 +353,8 @@ func (a *AdminSocket) Stop() error {
 	if a.listener != nil {
 		a.started = false
 		return a.listener.Close()
-	} else {
-		return nil
 	}
+	return nil
 }
 
 // listen is run by start and manages API connections.

src/crypto/crypto.go

@@ -17,12 +17,11 @@ import (
 	"crypto/rand"
 	"crypto/sha512"
 	"encoding/hex"
+	"sync"
 
 	"golang.org/x/crypto/curve25519"
 	"golang.org/x/crypto/ed25519"
 	"golang.org/x/crypto/nacl/box"
-
-	"github.com/yggdrasil-network/yggdrasil-go/src/util"
 )
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -225,29 +224,36 @@ func GetSharedKey(myPrivKey *BoxPrivKey,
 	return (*BoxSharedKey)(&shared)
 }
 
+// pool is used internally by BoxOpen and BoxSeal to avoid allocating temporary space
+var pool = sync.Pool{New: func() interface{} { return []byte(nil) }}
+
 // BoxOpen returns a message and true if it successfully opens a crypto box using the provided shared key and nonce.
+// The boxed input slice's backing array is reused for the unboxed output when possible.
 func BoxOpen(shared *BoxSharedKey,
 	boxed []byte,
 	nonce *BoxNonce) ([]byte, bool) {
-	out := util.GetBytes()
 	s := (*[BoxSharedKeyLen]byte)(shared)
 	n := (*[BoxNonceLen]byte)(nonce)
-	unboxed, success := box.OpenAfterPrecomputation(out, boxed, n, s)
+	temp := append(pool.Get().([]byte), boxed...)
+	unboxed, success := box.OpenAfterPrecomputation(boxed[:0], temp, n, s)
+	pool.Put(temp[:0])
 	return unboxed, success
 }
 
 // BoxSeal seals a crypto box using the provided shared key, returning the box and the nonce needed to decrypt it.
 // If nonce is nil, a random BoxNonce will be used and returned.
 // If nonce is non-nil, then nonce.Increment() will be called before using it, and the incremented BoxNonce is what is returned.
+// The unboxed input slice's backing array is reused for the boxed output when possible.
 func BoxSeal(shared *BoxSharedKey, unboxed []byte, nonce *BoxNonce) ([]byte, *BoxNonce) {
 	if nonce == nil {
 		nonce = NewBoxNonce()
 	}
 	nonce.Increment()
-	out := util.GetBytes()
 	s := (*[BoxSharedKeyLen]byte)(shared)
 	n := (*[BoxNonceLen]byte)(nonce)
-	boxed := box.SealAfterPrecomputation(out, unboxed, n, s)
+	temp := append(pool.Get().([]byte), unboxed...)
+	boxed := box.SealAfterPrecomputation(unboxed[:0], temp, n, s)
+	pool.Put(temp[:0])
 	return boxed, nonce
 }
 
@@ -272,7 +278,7 @@ func (n *BoxNonce) Increment() {
 	n[len(n)-1] += 2
 	for i := len(n) - 2; i >= 0; i-- {
 		if n[i+1] < oldNonce[i+1] {
-			n[i] += 1
+			n[i]++
 		}
 	}
 }

src/defaults/defaults_freebsd.go

@@ -10,7 +10,7 @@ func GetDefaults() platformDefaultParameters {
 		DefaultAdminListen: "unix:///var/run/yggdrasil.sock",
 
 		// Configuration (used for yggdrasilctl)
-		DefaultConfigFile: "/etc/yggdrasil.conf",
+		DefaultConfigFile: "/usr/local/etc/yggdrasil.conf",
 
 		// Multicast interfaces
 		DefaultMulticastInterfaces: []string{

src/defaults/defaults_openbsd.go

@@ -20,6 +20,6 @@ func GetDefaults() platformDefaultParameters {
 		// TUN/TAP
 		MaximumIfMTU:  16384,
 		DefaultIfMTU:  16384,
-		DefaultIfName: "/dev/tun0",
+		DefaultIfName: "tun0",
 	}
 }

src/multicast/multicast.go

@@ -21,16 +21,20 @@ import (
 // automatically.
 type Multicast struct {
 	phony.Inbox
-	core            *yggdrasil.Core
-	config          *config.NodeState
-	log             *log.Logger
-	sock            *ipv6.PacketConn
-	groupAddr       string
-	listeners       map[string]*listenerInfo
-	listenPort      uint16
-	isOpen          bool
-	announcer       *time.Timer
-	platformhandler *time.Timer
+	core        *yggdrasil.Core
+	config      *config.NodeState
+	log         *log.Logger
+	sock        *ipv6.PacketConn
+	groupAddr   string
+	listeners   map[string]*listenerInfo
+	listenPort  uint16
+	isOpen      bool
+	_interfaces map[string]interfaceInfo
+}
+
+type interfaceInfo struct {
+	iface net.Interface
+	addrs []net.Addr
 }
 
 type listenerInfo struct {
@@ -45,6 +49,7 @@ func (m *Multicast) Init(core *yggdrasil.Core, state *config.NodeState, log *log
 	m.config = state
 	m.log = log
 	m.listeners = make(map[string]*listenerInfo)
+	m._interfaces = make(map[string]interfaceInfo)
 	current := m.config.GetCurrent()
 	m.listenPort = current.LinkLocalTCPPort
 	m.groupAddr = "[ff02::114]:9001"
@@ -90,8 +95,8 @@ func (m *Multicast) _start() error {
 
 	m.isOpen = true
 	go m.listen()
-	m.Act(m, m.multicastStarted)
-	m.Act(m, m.announce)
+	m.Act(nil, m._multicastStarted)
+	m.Act(nil, m._announce)
 
 	return nil
 }
@@ -118,12 +123,6 @@ func (m *Multicast) Stop() error {
 func (m *Multicast) _stop() error {
 	m.log.Infoln("Stopping multicast module")
 	m.isOpen = false
-	if m.announcer != nil {
-		m.announcer.Stop()
-	}
-	if m.platformhandler != nil {
-		m.platformhandler.Stop()
-	}
 	if m.sock != nil {
 		m.sock.Close()
 	}
@@ -134,7 +133,7 @@ func (m *Multicast) _stop() error {
 // and then signals the various module goroutines to reconfigure themselves if
 // needed.
 func (m *Multicast) UpdateConfig(config *config.NodeConfig) {
-	m.Act(m, func() { m._updateConfig(config) })
+	m.Act(nil, func() { m._updateConfig(config) })
 }
 
 func (m *Multicast) _updateConfig(config *config.NodeConfig) {
@@ -156,10 +155,35 @@ func (m *Multicast) _updateConfig(config *config.NodeConfig) {
 	m.log.Debugln("Reloaded multicast configuration successfully")
 }
 
-// GetInterfaces returns the currently known/enabled multicast interfaces. It is
-// expected that UpdateInterfaces has been called at least once before calling
-// this method.
+func (m *Multicast) _updateInterfaces() {
+	interfaces := make(map[string]interfaceInfo)
+	intfs := m.getAllowedInterfaces()
+	for _, intf := range intfs {
+		addrs, err := intf.Addrs()
+		if err != nil {
+			m.log.Warnf("Failed up get addresses for interface %s: %s", intf.Name, err)
+			continue
+		}
+		interfaces[intf.Name] = interfaceInfo{
+			iface: intf,
+			addrs: addrs,
+		}
+	}
+	m._interfaces = interfaces
+}
+
 func (m *Multicast) Interfaces() map[string]net.Interface {
+	interfaces := make(map[string]net.Interface)
+	phony.Block(m, func() {
+		for _, info := range m._interfaces {
+			interfaces[info.iface.Name] = info.iface
+		}
+	})
+	return interfaces
+}
+
+// getAllowedInterfaces returns the currently known/enabled multicast interfaces.
+func (m *Multicast) getAllowedInterfaces() map[string]net.Interface {
 	interfaces := make(map[string]net.Interface)
 	// Get interface expressions from config
 	current := m.config.GetCurrent()
@@ -167,7 +191,9 @@ func (m *Multicast) Interfaces() map[string]net.Interface {
 	// Ask the system for network interfaces
 	allifaces, err := net.Interfaces()
 	if err != nil {
-		panic(err)
+		// Don't panic, since this may be from e.g. too many open files (from too much connection spam)
+		// TODO? log something
+		return nil
 	}
 	// Work out which interfaces to announce on
 	for _, iface := range allifaces {
@@ -198,7 +224,11 @@ func (m *Multicast) Interfaces() map[string]net.Interface {
 	return interfaces
 }
 
-func (m *Multicast) announce() {
+func (m *Multicast) _announce() {
+	if !m.isOpen {
+		return
+	}
+	m._updateInterfaces()
 	groupAddr, err := net.ResolveUDPAddr("udp6", m.groupAddr)
 	if err != nil {
 		panic(err)
@@ -207,7 +237,6 @@ func (m *Multicast) announce() {
 	if err != nil {
 		panic(err)
 	}
-	interfaces := m.Interfaces()
 	// There might be interfaces that we configured listeners for but are no
 	// longer up - if that's the case then we should stop the listeners
 	for name, info := range m.listeners {
@@ -219,7 +248,7 @@ func (m *Multicast) announce() {
 		}
 		// If the interface is no longer visible on the system then stop the
 		// listener, as another one will be started further down
-		if _, ok := interfaces[name]; !ok {
+		if _, ok := m._interfaces[name]; !ok {
 			stop()
 			continue
 		}
@@ -232,17 +261,13 @@ func (m *Multicast) announce() {
 			continue
 		}
 		// Find the interface that matches the listener
-		if intf, err := net.InterfaceByName(name); err == nil {
-			if addrs, err := intf.Addrs(); err == nil {
-				// Loop through the addresses attached to that listener and see if any
-				// of them match the current address of the listener
-				for _, addr := range addrs {
-					if ip, _, err := net.ParseCIDR(addr.String()); err == nil {
-						// Does the interface address match our listener address?
-						if ip.Equal(listenaddr.IP) {
-							found = true
-							break
-						}
+		if info, ok := m._interfaces[name]; ok {
+			for _, addr := range info.addrs {
+				if ip, _, err := net.ParseCIDR(addr.String()); err == nil {
+					// Does the interface address match our listener address?
+					if ip.Equal(listenaddr.IP) {
+						found = true
+						break
 					}
 				}
 			}
@@ -256,13 +281,9 @@ func (m *Multicast) announce() {
 	}
 	// Now that we have a list of valid interfaces from the operating system,
 	// we can start checking if we can send multicasts on them
-	for _, iface := range interfaces {
-		// Find interface addresses
-		addrs, err := iface.Addrs()
-		if err != nil {
-			panic(err)
-		}
-		for _, addr := range addrs {
+	for _, info := range m._interfaces {
+		iface := info.iface
+		for _, addr := range info.addrs {
 			addrIP, _, _ := net.ParseCIDR(addr.String())
 			// Ignore IPv4 addresses
 			if addrIP.To4() != nil {
@@ -312,8 +333,8 @@ func (m *Multicast) announce() {
 			break
 		}
 	}
-	m.announcer = time.AfterFunc(time.Second, func() {
-		m.Act(m, m.announce)
+	time.AfterFunc(time.Second, func() {
+		m.Act(nil, m._announce)
 	})
 }
 
@@ -351,7 +372,11 @@ func (m *Multicast) listen() {
 		if addr.IP.String() != from.IP.String() {
 			continue
 		}
-		if _, ok := m.Interfaces()[from.Zone]; ok {
+		var interfaces map[string]interfaceInfo
+		phony.Block(m, func() {
+			interfaces = m._interfaces
+		})
+		if _, ok := interfaces[from.Zone]; ok {
 			addr.Zone = ""
 			if err := m.core.CallPeer("tcp://"+addr.String(), from.Zone); err != nil {
 				m.log.Debugln("Call from multicast failed:", err)

src/multicast/multicast_darwin.go

@@ -31,16 +31,19 @@ import (
 
 var awdlGoroutineStarted bool
 
-func (m *Multicast) multicastStarted() {
+func (m *Multicast) _multicastStarted() {
+	if !m.isOpen {
+		return
+	}
 	C.StopAWDLBrowsing()
-	for intf := range m.Interfaces() {
+	for intf := range m._interfaces {
 		if intf == "awdl0" {
 			C.StartAWDLBrowsing()
 			break
 		}
 	}
-	m.platformhandler = time.AfterFunc(time.Minute, func() {
-		m.Act(m, m.multicastStarted)
+	time.AfterFunc(time.Minute, func() {
+		m.Act(nil, m._multicastStarted)
 	})
 }
 

src/multicast/multicast_other.go

@@ -4,7 +4,7 @@ package multicast
 
 import "syscall"
 
-func (m *Multicast) multicastStarted() {
+func (m *Multicast) _multicastStarted() {
 
 }
 

src/multicast/multicast_unix.go

@@ -5,7 +5,7 @@ package multicast
 import "syscall"
 import "golang.org/x/sys/unix"
 
-func (m *Multicast) multicastStarted() {
+func (m *Multicast) _multicastStarted() {
 
 }
 

src/multicast/multicast_windows.go

@@ -5,7 +5,7 @@ package multicast
 import "syscall"
 import "golang.org/x/sys/windows"
 
-func (m *Multicast) multicastStarted() {
+func (m *Multicast) _multicastStarted() {
 
 }
 

src/tuntap/admin.go

@@ -68,16 +68,14 @@ func (t *TunAdapter) SetupAdminHandlers(a *admin.AdminSocket) {
 	a.AddHandler("addLocalSubnet", []string{"subnet"}, func(in admin.Info) (admin.Info, error) {
 		if err := t.ckr.addLocalSubnet(in["subnet"].(string)); err == nil {
 			return admin.Info{"added": []string{in["subnet"].(string)}}, nil
-		} else {
-			return admin.Info{"not_added": []string{in["subnet"].(string)}}, errors.New("Failed to add source subnet")
 		}
+		return admin.Info{"not_added": []string{in["subnet"].(string)}}, errors.New("Failed to add source subnet")
 	})
 	a.AddHandler("addRemoteSubnet", []string{"subnet", "box_pub_key"}, func(in admin.Info) (admin.Info, error) {
 		if err := t.ckr.addRemoteSubnet(in["subnet"].(string), in["box_pub_key"].(string)); err == nil {
 			return admin.Info{"added": []string{fmt.Sprintf("%s via %s", in["subnet"].(string), in["box_pub_key"].(string))}}, nil
-		} else {
-			return admin.Info{"not_added": []string{fmt.Sprintf("%s via %s", in["subnet"].(string), in["box_pub_key"].(string))}}, errors.New("Failed to add route")
 		}
+		return admin.Info{"not_added": []string{fmt.Sprintf("%s via %s", in["subnet"].(string), in["box_pub_key"].(string))}}, errors.New("Failed to add route")
 	})
 	a.AddHandler("getSourceSubnets", []string{}, func(in admin.Info) (admin.Info, error) {
 		var subnets []string
@@ -104,15 +102,13 @@ func (t *TunAdapter) SetupAdminHandlers(a *admin.AdminSocket) {
 	a.AddHandler("removeLocalSubnet", []string{"subnet"}, func(in admin.Info) (admin.Info, error) {
 		if err := t.ckr.removeLocalSubnet(in["subnet"].(string)); err == nil {
 			return admin.Info{"removed": []string{in["subnet"].(string)}}, nil
-		} else {
-			return admin.Info{"not_removed": []string{in["subnet"].(string)}}, errors.New("Failed to remove source subnet")
 		}
+		return admin.Info{"not_removed": []string{in["subnet"].(string)}}, errors.New("Failed to remove source subnet")
 	})
 	a.AddHandler("removeRemoteSubnet", []string{"subnet", "box_pub_key"}, func(in admin.Info) (admin.Info, error) {
 		if err := t.ckr.removeRemoteSubnet(in["subnet"].(string), in["box_pub_key"].(string)); err == nil {
 			return admin.Info{"removed": []string{fmt.Sprintf("%s via %s", in["subnet"].(string), in["box_pub_key"].(string))}}, nil
-		} else {
-			return admin.Info{"not_removed": []string{fmt.Sprintf("%s via %s", in["subnet"].(string), in["box_pub_key"].(string))}}, errors.New("Failed to remove route")
 		}
+		return admin.Info{"not_removed": []string{fmt.Sprintf("%s via %s", in["subnet"].(string), in["box_pub_key"].(string))}}, errors.New("Failed to remove route")
 	})
 }

src/tuntap/conn.go

@@ -44,13 +44,11 @@ func (s *tunConn) _read(bs []byte) (err error) {
 	select {
 	case <-s.stop:
 		err = errors.New("session was already closed")
-		util.PutBytes(bs)
 		return
 	default:
 	}
 	if len(bs) == 0 {
 		err = errors.New("read packet with 0 size")
-		util.PutBytes(bs)
 		return
 	}
 	ipv4 := len(bs) > 20 && bs[0]&0xf0 == 0x40
@@ -107,7 +105,6 @@ func (s *tunConn) _read(bs []byte) (err error) {
 	}
 	if skip {
 		err = errors.New("address not allowed")
-		util.PutBytes(bs)
 		return
 	}
 	s.tun.writer.writeFrom(s, bs)
@@ -125,7 +122,6 @@ func (s *tunConn) _write(bs []byte) (err error) {
 	select {
 	case <-s.stop:
 		err = errors.New("session was already closed")
-		util.PutBytes(bs)
 		return
 	default:
 	}
@@ -183,7 +179,6 @@ func (s *tunConn) _write(bs []byte) (err error) {
 	}
 	if skip {
 		err = errors.New("address not allowed")
-		util.PutBytes(bs)
 		return
 	}
 	msg := yggdrasil.FlowKeyMessage{

src/tuntap/iface.go

@@ -3,9 +3,11 @@ package tuntap
 import (
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
-	"github.com/yggdrasil-network/yggdrasil-go/src/util"
 	"github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil"
 
+	"golang.org/x/net/icmp"
+	"golang.org/x/net/ipv6"
+
 	"github.com/Arceliar/phony"
 )
 
@@ -14,6 +16,7 @@ const TUN_OFFSET_BYTES = 4
 type tunWriter struct {
 	phony.Inbox
 	tun *TunAdapter
+	buf [TUN_OFFSET_BYTES + 65536]byte
 }
 
 func (w *tunWriter) writeFrom(from phony.Actor, b []byte) {
@@ -25,15 +28,13 @@ func (w *tunWriter) writeFrom(from phony.Actor, b []byte) {
 // write is pretty loose with the memory safety rules, e.g. it assumes it can
 // read w.tun.iface.IsTap() safely
 func (w *tunWriter) _write(b []byte) {
-	defer util.PutBytes(b)
 	var written int
 	var err error
 	n := len(b)
 	if n == 0 {
 		return
 	}
-	temp := append(util.ResizeBytes(util.GetBytes(), TUN_OFFSET_BYTES), b...)
-	defer util.PutBytes(temp)
+	temp := append(w.buf[:TUN_OFFSET_BYTES], b...)
 	written, err = w.tun.iface.Write(temp, TUN_OFFSET_BYTES)
 	if err != nil {
 		w.tun.Act(w, func() {
@@ -51,22 +52,23 @@ func (w *tunWriter) _write(b []byte) {
 type tunReader struct {
 	phony.Inbox
 	tun *TunAdapter
+	buf [TUN_OFFSET_BYTES + 65536]byte
 }
 
 func (r *tunReader) _read() {
 	// Get a slice to store the packet in
-	recvd := util.ResizeBytes(util.GetBytes(), int(r.tun.mtu)+TUN_OFFSET_BYTES)
 	// Wait for a packet to be delivered to us through the TUN adapter
-	n, err := r.tun.iface.Read(recvd, TUN_OFFSET_BYTES)
+	n, err := r.tun.iface.Read(r.buf[:], TUN_OFFSET_BYTES)
 	if n <= TUN_OFFSET_BYTES || err != nil {
 		r.tun.log.Errorln("Error reading TUN:", err)
 		ferr := r.tun.iface.Flush()
 		if ferr != nil {
 			r.tun.log.Errorln("Unable to flush packets:", ferr)
 		}
-		util.PutBytes(recvd)
 	} else {
-		r.tun.handlePacketFrom(r, recvd[TUN_OFFSET_BYTES:n+TUN_OFFSET_BYTES], err)
+		bs := make([]byte, n, n+crypto.BoxOverhead) // extra capacity for later...
+		copy(bs, r.buf[TUN_OFFSET_BYTES:n+TUN_OFFSET_BYTES])
+		r.tun.handlePacketFrom(r, bs, err)
 	}
 	if err == nil {
 		// Now read again
@@ -148,6 +150,16 @@ func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
 	}
 	if addrlen != 16 || (!dstAddr.IsValid() && !dstSnet.IsValid()) {
 		// Couldn't find this node's ygg IP
+		dlen := len(bs)
+		if dlen > 900 {
+			dlen = 900
+		}
+		ptb := &icmp.DstUnreach{
+			Data: bs[:dlen],
+		}
+		if packet, err := CreateICMPv6(bs[8:24], bs[24:40], ipv6.ICMPTypeDestinationUnreachable, 0, ptb); err == nil {
+			tun.writer.writeFrom(nil, packet)
+		}
 		return
 	}
 	// Do we have an active connection for this node address?
@@ -175,7 +187,6 @@ func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
 		_, known := tun.dials[dstString]
 		tun.dials[dstString] = append(tun.dials[dstString], bs)
 		for len(tun.dials[dstString]) > 32 {
-			util.PutBytes(tun.dials[dstString][0])
 			tun.dials[dstString] = tun.dials[dstString][1:]
 		}
 		if !known {
@@ -199,7 +210,6 @@ func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
 						tc.writeFrom(nil, packet)
 					}
 				})
-				return
 			}()
 		}
 	}

src/util/bytes_mobile.go

@@ -1,21 +0,0 @@
-//+build mobile
-
-package util
-
-import "runtime/debug"
-
-func init() {
-	debug.SetGCPercent(25)
-}
-
-// GetBytes always returns a nil slice on mobile platforms.
-func GetBytes() []byte {
-	return nil
-}
-
-// PutBytes does literally nothing on mobile platforms.
-// This is done rather than keeping a free list of bytes on platforms with memory constraints.
-// It's needed to help keep memory usage low enough to fall under the limits set for e.g. iOS NEPacketTunnelProvider apps.
-func PutBytes(bs []byte) {
-	return
-}

src/util/bytes_other.go

@@ -1,18 +0,0 @@
-//+build !mobile
-
-package util
-
-import "sync"
-
-// This is used to buffer recently used slices of bytes, to prevent allocations in the hot loops.
-var byteStore = sync.Pool{New: func() interface{} { return []byte(nil) }}
-
-// GetBytes returns a 0-length (possibly nil) slice of bytes from a free list, so it may have a larger capacity.
-func GetBytes() []byte {
-	return byteStore.Get().([]byte)[:0]
-}
-
-// PutBytes stores a slice in a free list, where it can potentially be reused to prevent future allocations.
-func PutBytes(bs []byte) {
-	byteStore.Put(bs)
-}

src/util/cancellation.go

@@ -54,12 +54,11 @@ func (c *cancellation) Cancel(err error) error {
 	defer c.mutex.Unlock()
 	if c.done {
 		return c.err
-	} else {
-		c.err = err
-		c.done = true
-		close(c.cancel)
-		return nil
 	}
+	c.err = err
+	c.done = true
+	close(c.cancel)
+	return nil
 }
 
 // Error returns the error provided to Cancel, or nil if no error has been provided.

src/util/util.go

@@ -30,9 +30,8 @@ func UnlockThread() {
 func ResizeBytes(bs []byte, length int) []byte {
 	if cap(bs) >= length {
 		return bs[:length]
-	} else {
-		return make([]byte, length)
 	}
+	return make([]byte, length)
 }
 
 // TimerStop stops a timer and makes sure the channel is drained, returns true if the timer was stopped before firing.

src/yggdrasil/api.go

@@ -110,7 +110,8 @@ type Session struct {
 // there is exactly one entry then this node is not connected to any other nodes
 // and is therefore isolated.
 func (c *Core) GetPeers() []Peer {
-	ports := c.peers.ports.Load().(map[switchPort]*peer)
+	var ports map[switchPort]*peer
+	phony.Block(&c.peers, func() { ports = c.peers.ports })
 	var peers []Peer
 	var ps []switchPort
 	for port := range ports {
@@ -122,10 +123,10 @@ func (c *Core) GetPeers() []Peer {
 		var info Peer
 		phony.Block(p, func() {
 			info = Peer{
-				Endpoint:   p.intf.name,
+				Endpoint:   p.intf.name(),
 				BytesSent:  p.bytesSent,
 				BytesRecvd: p.bytesRecvd,
-				Protocol:   p.intf.info.linkType,
+				Protocol:   p.intf.interfaceType(),
 				Port:       uint64(port),
 				Uptime:     time.Since(p.firstSeen),
 			}
@@ -143,10 +144,14 @@ func (c *Core) GetPeers() []Peer {
 // isolated or not connected to any peers.
 func (c *Core) GetSwitchPeers() []SwitchPeer {
 	var switchpeers []SwitchPeer
-	table := c.switchTable.table.Load().(lookupTable)
-	peers := c.peers.ports.Load().(map[switchPort]*peer)
+	var table *lookupTable
+	var ports map[switchPort]*peer
+	phony.Block(&c.peers, func() {
+		table = c.peers.table
+		ports = c.peers.ports
+	})
 	for _, elem := range table.elems {
-		peer, isIn := peers[elem.port]
+		peer, isIn := ports[elem.port]
 		if !isIn {
 			continue
 		}
@@ -158,8 +163,8 @@ func (c *Core) GetSwitchPeers() []SwitchPeer {
 				BytesSent:  peer.bytesSent,
 				BytesRecvd: peer.bytesRecvd,
 				Port:       uint64(elem.port),
-				Protocol:   peer.intf.info.linkType,
-				Endpoint:   peer.intf.info.remote,
+				Protocol:   peer.intf.interfaceType(),
+				Endpoint:   peer.intf.remote(),
 			}
 			copy(info.PublicKey[:], peer.box[:])
 		})
@@ -194,34 +199,6 @@ func (c *Core) GetDHT() []DHTEntry {
 	return dhtentries
 }
 
-// GetSwitchQueues returns information about the switch queues that are
-// currently in effect. These values can change within an instant.
-func (c *Core) GetSwitchQueues() SwitchQueues {
-	var switchqueues SwitchQueues
-	switchTable := &c.switchTable
-	getSwitchQueues := func() {
-		switchqueues = SwitchQueues{
-			Count:        uint64(len(switchTable.queues.bufs)),
-			Size:         switchTable.queues.size,
-			HighestCount: uint64(switchTable.queues.maxbufs),
-			HighestSize:  switchTable.queues.maxsize,
-			MaximumSize:  switchTable.queues.totalMaxSize,
-		}
-		for k, v := range switchTable.queues.bufs {
-			nexthop := switchTable.bestPortForCoords([]byte(k))
-			queue := SwitchQueue{
-				ID:      k,
-				Size:    v.size,
-				Packets: uint64(len(v.packets)),
-				Port:    uint64(nexthop),
-			}
-			switchqueues.Queues = append(switchqueues.Queues, queue)
-		}
-	}
-	phony.Block(&c.switchTable, getSwitchQueues)
-	return switchqueues
-}
-
 // GetSessions returns a list of open sessions from this node to other nodes.
 func (c *Core) GetSessions() []Session {
 	var sessions []Session
@@ -280,14 +257,14 @@ func (c *Core) ConnDialer() (*Dialer, error) {
 // "Listen" configuration item, e.g.
 // 		tcp://a.b.c.d:e
 func (c *Core) ListenTCP(uri string) (*TcpListener, error) {
-	return c.link.tcp.listen(uri, nil)
+	return c.links.tcp.listen(uri, nil)
 }
 
 // ListenTLS starts a new TLS listener. The input URI should match that of the
 // "Listen" configuration item, e.g.
 // 		tls://a.b.c.d:e
 func (c *Core) ListenTLS(uri string) (*TcpListener, error) {
-	return c.link.tcp.listen(uri, c.link.tcp.tls.forListener)
+	return c.links.tcp.listen(uri, c.links.tcp.tls.forListener)
 }
 
 // NodeID gets the node ID. This is derived from your router encryption keys.
@@ -324,8 +301,11 @@ func (c *Core) EncryptionPublicKey() string {
 // connected to any other nodes (effectively making you the root of a
 // single-node network).
 func (c *Core) Coords() []uint64 {
-	table := c.switchTable.table.Load().(lookupTable)
-	return wire_coordsBytestoUint64s(table.self.getCoords())
+	var coords []byte
+	phony.Block(&c.router, func() {
+		coords = c.router.table.self.getCoords()
+	})
+	return wire_coordsBytestoUint64s(coords)
 }
 
 // Address gets the IPv6 address of the Yggdrasil node. This is always a /128
@@ -468,12 +448,33 @@ func (c *Core) AddPeer(addr string, sintf string) error {
 	return nil
 }
 
-// RemovePeer is not implemented yet.
 func (c *Core) RemovePeer(addr string, sintf string) error {
-	// TODO: Implement a reverse of AddPeer, where we look up the port number
-	// based on the addr and sintf, disconnect it and then remove it from the
-	// peers list so we don't reconnect to it later
-	return errors.New("not implemented")
+	if sintf == "" {
+		for i, peer := range c.config.Current.Peers {
+			if peer == addr {
+				c.config.Current.Peers = append(c.config.Current.Peers[:i], c.config.Current.Peers[i+1:]...)
+				break
+			}
+		}
+	} else if _, ok := c.config.Current.InterfacePeers[sintf]; ok {
+		for i, peer := range c.config.Current.InterfacePeers[sintf] {
+			if peer == addr {
+				c.config.Current.InterfacePeers[sintf] = append(c.config.Current.InterfacePeers[sintf][:i], c.config.Current.InterfacePeers[sintf][i+1:]...)
+				break
+			}
+		}
+	}
+
+	c.peers.Act(nil, func() {
+		ports := c.peers.ports
+		for _, peer := range ports {
+			if addr == peer.intf.name() {
+				c.peers._removePeer(peer)
+			}
+		}
+	})
+
+	return nil
 }
 
 // CallPeer calls a peer once. This should be specified in the peer URI format,
@@ -483,13 +484,17 @@ func (c *Core) RemovePeer(addr string, sintf string) error {
 // This does not add the peer to the peer list, so if the connection drops, the
 // peer will not be called again automatically.
 func (c *Core) CallPeer(addr string, sintf string) error {
-	return c.link.call(addr, sintf)
+	return c.links.call(addr, sintf)
 }
 
 // DisconnectPeer disconnects a peer once. This should be specified as a port
 // number.
 func (c *Core) DisconnectPeer(port uint64) error {
-	c.peers.removePeer(switchPort(port))
+	c.peers.Act(nil, func() {
+		if p, isIn := c.peers.ports[switchPort(port)]; isIn {
+			p.Act(&c.peers, p._removeSelf)
+		}
+	})
 	return nil
 }
 

src/yggdrasil/conn.go

@@ -145,7 +145,8 @@ func (c *Conn) search() error {
 }
 
 // Used in session keep-alive traffic
-func (c *Conn) doSearch() {
+func (c *Conn) _doSearch() {
+	s := fmt.Sprintf("conn=%p", c)
 	routerWork := func() {
 		// Check to see if there is a search already matching the destination
 		sinfo, isIn := c.core.router.searches.searches[*c.nodeID]
@@ -153,7 +154,7 @@ func (c *Conn) doSearch() {
 			// Nothing was found, so create a new search
 			searchCompleted := func(sinfo *sessionInfo, e error) {}
 			sinfo = c.core.router.searches.newIterSearch(c.nodeID, c.nodeMask, searchCompleted)
-			c.core.log.Debugf("%s DHT search started: %p", c.String(), sinfo)
+			c.core.log.Debugf("%s DHT search started: %p", s, sinfo)
 			// Start the search
 			sinfo.startSearch()
 		}
@@ -166,10 +167,9 @@ func (c *Conn) _getDeadlineCancellation(t *time.Time) (util.Cancellation, bool)
 		// A deadline is set, so return a Cancellation that uses it
 		c := util.CancellationWithDeadline(c.session.cancel, *t)
 		return c, true
-	} else {
-		// No deadline was set, so just return the existing cancellation and a dummy value
-		return c.session.cancel, false
 	}
+	// No deadline was set, so just return the existing cancellation and a dummy value
+	return c.session.cancel, false
 }
 
 // SetReadCallback allows you to specify a function that will be called whenever
@@ -224,9 +224,8 @@ func (c *Conn) readNoCopy() ([]byte, error) {
 	case <-cancel.Finished():
 		if cancel.Error() == util.CancellationTimeoutError {
 			return nil, ConnError{errors.New("read timeout"), true, false, false, 0}
-		} else {
-			return nil, ConnError{errors.New("session closed"), false, false, true, 0}
 		}
+		return nil, ConnError{errors.New("session closed"), false, false, true, 0}
 	case bs := <-c.readBuffer:
 		return bs, nil
 	}
@@ -252,7 +251,6 @@ func (c *Conn) Read(b []byte) (int, error) {
 	}
 	// Copy results to the output slice and clean up
 	copy(b, bs)
-	util.PutBytes(bs)
 	// Return the number of bytes copied to the slice, along with any error
 	return n, err
 }
@@ -269,7 +267,7 @@ func (c *Conn) _write(msg FlowKeyMessage) error {
 		case time.Since(c.session.time) > 6*time.Second:
 			if c.session.time.Before(c.session.pingTime) && time.Since(c.session.pingTime) > 6*time.Second {
 				// TODO double check that the above condition is correct
-				c.doSearch()
+				c._doSearch()
 			} else {
 				c.session.ping(c.session) // TODO send from self if this becomes an actor
 			}
@@ -323,10 +321,11 @@ func (c *Conn) writeNoCopy(msg FlowKeyMessage) error {
 // returned.
 func (c *Conn) Write(b []byte) (int, error) {
 	written := len(b)
-	msg := FlowKeyMessage{Message: append(util.GetBytes(), b...)}
+	bs := make([]byte, 0, len(b)+crypto.BoxOverhead)
+	bs = append(bs, b...)
+	msg := FlowKeyMessage{Message: bs}
 	err := c.writeNoCopy(msg)
 	if err != nil {
-		util.PutBytes(msg.Message)
 		written = 0
 	}
 	return written, err

src/yggdrasil/core.go

@@ -29,7 +29,7 @@ type Core struct {
 	switchTable  switchTable
 	peers        peers
 	router       router
-	link         link
+	links        links
 	log          *log.Logger
 	addPeerTimer *time.Timer
 }
@@ -160,9 +160,12 @@ func (c *Core) _start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState
 	}
 
 	c.log.Infoln("Starting up...")
-	c._init()
+	if err := c._init(); err != nil {
+		c.log.Errorln("Failed to initialize core")
+		return nil, err
+	}
 
-	if err := c.link.init(c); err != nil {
+	if err := c.links.init(c); err != nil {
 		c.log.Errorln("Failed to start link interfaces")
 		return nil, err
 	}
@@ -194,9 +197,11 @@ func (c *Core) _stop() {
 	if c.addPeerTimer != nil {
 		c.addPeerTimer.Stop()
 	}
-	c.link.stop()
+	c.links.stop()
+	/* FIXME this deadlocks, need a waitgroup or something to coordinate shutdown
 	for _, peer := range c.GetPeers() {
 		c.DisconnectPeer(peer.Port)
 	}
+	*/
 	c.log.Infoln("Stopped")
 }

src/yggdrasil/dht.go

@@ -89,6 +89,11 @@ func (t *dht) reconfigure() {
 // Resets the DHT in response to coord changes.
 // This empties all info from the DHT and drops outstanding requests.
 func (t *dht) reset() {
+	for _, info := range t.table {
+		if t.isImportant(info) {
+			t.ping(info, nil)
+		}
+	}
 	t.reqs = make(map[dhtReqKey]time.Time)
 	t.table = make(map[crypto.NodeID]*dhtInfo)
 	t.imp = nil
@@ -144,12 +149,8 @@ func (t *dht) insert(info *dhtInfo) {
 
 // Insert a peer into the table if it hasn't been pinged lately, to keep peers from dropping
 func (t *dht) insertPeer(info *dhtInfo) {
-	oldInfo, isIn := t.table[*info.getNodeID()]
-	if !isIn || time.Since(oldInfo.recv) > dht_max_delay+30*time.Second {
-		// TODO? also check coords?
-		newInfo := *info // Insert a copy
-		t.insert(&newInfo)
-	}
+	t.insert(info)    // FIXME this resets timers / ping counts / etc, so it seems kind of dangerous
+	t.ping(info, nil) // This is a quick fix to the above, ping them immediately...
 }
 
 // Return true if first/second/third are (partially) ordered correctly.
@@ -186,11 +187,9 @@ func dht_ordered(first, second, third *crypto.NodeID) bool {
 // Update info about the node that sent the request.
 func (t *dht) handleReq(req *dhtReq) {
 	// Send them what they asked for
-	loc := t.router.core.switchTable.getLocator()
-	coords := loc.getCoords()
 	res := dhtRes{
 		Key:    t.router.core.boxPub,
-		Coords: coords,
+		Coords: t.router.table.self.getCoords(),
 		Dest:   req.Dest,
 		Infos:  t.lookup(&req.Dest, false),
 	}
@@ -302,11 +301,9 @@ func (t *dht) ping(info *dhtInfo, target *crypto.NodeID) {
 	if target == nil {
 		target = &t.nodeID
 	}
-	loc := t.router.core.switchTable.getLocator()
-	coords := loc.getCoords()
 	req := dhtReq{
 		Key:    t.router.core.boxPub,
-		Coords: coords,
+		Coords: t.router.table.self.getCoords(),
 		Dest:   *target,
 	}
 	t.sendReq(&req, info)
@@ -380,7 +377,7 @@ func (t *dht) getImportant() []*dhtInfo {
 		})
 		// Keep the ones that are no further than the closest seen so far
 		minDist := ^uint64(0)
-		loc := t.router.core.switchTable.getLocator()
+		loc := t.router.table.self
 		important := infos[:0]
 		for _, info := range infos {
 			dist := uint64(loc.dist(info.coords))
@@ -418,7 +415,7 @@ func (t *dht) isImportant(ninfo *dhtInfo) bool {
 	}
 	important := t.getImportant()
 	// Check if ninfo is of equal or greater importance to what we already know
-	loc := t.router.core.switchTable.getLocator()
+	loc := t.router.table.self
 	ndist := uint64(loc.dist(ninfo.coords))
 	minDist := ^uint64(0)
 	for _, info := range important {

src/yggdrasil/link.go

@@ -16,16 +16,17 @@ import (
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
+	"golang.org/x/net/proxy"
 
 	"github.com/Arceliar/phony"
 )
 
-type link struct {
-	core       *Core
-	mutex      sync.RWMutex // protects interfaces below
-	interfaces map[linkInfo]*linkInterface
-	tcp        tcp // TCP interface support
-	stopped    chan struct{}
+type links struct {
+	core    *Core
+	mutex   sync.RWMutex // protects links below
+	links   map[linkInfo]*link
+	tcp     tcp // TCP interface support
+	stopped chan struct{}
 	// TODO timeout (to remove from switch), read from config.ReadTimeout
 }
 
@@ -37,7 +38,7 @@ type linkInfo struct {
 	remote   string           // Remote name or address
 }
 
-type linkInterfaceMsgIO interface {
+type linkMsgIO interface {
 	readMsg() ([]byte, error)
 	writeMsgs([][]byte) (int, error)
 	close() error
@@ -46,31 +47,37 @@ type linkInterfaceMsgIO interface {
 	_recvMetaBytes() ([]byte, error)
 }
 
-type linkInterface struct {
-	name           string
-	link           *link
+type link struct {
+	lname          string
+	links          *links
 	peer           *peer
-	msgIO          linkInterfaceMsgIO
+	options        linkOptions
+	msgIO          linkMsgIO
 	info           linkInfo
 	incoming       bool
 	force          bool
 	closed         chan struct{}
-	reader         linkReader  // Reads packets, notifies this linkInterface, passes packets to switch
-	writer         linkWriter  // Writes packets, notifies this linkInterface
+	reader         linkReader  // Reads packets, notifies this link, passes packets to switch
+	writer         linkWriter  // Writes packets, notifies this link
 	phony.Inbox                // Protects the below
 	sendTimer      *time.Timer // Fires to signal that sending is blocked
 	keepAliveTimer *time.Timer // Fires to send keep-alive traffic
 	stallTimer     *time.Timer // Fires to signal that no incoming traffic (including keep-alive) has been seen
 	closeTimer     *time.Timer // Fires when the link has been idle so long we need to close it
-	inSwitch       bool        // True if the switch is tracking this link
-	stalled        bool        // True if we haven't been receiving any response traffic
-	unstalled      bool        // False if an idle notification to the switch hasn't been sent because we stalled (or are first starting up)
+	readUnblocked  bool        // True if we've sent a read message unblocking this peer in the switch
+	writeUnblocked bool        // True if we've sent a write message unblocking this peer in the swithc
+	shutdown       bool        // True if we're shutting down, avoids sending some messages that could race with new peers being crated in the same port
 }
 
-func (l *link) init(c *Core) error {
+type linkOptions struct {
+	pinnedCurve25519Keys map[crypto.BoxPubKey]struct{}
+	pinnedEd25519Keys    map[crypto.SigPubKey]struct{}
+}
+
+func (l *links) init(c *Core) error {
 	l.core = c
 	l.mutex.Lock()
-	l.interfaces = make(map[linkInfo]*linkInterface)
+	l.links = make(map[linkInfo]*link)
 	l.mutex.Unlock()
 	l.stopped = make(chan struct{})
 
@@ -82,30 +89,58 @@ func (l *link) init(c *Core) error {
 	return nil
 }
 
-func (l *link) reconfigure() {
+func (l *links) reconfigure() {
 	l.tcp.reconfigure()
 }
 
-func (l *link) call(uri string, sintf string) error {
+func (l *links) call(uri string, sintf string) error {
 	u, err := url.Parse(uri)
 	if err != nil {
 		return fmt.Errorf("peer %s is not correctly formatted (%s)", uri, err)
 	}
 	pathtokens := strings.Split(strings.Trim(u.Path, "/"), "/")
+	tcpOpts := tcpOptions{}
+	if pubkeys, ok := u.Query()["curve25519"]; ok && len(pubkeys) > 0 {
+		tcpOpts.pinnedCurve25519Keys = make(map[crypto.BoxPubKey]struct{})
+		for _, pubkey := range pubkeys {
+			if boxPub, err := hex.DecodeString(pubkey); err == nil {
+				var boxPubKey crypto.BoxPubKey
+				copy(boxPubKey[:], boxPub)
+				tcpOpts.pinnedCurve25519Keys[boxPubKey] = struct{}{}
+			}
+		}
+	}
+	if pubkeys, ok := u.Query()["ed25519"]; ok && len(pubkeys) > 0 {
+		tcpOpts.pinnedEd25519Keys = make(map[crypto.SigPubKey]struct{})
+		for _, pubkey := range pubkeys {
+			if sigPub, err := hex.DecodeString(pubkey); err == nil {
+				var sigPubKey crypto.SigPubKey
+				copy(sigPubKey[:], sigPub)
+				tcpOpts.pinnedEd25519Keys[sigPubKey] = struct{}{}
+			}
+		}
+	}
 	switch u.Scheme {
 	case "tcp":
-		l.tcp.call(u.Host, nil, sintf, nil)
+		l.tcp.call(u.Host, tcpOpts, sintf)
 	case "socks":
-		l.tcp.call(pathtokens[0], u.Host, sintf, nil)
+		tcpOpts.socksProxyAddr = u.Host
+		if u.User != nil {
+			tcpOpts.socksProxyAuth = &proxy.Auth{}
+			tcpOpts.socksProxyAuth.User = u.User.Username()
+			tcpOpts.socksProxyAuth.Password, _ = u.User.Password()
+		}
+		l.tcp.call(pathtokens[0], tcpOpts, sintf)
 	case "tls":
-		l.tcp.call(u.Host, nil, sintf, l.tcp.tls.forDialer)
+		tcpOpts.upgrade = l.tcp.tls.forDialer
+		l.tcp.call(u.Host, tcpOpts, sintf)
 	default:
 		return errors.New("unknown call scheme: " + u.Scheme)
 	}
 	return nil
 }
 
-func (l *link) listen(uri string) error {
+func (l *links) listen(uri string) error {
 	u, err := url.Parse(uri)
 	if err != nil {
 		return fmt.Errorf("listener %s is not correctly formatted (%s)", uri, err)
@@ -122,12 +157,13 @@ func (l *link) listen(uri string) error {
 	}
 }
 
-func (l *link) create(msgIO linkInterfaceMsgIO, name, linkType, local, remote string, incoming, force bool) (*linkInterface, error) {
+func (l *links) create(msgIO linkMsgIO, name, linkType, local, remote string, incoming, force bool, options linkOptions) (*link, error) {
 	// Technically anything unique would work for names, but let's pick something human readable, just for debugging
-	intf := linkInterface{
-		name:  name,
-		link:  l,
-		msgIO: msgIO,
+	intf := link{
+		lname:   name,
+		links:   l,
+		options: options,
+		msgIO:   msgIO,
 		info: linkInfo{
 			linkType: linkType,
 			local:    local,
@@ -137,12 +173,13 @@ func (l *link) create(msgIO linkInterfaceMsgIO, name, linkType, local, remote st
 		force:    force,
 	}
 	intf.writer.intf = &intf
+	intf.writer.worker = make(chan [][]byte, 1)
 	intf.reader.intf = &intf
 	intf.reader.err = make(chan error)
 	return &intf, nil
 }
 
-func (l *link) stop() error {
+func (l *links) stop() error {
 	close(l.stopped)
 	if err := l.tcp.stop(); err != nil {
 		return err
@@ -150,94 +187,114 @@ func (l *link) stop() error {
 	return nil
 }
 
-func (intf *linkInterface) handler() error {
+func (intf *link) handler() (chan struct{}, error) {
 	// TODO split some of this into shorter functions, so it's easier to read, and for the FIXME duplicate peer issue mentioned later
+	go func() {
+		for bss := range intf.writer.worker {
+			intf.msgIO.writeMsgs(bss)
+		}
+	}()
+	defer intf.writer.Act(nil, func() {
+		intf.writer.closed = true
+		close(intf.writer.worker)
+	})
 	myLinkPub, myLinkPriv := crypto.NewBoxKeys()
 	meta := version_getBaseMetadata()
-	meta.box = intf.link.core.boxPub
-	meta.sig = intf.link.core.sigPub
+	meta.box = intf.links.core.boxPub
+	meta.sig = intf.links.core.sigPub
 	meta.link = *myLinkPub
 	metaBytes := meta.encode()
 	// TODO timeouts on send/recv (goroutine for send/recv, channel select w/ timer)
 	var err error
 	if !util.FuncTimeout(func() { err = intf.msgIO._sendMetaBytes(metaBytes) }, 30*time.Second) {
-		return errors.New("timeout on metadata send")
+		return nil, errors.New("timeout on metadata send")
 	}
 	if err != nil {
-		return err
+		return nil, err
 	}
 	if !util.FuncTimeout(func() { metaBytes, err = intf.msgIO._recvMetaBytes() }, 30*time.Second) {
-		return errors.New("timeout on metadata recv")
+		return nil, errors.New("timeout on metadata recv")
 	}
 	if err != nil {
-		return err
+		return nil, err
 	}
 	meta = version_metadata{}
 	if !meta.decode(metaBytes) || !meta.check() {
-		return errors.New("failed to decode metadata")
+		return nil, errors.New("failed to decode metadata")
 	}
 	base := version_getBaseMetadata()
 	if meta.ver > base.ver || meta.ver == base.ver && meta.minorVer > base.minorVer {
-		intf.link.core.log.Errorln("Failed to connect to node: " + intf.name + " version: " + fmt.Sprintf("%d.%d", meta.ver, meta.minorVer))
-		return errors.New("failed to connect: wrong version")
+		intf.links.core.log.Errorln("Failed to connect to node: " + intf.lname + " version: " + fmt.Sprintf("%d.%d", meta.ver, meta.minorVer))
+		return nil, errors.New("failed to connect: wrong version")
+	}
+	// Check if the remote side matches the keys we expected. This is a bit of a weak
+	// check - in future versions we really should check a signature or something like that.
+	if pinned := intf.options.pinnedCurve25519Keys; pinned != nil {
+		if _, allowed := pinned[meta.box]; !allowed {
+			intf.links.core.log.Errorf("Failed to connect to node: %q sent curve25519 key that does not match pinned keys", intf.name)
+			return nil, fmt.Errorf("failed to connect: host sent curve25519 key that does not match pinned keys")
+		}
+	}
+	if pinned := intf.options.pinnedEd25519Keys; pinned != nil {
+		if _, allowed := pinned[meta.sig]; !allowed {
+			intf.links.core.log.Errorf("Failed to connect to node: %q sent ed25519 key that does not match pinned keys", intf.name)
+			return nil, fmt.Errorf("failed to connect: host sent ed25519 key that does not match pinned keys")
+		}
 	}
 	// Check if we're authorized to connect to this key / IP
-	if intf.incoming && !intf.force && !intf.link.core.peers.isAllowedEncryptionPublicKey(&meta.box) {
-		intf.link.core.log.Warnf("%s connection from %s forbidden: AllowedEncryptionPublicKeys does not contain key %s",
+	if intf.incoming && !intf.force && !intf.links.core.peers.isAllowedEncryptionPublicKey(&meta.box) {
+		intf.links.core.log.Warnf("%s connection from %s forbidden: AllowedEncryptionPublicKeys does not contain key %s",
 			strings.ToUpper(intf.info.linkType), intf.info.remote, hex.EncodeToString(meta.box[:]))
 		intf.msgIO.close()
-		return nil
+		return nil, nil
 	}
 	// Check if we already have a link to this node
 	intf.info.box = meta.box
 	intf.info.sig = meta.sig
-	intf.link.mutex.Lock()
-	if oldIntf, isIn := intf.link.interfaces[intf.info]; isIn {
-		intf.link.mutex.Unlock()
+	intf.links.mutex.Lock()
+	if oldIntf, isIn := intf.links.links[intf.info]; isIn {
+		intf.links.mutex.Unlock()
 		// FIXME we should really return an error and let the caller block instead
 		// That lets them do things like close connections on its own, avoid printing a connection message in the first place, etc.
-		intf.link.core.log.Debugln("DEBUG: found existing interface for", intf.name)
+		intf.links.core.log.Debugln("DEBUG: found existing interface for", intf.name)
 		intf.msgIO.close()
-		if !intf.incoming {
-			// Block outgoing connection attempts until the existing connection closes
-			<-oldIntf.closed
-		}
-		return nil
+		return oldIntf.closed, nil
 	} else {
 		intf.closed = make(chan struct{})
-		intf.link.interfaces[intf.info] = intf
+		intf.links.links[intf.info] = intf
 		defer func() {
-			intf.link.mutex.Lock()
-			delete(intf.link.interfaces, intf.info)
-			intf.link.mutex.Unlock()
+			intf.links.mutex.Lock()
+			delete(intf.links.links, intf.info)
+			intf.links.mutex.Unlock()
 			close(intf.closed)
 		}()
-		intf.link.core.log.Debugln("DEBUG: registered interface for", intf.name)
+		intf.links.core.log.Debugln("DEBUG: registered interface for", intf.name)
 	}
-	intf.link.mutex.Unlock()
+	intf.links.mutex.Unlock()
 	// Create peer
 	shared := crypto.GetSharedKey(myLinkPriv, &meta.link)
-	intf.peer = intf.link.core.peers.newPeer(&meta.box, &meta.sig, shared, intf, func() { intf.msgIO.close() })
+	phony.Block(&intf.links.core.peers, func() {
+		// FIXME don't use phony.Block, it's bad practice, even if it's safe here
+		intf.peer = intf.links.core.peers._newPeer(&meta.box, &meta.sig, shared, intf)
+	})
 	if intf.peer == nil {
-		return errors.New("failed to create peer")
+		return nil, errors.New("failed to create peer")
 	}
 	defer func() {
 		// More cleanup can go here
-		intf.link.core.peers.removePeer(intf.peer.port)
+		intf.Act(nil, func() {
+			intf.shutdown = true
+			intf.peer.Act(intf, intf.peer._removeSelf)
+		})
 	}()
-	intf.peer.out = func(msgs [][]byte) {
-		intf.writer.sendFrom(intf.peer, msgs, false)
-	}
-	intf.peer.linkOut = func(bs []byte) {
-		intf.writer.sendFrom(intf.peer, [][]byte{bs}, true)
-	}
 	themAddr := address.AddrForNodeID(crypto.GetNodeID(&intf.info.box))
 	themAddrString := net.IP(themAddr[:]).String()
 	themString := fmt.Sprintf("%s@%s", themAddrString, intf.info.remote)
-	intf.link.core.log.Infof("Connected %s: %s, source %s",
+	intf.links.core.log.Infof("Connected %s: %s, source %s",
 		strings.ToUpper(intf.info.linkType), themString, intf.info.local)
 	// Start things
 	go intf.peer.start()
+	intf.Act(nil, intf._notifyIdle)
 	intf.reader.Act(nil, intf.reader._read)
 	// Wait for the reader to finish
 	// TODO find a way to do this without keeping live goroutines around
@@ -245,7 +302,7 @@ func (intf *linkInterface) handler() error {
 	defer close(done)
 	go func() {
 		select {
-		case <-intf.link.stopped:
+		case <-intf.links.stopped:
 			intf.msgIO.close()
 		case <-done:
 		}
@@ -253,17 +310,71 @@ func (intf *linkInterface) handler() error {
 	err = <-intf.reader.err
 	// TODO don't report an error if it's just a 'use of closed network connection'
 	if err != nil {
-		intf.link.core.log.Infof("Disconnected %s: %s, source %s; error: %s",
+		intf.links.core.log.Infof("Disconnected %s: %s, source %s; error: %s",
 			strings.ToUpper(intf.info.linkType), themString, intf.info.local, err)
 	} else {
-		intf.link.core.log.Infof("Disconnected %s: %s, source %s",
+		intf.links.core.log.Infof("Disconnected %s: %s, source %s",
 			strings.ToUpper(intf.info.linkType), themString, intf.info.local)
 	}
-	return err
+	return nil, err
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 
+// link needs to match the linkInterface type needed by the peers
+
+type linkInterface interface {
+	out([][]byte)
+	linkOut([]byte)
+	close()
+	// These next ones are only used by the API
+	name() string
+	local() string
+	remote() string
+	interfaceType() string
+}
+
+func (intf *link) out(bss [][]byte) {
+	intf.Act(nil, func() {
+		// nil to prevent it from blocking if the link is somehow frozen
+		// this is safe because another packet won't be sent until the link notifies
+		//  the peer that it's ready for one
+		intf.writer.sendFrom(nil, bss)
+	})
+}
+
+func (intf *link) linkOut(bs []byte) {
+	intf.Act(nil, func() {
+		// nil to prevent it from blocking if the link is somehow frozen
+		// FIXME this is hypothetically not safe, the peer shouldn't be sending
+		//  additional packets until this one finishes, otherwise this could leak
+		//  memory if writing happens slower than link packets are generated...
+		//  that seems unlikely, so it's a lesser evil than deadlocking for now
+		intf.writer.sendFrom(nil, [][]byte{bs})
+	})
+}
+
+func (intf *link) close() {
+	intf.Act(nil, func() { intf.msgIO.close() })
+}
+
+func (intf *link) name() string {
+	return intf.lname
+}
+
+func (intf *link) local() string {
+	return intf.info.local
+}
+
+func (intf *link) remote() string {
+	return intf.info.remote
+}
+
+func (intf *link) interfaceType() string {
+	return intf.info.linkType
+}
+
+////////////////////////////////////////////////////////////////////////////////
 const (
 	sendTime      = 1 * time.Second    // How long to wait before deciding a send is blocked
 	keepAliveTime = 2 * time.Second    // How long to wait before sending a keep-alive response if we have no real traffic to send
@@ -272,118 +383,108 @@ const (
 )
 
 // notify the intf that we're currently sending
-func (intf *linkInterface) notifySending(size int, isLinkTraffic bool) {
+func (intf *link) notifySending(size int) {
 	intf.Act(&intf.writer, func() {
-		if !isLinkTraffic {
-			intf.inSwitch = false
-		}
 		intf.sendTimer = time.AfterFunc(sendTime, intf.notifyBlockedSend)
-		intf._cancelStallTimer()
+		if intf.keepAliveTimer != nil {
+			intf.keepAliveTimer.Stop()
+			intf.keepAliveTimer = nil
+		}
+		intf.peer.notifyBlocked(intf)
 	})
 }
 
-// called by an AfterFunc if we seem to be blocked in a send syscall for a long time
-func (intf *linkInterface) _notifySyscall() {
-	intf.link.core.switchTable.Act(intf, func() {
-		intf.link.core.switchTable._sendingIn(intf.peer.port)
-	})
-}
-
-// we just sent something, so cancel any pending timer to send keep-alive traffic
-func (intf *linkInterface) _cancelStallTimer() {
-	if intf.stallTimer != nil {
-		intf.stallTimer.Stop()
-		intf.stallTimer = nil
-	}
-}
-
 // This gets called from a time.AfterFunc, and notifies the switch that we appear
 // to have gotten blocked on a write, so the switch should start routing traffic
 // through other links, if alternatives exist
-func (intf *linkInterface) notifyBlockedSend() {
+func (intf *link) notifyBlockedSend() {
 	intf.Act(nil, func() {
 		if intf.sendTimer != nil {
 			//As far as we know, we're still trying to send, and the timer fired.
-			intf.link.core.switchTable.blockPeer(intf.peer.port)
+			intf.sendTimer.Stop()
+			intf.sendTimer = nil
+			if !intf.shutdown && intf.writeUnblocked {
+				intf.writeUnblocked = false
+				intf.links.core.switchTable.blockPeer(intf, intf.peer.port, true)
+			}
 		}
 	})
 }
 
 // notify the intf that we've finished sending, returning the peer to the switch
-func (intf *linkInterface) notifySent(size int, isLinkTraffic bool) {
+func (intf *link) notifySent(size int) {
 	intf.Act(&intf.writer, func() {
-		intf.sendTimer.Stop()
-		intf.sendTimer = nil
-		if !isLinkTraffic {
-			intf._notifySwitch()
+		if intf.sendTimer != nil {
+			intf.sendTimer.Stop()
+			intf.sendTimer = nil
 		}
+		if intf.keepAliveTimer != nil {
+			// TODO? unset this when we start sending, not when we finish...
+			intf.keepAliveTimer.Stop()
+			intf.keepAliveTimer = nil
+		}
+		intf._notifyIdle()
 		if size > 0 && intf.stallTimer == nil {
 			intf.stallTimer = time.AfterFunc(stallTime, intf.notifyStalled)
 		}
+		if !intf.shutdown && !intf.writeUnblocked {
+			intf.writeUnblocked = true
+			intf.links.core.switchTable.unblockPeer(intf, intf.peer.port, true)
+		}
 	})
 }
 
-// Notify the switch that we're ready for more traffic, assuming we're not in a stalled state
-func (intf *linkInterface) _notifySwitch() {
-	if !intf.inSwitch {
-		if intf.stalled {
-			intf.unstalled = false
-		} else {
-			intf.inSwitch = true
-			intf.link.core.switchTable.Act(intf, func() {
-				intf.link.core.switchTable._idleIn(intf.peer.port)
-			})
-		}
-	}
+// Notify the peer that we're ready for more traffic
+func (intf *link) _notifyIdle() {
+	intf.peer.Act(intf, intf.peer._handleIdle)
 }
 
 // Set the peer as stalled, to prevent them from returning to the switch until a read succeeds
-func (intf *linkInterface) notifyStalled() {
+func (intf *link) notifyStalled() {
 	intf.Act(nil, func() { // Sent from a time.AfterFunc
 		if intf.stallTimer != nil {
 			intf.stallTimer.Stop()
 			intf.stallTimer = nil
-			intf.stalled = true
-			intf.link.core.switchTable.blockPeer(intf.peer.port)
+			if !intf.shutdown && intf.readUnblocked {
+				intf.readUnblocked = false
+				intf.links.core.switchTable.blockPeer(intf, intf.peer.port, false)
+			}
 		}
 	})
 }
 
 // reset the close timer
-func (intf *linkInterface) notifyReading() {
+func (intf *link) notifyReading() {
 	intf.Act(&intf.reader, func() {
-		if intf.closeTimer != nil {
-			intf.closeTimer.Stop()
-		}
 		intf.closeTimer = time.AfterFunc(closeTime, func() { intf.msgIO.close() })
 	})
 }
 
 // wake up the link if it was stalled, and (if size > 0) prepare to send keep-alive traffic
-func (intf *linkInterface) notifyRead(size int) {
+func (intf *link) notifyRead(size int) {
 	intf.Act(&intf.reader, func() {
+		intf.closeTimer.Stop()
 		if intf.stallTimer != nil {
 			intf.stallTimer.Stop()
 			intf.stallTimer = nil
 		}
-		intf.stalled = false
-		if !intf.unstalled {
-			intf._notifySwitch()
-			intf.unstalled = true
+		if size > 0 && intf.keepAliveTimer == nil {
+			intf.keepAliveTimer = time.AfterFunc(keepAliveTime, intf.notifyDoKeepAlive)
 		}
-		if size > 0 && intf.stallTimer == nil {
-			intf.stallTimer = time.AfterFunc(keepAliveTime, intf.notifyDoKeepAlive)
+		if !intf.shutdown && !intf.readUnblocked {
+			intf.readUnblocked = true
+			intf.links.core.switchTable.unblockPeer(intf, intf.peer.port, false)
 		}
 	})
 }
 
 // We need to send keep-alive traffic now
-func (intf *linkInterface) notifyDoKeepAlive() {
+func (intf *link) notifyDoKeepAlive() {
 	intf.Act(nil, func() { // Sent from a time.AfterFunc
-		if intf.stallTimer != nil {
-			intf.stallTimer.Stop()
-			intf.stallTimer = nil
-			intf.writer.sendFrom(nil, [][]byte{nil}, true) // Empty keep-alive traffic
+		if intf.keepAliveTimer != nil {
+			intf.keepAliveTimer.Stop()
+			intf.keepAliveTimer = nil
+			intf.writer.sendFrom(nil, [][]byte{nil}) // Empty keep-alive traffic
 		}
 	})
 }
@@ -392,34 +493,23 @@ func (intf *linkInterface) notifyDoKeepAlive() {
 
 type linkWriter struct {
 	phony.Inbox
-	intf *linkInterface
+	intf   *link
+	worker chan [][]byte
+	closed bool
 }
 
-func (w *linkWriter) sendFrom(from phony.Actor, bss [][]byte, isLinkTraffic bool) {
+func (w *linkWriter) sendFrom(from phony.Actor, bss [][]byte) {
 	w.Act(from, func() {
+		if w.closed {
+			return
+		}
 		var size int
 		for _, bs := range bss {
 			size += len(bs)
 		}
-		w.intf.notifySending(size, isLinkTraffic)
-		// start a timer that will fire if we get stuck in writeMsgs for an oddly long time
-		var once sync.Once
-		timer := time.AfterFunc(time.Millisecond, func() {
-			// 1 ms is kind of arbitrary
-			// the rationale is that this should be very long compared to a syscall
-			// but it's still short compared to end-to-end latency or human perception
-			once.Do(func() {
-				w.intf.Act(nil, w.intf._notifySyscall)
-			})
-		})
-		w.intf.msgIO.writeMsgs(bss)
-		// Make sure we either stop the timer from doing anything or wait until it's done
-		once.Do(func() { timer.Stop() })
-		w.intf.notifySent(size, isLinkTraffic)
-		// Cleanup
-		for _, bs := range bss {
-			util.PutBytes(bs)
-		}
+		w.intf.notifySending(size)
+		w.worker <- bss
+		w.intf.notifySent(size)
 	})
 }
 
@@ -427,7 +517,7 @@ func (w *linkWriter) sendFrom(from phony.Actor, bss [][]byte, isLinkTraffic bool
 
 type linkReader struct {
 	phony.Inbox
-	intf *linkInterface
+	intf *link
 	err  chan error
 }
 

src/yggdrasil/nodeinfo.go

@@ -18,6 +18,7 @@ type nodeinfo struct {
 	myNodeInfo NodeInfoPayload
 	callbacks  map[crypto.BoxPubKey]nodeinfoCallback
 	cache      map[crypto.BoxPubKey]nodeinfoCached
+	table      *lookupTable
 }
 
 type nodeinfoCached struct {
@@ -136,15 +137,15 @@ func (m *nodeinfo) _setNodeInfo(given interface{}, privacy bool) error {
 			newnodeinfo[key] = value
 		}
 	}
-	if newjson, err := json.Marshal(newnodeinfo); err == nil {
+	newjson, err := json.Marshal(newnodeinfo)
+	if err == nil {
 		if len(newjson) > 16384 {
 			return errors.New("NodeInfo exceeds max length of 16384 bytes")
 		}
 		m.myNodeInfo = newjson
 		return nil
-	} else {
-		return err
 	}
+	return err
 }
 
 // Add nodeinfo into the cache for a node
@@ -187,9 +188,9 @@ func (m *nodeinfo) sendNodeInfo(key crypto.BoxPubKey, coords []byte, isResponse
 }
 
 func (m *nodeinfo) _sendNodeInfo(key crypto.BoxPubKey, coords []byte, isResponse bool) {
-	table := m.core.switchTable.table.Load().(lookupTable)
+	loc := m.table.self
 	nodeinfo := nodeinfoReqRes{
-		SendCoords: table.self.getCoords(),
+		SendCoords: loc.getCoords(),
 		IsResponse: isResponse,
 		NodeInfo:   m._getNodeInfo(),
 	}

src/yggdrasil/packetqueue.go

@@ -0,0 +1,118 @@
+package yggdrasil
+
+import (
+	"container/heap"
+	"time"
+)
+
+// TODO separate queues per e.g. traffic flow
+//  For now, we put everything in queue
+
+type pqStreamID string
+
+type pqPacketInfo struct {
+	packet []byte
+	time   time.Time
+}
+
+type pqStream struct {
+	id    pqStreamID
+	infos []pqPacketInfo
+	size  uint64
+}
+
+type packetQueue struct {
+	streams []pqStream
+	size    uint64
+}
+
+// drop will remove a packet from the queue, returning it to the pool
+//  returns true if a packet was removed, false otherwise
+func (q *packetQueue) drop() bool {
+	if q.size == 0 {
+		return false
+	}
+	var longestIdx int
+	for idx := range q.streams {
+		if q.streams[idx].size > q.streams[longestIdx].size {
+			longestIdx = idx
+		}
+	}
+	stream := q.streams[longestIdx]
+	info := stream.infos[0]
+	if len(stream.infos) > 1 {
+		stream.infos = stream.infos[1:]
+		stream.size -= uint64(len(info.packet))
+		q.streams[longestIdx] = stream
+		q.size -= uint64(len(info.packet))
+		heap.Fix(q, longestIdx)
+	} else {
+		heap.Remove(q, longestIdx)
+	}
+	pool_putBytes(info.packet)
+	return true
+}
+
+func (q *packetQueue) push(packet []byte) {
+	id := pqStreamID(peer_getPacketCoords(packet)) // just coords for now
+	info := pqPacketInfo{packet: packet, time: time.Now()}
+	for idx := range q.streams {
+		if q.streams[idx].id == id {
+			q.streams[idx].infos = append(q.streams[idx].infos, info)
+			q.streams[idx].size += uint64(len(packet))
+			q.size += uint64(len(packet))
+			return
+		}
+	}
+	stream := pqStream{id: id, size: uint64(len(packet))}
+	stream.infos = append(stream.infos, info)
+	heap.Push(q, stream)
+}
+
+func (q *packetQueue) pop() ([]byte, bool) {
+	if q.size > 0 {
+		stream := q.streams[0]
+		info := stream.infos[0]
+		if len(stream.infos) > 1 {
+			stream.infos = stream.infos[1:]
+			stream.size -= uint64(len(info.packet))
+			q.streams[0] = stream
+			q.size -= uint64(len(info.packet))
+			heap.Fix(q, 0)
+		} else {
+			heap.Remove(q, 0)
+		}
+		return info.packet, true
+	}
+	return nil, false
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+// Interface methods for packetQueue to satisfy heap.Interface
+
+func (q *packetQueue) Len() int {
+	return len(q.streams)
+}
+
+func (q *packetQueue) Less(i, j int) bool {
+	return q.streams[i].infos[0].time.Before(q.streams[j].infos[0].time)
+}
+
+func (q *packetQueue) Swap(i, j int) {
+	q.streams[i], q.streams[j] = q.streams[j], q.streams[i]
+}
+
+func (q *packetQueue) Push(x interface{}) {
+	stream := x.(pqStream)
+	q.streams = append(q.streams, stream)
+	q.size += stream.size
+}
+
+func (q *packetQueue) Pop() interface{} {
+	idx := len(q.streams) - 1
+	stream := q.streams[idx]
+	q.streams = q.streams[:idx]
+	q.size -= stream.size
+	return stream
+}

src/yggdrasil/peer.go

@@ -6,12 +6,9 @@ package yggdrasil
 
 import (
 	"encoding/hex"
-	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
-	"github.com/yggdrasil-network/yggdrasil-go/src/util"
 
 	"github.com/Arceliar/phony"
 )
@@ -21,17 +18,17 @@ import (
 // In most cases, this involves passing the packet to the handler for outgoing traffic to another peer.
 // In other cases, its link protocol traffic is used to build the spanning tree, in which case this checks signatures and passes the message along to the switch.
 type peers struct {
+	phony.Inbox
 	core  *Core
-	mutex sync.Mutex   // Synchronize writes to atomic
-	ports atomic.Value //map[switchPort]*peer, use CoW semantics
+	ports map[switchPort]*peer // use CoW semantics, share updated version with each peer
+	table *lookupTable         // Sent from switch, share updated version with each peer
 }
 
 // Initializes the peers struct.
 func (ps *peers) init(c *Core) {
-	ps.mutex.Lock()
-	defer ps.mutex.Unlock()
-	ps.putPorts(make(map[switchPort]*peer))
 	ps.core = c
+	ps.ports = make(map[switchPort]*peer)
+	ps.table = new(lookupTable)
 }
 
 func (ps *peers) reconfigure() {
@@ -80,54 +77,62 @@ func (ps *peers) getAllowedEncryptionPublicKeys() []string {
 	return ps.core.config.Current.AllowedEncryptionPublicKeys
 }
 
-// Atomically gets a map[switchPort]*peer of known peers.
-func (ps *peers) getPorts() map[switchPort]*peer {
-	return ps.ports.Load().(map[switchPort]*peer)
-}
-
-// Stores a map[switchPort]*peer (note that you should take a mutex before store operations to avoid conflicts with other nodes attempting to read/change/store at the same time).
-func (ps *peers) putPorts(ports map[switchPort]*peer) {
-	ps.ports.Store(ports)
-}
-
 // Information known about a peer, including their box/sig keys, precomputed shared keys (static and ephemeral) and a handler for their outgoing traffic
 type peer struct {
 	phony.Inbox
 	core       *Core
-	intf       *linkInterface
+	intf       linkInterface
 	port       switchPort
 	box        crypto.BoxPubKey
 	sig        crypto.SigPubKey
 	shared     crypto.BoxSharedKey
 	linkShared crypto.BoxSharedKey
 	endpoint   string
-	firstSeen  time.Time       // To track uptime for getPeers
-	linkOut    func([]byte)    // used for protocol traffic (bypasses the switch)
-	dinfo      *dhtInfo        // used to keep the DHT working
-	out        func([][]byte)  // Set up by whatever created the peers struct, used to send packets to other nodes
-	done       (chan struct{}) // closed to exit the linkLoop
-	close      func()          // Called when a peer is removed, to close the underlying connection, or via admin api
+	firstSeen  time.Time // To track uptime for getPeers
+	dinfo      *dhtInfo  // used to keep the DHT working
 	// The below aren't actually useful internally, they're just gathered for getPeers statistics
 	bytesSent  uint64
 	bytesRecvd uint64
+	ports      map[switchPort]*peer
+	table      *lookupTable
+	queue      packetQueue
+	max        uint64
+	seq        uint64 // this and idle are used to detect when to drop packets from queue
+	idle       bool
+	drop       bool // set to true if we're dropping packets from the queue
+}
+
+func (ps *peers) updateTables(from phony.Actor, table *lookupTable) {
+	ps.Act(from, func() {
+		ps.table = table
+		ps._updatePeers()
+	})
+}
+
+func (ps *peers) _updatePeers() {
+	ports := ps.ports
+	table := ps.table
+	for _, peer := range ps.ports {
+		p := peer // peer is mutated during iteration
+		p.Act(ps, func() {
+			p.ports = ports
+			p.table = table
+		})
+	}
 }
 
 // Creates a new peer with the specified box, sig, and linkShared keys, using the lowest unoccupied port number.
-func (ps *peers) newPeer(box *crypto.BoxPubKey, sig *crypto.SigPubKey, linkShared *crypto.BoxSharedKey, intf *linkInterface, closer func()) *peer {
+func (ps *peers) _newPeer(box *crypto.BoxPubKey, sig *crypto.SigPubKey, linkShared *crypto.BoxSharedKey, intf linkInterface) *peer {
 	now := time.Now()
 	p := peer{box: *box,
+		core:       ps.core,
+		intf:       intf,
 		sig:        *sig,
 		shared:     *crypto.GetSharedKey(&ps.core.boxPriv, box),
 		linkShared: *linkShared,
 		firstSeen:  now,
-		done:       make(chan struct{}),
-		close:      closer,
-		core:       ps.core,
-		intf:       intf,
 	}
-	ps.mutex.Lock()
-	defer ps.mutex.Unlock()
-	oldPorts := ps.getPorts()
+	oldPorts := ps.ports
 	newPorts := make(map[switchPort]*peer)
 	for k, v := range oldPorts {
 		newPorts[k] = v
@@ -139,63 +144,62 @@ func (ps *peers) newPeer(box *crypto.BoxPubKey, sig *crypto.SigPubKey, linkShare
 			break
 		}
 	}
-	ps.putPorts(newPorts)
+	ps.ports = newPorts
+	ps._updatePeers()
 	return &p
 }
 
-// Removes a peer for a given port, if one exists.
-func (ps *peers) removePeer(port switchPort) {
-	if port == 0 {
-		return
-	} // Can't remove self peer
-	phony.Block(&ps.core.router, func() {
-		ps.core.switchTable.forgetPeer(port)
+func (p *peer) _removeSelf() {
+	p.core.peers.Act(p, func() {
+		p.core.peers._removePeer(p)
 	})
-	ps.mutex.Lock()
-	oldPorts := ps.getPorts()
-	p, isIn := oldPorts[port]
+}
+
+// Removes a peer for a given port, if one exists.
+func (ps *peers) _removePeer(p *peer) {
+	if q := ps.ports[p.port]; p.port == 0 || q != p {
+		return
+	} // Can't remove self peer or nonexistant peer
+	ps.core.switchTable.forgetPeer(ps, p.port)
+	oldPorts := ps.ports
 	newPorts := make(map[switchPort]*peer)
 	for k, v := range oldPorts {
 		newPorts[k] = v
 	}
-	delete(newPorts, port)
-	ps.putPorts(newPorts)
-	ps.mutex.Unlock()
-	if isIn {
-		if p.close != nil {
-			p.close()
-		}
-		close(p.done)
-	}
+	delete(newPorts, p.port)
+	p.intf.close()
+	ps.ports = newPorts
+	ps._updatePeers()
 }
 
 // If called, sends a notification to each peer that they should send a new switch message.
 // Mainly called by the switch after an update.
 func (ps *peers) sendSwitchMsgs(from phony.Actor) {
-	ports := ps.getPorts()
-	for _, p := range ports {
-		if p.port == 0 {
-			continue
+	ps.Act(from, func() {
+		for _, peer := range ps.ports {
+			p := peer
+			if p.port == 0 {
+				continue
+			}
+			p.Act(ps, p._sendSwitchMsg)
 		}
-		p.Act(from, p._sendSwitchMsg)
-	}
+	})
+}
+
+func (ps *peers) updateDHT(from phony.Actor) {
+	ps.Act(from, func() {
+		for _, peer := range ps.ports {
+			p := peer
+			if p.port == 0 {
+				continue
+			}
+			p.Act(ps, p._updateDHT)
+		}
+	})
 }
 
 // This must be launched in a separate goroutine by whatever sets up the peer struct.
-// It handles link protocol traffic.
 func (p *peer) start() {
-	var updateDHT func()
-	updateDHT = func() {
-		phony.Block(p, func() {
-			select {
-			case <-p.done:
-			default:
-				p._updateDHT()
-				time.AfterFunc(time.Second, updateDHT)
-			}
-		})
-	}
-	updateDHT()
 	// Just for good measure, immediately send a switch message to this peer when we start
 	p.Act(nil, p._sendSwitchMsg)
 }
@@ -229,37 +233,81 @@ func (p *peer) _handlePacket(packet []byte) {
 	case wire_LinkProtocolTraffic:
 		p._handleLinkTraffic(packet)
 	default:
-		util.PutBytes(packet)
 	}
 }
 
+// Get the coords of a packet without decoding
+func peer_getPacketCoords(packet []byte) []byte {
+	_, pTypeLen := wire_decode_uint64(packet)
+	coords, _ := wire_decode_coords(packet[pTypeLen:])
+	return coords
+}
+
 // Called to handle traffic or protocolTraffic packets.
 // In either case, this reads from the coords of the packet header, does a switch lookup, and forwards to the next node.
 func (p *peer) _handleTraffic(packet []byte) {
-	table := p.core.switchTable.getTable()
-	if _, isIn := table.elems[p.port]; !isIn && p.port != 0 {
+	if _, isIn := p.table.elems[p.port]; !isIn && p.port != 0 {
 		// Drop traffic if the peer isn't in the switch
 		return
 	}
-	p.core.switchTable.packetInFrom(p, packet)
+	coords := peer_getPacketCoords(packet)
+	next := p.table.lookup(coords)
+	if nPeer, isIn := p.ports[next]; isIn {
+		nPeer.sendPacketFrom(p, packet)
+	}
+	//p.core.switchTable.packetInFrom(p, packet)
 }
 
-func (p *peer) sendPacketsFrom(from phony.Actor, packets [][]byte) {
+func (p *peer) sendPacketFrom(from phony.Actor, packet []byte) {
 	p.Act(from, func() {
-		p._sendPackets(packets)
+		p._sendPacket(packet)
 	})
 }
 
-// This just calls p.out(packet) for now.
-func (p *peer) _sendPackets(packets [][]byte) {
-	// Is there ever a case where something more complicated is needed?
-	// What if p.out blocks?
-	var size int
-	for _, packet := range packets {
-		size += len(packet)
+func (p *peer) _sendPacket(packet []byte) {
+	p.queue.push(packet)
+	if p.idle {
+		p.idle = false
+		p._handleIdle()
+	} else if p.drop {
+		for p.queue.size > p.max {
+			p.queue.drop()
+		}
 	}
-	p.bytesSent += uint64(size)
-	p.out(packets)
+}
+
+func (p *peer) _handleIdle() {
+	var packets [][]byte
+	var size uint64
+	for {
+		if packet, success := p.queue.pop(); success {
+			packets = append(packets, packet)
+			size += uint64(len(packet))
+		} else {
+			break
+		}
+	}
+	p.seq++
+	if len(packets) > 0 {
+		p.bytesSent += uint64(size)
+		p.intf.out(packets)
+		p.max = p.queue.size
+	} else {
+		p.idle = true
+	}
+	p.drop = false
+}
+
+func (p *peer) notifyBlocked(from phony.Actor) {
+	p.Act(from, func() {
+		seq := p.seq
+		p.Act(nil, func() {
+			if seq == p.seq {
+				p.drop = true
+				p.max = 2*p.queue.size + streamMsgSize
+			}
+		})
+	})
 }
 
 // This wraps the packet in the inner (ephemeral) and outer (permanent) crypto layers.
@@ -277,7 +325,7 @@ func (p *peer) _sendLinkPacket(packet []byte) {
 		Payload: bs,
 	}
 	packet = linkPacket.encode()
-	p.linkOut(packet)
+	p.intf.linkOut(packet)
 }
 
 // Decrypts the outer (permanent) and inner (ephemeral) crypto layers on link traffic.
@@ -307,13 +355,12 @@ func (p *peer) _handleLinkTraffic(bs []byte) {
 	case wire_SwitchMsg:
 		p._handleSwitchMsg(payload)
 	default:
-		util.PutBytes(bs)
 	}
 }
 
 // Gets a switchMsg from the switch, adds signed next-hop info for this peer, and sends it to them.
 func (p *peer) _sendSwitchMsg() {
-	msg := p.core.switchTable.getMsg()
+	msg := p.table.getMsg()
 	if msg == nil {
 		return
 	}
@@ -335,7 +382,8 @@ func (p *peer) _handleSwitchMsg(packet []byte) {
 		return
 	}
 	if len(msg.Hops) < 1 {
-		p.core.peers.removePeer(p.port)
+		p._removeSelf()
+		return
 	}
 	var loc switchLocator
 	prevKey := msg.Root
@@ -346,23 +394,31 @@ func (p *peer) _handleSwitchMsg(packet []byte) {
 		loc.coords = append(loc.coords, hop.Port)
 		bs := getBytesForSig(&hop.Next, &sigMsg)
 		if !crypto.Verify(&prevKey, bs, &hop.Sig) {
-			p.core.peers.removePeer(p.port)
+			p._removeSelf()
+			return
 		}
 		prevKey = hop.Next
 	}
-	p.core.switchTable.handleMsg(&msg, p.port)
-	if !p.core.switchTable.checkRoot(&msg) {
-		// Bad switch message
-		p.dinfo = nil
-		return
-	}
-	// Pass a message to the dht informing it that this peer (still) exists
-	loc.coords = loc.coords[:len(loc.coords)-1]
-	p.dinfo = &dhtInfo{
-		key:    p.box,
-		coords: loc.getCoords(),
-	}
-	p._updateDHT()
+	p.core.switchTable.Act(p, func() {
+		if !p.core.switchTable._checkRoot(&msg) {
+			// Bad switch message
+			p.Act(&p.core.switchTable, func() {
+				p.dinfo = nil
+			})
+		} else {
+			// handle the message
+			p.core.switchTable._handleMsg(&msg, p.port, false)
+			p.Act(&p.core.switchTable, func() {
+				// Pass a message to the dht informing it that this peer (still) exists
+				loc.coords = loc.coords[:len(loc.coords)-1]
+				p.dinfo = &dhtInfo{
+					key:    p.box,
+					coords: loc.getCoords(),
+				}
+				p._updateDHT()
+			})
+		}
+	})
 }
 
 // This generates the bytes that we sign or check the signature of for a switchMsg.

src/yggdrasil/pool.go

@@ -0,0 +1,20 @@
+package yggdrasil
+
+import "sync"
+
+// Used internally to reduce allocations in the hot loop
+//  I.e. packets being switched or between the crypto and the switch
+// For safety reasons, these must not escape this package
+var pool = sync.Pool{New: func() interface{} { return []byte(nil) }}
+
+func pool_getBytes(size int) []byte {
+	bs := pool.Get().([]byte)
+	if cap(bs) < size {
+		bs = make([]byte, size)
+	}
+	return bs[:size]
+}
+
+func pool_putBytes(bs []byte) {
+	pool.Put(bs)
+}

src/yggdrasil/router.go

@@ -29,7 +29,6 @@ import (
 
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
-	"github.com/yggdrasil-network/yggdrasil-go/src/util"
 
 	"github.com/Arceliar/phony"
 )
@@ -46,6 +45,9 @@ type router struct {
 	nodeinfo nodeinfo
 	searches searches
 	sessions sessions
+	intf     routerInterface
+	peer     *peer
+	table    *lookupTable // has a copy of our locator
 }
 
 // Initializes the router struct, which includes setting up channels to/from the adapter.
@@ -53,17 +55,15 @@ func (r *router) init(core *Core) {
 	r.core = core
 	r.addr = *address.AddrForNodeID(&r.dht.nodeID)
 	r.subnet = *address.SubnetForNodeID(&r.dht.nodeID)
-	self := linkInterface{
-		name: "(self)",
-		info: linkInfo{
-			local:    "(self)",
-			remote:   "(self)",
-			linkType: "self",
-		},
+	r.intf.router = r
+	phony.Block(&r.core.peers, func() {
+		// FIXME don't block here!
+		r.peer = r.core.peers._newPeer(&r.core.boxPub, &r.core.sigPub, &crypto.BoxSharedKey{}, &r.intf)
+	})
+	r.peer.Act(r, r.peer._handleIdle)
+	r.out = func(bs []byte) {
+		r.peer.handlePacketFrom(r, bs)
 	}
-	p := r.core.peers.newPeer(&r.core.boxPub, &r.core.sigPub, &crypto.BoxSharedKey{}, &self, nil)
-	p.out = func(packets [][]byte) { r.handlePackets(p, packets) }
-	r.out = func(bs []byte) { p.handlePacketFrom(r, bs) }
 	r.nodeinfo.init(r.core)
 	r.core.config.Mutex.RLock()
 	r.nodeinfo.setNodeInfo(r.core.config.Current.NodeInfo, r.core.config.Current.NodeInfoPrivacy)
@@ -73,6 +73,21 @@ func (r *router) init(core *Core) {
 	r.sessions.init(r)
 }
 
+func (r *router) updateTable(from phony.Actor, table *lookupTable) {
+	r.Act(from, func() {
+		r.table = table
+		r.nodeinfo.Act(r, func() {
+			r.nodeinfo.table = table
+		})
+		for _, ses := range r.sessions.sinfos {
+			sinfo := ses
+			sinfo.Act(r, func() {
+				sinfo.table = table
+			})
+		}
+	})
+}
+
 // Reconfigures the router and any child modules. This should only ever be run
 // by the router actor.
 func (r *router) reconfigure() {
@@ -97,15 +112,6 @@ func (r *router) start() error {
 	return nil
 }
 
-// In practice, the switch will call this with 1 packet
-func (r *router) handlePackets(from phony.Actor, packets [][]byte) {
-	r.Act(from, func() {
-		for _, packet := range packets {
-			r._handlePacket(packet)
-		}
-	})
-}
-
 // Insert a peer info into the dht, TODO? make the dht a separate actor
 func (r *router) insertPeer(from phony.Actor, info *dhtInfo) {
 	r.Act(from, func() {
@@ -126,7 +132,7 @@ func (r *router) reset(from phony.Actor) {
 func (r *router) doMaintenance() {
 	phony.Block(r, func() {
 		// Any periodic maintenance stuff goes here
-		r.core.switchTable.doMaintenance()
+		r.core.switchTable.doMaintenance(r)
 		r.dht.doMaintenance()
 		r.sessions.cleanup()
 	})
@@ -151,14 +157,12 @@ func (r *router) _handlePacket(packet []byte) {
 // Handles incoming traffic, i.e. encapuslated ordinary IPv6 packets.
 // Passes them to the crypto session worker to be decrypted and sent to the adapter.
 func (r *router) _handleTraffic(packet []byte) {
-	defer util.PutBytes(packet)
 	p := wire_trafficPacket{}
 	if !p.decode(packet) {
 		return
 	}
 	sinfo, isIn := r.sessions.getSessionForHandle(&p.Handle)
 	if !isIn {
-		util.PutBytes(p.Payload)
 		return
 	}
 	sinfo.recv(r, &p)
@@ -204,7 +208,6 @@ func (r *router) _handleProto(packet []byte) {
 	case wire_DHTLookupResponse:
 		r._handleDHTRes(bs, &p.FromKey)
 	default:
-		util.PutBytes(packet)
 	}
 }
 
@@ -252,3 +255,35 @@ func (r *router) _handleNodeInfo(bs []byte, fromKey *crypto.BoxPubKey) {
 	req.SendPermPub = *fromKey
 	r.nodeinfo.handleNodeInfo(r, &req)
 }
+
+////////////////////////////////////////////////////////////////////////////////
+
+// routerInterface is a helper that implements linkInterface
+type routerInterface struct {
+	router *router
+}
+
+func (intf *routerInterface) out(bss [][]byte) {
+	// Note that this is run in the peer's goroutine
+	intf.router.Act(intf.router.peer, func() {
+		for _, bs := range bss {
+			intf.router._handlePacket(bs)
+		}
+	})
+	// This should now immediately make the peer idle again
+	// So the self-peer shouldn't end up buffering anything
+	// We let backpressure act as a throttle instead
+	intf.router.peer._handleIdle()
+}
+
+func (intf *routerInterface) linkOut(_ []byte) {}
+
+func (intf *routerInterface) close() {}
+
+func (intf *routerInterface) name() string { return "(self)" }
+
+func (intf *routerInterface) local() string { return "(self)" }
+
+func (intf *routerInterface) remote() string { return "(self)" }
+
+func (intf *routerInterface) interfaceType() string { return "self" }

src/yggdrasil/search.go

@@ -4,15 +4,13 @@ package yggdrasil
 
 // The basic idea is as follows:
 //  We may know a NodeID (with a mask) and want to connect
-//  We begin a search by initializing a list of all nodes in our DHT, sorted by closest to the destination
-//  We then iteratively ping nodes from the search, marking each pinged node as visited
-//  We add any unvisited nodes from ping responses to the search, truncating to some maximum search size
-//  This stops when we either run out of nodes to ping (we hit a dead end where we can't make progress without going back), or we reach the destination
-//  A new search packet is sent immediately after receiving a response
-//  A new search packet is sent periodically, once per second, in case a packet was dropped (this slowly causes the search to become parallel if the search doesn't timeout but also doesn't finish within 1 second for whatever reason)
-
-// TODO?
-//  Some kind of max search steps, in case the node is offline, so we don't crawl through too much of the network looking for a destination that isn't there?
+//  We begin a search by sending a dht lookup to ourself
+//  Each time a node responds, we sort the results and filter to only include useful nodes
+//  We then periodically send a packet to the first node from the list (after re-filtering)
+//  This happens in parallel for each node that replies
+//  Meanwhile, we keep a list of the (up to) 16 closest nodes to the destination that we've visited
+//  We only consider an unvisited node useful if either the list isn't full or the unvisited node is closer to the destination than the furthest node on the list
+//  That gives the search some chance to recover if it hits a dead end where a node doesn't know everyone it should
 
 import (
 	"errors"
@@ -24,7 +22,8 @@ import (
 
 // This defines the time after which we time out a search (so it can restart).
 const search_RETRY_TIME = 3 * time.Second
-const search_STEP_TIME = 100 * time.Millisecond
+const search_STEP_TIME = time.Second
+const search_MAX_RESULTS = dht_lookup_size
 
 // Information about an ongoing search.
 // Includes the target NodeID, the bitmask to match it to an IP, and the list of nodes to visit / already visited.
@@ -33,7 +32,7 @@ type searchInfo struct {
 	dest     crypto.NodeID
 	mask     crypto.NodeID
 	time     time.Time
-	visited  crypto.NodeID // Closest address visited so far
+	visited  []*crypto.NodeID // Closest addresses visited so far
 	callback func(*sessionInfo, error)
 	// TODO context.Context for timeout and cancellation
 	send uint64 // log number of requests sent
@@ -75,6 +74,9 @@ func (s *searches) createSearch(dest *crypto.NodeID, mask *crypto.NodeID, callba
 // If there is, it adds the response info to the search and triggers a new search step.
 // If there's no ongoing search, or we if the dhtRes finished the search (it was from the target node), then don't do anything more.
 func (sinfo *searchInfo) handleDHTRes(res *dhtRes) {
+	if nfo := sinfo.searches.searches[sinfo.dest]; nfo != sinfo {
+		return // already done
+	}
 	if res != nil {
 		sinfo.recv++
 		if sinfo.checkDHTRes(res) {
@@ -105,16 +107,32 @@ func (sinfo *searchInfo) doSearchStep(infos []*dhtInfo) {
 // Get a list of search targets that are close enough to the destination to try
 // Requires an initial list as input
 func (sinfo *searchInfo) getAllowedInfos(infos []*dhtInfo) []*dhtInfo {
+	var temp []*dhtInfo
+	for _, info := range infos {
+		if false && len(sinfo.visited) < search_MAX_RESULTS {
+			// We're not full on results yet, so don't block anything yet
+		} else if !dht_ordered(&sinfo.dest, info.getNodeID(), sinfo.visited[len(sinfo.visited)-1]) {
+			// Too far away
+			continue
+		}
+		var known bool
+		for _, nfo := range sinfo.visited {
+			if *nfo == *info.getNodeID() {
+				known = true
+				break
+			}
+		}
+		if !known {
+			temp = append(temp, info)
+		}
+	}
+	infos = append(infos[:0], temp...) // restrict to only the allowed infos
 	sort.SliceStable(infos, func(i, j int) bool {
 		// Should return true if i is closer to the destination than j
 		return dht_ordered(&sinfo.dest, infos[i].getNodeID(), infos[j].getNodeID())
-	})
-	// Remove anything too far away to be useful
-	for idx, info := range infos {
-		if !dht_ordered(&sinfo.dest, info.getNodeID(), &sinfo.visited) {
-			infos = infos[:idx]
-			break
-		}
+	}) // Sort infos to start with the closest
+	if len(infos) > search_MAX_RESULTS {
+		infos = infos[:search_MAX_RESULTS] // Limit max number of infos
 	}
 	return infos
 }
@@ -143,11 +161,10 @@ func (sinfo *searchInfo) continueSearch(infos []*dhtInfo) {
 
 // Initially start a search
 func (sinfo *searchInfo) startSearch() {
-	loc := sinfo.searches.router.core.switchTable.getLocator()
 	var infos []*dhtInfo
 	infos = append(infos, &dhtInfo{
 		key:    sinfo.searches.router.core.boxPub,
-		coords: loc.getCoords(),
+		coords: sinfo.searches.router.table.self.getCoords(),
 	})
 	// Start the search by asking ourself, useful if we're the destination
 	sinfo.continueSearch(infos)
@@ -164,6 +181,7 @@ func (sinfo *searchInfo) startSearch() {
 			if elapsed > search_RETRY_TIME {
 				// cleanup
 				delete(sinfo.searches.searches, sinfo.dest)
+				sinfo.searches.router.core.log.Debugln("search timeout:", &sinfo.dest, sinfo.send, sinfo.recv)
 				sinfo.callback(nil, errors.New("search reached dead end"))
 				return
 			}
@@ -176,7 +194,7 @@ func (sinfo *searchInfo) startSearch() {
 // Calls create search, and initializes the iterative search parts of the struct before returning it.
 func (s *searches) newIterSearch(dest *crypto.NodeID, mask *crypto.NodeID, callback func(*sessionInfo, error)) *searchInfo {
 	sinfo := s.createSearch(dest, mask, callback)
-	sinfo.visited = s.router.dht.nodeID
+	sinfo.visited = append(sinfo.visited, &s.router.dht.nodeID)
 	return sinfo
 }
 
@@ -185,13 +203,29 @@ func (s *searches) newIterSearch(dest *crypto.NodeID, mask *crypto.NodeID, callb
 // Otherwise return false.
 func (sinfo *searchInfo) checkDHTRes(res *dhtRes) bool {
 	from := dhtInfo{key: res.Key, coords: res.Coords}
-	if *from.getNodeID() != sinfo.visited && dht_ordered(&sinfo.dest, from.getNodeID(), &sinfo.visited) {
-		// Closer to the destination, so update visited
-		sinfo.searches.router.core.log.Debugln("Updating search:", &sinfo.dest, from.getNodeID(), sinfo.send, sinfo.recv)
-		sinfo.visited = *from.getNodeID()
-		sinfo.time = time.Now()
-	}
 	them := from.getNodeID()
+	var known bool
+	for _, v := range sinfo.visited {
+		if *v == *them {
+			known = true
+			break
+		}
+	}
+	if !known {
+		if len(sinfo.visited) < search_MAX_RESULTS || dht_ordered(&sinfo.dest, them, sinfo.visited[len(sinfo.visited)-1]) {
+			// Closer to the destination than the threshold, so update visited
+			sinfo.searches.router.core.log.Debugln("Updating search:", &sinfo.dest, them, sinfo.send, sinfo.recv)
+			sinfo.visited = append(sinfo.visited, them)
+			sort.SliceStable(sinfo.visited, func(i, j int) bool {
+				// Should return true if i is closer to the destination than j
+				return dht_ordered(&sinfo.dest, sinfo.visited[i], sinfo.visited[j])
+			}) // Sort infos to start with the closest
+			if len(sinfo.visited) > search_MAX_RESULTS {
+				sinfo.visited = sinfo.visited[:search_MAX_RESULTS]
+			}
+			sinfo.time = time.Now()
+		}
+	}
 	var destMasked crypto.NodeID
 	var themMasked crypto.NodeID
 	for idx := 0; idx < crypto.NodeIDLen; idx++ {

src/yggdrasil/session.go

@@ -16,9 +16,6 @@ import (
 	"github.com/Arceliar/phony"
 )
 
-// Duration that we keep track of old nonces per session, to allow some out-of-order packet delivery
-const nonceWindow = time.Second
-
 // All the information we know about an active session.
 // This includes coords, permanent and ephemeral keys, handles and nonces, various sorts of timing information for timeout and maintenance, and some metadata for the admin API.
 type sessionInfo struct {
@@ -52,6 +49,7 @@ type sessionInfo struct {
 	cancel        util.Cancellation   // Used to terminate workers
 	conn          *Conn               // The associated Conn object
 	callbacks     []chan func()       // Finished work from crypto workers
+	table         *lookupTable        // table.self is a locator where we get our coords
 }
 
 // Represents a session ping/pong packet, and includes information like public keys, a session handle, coords, a timestamp to prevent replays, and the tun/tap MTU.
@@ -217,6 +215,7 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 	sinfo.myHandle = *crypto.NewHandle()
 	sinfo.theirAddr = *address.AddrForNodeID(crypto.GetNodeID(&sinfo.theirPermPub))
 	sinfo.theirSubnet = *address.SubnetForNodeID(crypto.GetNodeID(&sinfo.theirPermPub))
+	sinfo.table = ss.router.table
 	ss.sinfos[sinfo.myHandle] = &sinfo
 	ss.byTheirPerm[sinfo.theirPermPub] = &sinfo.myHandle
 	return &sinfo
@@ -266,8 +265,7 @@ func (ss *sessions) removeSession(sinfo *sessionInfo) {
 
 // Returns a session ping appropriate for the given session info.
 func (sinfo *sessionInfo) _getPing() sessionPing {
-	loc := sinfo.sessions.router.core.switchTable.getLocator()
-	coords := loc.getCoords()
+	coords := sinfo.table.self.getCoords()
 	ping := sessionPing{
 		SendPermPub: sinfo.sessions.router.core.boxPub,
 		Handle:      sinfo.myHandle,
@@ -393,14 +391,9 @@ func (sinfo *sessionInfo) _getMTU() MTU {
 	return sinfo.myMTU
 }
 
-// Checks if a packet's nonce is recent enough to fall within the window of allowed packets, and not already received.
+// Checks if a packet's nonce is newer than any previously received
 func (sinfo *sessionInfo) _nonceIsOK(theirNonce *crypto.BoxNonce) bool {
-	// The bitmask is to allow for some non-duplicate out-of-order packets
-	if theirNonce.Minus(&sinfo.theirNonce) > 0 {
-		// This is newer than the newest nonce we've seen
-		return true
-	}
-	return time.Since(sinfo.time) < nonceWindow
+	return theirNonce.Minus(&sinfo.theirNonce) > 0
 }
 
 // Updates the nonce mask by (possibly) shifting the bitmask and setting the bit corresponding to this nonce to 1, and then updating the most recent nonce
@@ -455,12 +448,9 @@ func (sinfo *sessionInfo) _recvPacket(p *wire_trafficPacket) {
 	select {
 	case <-sinfo.init:
 	default:
-		// TODO find a better way to drop things until initialized
-		util.PutBytes(p.Payload)
 		return
 	}
 	if !sinfo._nonceIsOK(&p.Nonce) {
-		util.PutBytes(p.Payload)
 		return
 	}
 	k := sinfo.sharedSesKey
@@ -470,11 +460,9 @@ func (sinfo *sessionInfo) _recvPacket(p *wire_trafficPacket) {
 	poolFunc := func() {
 		bs, isOK = crypto.BoxOpen(&k, p.Payload, &p.Nonce)
 		callback := func() {
-			util.PutBytes(p.Payload)
 			if !isOK || k != sinfo.sharedSesKey || !sinfo._nonceIsOK(&p.Nonce) {
 				// Either we failed to decrypt, or the session was updated, or we
 				// received this packet in the mean time
-				util.PutBytes(bs)
 				return
 			}
 			sinfo._updateNonce(&p.Nonce)
@@ -492,8 +480,6 @@ func (sinfo *sessionInfo) _send(msg FlowKeyMessage) {
 	select {
 	case <-sinfo.init:
 	default:
-		// TODO find a better way to drop things until initialized
-		util.PutBytes(msg.Message)
 		return
 	}
 	sinfo.bytesSent += uint64(len(msg.Message))
@@ -512,14 +498,8 @@ func (sinfo *sessionInfo) _send(msg FlowKeyMessage) {
 	ch := make(chan func(), 1)
 	poolFunc := func() {
 		p.Payload, _ = crypto.BoxSeal(&k, msg.Message, &p.Nonce)
+		packet := p.encode()
 		callback := func() {
-			// Encoding may block on a util.GetBytes(), so kept out of the worker pool
-			packet := p.encode()
-			// Cleanup
-			util.PutBytes(msg.Message)
-			util.PutBytes(p.Payload)
-			// Send the packet
-			// TODO replace this with a send to the peer struct if that becomes an actor
 			sinfo.sessions.router.Act(sinfo, func() {
 				sinfo.sessions.router.out(packet)
 			})

src/yggdrasil/simlink.go

@@ -0,0 +1,91 @@
+package yggdrasil
+
+import (
+	"errors"
+	"github.com/Arceliar/phony"
+)
+
+type Simlink struct {
+	phony.Inbox
+	rch     chan []byte
+	dest    *Simlink
+	link    *link
+	started bool
+}
+
+func (s *Simlink) readMsg() ([]byte, error) {
+	bs, ok := <-s.rch
+	if !ok {
+		return nil, errors.New("read from closed Simlink")
+	}
+	return bs, nil
+}
+
+func (s *Simlink) _recvMetaBytes() ([]byte, error) {
+	return s.readMsg()
+}
+
+func (s *Simlink) _sendMetaBytes(bs []byte) error {
+	_, err := s.writeMsgs([][]byte{bs})
+	return err
+}
+
+func (s *Simlink) close() error {
+	defer func() { recover() }()
+	close(s.rch)
+	return nil
+}
+
+func (s *Simlink) writeMsgs(msgs [][]byte) (int, error) {
+	if s.dest == nil {
+		return 0, errors.New("write to unpaired Simlink")
+	}
+	var size int
+	for _, msg := range msgs {
+		size += len(msg)
+		bs := append([]byte(nil), msg...)
+		phony.Block(s, func() {
+			s.dest.Act(s, func() {
+				defer func() { recover() }()
+				s.dest.rch <- bs
+			})
+		})
+	}
+	return size, nil
+}
+
+func (c *Core) NewSimlink() *Simlink {
+	s := &Simlink{rch: make(chan []byte, 1)}
+	n := "Simlink"
+	var err error
+	s.link, err = c.links.create(s, n, n, n, n, false, true, linkOptions{})
+	if err != nil {
+		panic(err)
+	}
+	return s
+}
+
+func (s *Simlink) SetDestination(dest *Simlink) error {
+	var err error
+	phony.Block(s, func() {
+		if s.dest != nil {
+			err = errors.New("destination already set")
+		} else {
+			s.dest = dest
+		}
+	})
+	return err
+}
+
+func (s *Simlink) Start() error {
+	var err error
+	phony.Block(s, func() {
+		if s.started {
+			err = errors.New("already started")
+		} else {
+			s.started = true
+			go s.link.handler()
+		}
+	})
+	return err
+}

src/yggdrasil/stream.go

@@ -6,12 +6,10 @@ import (
 	"fmt"
 	"io"
 	"net"
-
-	"github.com/yggdrasil-network/yggdrasil-go/src/util"
 )
 
 // Test that this matches the interface we expect
-var _ = linkInterfaceMsgIO(&stream{})
+var _ = linkMsgIO(&stream{})
 
 type stream struct {
 	rwc          io.ReadWriteCloser
@@ -46,6 +44,9 @@ func (s *stream) writeMsgs(bss [][]byte) (int, error) {
 	}
 	s.outputBuffer = buf[:0] // So we can reuse the same underlying array later
 	_, err := buf.WriteTo(s.rwc)
+	for _, bs := range bss {
+		pool_putBytes(bs)
+	}
 	// TODO only include number of bytes from bs *successfully* written?
 	return written, err
 }
@@ -112,7 +113,7 @@ func (s *stream) readMsgFromBuffer() ([]byte, error) {
 	if msgLen > streamMsgSize {
 		return nil, errors.New("oversized message")
 	}
-	msg := util.ResizeBytes(util.GetBytes(), int(msgLen))
+	msg := pool_getBytes(int(msgLen))
 	_, err = io.ReadFull(s.inputBuffer, msg)
 	return msg, err
 }

src/yggdrasil/switch.go

@@ -12,13 +12,9 @@ package yggdrasil
 //  A little annoying to do with constant changes from backpressure
 
 import (
-	"math/rand"
-	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
-	"github.com/yggdrasil-network/yggdrasil-go/src/util"
 
 	"github.com/Arceliar/phony"
 )
@@ -97,6 +93,20 @@ func (l *switchLocator) dist(dest []byte) int {
 	return dist
 }
 
+func (l *switchLocator) ldist(sl *switchLocator) int {
+	lca := -1
+	for idx := 0; idx < len(l.coords); idx++ {
+		if idx >= len(sl.coords) {
+			break
+		}
+		if l.coords[idx] != sl.coords[idx] {
+			break
+		}
+		lca = idx
+	}
+	return len(l.coords) + len(sl.coords) - 2*(lca+1)
+}
+
 // Gets coords in wire encoded format, with *no* length prefix.
 func (l *switchLocator) getCoords() []byte {
 	bs := make([]byte, 0, len(l.coords))
@@ -126,14 +136,19 @@ func (x *switchLocator) isAncestorOf(y *switchLocator) bool {
 
 // Information about a peer, used by the switch to build the tree and eventually make routing decisions.
 type peerInfo struct {
-	key     crypto.SigPubKey      // ID of this peer
-	locator switchLocator         // Should be able to respond with signatures upon request
-	degree  uint64                // Self-reported degree
-	time    time.Time             // Time this node was last seen
-	faster  map[switchPort]uint64 // Counter of how often a node is faster than the current parent, penalized extra if slower
-	port    switchPort            // Interface number of this peer
-	msg     switchMsg             // The wire switchMsg used
-	blocked bool                  // True if the link is blocked, used to avoid parenting a blocked link
+	key        crypto.SigPubKey      // ID of this peer
+	locator    switchLocator         // Should be able to respond with signatures upon request
+	degree     uint64                // Self-reported degree
+	time       time.Time             // Time this node was last seen
+	faster     map[switchPort]uint64 // Counter of how often a node is faster than the current parent, penalized extra if slower
+	port       switchPort            // Interface number of this peer
+	msg        switchMsg             // The wire switchMsg used
+	readBlock  bool                  // True if the link notified us of a read that blocked too long
+	writeBlock bool                  // True of the link notified us of a write that blocked too long
+}
+
+func (pinfo *peerInfo) blocked() bool {
+	return pinfo.readBlock || pinfo.writeBlock
 }
 
 // This is just a uint64 with a named type for clarity reasons.
@@ -144,12 +159,15 @@ type tableElem struct {
 	port    switchPort
 	locator switchLocator
 	time    time.Time
+	next    map[switchPort]*tableElem
 }
 
 // This is the subset of the information about all peers needed to make routing decisions, and it stored separately in an atomically accessed table, which gets hammered in the "hot loop" of the routing logic (see: peer.handleTraffic in peers.go).
 type lookupTable struct {
-	self  switchLocator
-	elems map[switchPort]tableElem
+	self   switchLocator
+	elems  map[switchPort]tableElem // all switch peers, just for sanity checks + API/debugging
+	_start tableElem                // used for lookups
+	_msg   switchMsg
 }
 
 // This is switch information which is mutable and needs to be modified by other goroutines, but is not accessed atomically.
@@ -158,7 +176,6 @@ type switchData struct {
 	// All data that's mutable and used by exported Table methods
 	// To be read/written with atomic.Value Store/Load calls
 	locator switchLocator
-	seq     uint64 // Sequence number, reported to peers, so they know about changes
 	peers   map[switchPort]peerInfo
 	msg     *switchMsg
 }
@@ -167,17 +184,11 @@ type switchData struct {
 type switchTable struct {
 	core        *Core
 	key         crypto.SigPubKey           // Our own key
+	phony.Inbox                            // Owns the below
 	time        time.Time                  // Time when locator.tstamp was last updated
 	drop        map[crypto.SigPubKey]int64 // Tstamp associated with a dropped root
-	mutex       sync.RWMutex               // Lock for reads/writes of switchData
 	parent      switchPort                 // Port of whatever peer is our parent, or self if we're root
 	data        switchData                 //
-	updater     atomic.Value               // *sync.Once
-	table       atomic.Value               // lookupTable
-	phony.Inbox                            // Owns the below
-	queues      switch_buffers             // Queues - not atomic so ONLY use through the actor
-	idle        map[switchPort]struct{}    // idle peers - not atomic so ONLY use through the actor
-	sending     map[switchPort]struct{}    // peers known to be blocked in a send (somehow)
 }
 
 // Minimum allowed total size of switch queues.
@@ -191,47 +202,27 @@ func (t *switchTable) init(core *Core) {
 	locator := switchLocator{root: t.key, tstamp: now.Unix()}
 	peers := make(map[switchPort]peerInfo)
 	t.data = switchData{locator: locator, peers: peers}
-	t.updater.Store(&sync.Once{})
-	t.table.Store(lookupTable{})
 	t.drop = make(map[crypto.SigPubKey]int64)
-	phony.Block(t, func() {
-		core.config.Mutex.RLock()
-		if core.config.Current.SwitchOptions.MaxTotalQueueSize > SwitchQueueTotalMinSize {
-			t.queues.totalMaxSize = core.config.Current.SwitchOptions.MaxTotalQueueSize
-		} else {
-			t.queues.totalMaxSize = SwitchQueueTotalMinSize
-		}
-		core.config.Mutex.RUnlock()
-		t.queues.bufs = make(map[string]switch_buffer)
-		t.idle = make(map[switchPort]struct{})
-		t.sending = make(map[switchPort]struct{})
-	})
+	phony.Block(t, t._updateTable)
 }
 
 func (t *switchTable) reconfigure() {
 	// This is where reconfiguration would go, if we had anything useful to do.
-	t.core.link.reconfigure()
+	t.core.links.reconfigure()
 	t.core.peers.reconfigure()
 }
 
-// Safely gets a copy of this node's locator.
-func (t *switchTable) getLocator() switchLocator {
-	t.mutex.RLock()
-	defer t.mutex.RUnlock()
-	return t.data.locator.clone()
-}
-
 // Regular maintenance to possibly timeout/reset the root and similar.
-func (t *switchTable) doMaintenance() {
-	// Periodic maintenance work to keep things internally consistent
-	t.mutex.Lock()         // Write lock
-	defer t.mutex.Unlock() // Release lock when we're done
-	t.cleanRoot()
-	t.cleanDropped()
+func (t *switchTable) doMaintenance(from phony.Actor) {
+	t.Act(from, func() {
+		// Periodic maintenance work to keep things internally consistent
+		t._cleanRoot()
+		t._cleanDropped()
+	})
 }
 
 // Updates the root periodically if it is ourself, or promotes ourself to root if we're better than the current root or if the current root has timed out.
-func (t *switchTable) cleanRoot() {
+func (t *switchTable) _cleanRoot() {
 	// TODO rethink how this is done?...
 	// Get rid of the root if it looks like its timed out
 	now := time.Now()
@@ -255,59 +246,79 @@ func (t *switchTable) cleanRoot() {
 		t.parent = switchPort(0)
 		t.time = now
 		if t.data.locator.root != t.key {
-			t.data.seq++
-			t.updater.Store(&sync.Once{})
-			t.core.router.reset(nil)
+			defer t.core.router.reset(nil)
 		}
 		t.data.locator = switchLocator{root: t.key, tstamp: now.Unix()}
+		t._updateTable() // updates base copy of switch msg in lookupTable
 		t.core.peers.sendSwitchMsgs(t)
 	}
 }
 
 // Blocks and, if possible, unparents a peer
-func (t *switchTable) blockPeer(port switchPort) {
-	t.mutex.Lock()
-	defer t.mutex.Unlock()
-	peer, isIn := t.data.peers[port]
-	if !isIn {
-		return
-	}
-	peer.blocked = true
-	t.data.peers[port] = peer
-	if port != t.parent {
-		return
-	}
-	t.parent = 0
-	for _, info := range t.data.peers {
-		if info.port == port {
-			continue
+func (t *switchTable) blockPeer(from phony.Actor, port switchPort, isWrite bool) {
+	t.Act(from, func() {
+		peer, isIn := t.data.peers[port]
+		switch {
+		case isIn && !isWrite && !peer.readBlock:
+			peer.readBlock = true
+		case isIn && isWrite && !peer.writeBlock:
+			peer.writeBlock = true
+		default:
+			return
 		}
-		t.unlockedHandleMsg(&info.msg, info.port, true)
-	}
-	t.unlockedHandleMsg(&peer.msg, peer.port, true)
+		t.data.peers[port] = peer
+		defer t._updateTable()
+		if port != t.parent {
+			return
+		}
+		t.parent = 0
+		for _, info := range t.data.peers {
+			if info.port == port {
+				continue
+			}
+			t._handleMsg(&info.msg, info.port, true)
+		}
+		t._handleMsg(&peer.msg, peer.port, true)
+	})
+}
+
+func (t *switchTable) unblockPeer(from phony.Actor, port switchPort, isWrite bool) {
+	t.Act(from, func() {
+		peer, isIn := t.data.peers[port]
+		switch {
+		case isIn && !isWrite && peer.readBlock:
+			peer.readBlock = false
+		case isIn && isWrite && peer.writeBlock:
+			peer.writeBlock = false
+		default:
+			return
+		}
+		t.data.peers[port] = peer
+		t._updateTable()
+	})
 }
 
 // Removes a peer.
 // Must be called by the router actor with a lambda that calls this.
 // If the removed peer was this node's parent, it immediately tries to find a new parent.
-func (t *switchTable) forgetPeer(port switchPort) {
-	t.mutex.Lock()
-	defer t.mutex.Unlock()
-	delete(t.data.peers, port)
-	t.updater.Store(&sync.Once{})
-	if port != t.parent {
-		return
-	}
-	t.parent = 0
-	for _, info := range t.data.peers {
-		t.unlockedHandleMsg(&info.msg, info.port, true)
-	}
+func (t *switchTable) forgetPeer(from phony.Actor, port switchPort) {
+	t.Act(from, func() {
+		delete(t.data.peers, port)
+		defer t._updateTable()
+		if port != t.parent {
+			return
+		}
+		t.parent = 0
+		for _, info := range t.data.peers {
+			t._handleMsg(&info.msg, info.port, true)
+		}
+	})
 }
 
 // Dropped is a list of roots that are better than the current root, but stopped sending new timestamps.
 // If we switch to a new root, and that root is better than an old root that previously timed out, then we can clean up the old dropped root infos.
 // This function is called periodically to do that cleanup.
-func (t *switchTable) cleanDropped() {
+func (t *switchTable) _cleanDropped() {
 	// TODO? only call this after root changes, not periodically
 	for root := range t.drop {
 		if !firstIsBetter(&root, &t.data.locator.root) {
@@ -333,9 +344,7 @@ type switchMsgHop struct {
 }
 
 // This returns a *switchMsg to a copy of this node's current switchMsg, which can safely have additional information appended to Hops and sent to a peer.
-func (t *switchTable) getMsg() *switchMsg {
-	t.mutex.RLock()
-	defer t.mutex.RUnlock()
+func (t *switchTable) _getMsg() *switchMsg {
 	if t.parent == 0 {
 		return &switchMsg{Root: t.key, TStamp: t.data.locator.tstamp}
 	} else if parent, isIn := t.data.peers[t.parent]; isIn {
@@ -347,14 +356,18 @@ func (t *switchTable) getMsg() *switchMsg {
 	}
 }
 
+func (t *lookupTable) getMsg() *switchMsg {
+	msg := t._msg
+	msg.Hops = append([]switchMsgHop(nil), t._msg.Hops...)
+	return &msg
+}
+
 // This function checks that the root information in a switchMsg is OK.
 // In particular, that the root is better, or else the same as the current root but with a good timestamp, and that this root+timestamp haven't been dropped due to timeout.
-func (t *switchTable) checkRoot(msg *switchMsg) bool {
+func (t *switchTable) _checkRoot(msg *switchMsg) bool {
 	// returns false if it's a dropped root, not a better root, or has an older timestamp
 	// returns true otherwise
 	// used elsewhere to keep inserting peers into the dht only if root info is OK
-	t.mutex.RLock()
-	defer t.mutex.RUnlock()
 	dropTstamp, isIn := t.drop[msg.Root]
 	switch {
 	case isIn && dropTstamp >= msg.TStamp:
@@ -370,20 +383,13 @@ func (t *switchTable) checkRoot(msg *switchMsg) bool {
 	}
 }
 
-// This is a mutexed wrapper to unlockedHandleMsg, and is called by the peer structs in peers.go to pass a switchMsg for that peer into the switch.
-func (t *switchTable) handleMsg(msg *switchMsg, fromPort switchPort) {
-	t.mutex.Lock()
-	defer t.mutex.Unlock()
-	t.unlockedHandleMsg(msg, fromPort, false)
-}
-
 // This updates the switch with information about a peer.
 // Then the tricky part, it decides if it should update our own locator as a result.
 // That happens if this node is already our parent, or is advertising a better root, or is advertising a better path to the same root, etc...
 // There are a lot of very delicate order sensitive checks here, so its' best to just read the code if you need to understand what it's doing.
 // It's very important to not change the order of the statements in the case function unless you're absolutely sure that it's safe, including safe if used alongside nodes that used the previous order.
 // Set the third arg to true if you're reprocessing an old message, e.g. to find a new parent after one disconnects, to avoid updating some timing related things.
-func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort, reprocessing bool) {
+func (t *switchTable) _handleMsg(msg *switchMsg, fromPort switchPort, reprocessing bool) {
 	// TODO directly use a switchMsg instead of switchMessage + sigs
 	now := time.Now()
 	// Set up the sender peerInfo
@@ -397,6 +403,9 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort, rep
 		sender.key = prevKey
 		prevKey = hop.Next
 	}
+	if sender.key == t.key {
+		return // Don't peer with ourself via different interfaces
+	}
 	sender.msg = *msg
 	sender.port = fromPort
 	sender.time = now
@@ -426,7 +435,8 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort, rep
 	if reprocessing {
 		sender.faster = oldSender.faster
 		sender.time = oldSender.time
-		sender.blocked = oldSender.blocked
+		sender.readBlock = oldSender.readBlock
+		sender.writeBlock = oldSender.writeBlock
 	} else {
 		sender.faster = make(map[switchPort]uint64, len(oldSender.faster))
 		for port, peer := range t.data.peers {
@@ -449,6 +459,9 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort, rep
 			}
 		}
 	}
+	if sender.blocked() != oldSender.blocked() {
+		doUpdate = true
+	}
 	// Update sender
 	t.data.peers[fromPort] = sender
 	// Decide if we should also update our root info to make the sender our parent
@@ -486,10 +499,10 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort, rep
 	case sender.faster[t.parent] >= switch_faster_threshold:
 		// The is reliably faster than the current parent.
 		updateRoot = true
-	case !sender.blocked && oldParent.blocked:
+	case !sender.blocked() && oldParent.blocked():
 		// Replace a blocked parent
 		updateRoot = true
-	case reprocessing && sender.blocked && !oldParent.blocked:
+	case reprocessing && sender.blocked() && !oldParent.blocked():
 		// Don't replace an unblocked parent when reprocessing
 	case reprocessing && sender.faster[t.parent] > oldParent.faster[sender.port]:
 		// The sender seems to be reliably faster than the current parent, so switch to them instead.
@@ -506,75 +519,109 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort, rep
 			if peer.port == sender.port {
 				continue
 			}
-			t.unlockedHandleMsg(&peer.msg, peer.port, true)
+			t._handleMsg(&peer.msg, peer.port, true)
 		}
 		// Process the sender last, to avoid keeping them as a parent if at all possible.
-		t.unlockedHandleMsg(&sender.msg, sender.port, true)
+		t._handleMsg(&sender.msg, sender.port, true)
 	case now.Sub(t.time) < switch_throttle:
 		// We've already gotten an update from this root recently, so ignore this one to avoid flooding.
 	case sender.locator.tstamp > t.data.locator.tstamp:
 		// The timestamp was updated, so we need to update locally and send to our peers.
 		updateRoot = true
 	}
+	// Note that we depend on the LIFO order of the stack of defers here...
 	if updateRoot {
+		doUpdate = true
 		if !equiv(&sender.locator, &t.data.locator) {
-			doUpdate = true
-			t.data.seq++
-			t.core.router.reset(nil)
+			defer t.core.router.reset(t)
 		}
 		if t.data.locator.tstamp != sender.locator.tstamp {
 			t.time = now
 		}
 		t.data.locator = sender.locator
 		t.parent = sender.port
-		t.core.peers.sendSwitchMsgs(t)
+		defer t.core.peers.sendSwitchMsgs(t)
 	}
-	if true || doUpdate {
-		t.updater.Store(&sync.Once{})
+	if doUpdate {
+		t._updateTable()
 	}
-	return
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 
-// The rest of these are related to the switch worker
+// The rest of these are related to the switch lookup table
 
-// This is called via a sync.Once to update the atomically readable subset of switch information that gets used for routing decisions.
-func (t *switchTable) updateTable() {
-	// WARNING this should only be called from within t.data.updater.Do()
-	//  It relies on the sync.Once for synchronization with messages and lookups
-	// TODO use a pre-computed faster lookup table
-	//  Instead of checking distance for every destination every time
-	//  Array of structs, indexed by first coord that differs from self
-	//  Each struct has stores the best port to forward to, and a next coord map
-	//  Move to struct, then iterate over coord maps until you dead end
-	//  The last port before the dead end should be the closest
-	t.mutex.RLock()
-	defer t.mutex.RUnlock()
+func (t *switchTable) _updateTable() {
 	newTable := lookupTable{
 		self:  t.data.locator.clone(),
 		elems: make(map[switchPort]tableElem, len(t.data.peers)),
+		_msg:  *t._getMsg(),
 	}
+	newTable._init()
 	for _, pinfo := range t.data.peers {
-		//if !pinfo.forward { continue }
-		if pinfo.locator.root != newTable.self.root {
+		if pinfo.blocked() || pinfo.locator.root != newTable.self.root {
 			continue
 		}
 		loc := pinfo.locator.clone()
 		loc.coords = loc.coords[:len(loc.coords)-1] // Remove the them->self link
-		newTable.elems[pinfo.port] = tableElem{
+		elem := tableElem{
 			locator: loc,
 			port:    pinfo.port,
 			time:    pinfo.time,
 		}
+		newTable._insert(&elem)
+		newTable.elems[pinfo.port] = elem
 	}
-	t.table.Store(newTable)
+	t.core.peers.updateTables(t, &newTable)
+	t.core.router.updateTable(t, &newTable)
 }
 
-// Returns a copy of the atomically-updated table used for switch lookups
-func (t *switchTable) getTable() lookupTable {
-	t.updater.Load().(*sync.Once).Do(t.updateTable)
-	return t.table.Load().(lookupTable)
+func (t *lookupTable) _init() {
+	// WARNING: this relies on the convention that the self port is 0
+	self := tableElem{locator: t.self} // create self elem
+	t._start = self                    // initialize _start to self
+	t._insert(&self)                   // insert self into table
+}
+
+func (t *lookupTable) _insert(elem *tableElem) {
+	// This is a helper that should only be run during _updateTable
+	here := &t._start
+	for idx := 0; idx <= len(elem.locator.coords); idx++ {
+		refLoc := here.locator
+		refLoc.coords = refLoc.coords[:idx] // Note that this is length idx (starts at length 0)
+		oldDist := refLoc.ldist(&here.locator)
+		newDist := refLoc.ldist(&elem.locator)
+		var update bool
+		switch {
+		case newDist < oldDist: // new elem is closer to this point in the tree
+			update = true
+		case newDist > oldDist: // new elem is too far
+		case elem.locator.tstamp > refLoc.tstamp: // new elem has a closer timestamp
+			update = true
+		case elem.locator.tstamp < refLoc.tstamp: // new elem's timestamp is too old
+		case elem.time.Before(here.time): // same dist+timestamp, but new elem delivered it faster
+			update = true
+		}
+		if update {
+			here.port = elem.port
+			here.locator = elem.locator
+			here.time = elem.time
+			// Problem: here is a value, so this doesn't actually update anything...
+		}
+		if idx < len(elem.locator.coords) {
+			if here.next == nil {
+				here.next = make(map[switchPort]*tableElem)
+			}
+			var next *tableElem
+			var ok bool
+			if next, ok = here.next[elem.locator.coords[idx]]; !ok {
+				nextVal := *elem
+				next = &nextVal
+				here.next[next.locator.coords[idx]] = next
+			}
+			here = next
+		}
+	}
 }
 
 // Starts the switch worker
@@ -584,306 +631,17 @@ func (t *switchTable) start() error {
 	return nil
 }
 
-type closerInfo struct {
-	elem tableElem
-	dist int
-}
-
-// Return a map of ports onto distance, keeping only ports closer to the destination than this node
-// If the map is empty (or nil), then no peer is closer
-func (t *switchTable) getCloser(dest []byte) []closerInfo {
-	table := t.getTable()
-	myDist := table.self.dist(dest)
-	if myDist == 0 {
-		// Skip the iteration step if it's impossible to be closer
-		return nil
-	}
-	t.queues.closer = t.queues.closer[:0]
-	for _, info := range table.elems {
-		dist := info.locator.dist(dest)
-		if dist < myDist {
-			t.queues.closer = append(t.queues.closer, closerInfo{info, dist})
-		}
-	}
-	return t.queues.closer
-}
-
-// Returns true if the peer is closer to the destination than ourself
-func (t *switchTable) portIsCloser(dest []byte, port switchPort) bool {
-	table := t.getTable()
-	if info, isIn := table.elems[port]; isIn {
-		theirDist := info.locator.dist(dest)
-		myDist := table.self.dist(dest)
-		return theirDist < myDist
-	} else {
-		return false
-	}
-}
-
-// Get the coords of a packet without decoding
-func switch_getPacketCoords(packet []byte) []byte {
-	_, pTypeLen := wire_decode_uint64(packet)
-	coords, _ := wire_decode_coords(packet[pTypeLen:])
-	return coords
-}
-
-// Returns a unique string for each stream of traffic
-// Equal to coords
-// The sender may append arbitrary info to the end of coords (as long as it's begins with a 0x00) to designate separate traffic streams
-// Currently, it's the IPv6 next header type and the first 2 uint16 of the next header
-// This is equivalent to the TCP/UDP protocol numbers and the source / dest ports
-// TODO figure out if something else would make more sense (other transport protocols?)
-func switch_getPacketStreamID(packet []byte) string {
-	return string(switch_getPacketCoords(packet))
-}
-
-// Returns the flowlabel from a given set of coords
-func switch_getFlowLabelFromCoords(in []byte) []byte {
-	for i, v := range in {
-		if v == 0 {
-			return in[i+1:]
-		}
-	}
-	return []byte{}
-}
-
-// Find the best port for a given set of coords
-func (t *switchTable) bestPortForCoords(coords []byte) switchPort {
-	table := t.getTable()
-	var best switchPort
-	bestDist := table.self.dist(coords)
-	for to, elem := range table.elems {
-		dist := elem.locator.dist(coords)
-		if !(dist < bestDist) {
-			continue
-		}
-		best = to
-		bestDist = dist
-	}
-	return best
-}
-
-// Handle an incoming packet
-// Either send it to ourself, or to the first idle peer that's free
-// Returns true if the packet has been handled somehow, false if it should be queued
-func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]struct{}, sending map[switchPort]struct{}) bool {
-	coords := switch_getPacketCoords(packet)
-	closer := t.getCloser(coords)
-	if len(closer) == 0 {
-		// TODO? call the router directly, and remove the whole concept of a self peer?
-		self := t.core.peers.getPorts()[0]
-		self.sendPacketsFrom(t, [][]byte{packet})
-		return true
-	}
-	var best *closerInfo
-	ports := t.core.peers.getPorts()
-	for _, cinfo := range closer {
-		to := ports[cinfo.elem.port]
-		//_, isIdle := idle[cinfo.elem.port]
-		_, isSending := sending[cinfo.elem.port]
-		var update bool
-		switch {
-		case to == nil:
-			// no port was found, ignore it
-		case isSending:
-			// the port is busy, ignore it
-		case best == nil:
-			// this is the first idle port we've found, so select it until we find a
-			// better candidate port to use instead
-			update = true
-		case cinfo.dist < best.dist:
-			// the port takes a shorter path/is more direct than our current
-			// candidate, so select that instead
-			update = true
-		case cinfo.dist > best.dist:
-			// the port takes a longer path/is less direct than our current candidate,
-			// ignore it
-		case cinfo.elem.locator.tstamp > best.elem.locator.tstamp:
-			// has a newer tstamp from the root, so presumably a better path
-			update = true
-		case cinfo.elem.locator.tstamp < best.elem.locator.tstamp:
-			// has a n older tstamp, so presumably a worse path
-		case cinfo.elem.time.Before(best.elem.time):
-			// same tstamp, but got it earlier, so presumably a better path
-			//t.core.log.Println("DEBUG new best:", best.elem.time, cinfo.elem.time)
-			update = true
-		default:
-			// the search for a port has finished
-		}
-		if update {
-			b := cinfo // because cinfo gets mutated by the iteration
-			best = &b
-		}
-	}
-	if best != nil {
-		if _, isIdle := idle[best.elem.port]; isIdle {
-			delete(idle, best.elem.port)
-			ports[best.elem.port].sendPacketsFrom(t, [][]byte{packet})
-			return true
-		}
-	}
-	// Didn't find anyone idle to send it to
-	return false
-}
-
-// Info about a buffered packet
-type switch_packetInfo struct {
-	bytes []byte
-	time  time.Time // Timestamp of when the packet arrived
-}
-
-// Used to keep track of buffered packets
-type switch_buffer struct {
-	packets []switch_packetInfo // Currently buffered packets, which may be dropped if it grows too large
-	size    uint64              // Total queue size in bytes
-}
-
-type switch_buffers struct {
-	totalMaxSize uint64
-	bufs         map[string]switch_buffer // Buffers indexed by StreamID
-	size         uint64                   // Total size of all buffers, in bytes
-	maxbufs      int
-	maxsize      uint64
-	closer       []closerInfo // Scratch space
-}
-
-func (b *switch_buffers) _cleanup(t *switchTable) {
-	for streamID, buf := range b.bufs {
-		// Remove queues for which we have no next hop
-		packet := buf.packets[0]
-		coords := switch_getPacketCoords(packet.bytes)
-		if len(t.getCloser(coords)) == 0 {
-			for _, packet := range buf.packets {
-				util.PutBytes(packet.bytes)
-			}
-			b.size -= buf.size
-			delete(b.bufs, streamID)
-		}
-	}
-
-	for b.size > b.totalMaxSize {
-		// Drop a random queue
-		target := rand.Uint64() % b.size
-		var size uint64 // running total
-		for streamID, buf := range b.bufs {
-			size += buf.size
-			if size < target {
-				continue
-			}
-			var packet switch_packetInfo
-			packet, buf.packets = buf.packets[0], buf.packets[1:]
-			buf.size -= uint64(len(packet.bytes))
-			b.size -= uint64(len(packet.bytes))
-			util.PutBytes(packet.bytes)
-			if len(buf.packets) == 0 {
-				delete(b.bufs, streamID)
-			} else {
-				// Need to update the map, since buf was retrieved by value
-				b.bufs[streamID] = buf
-			}
-			break
-		}
-	}
-}
-
-// Handles incoming idle notifications
-// Loops over packets and sends the newest one that's OK for this peer to send
-// Returns true if the peer is no longer idle, false if it should be added to the idle list
-func (t *switchTable) _handleIdle(port switchPort) bool {
-	// TODO? only send packets for which this is the best next hop that isn't currently blocked sending
-	to := t.core.peers.getPorts()[port]
-	if to == nil {
-		return true
-	}
-	var packets [][]byte
-	var psize int
-	t.queues._cleanup(t)
-	now := time.Now()
-	for psize < 65535 {
-		var best string
-		var bestPriority float64
-		for streamID, buf := range t.queues.bufs {
-			// Filter over the streams that this node is closer to
-			// Keep the one with the smallest queue
-			packet := buf.packets[0]
-			coords := switch_getPacketCoords(packet.bytes)
-			priority := float64(now.Sub(packet.time)) / float64(buf.size)
-			if priority >= bestPriority && t.portIsCloser(coords, port) {
-				best = streamID
-				bestPriority = priority
-			}
-		}
-		if best != "" {
-			buf := t.queues.bufs[best]
-			var packet switch_packetInfo
-			// TODO decide if this should be LIFO or FIFO
-			packet, buf.packets = buf.packets[0], buf.packets[1:]
-			buf.size -= uint64(len(packet.bytes))
-			t.queues.size -= uint64(len(packet.bytes))
-			if len(buf.packets) == 0 {
-				delete(t.queues.bufs, best)
-			} else {
-				// Need to update the map, since buf was retrieved by value
-				t.queues.bufs[best] = buf
-			}
-			packets = append(packets, packet.bytes)
-			psize += len(packet.bytes)
+func (t *lookupTable) lookup(coords []byte) switchPort {
+	var offset int
+	here := &t._start
+	for offset < len(coords) {
+		port, l := wire_decode_uint64(coords[offset:])
+		offset += l
+		if next, ok := here.next[switchPort(port)]; ok {
+			here = next
 		} else {
-			// Finished finding packets
 			break
 		}
 	}
-	if len(packets) > 0 {
-		to.sendPacketsFrom(t, packets)
-		return true
-	}
-	return false
-}
-
-func (t *switchTable) packetInFrom(from phony.Actor, bytes []byte) {
-	t.Act(from, func() {
-		t._packetIn(bytes)
-	})
-}
-
-func (t *switchTable) _packetIn(bytes []byte) {
-	// Try to send it somewhere (or drop it if it's corrupt or at a dead end)
-	if !t._handleIn(bytes, t.idle, t.sending) {
-		// There's nobody free to take it right now, so queue it for later
-		packet := switch_packetInfo{bytes, time.Now()}
-		streamID := switch_getPacketStreamID(packet.bytes)
-		buf, bufExists := t.queues.bufs[streamID]
-		buf.packets = append(buf.packets, packet)
-		buf.size += uint64(len(packet.bytes))
-		t.queues.size += uint64(len(packet.bytes))
-		// Keep a track of the max total queue size
-		if t.queues.size > t.queues.maxsize {
-			t.queues.maxsize = t.queues.size
-		}
-		t.queues.bufs[streamID] = buf
-		if !bufExists {
-			// Keep a track of the max total queue count. Only recalculate this
-			// when the queue is new because otherwise repeating len(dict) might
-			// cause unnecessary processing overhead
-			if len(t.queues.bufs) > t.queues.maxbufs {
-				t.queues.maxbufs = len(t.queues.bufs)
-			}
-		}
-		t.queues._cleanup(t)
-	}
-}
-
-func (t *switchTable) _idleIn(port switchPort) {
-	// Try to find something to send to this peer
-	delete(t.sending, port)
-	if !t._handleIdle(port) {
-		// Didn't find anything ready to send yet, so stay idle
-		t.idle[port] = struct{}{}
-	}
-}
-
-func (t *switchTable) _sendingIn(port switchPort) {
-	if _, isIn := t.idle[port]; !isIn {
-		t.sending[port] = struct{}{}
-	}
+	return here.port
 }

src/yggdrasil/tcp.go

@@ -25,6 +25,7 @@ import (
 
 	"golang.org/x/net/proxy"
 
+	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
 )
 
@@ -33,7 +34,7 @@ const tcp_ping_interval = (default_timeout * 2 / 3)
 
 // The TCP listener and information about active TCP connections, to avoid duplication.
 type tcp struct {
-	link      *link
+	links     *links
 	waitgroup sync.WaitGroup
 	mutex     sync.Mutex // Protecting the below
 	listeners map[string]*TcpListener
@@ -57,6 +58,14 @@ type TcpUpgrade struct {
 	name    string
 }
 
+type tcpOptions struct {
+	linkOptions
+	upgrade        *TcpUpgrade
+	socksProxyAddr string
+	socksProxyAuth *proxy.Auth
+	socksPeerAddr  string
+}
+
 func (l *TcpListener) Stop() {
 	defer func() { recover() }()
 	close(l.stop)
@@ -86,8 +95,8 @@ func (t *tcp) getAddr() *net.TCPAddr {
 }
 
 // Initializes the struct.
-func (t *tcp) init(l *link) error {
-	t.link = l
+func (t *tcp) init(l *links) error {
+	t.links = l
 	t.tls.init(t)
 	t.mutex.Lock()
 	t.calls = make(map[string]struct{})
@@ -95,9 +104,9 @@ func (t *tcp) init(l *link) error {
 	t.listeners = make(map[string]*TcpListener)
 	t.mutex.Unlock()
 
-	t.link.core.config.Mutex.RLock()
-	defer t.link.core.config.Mutex.RUnlock()
-	for _, listenaddr := range t.link.core.config.Current.Listen {
+	t.links.core.config.Mutex.RLock()
+	defer t.links.core.config.Mutex.RUnlock()
+	for _, listenaddr := range t.links.core.config.Current.Listen {
 		switch listenaddr[:6] {
 		case "tcp://":
 			if _, err := t.listen(listenaddr[6:], nil); err != nil {
@@ -108,7 +117,7 @@ func (t *tcp) init(l *link) error {
 				return err
 			}
 		default:
-			t.link.core.log.Errorln("Failed to add listener: listener", listenaddr, "is not correctly formatted, ignoring")
+			t.links.core.log.Errorln("Failed to add listener: listener", listenaddr, "is not correctly formatted, ignoring")
 		}
 	}
 
@@ -126,35 +135,35 @@ func (t *tcp) stop() error {
 }
 
 func (t *tcp) reconfigure() {
-	t.link.core.config.Mutex.RLock()
-	added := util.Difference(t.link.core.config.Current.Listen, t.link.core.config.Previous.Listen)
-	deleted := util.Difference(t.link.core.config.Previous.Listen, t.link.core.config.Current.Listen)
-	t.link.core.config.Mutex.RUnlock()
+	t.links.core.config.Mutex.RLock()
+	added := util.Difference(t.links.core.config.Current.Listen, t.links.core.config.Previous.Listen)
+	deleted := util.Difference(t.links.core.config.Previous.Listen, t.links.core.config.Current.Listen)
+	t.links.core.config.Mutex.RUnlock()
 	if len(added) > 0 || len(deleted) > 0 {
 		for _, a := range added {
 			switch a[:6] {
 			case "tcp://":
 				if _, err := t.listen(a[6:], nil); err != nil {
-					t.link.core.log.Errorln("Error adding TCP", a[6:], "listener:", err)
+					t.links.core.log.Errorln("Error adding TCP", a[6:], "listener:", err)
 				}
 			case "tls://":
 				if _, err := t.listen(a[6:], t.tls.forListener); err != nil {
-					t.link.core.log.Errorln("Error adding TLS", a[6:], "listener:", err)
+					t.links.core.log.Errorln("Error adding TLS", a[6:], "listener:", err)
 				}
 			default:
-				t.link.core.log.Errorln("Failed to add listener: listener", a, "is not correctly formatted, ignoring")
+				t.links.core.log.Errorln("Failed to add listener: listener", a, "is not correctly formatted, ignoring")
 			}
 		}
 		for _, d := range deleted {
 			if d[:6] != "tcp://" && d[:6] != "tls://" {
-				t.link.core.log.Errorln("Failed to delete listener: listener", d, "is not correctly formatted, ignoring")
+				t.links.core.log.Errorln("Failed to delete listener: listener", d, "is not correctly formatted, ignoring")
 				continue
 			}
 			t.mutex.Lock()
 			if listener, ok := t.listeners[d[6:]]; ok {
 				t.mutex.Unlock()
 				listener.Stop()
-				t.link.core.log.Infoln("Stopped TCP listener:", d[6:])
+				t.links.core.log.Infoln("Stopped TCP listener:", d[6:])
 			} else {
 				t.mutex.Unlock()
 			}
@@ -196,19 +205,18 @@ func (t *tcp) listener(l *TcpListener, listenaddr string) {
 		t.mutex.Unlock()
 		l.Listener.Close()
 		return
-	} else {
-		t.listeners[listenaddr] = l
-		t.mutex.Unlock()
 	}
+	t.listeners[listenaddr] = l
+	t.mutex.Unlock()
 	// And here we go!
 	defer func() {
-		t.link.core.log.Infoln("Stopping TCP listener on:", l.Listener.Addr().String())
+		t.links.core.log.Infoln("Stopping TCP listener on:", l.Listener.Addr().String())
 		l.Listener.Close()
 		t.mutex.Lock()
 		delete(t.listeners, listenaddr)
 		t.mutex.Unlock()
 	}()
-	t.link.core.log.Infoln("Listening for TCP on:", l.Listener.Addr().String())
+	t.links.core.log.Infoln("Listening for TCP on:", l.Listener.Addr().String())
 	go func() {
 		<-l.stop
 		l.Listener.Close()
@@ -217,11 +225,20 @@ func (t *tcp) listener(l *TcpListener, listenaddr string) {
 	for {
 		sock, err := l.Listener.Accept()
 		if err != nil {
-			t.link.core.log.Errorln("Failed to accept connection:", err)
-			return
+			t.links.core.log.Errorln("Failed to accept connection:", err)
+			select {
+			case <-l.stop:
+				return
+			default:
+			}
+			time.Sleep(time.Second) // So we don't busy loop
+			continue
 		}
 		t.waitgroup.Add(1)
-		go t.handler(sock, true, nil, l.upgrade)
+		options := tcpOptions{
+			upgrade: l.upgrade,
+		}
+		go t.handler(sock, true, options)
 	}
 }
 
@@ -239,12 +256,12 @@ func (t *tcp) startCalling(saddr string) bool {
 // If the dial is successful, it launches the handler.
 // When finished, it removes the outgoing call, so reconnection attempts can be made later.
 // This all happens in a separate goroutine that it spawns.
-func (t *tcp) call(saddr string, options interface{}, sintf string, upgrade *TcpUpgrade) {
+func (t *tcp) call(saddr string, options tcpOptions, sintf string) {
 	go func() {
 		callname := saddr
 		callproto := "TCP"
-		if upgrade != nil {
-			callproto = strings.ToUpper(upgrade.name)
+		if options.upgrade != nil {
+			callproto = strings.ToUpper(options.upgrade.name)
 		}
 		if sintf != "" {
 			callname = fmt.Sprintf("%s/%s/%s", callproto, saddr, sintf)
@@ -263,17 +280,16 @@ func (t *tcp) call(saddr string, options interface{}, sintf string, upgrade *Tcp
 		}()
 		var conn net.Conn
 		var err error
-		socksaddr, issocks := options.(string)
-		if issocks {
+		if options.socksProxyAddr != "" {
 			if sintf != "" {
 				return
 			}
-			dialerdst, er := net.ResolveTCPAddr("tcp", socksaddr)
+			dialerdst, er := net.ResolveTCPAddr("tcp", options.socksProxyAddr)
 			if er != nil {
 				return
 			}
 			var dialer proxy.Dialer
-			dialer, err = proxy.SOCKS5("tcp", dialerdst.String(), nil, proxy.Direct)
+			dialer, err = proxy.SOCKS5("tcp", dialerdst.String(), options.socksProxyAuth, proxy.Direct)
 			if err != nil {
 				return
 			}
@@ -282,7 +298,10 @@ func (t *tcp) call(saddr string, options interface{}, sintf string, upgrade *Tcp
 				return
 			}
 			t.waitgroup.Add(1)
-			t.handler(conn, false, saddr, nil)
+			options.socksPeerAddr = conn.RemoteAddr().String()
+			if ch := t.handler(conn, false, options); ch != nil {
+				<-ch
+			}
 		} else {
 			dst, err := net.ResolveTCPAddr("tcp", saddr)
 			if err != nil {
@@ -344,40 +363,41 @@ func (t *tcp) call(saddr string, options interface{}, sintf string, upgrade *Tcp
 			}
 			conn, err = dialer.Dial("tcp", dst.String())
 			if err != nil {
-				t.link.core.log.Debugf("Failed to dial %s: %s", callproto, err)
+				t.links.core.log.Debugf("Failed to dial %s: %s", callproto, err)
 				return
 			}
 			t.waitgroup.Add(1)
-			t.handler(conn, false, nil, upgrade)
+			if ch := t.handler(conn, false, options); ch != nil {
+				<-ch
+			}
 		}
 	}()
 }
 
-func (t *tcp) handler(sock net.Conn, incoming bool, options interface{}, upgrade *TcpUpgrade) {
+func (t *tcp) handler(sock net.Conn, incoming bool, options tcpOptions) chan struct{} {
 	defer t.waitgroup.Done() // Happens after sock.close
 	defer sock.Close()
 	t.setExtraOptions(sock)
 	var upgraded bool
-	if upgrade != nil {
+	if options.upgrade != nil {
 		var err error
-		if sock, err = upgrade.upgrade(sock); err != nil {
-			t.link.core.log.Errorln("TCP handler upgrade failed:", err)
-			return
-		} else {
-			upgraded = true
+		if sock, err = options.upgrade.upgrade(sock); err != nil {
+			t.links.core.log.Errorln("TCP handler upgrade failed:", err)
+			return nil
 		}
+		upgraded = true
 	}
 	stream := stream{}
 	stream.init(sock)
 	var name, proto, local, remote string
-	if socksaddr, issocks := options.(string); issocks {
-		name = "socks://" + sock.RemoteAddr().String() + "/" + socksaddr
+	if options.socksProxyAddr != "" {
+		name = "socks://" + sock.RemoteAddr().String() + "/" + options.socksPeerAddr
 		proto = "socks"
 		local, _, _ = net.SplitHostPort(sock.LocalAddr().String())
-		remote, _, _ = net.SplitHostPort(socksaddr)
+		remote, _, _ = net.SplitHostPort(options.socksPeerAddr)
 	} else {
 		if upgraded {
-			proto = upgrade.name
+			proto = options.upgrade.name
 			name = proto + "://" + sock.RemoteAddr().String()
 		} else {
 			proto = "tcp"
@@ -386,13 +406,30 @@ func (t *tcp) handler(sock net.Conn, incoming bool, options interface{}, upgrade
 		local, _, _ = net.SplitHostPort(sock.LocalAddr().String())
 		remote, _, _ = net.SplitHostPort(sock.RemoteAddr().String())
 	}
+	localIP := net.ParseIP(local)
+	if localIP = localIP.To16(); localIP != nil {
+		var laddr address.Address
+		var lsubnet address.Subnet
+		copy(laddr[:], localIP)
+		copy(lsubnet[:], localIP)
+		if laddr.IsValid() || lsubnet.IsValid() {
+			// The local address is with the network address/prefix range
+			// This would route ygg over ygg, which we don't want
+			// FIXME ideally this check should happen outside of the core library
+			//  Maybe dial/listen at the application level
+			//  Then pass a net.Conn to the core library (after these kinds of checks are done)
+			t.links.core.log.Debugln("Dropping ygg-tunneled connection", local, remote)
+			return nil
+		}
+	}
 	force := net.ParseIP(strings.Split(remote, "%")[0]).IsLinkLocalUnicast()
-	link, err := t.link.core.link.create(&stream, name, proto, local, remote, incoming, force)
+	link, err := t.links.create(&stream, name, proto, local, remote, incoming, force, options.linkOptions)
 	if err != nil {
-		t.link.core.log.Println(err)
+		t.links.core.log.Println(err)
 		panic(err)
 	}
-	t.link.core.log.Debugln("DEBUG: starting handler for", name)
-	err = link.handler()
-	t.link.core.log.Debugln("DEBUG: stopped handler for", name, err)
+	t.links.core.log.Debugln("DEBUG: starting handler for", name)
+	ch, err := link.handler()
+	t.links.core.log.Debugln("DEBUG: stopped handler for", name, err)
+	return ch
 }

src/yggdrasil/tcp_linux.go

@@ -20,10 +20,10 @@ func (t *tcp) tcpContext(network, address string, c syscall.RawConn) error {
 
 	// Log any errors
 	if bbr != nil {
-		t.link.core.log.Debugln("Failed to set tcp_congestion_control to bbr for socket, SetsockoptString error:", bbr)
+		t.links.core.log.Debugln("Failed to set tcp_congestion_control to bbr for socket, SetsockoptString error:", bbr)
 	}
 	if control != nil {
-		t.link.core.log.Debugln("Failed to set tcp_congestion_control to bbr for socket, Control error:", control)
+		t.links.core.log.Debugln("Failed to set tcp_congestion_control to bbr for socket, Control error:", control)
 	}
 
 	// Return nil because errors here are not considered fatal for the connection, it just means congestion control is suboptimal
@@ -38,7 +38,7 @@ func (t *tcp) getControl(sintf string) func(string, string, syscall.RawConn) err
 		}
 		c.Control(btd)
 		if err != nil {
-			t.link.core.log.Debugln("Failed to set SO_BINDTODEVICE:", sintf)
+			t.links.core.log.Debugln("Failed to set SO_BINDTODEVICE:", sintf)
 		}
 		return t.tcpContext(network, address, c)
 	}

src/yggdrasil/tls.go

@@ -34,7 +34,7 @@ func (t *tcptls) init(tcp *tcp) {
 	}
 
 	edpriv := make(ed25519.PrivateKey, ed25519.PrivateKeySize)
-	copy(edpriv[:], tcp.link.core.sigPriv[:])
+	copy(edpriv[:], tcp.links.core.sigPriv[:])
 
 	certBuf := &bytes.Buffer{}
 
@@ -42,7 +42,7 @@ func (t *tcptls) init(tcp *tcp) {
 	pubtemp := x509.Certificate{
 		SerialNumber: big.NewInt(1),
 		Subject: pkix.Name{
-			CommonName: hex.EncodeToString(tcp.link.core.sigPub[:]),
+			CommonName: hex.EncodeToString(tcp.links.core.sigPub[:]),
 		},
 		NotBefore:             time.Now(),
 		NotAfter:              time.Now().Add(time.Hour * 24 * 365),

src/yggdrasil/version.go

@@ -28,11 +28,11 @@ func version_getBaseMetadata() version_metadata {
 	}
 }
 
-// Gest the length of the metadata for this version, used to know how many bytes to read from the start of a connection.
+// Gets the length of the metadata for this version, used to know how many bytes to read from the start of a connection.
 func version_getMetaLength() (mlen int) {
 	mlen += 4                   // meta
-	mlen += 1                   // ver, as long as it's < 127, which it is in this version
-	mlen += 1                   // minorVer, as long as it's < 127, which it is in this version
+	mlen++                      // ver, as long as it's < 127, which it is in this version
+	mlen++                      // minorVer, as long as it's < 127, which it is in this version
 	mlen += crypto.BoxPubKeyLen // box
 	mlen += crypto.SigPubKeyLen // sig
 	mlen += crypto.BoxPubKeyLen // link

src/yggdrasil/wire.go

@@ -9,7 +9,6 @@ package yggdrasil
 
 import (
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
-	"github.com/yggdrasil-network/yggdrasil-go/src/util"
 )
 
 const (
@@ -230,8 +229,9 @@ type wire_trafficPacket struct {
 }
 
 // Encodes a wire_trafficPacket into its wire format.
+// The returned slice was taken from the pool.
 func (p *wire_trafficPacket) encode() []byte {
-	bs := util.GetBytes()
+	bs := pool_getBytes(0)
 	bs = wire_put_uint64(wire_Traffic, bs)
 	bs = wire_put_coords(p.Coords, bs)
 	bs = append(bs, p.Handle[:]...)
@@ -241,7 +241,9 @@ func (p *wire_trafficPacket) encode() []byte {
 }
 
 // Decodes an encoded wire_trafficPacket into the struct, returning true if successful.
+// Either way, the argument slice is added to the pool.
 func (p *wire_trafficPacket) decode(bs []byte) bool {
+	defer pool_putBytes(bs)
 	var pType uint64
 	switch {
 	case !wire_chop_uint64(&pType, &bs):
@@ -255,7 +257,7 @@ func (p *wire_trafficPacket) decode(bs []byte) bool {
 	case !wire_chop_slice(p.Nonce[:], &bs):
 		return false
 	}
-	p.Payload = append(util.GetBytes(), bs...)
+	p.Payload = append(p.Payload, bs...)
 	return true
 }