Merge pull request #770 from yggdrasil-network/develop

Version 0.3.16

.circleci/config.yml

@@ -3,9 +3,22 @@
 # Check https://circleci.com/docs/2.0/language-go/ for more details
 version: 2.1
 jobs:
+  lint:
+    docker:
+      - image: circleci/golang:1.16
+
+    steps:
+      - checkout
+
+      - run:
+          name: Run golangci-lint
+          command: |
+              go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.31.0
+              golangci-lint run
+
   build-linux:
     docker:
-      - image: circleci/golang:1.13.3
+      - image: circleci/golang:1.16
 
     steps:
       - checkout
@@ -106,11 +119,11 @@ jobs:
               echo -e "Host *\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
 
       - run:
-          name: Install Go 1.13.3
+          name: Install Go 1.16
           command: |
               cd /tmp
-              curl -LO https://dl.google.com/go/go1.13.3.darwin-amd64.pkg
-              sudo installer -pkg /tmp/go1.13.3.darwin-amd64.pkg -target /
+              curl -LO https://dl.google.com/go/go1.16.darwin-amd64.pkg
+              sudo installer -pkg /tmp/go1.16.darwin-amd64.pkg -target /
 
       #- run:
       #    name: Install Gomobile
@@ -144,9 +157,46 @@ jobs:
           paths:
             - upload
 
+  build-windows:
+    docker:
+      - image: circleci/golang:1.16
+
+    steps:
+      - checkout
+
+      - run:
+          name: Create artifact upload directory and set variables
+          command: |
+              mkdir /tmp/upload
+              echo 'export CINAME=$(sh contrib/semver/name.sh)' >> $BASH_ENV
+              echo 'export CIVERSION=$(sh contrib/semver/version.sh --bare)' >> $BASH_ENV
+              git config --global user.email "$(git log --format='%ae' HEAD -1)";
+              git config --global user.name "$(git log --format='%an' HEAD -1)";
+
+      - run:
+          name: Install tools
+          command: |
+              sudo apt-get update
+              sudo apt-get -y install msitools wixl
+
+      - run:
+          name: Build for Windows
+          command: |
+              rm -f {yggdrasil,yggdrasilctl}
+              GOOS=windows GOARCH=amd64 ./build && mv yggdrasil.exe /tmp/upload/$CINAME-$CIVERSION-windows-amd64.exe && mv yggdrasilctl.exe /tmp/upload/$CINAME-$CIVERSION-yggdrasilctl-windows-amd64.exe;
+              GOOS=windows GOARCH=386 ./build && mv yggdrasil.exe /tmp/upload/$CINAME-$CIVERSION-windows-i386.exe && mv yggdrasilctl.exe /tmp/upload/$CINAME-$CIVERSION-yggdrasilctl-windows-i386.exe;
+              bash contrib/msi/build-msi.sh x64
+              bash contrib/msi/build-msi.sh x86
+              mv *.msi /tmp/upload
+
+      - persist_to_workspace:
+          root: /tmp
+          paths:
+            - upload
+
   build-other:
     docker:
-      - image: circleci/golang:1.13.3
+      - image: circleci/golang:1.16
 
     steps:
       - checkout
@@ -174,20 +224,6 @@ jobs:
               GOOS=freebsd GOARCH=amd64 ./build && mv yggdrasil /tmp/upload/$CINAME-$CIVERSION-freebsd-amd64 && mv yggdrasilctl /tmp/upload/$CINAME-$CIVERSION-yggdrasilctl-freebsd-amd64;
               GOOS=freebsd GOARCH=386 ./build && mv yggdrasil /tmp/upload/$CINAME-$CIVERSION-freebsd-i386 && mv yggdrasilctl /tmp/upload/$CINAME-$CIVERSION-yggdrasilctl-freebsd-i386;
 
-      - run:
-          name: Build for NetBSD
-          command: |
-              rm -f {yggdrasil,yggdrasilctl}
-              GOOS=netbsd GOARCH=amd64 ./build && mv yggdrasil /tmp/upload/$CINAME-$CIVERSION-netbsd-amd64 && mv yggdrasilctl /tmp/upload/$CINAME-$CIVERSION-yggdrasilctl-netbsd-amd64;
-              GOOS=netbsd GOARCH=386 ./build && mv yggdrasil /tmp/upload/$CINAME-$CIVERSION-netbsd-i386 && mv yggdrasilctl /tmp/upload/$CINAME-$CIVERSION-yggdrasilctl-netbsd-i386;
-
-      - run:
-          name: Build for Windows
-          command: |
-              rm -f {yggdrasil,yggdrasilctl}
-              GOOS=windows GOARCH=amd64 ./build && mv yggdrasil.exe /tmp/upload/$CINAME-$CIVERSION-windows-amd64.exe && mv yggdrasilctl.exe /tmp/upload/$CINAME-$CIVERSION-yggdrasilctl-windows-amd64.exe;
-              GOOS=windows GOARCH=386 ./build && mv yggdrasil.exe /tmp/upload/$CINAME-$CIVERSION-windows-i386.exe && mv yggdrasilctl.exe /tmp/upload/$CINAME-$CIVERSION-yggdrasilctl-windows-i386.exe;
-
       - persist_to_workspace:
           root: /tmp
           paths:
@@ -208,11 +244,14 @@ workflows:
   version: 2.1
   build:
     jobs:
+      - lint
       - build-linux
       - build-macos
+      - build-windows
       - build-other
       - upload:
           requires:
             - build-linux
             - build-macos
+            - build-windows
             - build-other

.golangci.yml

@@ -0,0 +1,10 @@
+run:
+  build-tags:
+    - lint
+  issues-exit-code: 0 # TODO: change this to 1 when we want it to fail builds
+  skip-dirs:
+    - contrib/
+    - misc/
+linters:
+  disable:
+    - gocyclo

CHANGELOG.md

@@ -25,6 +25,84 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - in case of vulnerabilities.
 -->
 
+## [0.3.16] - 2021-03-18
+### Added
+- New simulation code under `cmd/yggdrasilsim` (work-in-progress)
+
+### Changed
+- Multi-threading in the switch
+  - Swich lookups happen independently for each (incoming) peer connection, instead of being funneled to a single dedicated switch worker
+  - Packets are queued for each (outgoing) peer connection, instead of being handled by a single dedicated switch worker
+- Queue logic rewritten
+  - Heap structure per peer that traffic is routed to, with one FIFO queue per traffic flow
+  - The total size of each heap is configured automatically (we basically queue packets until we think we're blocked on a socket write)
+  - When adding to a full heap, the oldest packet from the largest queue is dropped
+  - Packets are popped from the queue in FIFO order (oldest packet from among all queues in the heap) to prevent packet reordering at the session level
+- Removed global `sync.Pool` of `[]byte`
+  - Local `sync.Pool`s are used in the hot loops, but not exported, to avoid memory corruption if libraries are reused by other projects
+  - This may increase allocations (and slightly reduce speed in CPU-bound benchmarks) when interacting with the tun/tap device, but traffic forwarded at the switch layer should be unaffected
+- Upgrade dependencies
+- Upgrade build to Go 1.16
+
+### Fixed
+- Fixed a bug where the connection listener could exit prematurely due to resoruce exhaustion (if e.g. too many connections were opened)
+- Fixed DefaultIfName for OpenBSD (`/dev/tun0` -> `tun0`)
+- Fixed an issue where a peer could sometimes never be added to the switch
+- Fixed a goroutine leak that could occur if a peer with an open connection continued to spam additional connection attempts
+
+## [0.3.15] - 2020-09-27
+### Added
+- Support for pinning remote public keys in peering strings has been added, e.g.
+  - By signing public key: `tcp://host:port?ed25519=key`
+  - By encryption public key: `tcp://host:port?curve25519=key`
+  - By both: `tcp://host:port?ed25519=key&curve25519=key`
+  - By multiple, in case of DNS round-robin or similar: `tcp://host:port?curve25519=key&curve25519=key&ed25519=key&ed25519=key`
+- Some checks to prevent Yggdrasil-over-Yggdrasil peerings have been added
+- Added support for SOCKS proxy authentication, e.g. `socks://user@password:host/...`
+
+### Fixed
+- Some bugs in the multicast code that could cause unnecessary CPU usage have been fixed
+- A possible multicast deadlock on macOS when enumerating interfaces has been fixed
+- A deadlock in the connection code has been fixed
+- Updated HJSON dependency that caused some build problems
+
+### Changed
+- `DisconnectPeer` and `RemovePeer` have been separated and implemented properly now
+- Less nodes are stored in the DHT now, reducing ambient network traffic and possible instability
+- Default config file for FreeBSD is now at `/usr/local/etc/yggdrasil.conf` instead of `/etc/yggdrasil.conf`
+
+## [0.3.14] - 2020-03-28
+### Fixed
+- Fixes a memory leak that may occur if packets are incorrectly never removed from a switch queue
+
+### Changed
+- Make DHT searches a bit more reliable by tracking the 16 most recently visited nodes
+
+## [0.3.13] - 2020-02-21
+### Added
+- Support for the Wireguard TUN driver, which now replaces Water and provides far better support and performance on Windows
+- Windows `.msi` installer files are now supported (bundling the Wireguard TUN driver)
+- NodeInfo code is now actorised, should be more reliable
+- The DHT now tries to store the two closest nodes in either direction instead of one, such that if a node goes offline, the replacement is already known
+- The Yggdrasil API now supports dialing a remote node using the public key instead of the Node ID
+
+### Changed
+- The `-loglevel` command line parameter is now cumulative and automatically includes all levels below the one specified
+- DHT search code has been significantly simplified and processes rumoured nodes in parallel, speeding up search time
+- DHT search results are now sorted
+- The systemd service now handles configuration generation in a different unit
+- The Yggdrasil API now returns public keys instead of node IDs when querying for local and remote addresses
+
+### Fixed
+- The multicast code no longer panics when shutting down the node
+- A potential OOB error when calculating IPv4 flow labels (when tunnel routing is enabled) has been fixed
+- A bug resulting in incorrect idle notifications in the switch should now be fixed
+- MTUs are now using a common datatype throughout the codebase
+
+### Removed
+- TAP mode has been removed entirely, since it is no longer supported with the Wireguard TUN package. Please note that if you are using TAP mode, you may need to revise your config!
+- NetBSD support has been removed until the Wireguard TUN package supports NetBSD
+
 ## [0.3.12] - 2019-11-24
 ### Added
 - New API functions `SetMaximumSessionMTU` and `GetMaximumSessionMTU`
@@ -66,7 +144,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Changed
 - On recent Linux kernels, Yggdrasil will now set the `tcp_congestion_control` algorithm used for its own TCP sockets to [BBR](https://github.com/google/bbr), which reduces latency under load
-- The systemd service configuration in `contrib` (and, by extension, some of our packages) now attemps to load the `tun` module, in case TUN/TAP support is available but not loaded, and it restricts Yggdrasil to the `CAP_NET_ADMIN` capability for managing the TUN/TAP adapter, rather than letting it do whatever the (typically `root`) user can do
+- The systemd service configuration in `contrib` (and, by extension, some of our packages) now attempts to load the `tun` module, in case TUN/TAP support is available but not loaded, and it restricts Yggdrasil to the `CAP_NET_ADMIN` capability for managing the TUN/TAP adapter, rather than letting it do whatever the (typically `root`) user can do
 
 ### Fixed
 - The `yggdrasil.Conn.RemoteAddr()` function no longer blocks, fixing a deadlock when CKR is used while under heavy load
@@ -90,7 +168,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Some minor memory leaks in the switch have been fixed, which improves memory usage on mobile builds
 - A memory leak in the add-peer loop has been fixed
 - The admin socket now reports the correct URI strings for SOCKS peers in `getPeers`
-- A race condition when dialling a remote node by both the node address and routed prefix simultaneously has been fixed
+- A race condition when dialing a remote node by both the node address and routed prefix simultaneously has been fixed
 - A race condition between the router and the dial code resulting in a panic has been fixed
 - A panic which could occur when the TUN/TAP interface disappears (e.g. during soft-shutdown) has been fixed
 - A bug in the semantic versioning script which accompanies Yggdrasil for builds has been fixed
@@ -180,7 +258,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ## [0.3.4] - 2019-03-12
 ### Added
 - Support for multiple listeners (although currently only TCP listeners are supported)
-- New multicast behaviour where each multicast interface is given it's own link-local listener and does not depend on the `Listen` configuration
+- New multicast behaviour where each multicast interface is given its own link-local listener and does not depend on the `Listen` configuration
 - Blocking detection in the switch to avoid parenting a blocked peer
 - Support for adding and removing listeners and multicast interfaces when reloading configuration during runtime
 - Yggdrasil will now attempt to clean up UNIX admin sockets on startup if left behind by a previous crash
@@ -374,7 +452,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Wire format changes (backwards incompatible).
 - Less maintenance traffic per peer.
 - Exponential back-off for DHT maintenance traffic (less maintenance traffic for known good peers).
-- Iterative DHT (added some time between v0.1.0 and here).
+- Iterative DHT (added sometime between v0.1.0 and here).
 - Use local queue sizes for a sort of local-only backpressure routing, instead of the removed bandwidth estimates, when deciding where to send a packet.
 
 ### Removed

README.md

@@ -26,7 +26,7 @@ some of the below:
 - Linux
   - `.deb` and `.rpm` packages are built by CI for Debian and Red Hat-based
     distributions
-  - Void and Arch packages also available within their respective repositories
+  - Arch, Nix, Void packages also available within their respective repositories
 - macOS
   - `.pkg` packages are built by CI
 - Ubiquiti EdgeOS
@@ -34,7 +34,6 @@ some of the below:
 - Windows
 - FreeBSD
 - OpenBSD
-- NetBSD
 - OpenWrt
 
 Please see our [Platforms](https://yggdrasil-network.github.io/platforms.html) pages for more
@@ -49,7 +48,7 @@ You may also find other platform-specific wrappers, scripts or tools in the
 If you want to build from source, as opposed to installing one of the pre-built
 packages:
 
-1. Install [Go](https://golang.org) (requires Go 1.13 or later)
+1. Install [Go](https://golang.org) (requires Go 1.16 or later)
 2. Clone this repository
 2. Run `./build`
 

build

@@ -9,7 +9,7 @@ PKGVER=${PKGVER:-$(sh contrib/semver/version.sh --bare)}
 LDFLAGS="-X $PKGSRC.buildName=$PKGNAME -X $PKGSRC.buildVersion=$PKGVER"
 ARGS="-v"
 
-while getopts "uaitc:l:dro:" option
+while getopts "uaitc:l:dro:p" option
 do
   case "$option"
   in
@@ -22,6 +22,7 @@ do
   d) ARGS="$ARGS -tags debug" DEBUG=true;;
   r) ARGS="$ARGS -race";;
   o) ARGS="$ARGS -o $OPTARG";;
+  p) ARGS="$ARGS -buildmode=pie";;
   esac
 done
 

cmd/genkeys/main.go

@@ -3,7 +3,7 @@
 This file generates crypto keys.
 It prints out a new set of keys each time if finds a "better" one.
 By default, "better" means a higher NodeID (-> higher IP address).
-This is because the IP address format can compress leading 1s in the address, to incrase the number of ID bits in the address.
+This is because the IP address format can compress leading 1s in the address, to increase the number of ID bits in the address.
 
 If run with the "-sig" flag, it generates signing keys instead.
 A "better" signing key means one with a higher TreeID.
@@ -51,7 +51,7 @@ func main() {
 
 	for {
 		newKey := <-newKeys
-		if isBetter(currentBest[:], newKey.id[:]) || len(currentBest) == 0 {
+		if isBetter(currentBest, newKey.id[:]) || len(currentBest) == 0 {
 			currentBest = newKey.id
 			for _, channel := range threadChannels {
 				select {
@@ -61,13 +61,13 @@ func main() {
 			fmt.Println("--------------------------------------------------------------------------------")
 			switch {
 			case *doSig:
-				fmt.Println("sigPriv:", hex.EncodeToString(newKey.priv[:]))
-				fmt.Println("sigPub:", hex.EncodeToString(newKey.pub[:]))
-				fmt.Println("TreeID:", hex.EncodeToString(newKey.id[:]))
+				fmt.Println("sigPriv:", hex.EncodeToString(newKey.priv))
+				fmt.Println("sigPub:", hex.EncodeToString(newKey.pub))
+				fmt.Println("TreeID:", hex.EncodeToString(newKey.id))
 			default:
-				fmt.Println("boxPriv:", hex.EncodeToString(newKey.priv[:]))
-				fmt.Println("boxPub:", hex.EncodeToString(newKey.pub[:]))
-				fmt.Println("NodeID:", hex.EncodeToString(newKey.id[:]))
+				fmt.Println("boxPriv:", hex.EncodeToString(newKey.priv))
+				fmt.Println("boxPub:", hex.EncodeToString(newKey.pub))
+				fmt.Println("NodeID:", hex.EncodeToString(newKey.id))
 				fmt.Println("IP:", newKey.ip)
 			}
 		}
@@ -76,11 +76,8 @@ func main() {
 
 func isBetter(oldID, newID []byte) bool {
 	for idx := range oldID {
-		if newID[idx] > oldID[idx] {
-			return true
-		}
-		if newID[idx] < oldID[idx] {
-			return false
+		if newID[idx] != oldID[idx] {
+			return newID[idx] > oldID[idx]
 		}
 	}
 	return false

cmd/yggdrasil/main.go

@@ -60,8 +60,8 @@ func readConfig(useconf *bool, useconffile *string, normaliseconf *bool) *config
 	// throwing everywhere when it's converting things into UTF-16 for the hell
 	// of it - remove it and decode back down into UTF-8. This is necessary
 	// because hjson doesn't know what to do with UTF-16 and will panic
-	if bytes.Compare(conf[0:2], []byte{0xFF, 0xFE}) == 0 ||
-		bytes.Compare(conf[0:2], []byte{0xFE, 0xFF}) == 0 {
+	if bytes.Equal(conf[0:2], []byte{0xFF, 0xFE}) ||
+		bytes.Equal(conf[0:2], []byte{0xFE, 0xFF}) {
 		utf := unicode.UTF16(unicode.BigEndian, unicode.UseBOM)
 		decoder := utf.NewDecoder()
 		conf, err = decoder.Bytes(conf)
@@ -132,6 +132,32 @@ func doGenconf(isjson bool) string {
 	return string(bs)
 }
 
+func setLogLevel(loglevel string, logger *log.Logger) {
+	levels := [...]string{"error", "warn", "info", "debug", "trace"}
+	loglevel = strings.ToLower(loglevel)
+
+	contains := func() bool {
+		for _, l := range levels {
+			if l == loglevel {
+				return true
+			}
+		}
+		return false
+	}
+
+	if !contains() { // set default log level
+		logger.Infoln("Loglevel parse failed. Set default level(info)")
+		loglevel = "info"
+	}
+
+	for _, l := range levels {
+		logger.EnableLevel(l)
+		if l == loglevel {
+			break
+		}
+	}
+}
+
 // The main function is responsible for configuring and starting Yggdrasil.
 func main() {
 	// Configure the command line parameters.
@@ -142,10 +168,10 @@ func main() {
 	confjson := flag.Bool("json", false, "print configuration from -genconf or -normaliseconf as JSON instead of HJSON")
 	autoconf := flag.Bool("autoconf", false, "automatic mode (dynamic IP, peer with IPv6 neighbors)")
 	ver := flag.Bool("version", false, "prints the version of this build")
-	logging := flag.String("logging", "info,warn,error", "comma-separated list of logging levels to enable")
 	logto := flag.String("logto", "stdout", "file path to log to, \"syslog\" or \"stdout\"")
 	getaddr := flag.Bool("address", false, "returns the IPv6 address as derived from the supplied configuration")
 	getsnet := flag.Bool("subnet", false, "returns the IPv6 subnet as derived from the supplied configuration")
+	loglevel := flag.String("loglevel", "info", "loglevel to enable")
 	flag.Parse()
 
 	var cfg *config.NodeConfig
@@ -196,7 +222,7 @@ func main() {
 	getNodeID := func() *crypto.NodeID {
 		if pubkey, err := hex.DecodeString(cfg.EncryptionPublicKey); err == nil {
 			var box crypto.BoxPubKey
-			copy(box[:], pubkey[:])
+			copy(box[:], pubkey)
 			return crypto.GetNodeID(&box)
 		}
 		return nil
@@ -239,20 +265,9 @@ func main() {
 		logger = log.New(os.Stdout, "", log.Flags())
 		logger.Warnln("Logging defaulting to stdout")
 	}
-	//logger.EnableLevel("error")
-	//logger.EnableLevel("warn")
-	//logger.EnableLevel("info")
-	if levels := strings.Split(*logging, ","); len(levels) > 0 {
-		for _, level := range levels {
-			l := strings.TrimSpace(level)
-			switch l {
-			case "error", "warn", "info", "trace", "debug":
-				logger.EnableLevel(l)
-			default:
-				continue
-			}
-		}
-	}
+
+	setLogLevel(*loglevel, logger)
+
 	// Setup the Yggdrasil node itself. The node{} type includes a Core, so we
 	// don't need to create this manually.
 	n := node{}
@@ -313,9 +328,9 @@ func main() {
 	// deferred Stop function above will run which will shut down TUN/TAP.
 	for {
 		select {
-		case _ = <-c:
+		case <-c:
 			goto exit
-		case _ = <-r:
+		case <-r:
 			if *useconffile != "" {
 				cfg = readConfig(useconf, useconffile, normaliseconf)
 				logger.Infoln("Reloading configuration from", *useconffile)

cmd/yggdrasilctl/main.go

@@ -25,14 +25,20 @@ import (
 type admin_info map[string]interface{}
 
 func main() {
+	// makes sure we can use defer and still return an error code to the OS
+	os.Exit(run())
+}
+
+func run() int {
 	logbuffer := &bytes.Buffer{}
 	logger := log.New(logbuffer, "", log.Flags())
-	defer func() {
+	defer func() int {
 		if r := recover(); r != nil {
 			logger.Println("Fatal error:", r)
 			fmt.Print(logbuffer)
-			os.Exit(1)
+			return 1
 		}
+		return 0
 	}()
 
 	endpoint := defaults.GetDefaults().DefaultAdminListen
@@ -62,18 +68,18 @@ func main() {
 		fmt.Println("Build name:", version.BuildName())
 		fmt.Println("Build version:", version.BuildVersion())
 		fmt.Println("To get the version number of the running Yggdrasil node, run", os.Args[0], "getSelf")
-		return
+		return 0
 	}
 
 	if len(args) == 0 {
 		flag.Usage()
-		return
+		return 0
 	}
 
 	if *server == endpoint {
 		if config, err := ioutil.ReadFile(defaults.GetDefaults().DefaultConfigFile); err == nil {
-			if bytes.Compare(config[0:2], []byte{0xFF, 0xFE}) == 0 ||
-				bytes.Compare(config[0:2], []byte{0xFE, 0xFF}) == 0 {
+			if bytes.Equal(config[0:2], []byte{0xFF, 0xFE}) ||
+				bytes.Equal(config[0:2], []byte{0xFE, 0xFF}) {
 				utf := unicode.UTF16(unicode.BigEndian, unicode.UseBOM)
 				decoder := utf.NewDecoder()
 				config, err = decoder.Bytes(config)
@@ -176,15 +182,15 @@ func main() {
 			} else {
 				fmt.Println("Admin socket returned an error but didn't specify any error text")
 			}
-			os.Exit(1)
+			return 1
 		}
 		if _, ok := recv["request"]; !ok {
 			fmt.Println("Missing request in response (malformed response?)")
-			os.Exit(1)
+			return 1
 		}
 		if _, ok := recv["response"]; !ok {
 			fmt.Println("Missing response body (malformed response?)")
-			os.Exit(1)
+			return 1
 		}
 		req := recv["request"].(map[string]interface{})
 		res := recv["response"].(map[string]interface{})
@@ -193,7 +199,7 @@ func main() {
 			if json, err := json.MarshalIndent(res, "", "  "); err == nil {
 				fmt.Println(string(json))
 			}
-			os.Exit(0)
+			return 0
 		}
 
 		switch strings.ToLower(req["request"].(string)) {
@@ -434,7 +440,7 @@ func main() {
 	}
 
 	if v, ok := recv["status"]; ok && v != "success" {
-		os.Exit(1)
+		return 1
 	}
-	os.Exit(0)
+	return 0
 }

cmd/yggdrasilsim/dial.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"fmt"
+	"sort"
+	"time"
+
+	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
+)
+
+func doListen(recvNode *simNode) {
+	// TODO be able to stop the listeners somehow so they don't leak across different tests
+	for {
+		c, err := recvNode.listener.Accept()
+		if err != nil {
+			panic(err)
+		}
+		c.Close()
+	}
+}
+
+func dialTest(sendNode, recvNode *simNode) {
+	if sendNode.id == recvNode.id {
+		fmt.Println("Skipping dial to self")
+		return
+	}
+	var mask crypto.NodeID
+	for idx := range mask {
+		mask[idx] = 0xff
+	}
+	for {
+		c, err := sendNode.dialer.DialByNodeIDandMask(nil, &recvNode.nodeID, &mask)
+		if c != nil {
+			c.Close()
+			return
+		}
+		if err != nil {
+			fmt.Println("Dial failed:", err)
+		}
+		time.Sleep(time.Second)
+	}
+}
+
+func dialStore(store nodeStore) {
+	var nodeIdxs []int
+	for idx, n := range store {
+		nodeIdxs = append(nodeIdxs, idx)
+		go doListen(n)
+	}
+	sort.Slice(nodeIdxs, func(i, j int) bool {
+		return nodeIdxs[i] < nodeIdxs[j]
+	})
+	for _, idx := range nodeIdxs {
+		sendNode := store[idx]
+		for _, jdx := range nodeIdxs {
+			recvNode := store[jdx]
+			fmt.Printf("Dialing from node %d to node %d / %d...\n", idx, jdx, len(store))
+			dialTest(sendNode, recvNode)
+		}
+	}
+}

cmd/yggdrasilsim/main.go

@@ -0,0 +1,6 @@
+package main
+
+func main() {
+	store := makeStoreSquareGrid(4)
+	dialStore(store)
+}

cmd/yggdrasilsim/node.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"io/ioutil"
+
+	"github.com/gologme/log"
+
+	"github.com/yggdrasil-network/yggdrasil-go/src/config"
+	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
+	"github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil"
+)
+
+type simNode struct {
+	core     yggdrasil.Core
+	id       int
+	nodeID   crypto.NodeID
+	dialer   *yggdrasil.Dialer
+	listener *yggdrasil.Listener
+}
+
+func newNode(id int) *simNode {
+	n := simNode{id: id}
+	n.core.Start(config.GenerateConfig(), log.New(ioutil.Discard, "", 0))
+	n.nodeID = *n.core.NodeID()
+	n.dialer, _ = n.core.ConnDialer()
+	n.listener, _ = n.core.ConnListen()
+	return &n
+}

cmd/yggdrasilsim/store.go

@@ -0,0 +1,41 @@
+package main
+
+type nodeStore map[int]*simNode
+
+func makeStoreSingle() nodeStore {
+	s := make(nodeStore)
+	s[0] = newNode(0)
+	return s
+}
+
+func linkNodes(a *simNode, b *simNode) {
+	la := a.core.NewSimlink()
+	lb := b.core.NewSimlink()
+	la.SetDestination(lb)
+	lb.SetDestination(la)
+	la.Start()
+	lb.Start()
+}
+
+func makeStoreSquareGrid(sideLength int) nodeStore {
+	store := make(nodeStore)
+	nNodes := sideLength * sideLength
+	idxs := make([]int, 0, nNodes)
+	// TODO shuffle nodeIDs
+	for idx := 1; idx <= nNodes; idx++ {
+		idxs = append(idxs, idx)
+	}
+	for _, idx := range idxs {
+		n := newNode(idx)
+		store[idx] = n
+	}
+	for idx := 0; idx < nNodes; idx++ {
+		if (idx % sideLength) != 0 {
+			linkNodes(store[idxs[idx]], store[idxs[idx-1]])
+		}
+		if idx >= sideLength {
+			linkNodes(store[idxs[idx]], store[idxs[idx-sideLength]])
+		}
+	}
+	return store
+}

contrib/ansible/genkeys.go

@@ -12,6 +12,7 @@ import (
 	"net"
 	"os"
 
+	"github.com/cheggaaa/pb/v3"
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 )
@@ -29,6 +30,8 @@ type keySet struct {
 func main() {
 	flag.Parse()
 
+	bar := pb.StartNew(*keyTries*2 + *numHosts)
+
 	if *numHosts > *keyTries {
 		println("Can't generate less keys than hosts.")
 		return
@@ -37,21 +40,25 @@ func main() {
 	var encryptionKeys []keySet
 	for i := 0; i < *numHosts+1; i++ {
 		encryptionKeys = append(encryptionKeys, newBoxKey())
+		bar.Increment()
 	}
 	encryptionKeys = sortKeySetArray(encryptionKeys)
 	for i := 0; i < *keyTries-*numHosts-1; i++ {
 		encryptionKeys[0] = newBoxKey()
 		encryptionKeys = bubbleUpTo(encryptionKeys, 0)
+		bar.Increment()
 	}
 
 	var signatureKeys []keySet
 	for i := 0; i < *numHosts+1; i++ {
 		signatureKeys = append(signatureKeys, newSigKey())
+		bar.Increment()
 	}
 	signatureKeys = sortKeySetArray(signatureKeys)
 	for i := 0; i < *keyTries-*numHosts-1; i++ {
 		signatureKeys[0] = newSigKey()
 		signatureKeys = bubbleUpTo(signatureKeys, 0)
+		bar.Increment()
 	}
 
 	os.MkdirAll("host_vars", 0755)
@@ -76,7 +83,9 @@ func main() {
 		defer file.Close()
 		file.WriteString(fmt.Sprintf("vault_yggdrasil_encryption_private_key: %v\n", hex.EncodeToString(encryptionKeys[i].priv)))
 		file.WriteString(fmt.Sprintf("vault_yggdrasil_signing_private_key: %v\n", hex.EncodeToString(signatureKeys[i].priv)))
+		bar.Increment()
 	}
+	bar.Finish()
 }
 
 func newBoxKey() keySet {

contrib/apparmor/usr.bin.yggdrasil

@@ -1,23 +1,17 @@
-# Last Modified: Sat Mar  9 06:08:02 2019
+# Last Modified: Fri Oct 30 11:33:31 2020
 #include <tunables/global>
 
 /usr/bin/yggdrasil {
   #include <abstractions/base>
+  #include <abstractions/nameservice>
 
   capability net_admin,
+  capability net_raw,
 
-  network inet stream,
-  network inet dgram,
-  network inet6 dgram,
-  network inet6 stream,
-  network netlink raw,
-
-  /lib/@{multiarch}/ld-*.so mr,
-  /proc/sys/net/core/somaxconn r,
   /dev/net/tun rw,
+  /proc/sys/net/core/somaxconn r,
+  /sys/kernel/mm/transparent_hugepage/hpage_pmd_size r,
 
-  /usr/bin/yggdrasil mr,
   /etc/yggdrasil.conf rw,
   /run/yggdrasil.sock rw,
-
 }

contrib/config/yggdrasilconf.go

@@ -1,97 +0,0 @@
-package main
-
-/*
-This is a small utility that is designed to accompany the vyatta-yggdrasil
-package. It takes a HJSON configuration file, makes changes to it based on
-the command line arguments, and then spits out an updated file.
-*/
-
-import (
-	"bytes"
-	"encoding/json"
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"strconv"
-
-	"github.com/hjson/hjson-go"
-	"golang.org/x/text/encoding/unicode"
-
-	"github.com/yggdrasil-network/yggdrasil-go/src/config"
-)
-
-type nodeConfig = config.NodeConfig
-
-func main() {
-	useconffile := flag.String("useconffile", "/etc/yggdrasil.conf", "update config at specified file path")
-	flag.Parse()
-	cfg := nodeConfig{}
-	var config []byte
-	var err error
-	config, err = ioutil.ReadFile(*useconffile)
-	if err != nil {
-		panic(err)
-	}
-	if bytes.Compare(config[0:2], []byte{0xFF, 0xFE}) == 0 ||
-		bytes.Compare(config[0:2], []byte{0xFE, 0xFF}) == 0 {
-		utf := unicode.UTF16(unicode.BigEndian, unicode.UseBOM)
-		decoder := utf.NewDecoder()
-		config, err = decoder.Bytes(config)
-		if err != nil {
-			panic(err)
-		}
-	}
-	var dat map[string]interface{}
-	if err := hjson.Unmarshal(config, &dat); err != nil {
-		panic(err)
-	}
-	confJson, err := json.Marshal(dat)
-	if err != nil {
-		panic(err)
-	}
-	json.Unmarshal(confJson, &cfg)
-	switch flag.Arg(0) {
-	case "setMTU":
-		cfg.IfMTU, err = strconv.Atoi(flag.Arg(1))
-		if err != nil {
-			cfg.IfMTU = 1280
-		}
-		if mtu, _ := strconv.Atoi(flag.Arg(1)); mtu < 1280 {
-			cfg.IfMTU = 1280
-		}
-	case "setIfName":
-		cfg.IfName = flag.Arg(1)
-	case "setListen":
-		cfg.Listen = flag.Arg(1)
-	case "setAdminListen":
-		cfg.AdminListen = flag.Arg(1)
-	case "setIfTapMode":
-		if flag.Arg(1) == "true" {
-			cfg.IfTAPMode = true
-		} else {
-			cfg.IfTAPMode = false
-		}
-	case "addPeer":
-		found := false
-		for _, v := range cfg.Peers {
-			if v == flag.Arg(1) {
-				found = true
-			}
-		}
-		if !found {
-			cfg.Peers = append(cfg.Peers, flag.Arg(1))
-		}
-	case "removePeer":
-		for k, v := range cfg.Peers {
-			if v == flag.Arg(1) {
-				cfg.Peers = append(cfg.Peers[:k], cfg.Peers[k+1:]...)
-			}
-		}
-	}
-	bs, err := hjson.Marshal(cfg)
-	if err != nil {
-		panic(err)
-	}
-	fmt.Println(string(bs))
-	return
-}

contrib/deb/generate.sh

@@ -83,7 +83,7 @@ then
   echo "Backing up configuration file to /var/backups/yggdrasil.conf.`date +%Y%m%d`"
   cp /etc/yggdrasil.conf /var/backups/yggdrasil.conf.`date +%Y%m%d`
   echo "Normalising and updating /etc/yggdrasil.conf"
-  /usr/bin/yggdrasil -useconffile /var/backups/yggdrasil.conf.`date +%Y%m%d` -normaliseconf > /etc/yggdrasil.conf
+  /usr/bin/yggdrasil -useconf -normaliseconf < /var/backups/yggdrasil.conf.`date +%Y%m%d` > /etc/yggdrasil.conf
   chgrp yggdrasil /etc/yggdrasil.conf
 
   if command -v systemctl >/dev/null; then
@@ -94,7 +94,7 @@ then
 else
   echo "Generating initial configuration file /etc/yggdrasil.conf"
   echo "Please familiarise yourself with this file before starting Yggdrasil"
-  /usr/bin/yggdrasil -genconf > /etc/yggdrasil.conf
+  sh -c 'umask 0027 && /usr/bin/yggdrasil -genconf > /etc/yggdrasil.conf'
   chgrp yggdrasil /etc/yggdrasil.conf
 fi
 EOF
@@ -110,11 +110,12 @@ EOF
 
 cp yggdrasil /tmp/$PKGNAME/usr/bin/
 cp yggdrasilctl /tmp/$PKGNAME/usr/bin/
-cp contrib/systemd/yggdrasil.service /tmp/$PKGNAME/etc/systemd/system/
+cp contrib/systemd/*.service /tmp/$PKGNAME/etc/systemd/system/
 
 tar -czvf /tmp/$PKGNAME/data.tar.gz -C /tmp/$PKGNAME/ \
   usr/bin/yggdrasil usr/bin/yggdrasilctl \
-  etc/systemd/system/yggdrasil.service
+  etc/systemd/system/yggdrasil.service \
+  etc/systemd/system/yggdrasil-default-config.service
 tar -czvf /tmp/$PKGNAME/control.tar.gz -C /tmp/$PKGNAME/debian .
 echo 2.0 > /tmp/$PKGNAME/debian-binary
 

contrib/docker/Dockerfile

@@ -5,13 +5,14 @@ WORKDIR /src
 
 ENV CGO_ENABLED=0
 
-RUN apk add git && ./build 
+RUN apk add git && ./build && go build -o /src/genkeys cmd/genkeys/main.go
 
 FROM docker.io/alpine
 LABEL maintainer="Christer Waren/CWINFO <christer.waren@cwinfo.org>"
 
 COPY --from=builder /src/yggdrasil /usr/bin/yggdrasil
 COPY --from=builder /src/yggdrasilctl /usr/bin/yggdrasilctl
+COPY --from=builder /src/genkeys /usr/bin/genkeys
 COPY contrib/docker/entrypoint.sh /usr/bin/entrypoint.sh
 
 # RUN addgroup -g 1000 -S yggdrasil-network \

contrib/msi/build-msi.sh

@@ -0,0 +1,200 @@
+#!/bin/bash
+
+# This script generates an MSI file for Yggdrasil for a given architecture. It
+# needs to run on Linux or macOS with Go 1.16, wixl and msitools installed.
+#
+# Author: Neil Alexander <neilalexander@users.noreply.github.com>
+
+# Get arch from command line if given
+PKGARCH=$1
+if [ "${PKGARCH}" == "" ];
+then
+  echo "tell me the architecture: x86, x64 or arm"
+  exit 1
+fi
+
+# Get the rest of the repository history. This is needed within Appveyor because
+# otherwise we don't get all of the branch histories and therefore the semver
+# scripts don't work properly.
+if [ "${APPVEYOR_PULL_REQUEST_HEAD_REPO_BRANCH}" != "" ];
+then
+  git fetch --all
+  git checkout ${APPVEYOR_PULL_REQUEST_HEAD_REPO_BRANCH}
+elif [ "${APPVEYOR_REPO_BRANCH}" != "" ];
+then
+  git fetch --all
+  git checkout ${APPVEYOR_REPO_BRANCH}
+fi
+
+# Build Yggdrasil!
+[ "${PKGARCH}" == "x64" ] && GOOS=windows GOARCH=amd64 CGO_ENABLED=0 ./build -p -l "-aslr"
+[ "${PKGARCH}" == "x86" ] && GOOS=windows GOARCH=386 CGO_ENABLED=0 ./build -p -l "-aslr"
+[ "${PKGARCH}" == "arm" ] && GOOS=windows GOARCH=arm CGO_ENABLED=0 ./build -p -l "-aslr"
+#[ "${PKGARCH}" == "arm64" ] && GOOS=windows GOARCH=arm64 CGO_ENABLED=0 ./build
+
+# Create the postinstall script
+cat > updateconfig.bat << EOF
+if not exist %ALLUSERSPROFILE%\\Yggdrasil (
+  mkdir %ALLUSERSPROFILE%\\Yggdrasil
+)
+if not exist %ALLUSERSPROFILE%\\Yggdrasil\\yggdrasil.conf (
+  if exist yggdrasil.exe (
+    if not exist %ALLUSERSPROFILE%\\Yggdrasil\\yggdrasil.conf (
+      yggdrasil.exe -genconf > %ALLUSERSPROFILE%\\Yggdrasil\\yggdrasil.conf
+    )
+  )
+)
+EOF
+
+# Work out metadata for the package info
+PKGNAME=$(sh contrib/semver/name.sh)
+PKGVERSION=$(sh contrib/semver/version.sh --bare)
+PKGVERSIONMS=$(echo $PKGVERSION | tr - .)
+[ "${PKGARCH}" == "x64" ] && \
+  PKGGUID="77757838-1a23-40a5-a720-c3b43e0260cc" PKGINSTFOLDER="ProgramFiles64Folder" || \
+  PKGGUID="54a3294e-a441-4322-aefb-3bb40dd022bb" PKGINSTFOLDER="ProgramFilesFolder"
+
+# Download the Wintun driver
+curl -o wintun.zip https://www.wintun.net/builds/wintun-0.10.2.zip
+unzip wintun.zip
+if [ $PKGARCH = "x64" ]; then
+  PKGWINTUNDLL=wintun/bin/amd64/wintun.dll
+elif [ $PKGARCH = "x86" ]; then
+  PKGWINTUNDLL=wintun/bin/x86/wintun.dll
+elif [ $PKGARCH = "arm" ]; then
+  PKGWINTUNDLL=wintun/bin/arm/wintun.dll
+#elif [ $PKGARCH = "arm64" ]; then
+#  PKGWINTUNDLL=wintun/bin/arm64/wintun.dll
+else
+  echo "wasn't sure which architecture to get wintun for"
+  exit 1
+fi
+
+if [ $PKGNAME != "master" ]; then
+  PKGDISPLAYNAME="Yggdrasil Network (${PKGNAME} branch)"
+else
+  PKGDISPLAYNAME="Yggdrasil Network"
+fi
+
+# Generate the wix.xml file
+cat > wix.xml << EOF
+<?xml version="1.0" encoding="windows-1252"?>
+<Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
+  <Product
+    Name="${PKGDISPLAYNAME}"
+    Id="*"
+    UpgradeCode="${PKGGUID}"
+    Language="1033"
+    Codepage="1252"
+    Version="${PKGVERSIONMS}"
+    Platform="${PKGARCH}"
+    Manufacturer="github.com/yggdrasil-network">
+
+    <Package
+      Id="*"
+      Keywords="Installer"
+      Description="Yggdrasil Network Installer"
+      Comments="Yggdrasil Network standalone router for Windows."
+      Manufacturer="github.com/yggdrasil-network"
+      InstallerVersion="200"
+      InstallScope="perMachine"
+      Languages="1033"
+      Compressed="yes"
+      Platform="${PKGARCH}"
+      SummaryCodepage="1252" />
+
+    <MajorUpgrade
+      AllowDowngrades="yes" />
+
+    <Media
+      Id="1"
+      Cabinet="Media.cab"
+      EmbedCab="yes"
+      CompressionLevel="high" />
+
+    <Directory Id="TARGETDIR" Name="SourceDir">
+      <Directory Id="${PKGINSTFOLDER}" Name="PFiles">
+        <Directory Id="YggdrasilInstallFolder" Name="Yggdrasil">
+
+          <Component Id="MainExecutable" Guid="c2119231-2aa3-4962-867a-9759c87beb24">
+            <File
+              Id="Yggdrasil"
+              Name="yggdrasil.exe"
+              DiskId="1"
+              Source="yggdrasil.exe"
+              KeyPath="yes" />
+
+            <File
+              Id="Wintun"
+              Name="wintun.dll"
+              DiskId="1"
+              Source="${PKGWINTUNDLL}" />
+
+            <ServiceInstall
+              Id="ServiceInstaller"
+              Account="LocalSystem"
+              Description="Yggdrasil Network router process"
+              DisplayName="Yggdrasil Service"
+              ErrorControl="normal"
+              LoadOrderGroup="NetworkProvider"
+              Name="Yggdrasil"
+              Start="auto"
+              Type="ownProcess"
+              Arguments='-useconffile "%ALLUSERSPROFILE%\\Yggdrasil\\yggdrasil.conf" -logto "%ALLUSERSPROFILE%\\Yggdrasil\\yggdrasil.log"'
+              Vital="yes" />
+
+            <ServiceControl
+              Id="ServiceControl"
+              Name="yggdrasil"
+              Start="install"
+              Stop="both"
+              Remove="uninstall" />
+          </Component>
+
+          <Component Id="CtrlExecutable" Guid="a916b730-974d-42a1-b687-d9d504cbb86a">
+            <File
+              Id="Yggdrasilctl"
+              Name="yggdrasilctl.exe"
+              DiskId="1"
+              Source="yggdrasilctl.exe"
+              KeyPath="yes"/>
+          </Component>
+
+          <Component Id="ConfigScript" Guid="64a3733b-c98a-4732-85f3-20cd7da1a785">
+            <File
+              Id="Configbat"
+              Name="updateconfig.bat"
+              DiskId="1"
+              Source="updateconfig.bat"
+              KeyPath="yes"/>
+          </Component>
+        </Directory>
+      </Directory>
+    </Directory>
+
+    <Feature Id="YggdrasilFeature" Title="Yggdrasil" Level="1">
+      <ComponentRef Id="MainExecutable" />
+      <ComponentRef Id="CtrlExecutable" />
+      <ComponentRef Id="ConfigScript" />
+    </Feature>
+
+    <CustomAction
+      Id="UpdateGenerateConfig"
+      Directory="YggdrasilInstallFolder"
+      ExeCommand="cmd.exe /c updateconfig.bat"
+      Execute="deferred"
+      Return="check"
+      Impersonate="yes" />
+
+    <InstallExecuteSequence>
+      <Custom
+        Action="UpdateGenerateConfig"
+        Before="StartServices" />
+    </InstallExecuteSequence>
+
+  </Product>
+</Wix>
+EOF
+
+# Generate the MSI
+wixl -v wix.xml -a ${PKGARCH} -o ${PKGNAME}-${PKGVERSION}-${PKGARCH}.msi

contrib/systemd/yggdrasil-default-config.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=yggdrasil default config generator
+ConditionPathExists=|!/etc/yggdrasil.conf
+ConditionFileNotEmpty=|!/etc/yggdrasil.conf
+Wants=local-fs.target
+After=local-fs.target
+
+[Service]
+Type=oneshot
+Group=yggdrasil
+StandardOutput=file:/etc/yggdrasil.conf
+ExecStart=/usr/bin/yggdrasil -genconf
+ExecStartPost=/usr/bin/chmod 0640 /etc/yggdrasil.conf

contrib/systemd/yggdrasil.service

@@ -1,20 +1,17 @@
 [Unit]
 Description=yggdrasil
 Wants=network.target
+Wants=yggdrasil-default-config.service
 After=network.target
+After=yggdrasil-default-config.service
 
 [Service]
 Group=yggdrasil
 ProtectHome=true
 ProtectSystem=true
 SyslogIdentifier=yggdrasil
-CapabilityBoundingSet=CAP_NET_ADMIN
+CapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW
 ExecStartPre=+-/sbin/modprobe tun
-ExecStartPre=/bin/sh -ec "if ! test -s /etc/yggdrasil.conf; \
-                then umask 077; \
-                yggdrasil -genconf > /etc/yggdrasil.conf; \
-                echo 'WARNING: A new /etc/yggdrasil.conf file has been generated.'; \
-            fi"
 ExecStart=/usr/bin/yggdrasil -useconffile /etc/yggdrasil.conf
 ExecReload=/bin/kill -HUP $MAINPID
 Restart=always

doc/Whitepaper.md

@@ -65,12 +65,12 @@ Given the coordinates of any two nodes, it is possible to calculate the length o
 Traffic is forwarded using a [greedy routing](https://en.wikipedia.org/wiki/Small-world_routing#Greedy_routing) scheme, where each node forwards the packet to a one-hop neighbor that is closer to the destination (according to this distance metric) than the current node.
 In particular, when a packet needs to be forwarded, a node will forward it to whatever peer is closest to the destination in the greedy [metric space](https://en.wikipedia.org/wiki/Metric_space) used by the network, provided that the peer is closer to the destination than the current node.
 
-If no closer peers are idle, then the packet is queued in FIFO order, with separate queues per destination coords (currently, as a bit of a hack, IPv6 flow labels are embedeed after the end of the significant part of the coords, so queues distinguish between different traffic streams with the same destination).
+If no closer peers are idle, then the packet is queued in FIFO order, with separate queues per destination coords (currently, as a bit of a hack, IPv6 flow labels are embedded after the end of the significant part of the coords, so queues distinguish between different traffic streams with the same destination).
 Whenever the node finishes forwarding a packet to a peer, it checks the queues, and will forward the first packet from the queue with the maximum `<age of first packet>/<queue size in bytes>`, i.e. the bandwidth the queue is attempting to use, subject to the constraint that the peer is a valid next hop (i.e. closer to the destination than the current node).
 If no non-empty queue is available, then the peer is added to the idle set, forward packets when the need arises.
 
 This acts as a crude approximation of backpressure routing, where the remote queue sizes are assumed to be equal to the distance of a node from a destination (rather than communicating queue size information), and packets are never forwarded "backwards" through the network, but congestion on a local link is routed around when possible.
-The queue selection strategy behaves similar to shortest-queue-first, in that a larger fration of available bandwith to sessions that attempt to use less bandwidth, and is loosely based on the rationale behind some proposed solutions to the [cake-cutting](https://en.wikipedia.org/wiki/Fair_cake-cutting) problem.
+The queue selection strategy behaves similar to shortest-queue-first, in that a larger fraction of available bandwidth to sessions that attempt to use less bandwidth, and is loosely based on the rationale behind some proposed solutions to the [cake-cutting](https://en.wikipedia.org/wiki/Fair_cake-cutting) problem.
 
 The queue size is limited to 4 MB. If a packet is added to a queue and the total size of all queues is larger than this threshold, then a random queue is selected (with odds proportional to relative queue sizes), and the first packet from that queue is dropped, with the process repeated until the total queue size drops below the allowed threshold.
 

go.mod

@@ -1,21 +1,24 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
-go 1.13
+go 1.16
 
 require (
-	github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0
-	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
+	github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979
+	github.com/cheggaaa/pb/v3 v3.0.6
+	github.com/fatih/color v1.10.0 // indirect
+	github.com/gologme/log v1.2.0
 	github.com/hashicorp/go-syslog v1.0.0
-	github.com/hjson/hjson-go v3.0.1-0.20190209023717-9147687966d9+incompatible
-	github.com/kardianos/minwinsvc v0.0.0-20151122163309-cad6b2b879b0
-	github.com/mitchellh/mapstructure v1.1.2
-	github.com/songgao/packets v0.0.0-20160404182456-549a10cd4091
-	github.com/songgao/water v0.0.0-20190725173103-fd331bda3f4b // indirect
-	github.com/vishvananda/netlink v1.0.0
-	github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f // indirect
-	github.com/yggdrasil-network/water v0.0.0-20190812103929-c83fe40250f8
-	golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550
-	golang.org/x/net v0.0.0-20191021144547-ec77196f6094
-	golang.org/x/sys v0.0.0-20191024172528-b4ff53e7a1cb
-	golang.org/x/text v0.3.2
+	github.com/hjson/hjson-go v3.1.0+incompatible
+	github.com/kardianos/minwinsvc v1.0.0
+	github.com/mattn/go-runewidth v0.0.10 // indirect
+	github.com/mitchellh/mapstructure v1.4.1
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/vishvananda/netlink v1.1.0
+	github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
+	golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
+	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110
+	golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b
+	golang.org/x/text v0.3.6-0.20210220033129-8f690f22cf1c
+	golang.zx2c4.com/wireguard v0.0.0-20210306175010-7e3b8371a1bf
+	golang.zx2c4.com/wireguard/windows v0.3.8
 )

go.sum

@@ -1,36 +1,72 @@
-github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0 h1:p3puK8Sl2xK+2FnnIvY/C0N1aqJo2kbEsdAzU+Tnv48=
-github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
-github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
-github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
+github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 h1:WndgpSW13S32VLQ3ugUxx2EnnWmgba1kCqPkd4Gk1yQ=
+github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
+github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
+github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
+github.com/cheggaaa/pb/v3 v3.0.6 h1:ULPm1wpzvj60FvmCrX7bIaB80UgbhI+zSaQJKRfCbAs=
+github.com/cheggaaa/pb/v3 v3.0.6/go.mod h1:X1L61/+36nz9bjIsrDU52qHKOQukUQe2Ge+YvGuquCw=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
+github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
+github.com/gologme/log v1.2.0 h1:Ya5Ip/KD6FX7uH0S31QO87nCCSucKtF44TLbTtO7V4c=
+github.com/gologme/log v1.2.0/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
-github.com/hjson/hjson-go v3.0.1-0.20190209023717-9147687966d9+incompatible h1:bLQ2Ve+eW65id3b8xEMQiAwJT4qGZeywAEMLvXjznvw=
-github.com/hjson/hjson-go v3.0.1-0.20190209023717-9147687966d9+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
-github.com/kardianos/minwinsvc v0.0.0-20151122163309-cad6b2b879b0 h1:YnZmFjg0Nvk8851WTVWlqMC1ecJH07Ctz+Ezxx4u54g=
-github.com/kardianos/minwinsvc v0.0.0-20151122163309-cad6b2b879b0/go.mod h1:rUi0/YffDo1oXBOGn1KRq7Fr07LX48XEBecQnmwjsAo=
-github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
-github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/songgao/packets v0.0.0-20160404182456-549a10cd4091 h1:1zN6ImoqhSJhN8hGXFaJlSC8msLmIbX8bFqOfWLKw0w=
-github.com/songgao/packets v0.0.0-20160404182456-549a10cd4091/go.mod h1:N20Z5Y8oye9a7HmytmZ+tr8Q2vlP0tAHP13kTHzwvQY=
-github.com/songgao/water v0.0.0-20190725173103-fd331bda3f4b h1:+y4hCMc/WKsDbAPsOQZgBSaSZ26uh2afyaWeVg/3s/c=
-github.com/songgao/water v0.0.0-20190725173103-fd331bda3f4b/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E=
-github.com/vishvananda/netlink v1.0.0 h1:bqNY2lgheFIu1meHUFSH3d7vG93AFyqg3oGbJCOJgSM=
-github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
-github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f h1:nBX3nTcmxEtHSERBJaIo1Qa26VwRaopnZmfDQUXsF4I=
-github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
-github.com/yggdrasil-network/water v0.0.0-20190812103929-c83fe40250f8 h1:YY9Pg2BEp0jeUVU60svTOaDr+fs1ySC9RbdC1Qc6wOw=
-github.com/yggdrasil-network/water v0.0.0-20190812103929-c83fe40250f8/go.mod h1:R0SBCsugm+Sf1katgTb2t7GXMm+nRIv43tM4VDZbaOs=
+github.com/hjson/hjson-go v3.1.0+incompatible h1:DY/9yE8ey8Zv22bY+mHV1uk2yRy0h8tKhZ77hEdi0Aw=
+github.com/hjson/hjson-go v3.1.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
+github.com/kardianos/minwinsvc v1.0.0 h1:+JfAi8IBJna0jY2dJGZqi7o15z13JelFIklJCAENALA=
+github.com/kardianos/minwinsvc v1.0.0/go.mod h1:Bgd0oc+D0Qo3bBytmNtyRKVlp85dAloLKhfxanPFFRc=
+github.com/lxn/walk v0.0.0-20210112085537-c389da54e794/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ=
+github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg=
+github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
+github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
+github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
+github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
+github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20191021144547-ec77196f6094 h1:5O4U9trLjNpuhpynaDsqwCk+Tw6seqJz1EbqbnzHrc8=
-golang.org/x/net v0.0.0-20191021144547-ec77196f6094/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191024172528-b4ff53e7a1cb h1:ZxSglHghKPYD8WDeRUzRJrUJtDF0PxsTUSxyqr9/5BI=
-golang.org/x/sys v0.0.0-20191024172528-b4ff53e7a1cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210225014209-683adc9d29d7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210305215415-5cdee2b1b5a0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b h1:ggRgirZABFolTmi3sn6Ivd9SipZwLedQ5wR0aAKnFxU=
+golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6-0.20210220033129-8f690f22cf1c h1:SW/oilbeWd6f32u3ZvuYGqZ+wivcp//I3Dy/gByk7Wk=
+golang.org/x/text v0.3.6-0.20210220033129-8f690f22cf1c/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.zx2c4.com/wireguard v0.0.0-20210225140808-70b7b7158fc9/go.mod h1:39ZQQ95hUxDxT7opsWy/rtfgvXXc8s30qfZ02df69Fo=
+golang.zx2c4.com/wireguard v0.0.0-20210306175010-7e3b8371a1bf h1:AtdIMfzvVNPXN4kVY/yWS8mvpQogSwtCRJk2y/LBPpg=
+golang.zx2c4.com/wireguard v0.0.0-20210306175010-7e3b8371a1bf/go.mod h1:ojGPy+9W6ZSM8anL+xC67fvh8zPQJwA6KpFOHyDWLX4=
+golang.zx2c4.com/wireguard/windows v0.3.8 h1:FvfBEhdZZTwthLuPHdyP6zpivYL3enopxd4XpggAufM=
+golang.zx2c4.com/wireguard/windows v0.3.8/go.mod h1:lm7dxHcBuzMNq706Ge1tZKZKw4+19vG9dLOhoDX05HQ=

misc/sim/treesim-forward.py

@@ -793,7 +793,7 @@ def timelineDimesTest():
     store = makeStoreDimesEdges(path, bestRoot)
     rootID = "R" + bestRoot[1:]
     assert rootID in store
-    # Don't forget to set random seed before setitng times
+    # Don't forget to set random seed before setting times
     # To make results reproducible
     nodeIDs = sorted(store.keys())
     random.seed(12345)

misc/sim/treesim.go

@@ -1,20 +1,24 @@
+// +build !lint
+
 package main
 
-import "fmt"
-import "bufio"
-import "os"
-import "strings"
-import "strconv"
-import "time"
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"os"
+	"runtime"
+	"runtime/pprof"
+	"strconv"
+	"strings"
+	"time"
 
-import "runtime"
-import "runtime/pprof"
-import "flag"
+	"github.com/gologme/log"
 
-import "github.com/gologme/log"
+	. "github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil"
 
-import . "github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil"
-import . "github.com/yggdrasil-network/yggdrasil-go/src/crypto"
+	. "github.com/yggdrasil-network/yggdrasil-go/src/crypto"
+)
 
 ////////////////////////////////////////////////////////////////////////////////
 
@@ -424,7 +428,7 @@ func main() {
 	//*/
 	startNetwork(kstore)
 	//time.Sleep(10*time.Second)
-	// Note that testPaths only works if pressure is turend off
+	// Note that testPaths only works if pressure is turned off
 	//  Otherwise congestion can lead to routing loops?
 	for finished := false; !finished; {
 		finished = testPaths(kstore)

src/address/address.go

@@ -15,9 +15,9 @@ type Address [16]byte
 type Subnet [8]byte
 
 // GetPrefix returns the address prefix used by yggdrasil.
-// The current implementation requires this to be a muliple of 8 bits + 7 bits.
+// The current implementation requires this to be a multiple of 8 bits + 7 bits.
 // The 8th bit of the last byte is used to signal nodes (0) or /64 prefixes (1).
-// Nodes that configure this differently will be unable to communicate with eachother using IP packets, though routing and the DHT machinery *should* still work.
+// Nodes that configure this differently will be unable to communicate with each other using IP packets, though routing and the DHT machinery *should* still work.
 func GetPrefix() [1]byte {
 	return [...]byte{0x02}
 }

src/admin/admin.go

@@ -53,7 +53,7 @@ func (a *AdminSocket) AddHandler(name string, args []string, handlerfunc func(In
 	return nil
 }
 
-// init runs the initial admin setup.
+// Init runs the initial admin setup.
 func (a *AdminSocket) Init(c *yggdrasil.Core, state *config.NodeState, log *log.Logger, options interface{}) error {
 	a.core = c
 	a.log = log
@@ -181,32 +181,57 @@ func (a *AdminSocket) SetupAdminHandlers(na *AdminSocket) {
 					in["uri"].(string),
 				},
 			}, nil
-		} else {
-			return Info{
-				"not_added": []string{
-					in["uri"].(string),
-				},
-			}, errors.New("Failed to add peer")
 		}
+		return Info{
+			"not_added": []string{
+				in["uri"].(string),
+			},
+		}, errors.New("Failed to add peer")
 	})
-	a.AddHandler("removePeer", []string{"port"}, func(in Info) (Info, error) {
+	a.AddHandler("disconnectPeer", []string{"port"}, func(in Info) (Info, error) {
 		port, err := strconv.ParseInt(fmt.Sprint(in["port"]), 10, 64)
 		if err != nil {
 			return Info{}, err
 		}
 		if a.core.DisconnectPeer(uint64(port)) == nil {
 			return Info{
-				"removed": []string{
+				"disconnected": []string{
 					fmt.Sprint(port),
 				},
 			}, nil
 		} else {
 			return Info{
-				"not_removed": []string{
+				"not_disconnected": []string{
 					fmt.Sprint(port),
 				},
+			}, errors.New("Failed to disconnect peer")
+		}
+	})
+	a.AddHandler("removePeer", []string{"uri", "[interface]"}, func(in Info) (Info, error) {
+		// Set sane defaults
+		intf := ""
+		// Has interface been specified?
+		if itf, ok := in["interface"]; ok {
+			intf = itf.(string)
+		}
+		if a.core.RemovePeer(in["uri"].(string), intf) == nil {
+			return Info{
+				"removed": []string{
+					in["uri"].(string),
+				},
+			}, nil
+		} else {
+			return Info{
+				"not_removed": []string{
+					in["uri"].(string),
+				},
 			}, errors.New("Failed to remove peer")
 		}
+		return Info{
+			"not_removed": []string{
+				in["uri"].(string),
+			},
+		}, errors.New("Failed to remove peer")
 	})
 	a.AddHandler("getAllowedEncryptionPublicKeys", []string{}, func(in Info) (Info, error) {
 		return Info{"allowed_box_pubs": a.core.GetAllowedEncryptionPublicKeys()}, nil
@@ -218,13 +243,12 @@ func (a *AdminSocket) SetupAdminHandlers(na *AdminSocket) {
 					in["box_pub_key"].(string),
 				},
 			}, nil
-		} else {
-			return Info{
-				"not_added": []string{
-					in["box_pub_key"].(string),
-				},
-			}, errors.New("Failed to add allowed key")
 		}
+		return Info{
+			"not_added": []string{
+				in["box_pub_key"].(string),
+			},
+		}, errors.New("Failed to add allowed key")
 	})
 	a.AddHandler("removeAllowedEncryptionPublicKey", []string{"box_pub_key"}, func(in Info) (Info, error) {
 		if a.core.RemoveAllowedEncryptionPublicKey(in["box_pub_key"].(string)) == nil {
@@ -233,13 +257,12 @@ func (a *AdminSocket) SetupAdminHandlers(na *AdminSocket) {
 					in["box_pub_key"].(string),
 				},
 			}, nil
-		} else {
-			return Info{
-				"not_removed": []string{
-					in["box_pub_key"].(string),
-				},
-			}, errors.New("Failed to remove allowed key")
 		}
+		return Info{
+			"not_removed": []string{
+				in["box_pub_key"].(string),
+			},
+		}, errors.New("Failed to remove allowed key")
 	})
 	a.AddHandler("dhtPing", []string{"box_pub_key", "coords", "[target]"}, func(in Info) (Info, error) {
 		var reserr error
@@ -250,10 +273,10 @@ func (a *AdminSocket) SetupAdminHandlers(na *AdminSocket) {
 		coords := util.DecodeCoordString(in["coords"].(string))
 		var boxPubKey crypto.BoxPubKey
 		if b, err := hex.DecodeString(in["box_pub_key"].(string)); err == nil {
-			copy(boxPubKey[:], b[:])
+			copy(boxPubKey[:], b)
 			if n, err := hex.DecodeString(in["target"].(string)); err == nil {
 				var targetNodeID crypto.NodeID
-				copy(targetNodeID[:], n[:])
+				copy(targetNodeID[:], n)
 				result, reserr = a.core.DHTPing(boxPubKey, coords, &targetNodeID)
 			} else {
 				result, reserr = a.core.DHTPing(boxPubKey, coords, nil)
@@ -287,14 +310,13 @@ func (a *AdminSocket) SetupAdminHandlers(na *AdminSocket) {
 			var jsoninfo interface{}
 			if err := json.Unmarshal(nodeinfo, &jsoninfo); err != nil {
 				return Info{}, err
-			} else {
-				return Info{"nodeinfo": jsoninfo}, nil
 			}
+			return Info{"nodeinfo": jsoninfo}, nil
 		} else if in["box_pub_key"] == nil || in["coords"] == nil {
 			return Info{}, errors.New("Expecting both box_pub_key and coords")
 		} else {
 			if b, err := hex.DecodeString(in["box_pub_key"].(string)); err == nil {
-				copy(boxPubKey[:], b[:])
+				copy(boxPubKey[:], b)
 			} else {
 				return Info{}, err
 			}
@@ -305,12 +327,10 @@ func (a *AdminSocket) SetupAdminHandlers(na *AdminSocket) {
 			var m map[string]interface{}
 			if err = json.Unmarshal(result, &m); err == nil {
 				return Info{"nodeinfo": m}, nil
-			} else {
-				return Info{}, err
 			}
-		} else {
 			return Info{}, err
 		}
+		return Info{}, err
 	})
 }
 
@@ -333,9 +353,8 @@ func (a *AdminSocket) Stop() error {
 	if a.listener != nil {
 		a.started = false
 		return a.listener.Close()
-	} else {
-		return nil
 	}
+	return nil
 }
 
 // listen is run by start and manages API connections.

src/config/config.go

@@ -5,7 +5,7 @@ Yggdrasil node.
 The configuration contains, amongst other things, encryption keys which are used
 to derive a node's identity, information about peerings and node information
 that is shared with the network. There are also some module-specific options
-related to TUN/TAP, multicast and the admin socket.
+related to TUN, multicast and the admin socket.
 
 In order for a node to maintain the same identity across restarts, you should
 persist the configuration onto the filesystem or into some configuration storage
@@ -22,8 +22,11 @@ import (
 
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/defaults"
+	"github.com/yggdrasil-network/yggdrasil-go/src/types"
 )
 
+type MTU = types.MTU
+
 // NodeState represents the active and previous configuration of an Yggdrasil
 // node. A NodeState object is returned when starting an Yggdrasil node. Note
 // that this structure and related functions are likely to disappear soon.
@@ -70,9 +73,8 @@ type NodeConfig struct {
 	SigningPublicKey            string                 `comment:"Your public signing key. You should not ordinarily need to share\nthis with anyone."`
 	SigningPrivateKey           string                 `comment:"Your private signing key. DO NOT share this with anyone!"`
 	LinkLocalTCPPort            uint16                 `comment:"The port number to be used for the link-local TCP listeners for the\nconfigured MulticastInterfaces. This option does not affect listeners\nspecified in the Listen option. Unless you plan to firewall link-local\ntraffic, it is best to leave this as the default value of 0. This\noption cannot currently be changed by reloading config during runtime."`
-	IfName                      string                 `comment:"Local network interface name for TUN/TAP adapter, or \"auto\" to select\nan interface automatically, or \"none\" to run without TUN/TAP."`
-	IfTAPMode                   bool                   `comment:"Set local network interface to TAP mode rather than TUN mode if\nsupported by your platform - option will be ignored if not."`
-	IfMTU                       int                    `comment:"Maximux Transmission Unit (MTU) size for your local TUN/TAP interface.\nDefault is the largest supported size for your platform. The lowest\npossible value is 1280."`
+	IfName                      string                 `comment:"Local network interface name for TUN adapter, or \"auto\" to select\nan interface automatically, or \"none\" to run without TUN."`
+	IfMTU                       MTU                    `comment:"Maximum Transmission Unit (MTU) size for your local TUN interface.\nDefault is the largest supported size for your platform. The lowest\npossible value is 1280."`
 	SessionFirewall             SessionFirewall        `comment:"The session firewall controls who can send/receive network traffic\nto/from. This is useful if you want to protect this node without\nresorting to using a real firewall. This does not affect traffic\nbeing routed via this node to somewhere else. Rules are prioritised as\nfollows: blacklist, whitelist, always allow outgoing, direct, remote."`
 	TunnelRouting               TunnelRouting          `comment:"Allow tunneling non-Yggdrasil traffic over Yggdrasil. This effectively\nallows you to use Yggdrasil to route to, or to bridge other networks,\nsimilar to a VPN tunnel. Tunnelling works between any two nodes and\ndoes not require them to be directly peered."`
 	SwitchOptions               SwitchOptions          `comment:"Advanced options for tuning the switch. Normally you will not need\nto edit these options."`
@@ -127,7 +129,6 @@ func GenerateConfig() *NodeConfig {
 	cfg.MulticastInterfaces = defaults.GetDefaults().DefaultMulticastInterfaces
 	cfg.IfName = defaults.GetDefaults().DefaultIfName
 	cfg.IfMTU = defaults.GetDefaults().DefaultIfMTU
-	cfg.IfTAPMode = defaults.GetDefaults().DefaultIfTAPMode
 	cfg.SessionFirewall.Enable = false
 	cfg.SessionFirewall.AllowFromDirect = true
 	cfg.SessionFirewall.AllowFromRemote = true

src/crypto/crypto.go

@@ -17,12 +17,11 @@ import (
 	"crypto/rand"
 	"crypto/sha512"
 	"encoding/hex"
+	"sync"
 
 	"golang.org/x/crypto/curve25519"
 	"golang.org/x/crypto/ed25519"
 	"golang.org/x/crypto/nacl/box"
-
-	"github.com/yggdrasil-network/yggdrasil-go/src/util"
 )
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -194,6 +193,16 @@ type BoxSharedKey [BoxSharedKeyLen]byte
 // BoxNonce is the nonce used in NaCl-like crypto "box" operations (curve25519+xsalsa20+poly1305), and must not be reused for different messages encrypted using the same BoxSharedKey.
 type BoxNonce [BoxNonceLen]byte
 
+// String returns a string representation of the "box" key.
+func (k BoxPubKey) String() string {
+	return hex.EncodeToString(k[:])
+}
+
+// Network returns "curve25519" for "box" keys.
+func (n BoxPubKey) Network() string {
+	return "curve25519"
+}
+
 // NewBoxKeys generates a new pair of public/private crypto box keys.
 func NewBoxKeys() (*BoxPubKey, *BoxPrivKey) {
 	pubBytes, privBytes, err := box.GenerateKey(rand.Reader)
@@ -215,29 +224,36 @@ func GetSharedKey(myPrivKey *BoxPrivKey,
 	return (*BoxSharedKey)(&shared)
 }
 
-// BoxOpen returns a message and true if it successfull opens a crypto box using the provided shared key and nonce.
+// pool is used internally by BoxOpen and BoxSeal to avoid allocating temporary space
+var pool = sync.Pool{New: func() interface{} { return []byte(nil) }}
+
+// BoxOpen returns a message and true if it successfully opens a crypto box using the provided shared key and nonce.
+// The boxed input slice's backing array is reused for the unboxed output when possible.
 func BoxOpen(shared *BoxSharedKey,
 	boxed []byte,
 	nonce *BoxNonce) ([]byte, bool) {
-	out := util.GetBytes()
 	s := (*[BoxSharedKeyLen]byte)(shared)
 	n := (*[BoxNonceLen]byte)(nonce)
-	unboxed, success := box.OpenAfterPrecomputation(out, boxed, n, s)
+	temp := append(pool.Get().([]byte), boxed...)
+	unboxed, success := box.OpenAfterPrecomputation(boxed[:0], temp, n, s)
+	pool.Put(temp[:0])
 	return unboxed, success
 }
 
 // BoxSeal seals a crypto box using the provided shared key, returning the box and the nonce needed to decrypt it.
 // If nonce is nil, a random BoxNonce will be used and returned.
 // If nonce is non-nil, then nonce.Increment() will be called before using it, and the incremented BoxNonce is what is returned.
+// The unboxed input slice's backing array is reused for the boxed output when possible.
 func BoxSeal(shared *BoxSharedKey, unboxed []byte, nonce *BoxNonce) ([]byte, *BoxNonce) {
 	if nonce == nil {
 		nonce = NewBoxNonce()
 	}
 	nonce.Increment()
-	out := util.GetBytes()
 	s := (*[BoxSharedKeyLen]byte)(shared)
 	n := (*[BoxNonceLen]byte)(nonce)
-	boxed := box.SealAfterPrecomputation(out, unboxed, n, s)
+	temp := append(pool.Get().([]byte), unboxed...)
+	boxed := box.SealAfterPrecomputation(unboxed[:0], temp, n, s)
+	pool.Put(temp[:0])
 	return boxed, nonce
 }
 
@@ -262,7 +278,7 @@ func (n *BoxNonce) Increment() {
 	n[len(n)-1] += 2
 	for i := len(n) - 2; i >= 0; i-- {
 		if n[i+1] < oldNonce[i+1] {
-			n[i] += 1
+			n[i]++
 		}
 	}
 }

src/defaults/defaults.go

@@ -1,5 +1,7 @@
 package defaults
 
+import "github.com/yggdrasil-network/yggdrasil-go/src/types"
+
 // Defines which parameters are expected by default for configuration on a
 // specific platform. These values are populated in the relevant defaults_*.go
 // for the platform being targeted. They must be set.
@@ -14,8 +16,7 @@ type platformDefaultParameters struct {
 	DefaultMulticastInterfaces []string
 
 	// TUN/TAP
-	MaximumIfMTU     int
-	DefaultIfMTU     int
-	DefaultIfName    string
-	DefaultIfTAPMode bool
+	MaximumIfMTU  types.MTU
+	DefaultIfMTU  types.MTU
+	DefaultIfName string
 }

src/defaults/defaults_darwin.go

@@ -19,9 +19,8 @@ func GetDefaults() platformDefaultParameters {
 		},
 
 		// TUN/TAP
-		MaximumIfMTU:     65535,
-		DefaultIfMTU:     65535,
-		DefaultIfName:    "auto",
-		DefaultIfTAPMode: false,
+		MaximumIfMTU:  65535,
+		DefaultIfMTU:  65535,
+		DefaultIfName: "auto",
 	}
 }

src/defaults/defaults_freebsd.go

@@ -10,7 +10,7 @@ func GetDefaults() platformDefaultParameters {
 		DefaultAdminListen: "unix:///var/run/yggdrasil.sock",
 
 		// Configuration (used for yggdrasilctl)
-		DefaultConfigFile: "/etc/yggdrasil.conf",
+		DefaultConfigFile: "/usr/local/etc/yggdrasil.conf",
 
 		// Multicast interfaces
 		DefaultMulticastInterfaces: []string{
@@ -18,9 +18,8 @@ func GetDefaults() platformDefaultParameters {
 		},
 
 		// TUN/TAP
-		MaximumIfMTU:     32767,
-		DefaultIfMTU:     32767,
-		DefaultIfName:    "/dev/tap0",
-		DefaultIfTAPMode: true,
+		MaximumIfMTU:  32767,
+		DefaultIfMTU:  32767,
+		DefaultIfName: "/dev/tun0",
 	}
 }

src/defaults/defaults_linux.go

@@ -18,9 +18,8 @@ func GetDefaults() platformDefaultParameters {
 		},
 
 		// TUN/TAP
-		MaximumIfMTU:     65535,
-		DefaultIfMTU:     65535,
-		DefaultIfName:    "auto",
-		DefaultIfTAPMode: false,
+		MaximumIfMTU:  65535,
+		DefaultIfMTU:  65535,
+		DefaultIfName: "auto",
 	}
 }

src/defaults/defaults_netbsd.go

@@ -1,26 +0,0 @@
-// +build netbsd
-
-package defaults
-
-// Sane defaults for the BSD platforms. The "default" options may be
-// may be replaced by the running configuration.
-func GetDefaults() platformDefaultParameters {
-	return platformDefaultParameters{
-		// Admin
-		DefaultAdminListen: "unix:///var/run/yggdrasil.sock",
-
-		// Configuration (used for yggdrasilctl)
-		DefaultConfigFile: "/etc/yggdrasil.conf",
-
-		// Multicast interfaces
-		DefaultMulticastInterfaces: []string{
-			".*",
-		},
-
-		// TUN/TAP
-		MaximumIfMTU:     9000,
-		DefaultIfMTU:     9000,
-		DefaultIfName:    "/dev/tap0",
-		DefaultIfTAPMode: true,
-	}
-}

src/defaults/defaults_openbsd.go

@@ -18,9 +18,8 @@ func GetDefaults() platformDefaultParameters {
 		},
 
 		// TUN/TAP
-		MaximumIfMTU:     16384,
-		DefaultIfMTU:     16384,
-		DefaultIfName:    "/dev/tap0",
-		DefaultIfTAPMode: true,
+		MaximumIfMTU:  16384,
+		DefaultIfMTU:  16384,
+		DefaultIfName: "tun0",
 	}
 }

src/defaults/defaults_other.go

@@ -1,4 +1,4 @@
-// +build !linux,!darwin,!windows,!openbsd,!freebsd,!netbsd
+// +build !linux,!darwin,!windows,!openbsd,!freebsd
 
 package defaults
 
@@ -18,9 +18,8 @@ func GetDefaults() platformDefaultParameters {
 		},
 
 		// TUN/TAP
-		MaximumIfMTU:     65535,
-		DefaultIfMTU:     65535,
-		DefaultIfName:    "none",
-		DefaultIfTAPMode: false,
+		MaximumIfMTU:  65535,
+		DefaultIfMTU:  65535,
+		DefaultIfName: "none",
 	}
 }

src/defaults/defaults_windows.go

@@ -18,9 +18,8 @@ func GetDefaults() platformDefaultParameters {
 		},
 
 		// TUN/TAP
-		MaximumIfMTU:     65535,
-		DefaultIfMTU:     65535,
-		DefaultIfName:    "auto",
-		DefaultIfTAPMode: true,
+		MaximumIfMTU:  65535,
+		DefaultIfMTU:  65535,
+		DefaultIfName: "Yggdrasil",
 	}
 }

src/multicast/multicast.go

@@ -21,16 +21,20 @@ import (
 // automatically.
 type Multicast struct {
 	phony.Inbox
-	core            *yggdrasil.Core
-	config          *config.NodeState
-	log             *log.Logger
-	sock            *ipv6.PacketConn
-	groupAddr       string
-	listeners       map[string]*listenerInfo
-	listenPort      uint16
-	isOpen          bool
-	announcer       *time.Timer
-	platformhandler *time.Timer
+	core        *yggdrasil.Core
+	config      *config.NodeState
+	log         *log.Logger
+	sock        *ipv6.PacketConn
+	groupAddr   string
+	listeners   map[string]*listenerInfo
+	listenPort  uint16
+	isOpen      bool
+	_interfaces map[string]interfaceInfo
+}
+
+type interfaceInfo struct {
+	iface net.Interface
+	addrs []net.Addr
 }
 
 type listenerInfo struct {
@@ -45,6 +49,7 @@ func (m *Multicast) Init(core *yggdrasil.Core, state *config.NodeState, log *log
 	m.config = state
 	m.log = log
 	m.listeners = make(map[string]*listenerInfo)
+	m._interfaces = make(map[string]interfaceInfo)
 	current := m.config.GetCurrent()
 	m.listenPort = current.LinkLocalTCPPort
 	m.groupAddr = "[ff02::114]:9001"
@@ -90,8 +95,8 @@ func (m *Multicast) _start() error {
 
 	m.isOpen = true
 	go m.listen()
-	m.Act(m, m.multicastStarted)
-	m.Act(m, m.announce)
+	m.Act(nil, m._multicastStarted)
+	m.Act(nil, m._announce)
 
 	return nil
 }
@@ -112,18 +117,12 @@ func (m *Multicast) Stop() error {
 		err = m._stop()
 	})
 	m.log.Debugln("Stopped multicast module")
-	return nil
+	return err
 }
 
 func (m *Multicast) _stop() error {
 	m.log.Infoln("Stopping multicast module")
 	m.isOpen = false
-	if m.announcer != nil {
-		m.announcer.Stop()
-	}
-	if m.platformhandler != nil {
-		m.platformhandler.Stop()
-	}
 	if m.sock != nil {
 		m.sock.Close()
 	}
@@ -134,7 +133,7 @@ func (m *Multicast) _stop() error {
 // and then signals the various module goroutines to reconfigure themselves if
 // needed.
 func (m *Multicast) UpdateConfig(config *config.NodeConfig) {
-	m.Act(m, func() { m._updateConfig(config) })
+	m.Act(nil, func() { m._updateConfig(config) })
 }
 
 func (m *Multicast) _updateConfig(config *config.NodeConfig) {
@@ -156,10 +155,35 @@ func (m *Multicast) _updateConfig(config *config.NodeConfig) {
 	m.log.Debugln("Reloaded multicast configuration successfully")
 }
 
-// GetInterfaces returns the currently known/enabled multicast interfaces. It is
-// expected that UpdateInterfaces has been called at least once before calling
-// this method.
+func (m *Multicast) _updateInterfaces() {
+	interfaces := make(map[string]interfaceInfo)
+	intfs := m.getAllowedInterfaces()
+	for _, intf := range intfs {
+		addrs, err := intf.Addrs()
+		if err != nil {
+			m.log.Warnf("Failed up get addresses for interface %s: %s", intf.Name, err)
+			continue
+		}
+		interfaces[intf.Name] = interfaceInfo{
+			iface: intf,
+			addrs: addrs,
+		}
+	}
+	m._interfaces = interfaces
+}
+
 func (m *Multicast) Interfaces() map[string]net.Interface {
+	interfaces := make(map[string]net.Interface)
+	phony.Block(m, func() {
+		for _, info := range m._interfaces {
+			interfaces[info.iface.Name] = info.iface
+		}
+	})
+	return interfaces
+}
+
+// getAllowedInterfaces returns the currently known/enabled multicast interfaces.
+func (m *Multicast) getAllowedInterfaces() map[string]net.Interface {
 	interfaces := make(map[string]net.Interface)
 	// Get interface expressions from config
 	current := m.config.GetCurrent()
@@ -167,7 +191,9 @@ func (m *Multicast) Interfaces() map[string]net.Interface {
 	// Ask the system for network interfaces
 	allifaces, err := net.Interfaces()
 	if err != nil {
-		panic(err)
+		// Don't panic, since this may be from e.g. too many open files (from too much connection spam)
+		// TODO? log something
+		return nil
 	}
 	// Work out which interfaces to announce on
 	for _, iface := range allifaces {
@@ -198,7 +224,11 @@ func (m *Multicast) Interfaces() map[string]net.Interface {
 	return interfaces
 }
 
-func (m *Multicast) announce() {
+func (m *Multicast) _announce() {
+	if !m.isOpen {
+		return
+	}
+	m._updateInterfaces()
 	groupAddr, err := net.ResolveUDPAddr("udp6", m.groupAddr)
 	if err != nil {
 		panic(err)
@@ -207,7 +237,6 @@ func (m *Multicast) announce() {
 	if err != nil {
 		panic(err)
 	}
-	interfaces := m.Interfaces()
 	// There might be interfaces that we configured listeners for but are no
 	// longer up - if that's the case then we should stop the listeners
 	for name, info := range m.listeners {
@@ -219,7 +248,7 @@ func (m *Multicast) announce() {
 		}
 		// If the interface is no longer visible on the system then stop the
 		// listener, as another one will be started further down
-		if _, ok := interfaces[name]; !ok {
+		if _, ok := m._interfaces[name]; !ok {
 			stop()
 			continue
 		}
@@ -232,17 +261,13 @@ func (m *Multicast) announce() {
 			continue
 		}
 		// Find the interface that matches the listener
-		if intf, err := net.InterfaceByName(name); err == nil {
-			if addrs, err := intf.Addrs(); err == nil {
-				// Loop through the addresses attached to that listener and see if any
-				// of them match the current address of the listener
-				for _, addr := range addrs {
-					if ip, _, err := net.ParseCIDR(addr.String()); err == nil {
-						// Does the interface address match our listener address?
-						if ip.Equal(listenaddr.IP) {
-							found = true
-							break
-						}
+		if info, ok := m._interfaces[name]; ok {
+			for _, addr := range info.addrs {
+				if ip, _, err := net.ParseCIDR(addr.String()); err == nil {
+					// Does the interface address match our listener address?
+					if ip.Equal(listenaddr.IP) {
+						found = true
+						break
 					}
 				}
 			}
@@ -256,13 +281,9 @@ func (m *Multicast) announce() {
 	}
 	// Now that we have a list of valid interfaces from the operating system,
 	// we can start checking if we can send multicasts on them
-	for _, iface := range interfaces {
-		// Find interface addresses
-		addrs, err := iface.Addrs()
-		if err != nil {
-			panic(err)
-		}
-		for _, addr := range addrs {
+	for _, info := range m._interfaces {
+		iface := info.iface
+		for _, addr := range info.addrs {
 			addrIP, _, _ := net.ParseCIDR(addr.String())
 			// Ignore IPv4 addresses
 			if addrIP.To4() != nil {
@@ -312,8 +333,8 @@ func (m *Multicast) announce() {
 			break
 		}
 	}
-	m.announcer = time.AfterFunc(time.Second, func() {
-		m.Act(m, m.announce)
+	time.AfterFunc(time.Second, func() {
+		m.Act(nil, m._announce)
 	})
 }
 
@@ -351,7 +372,11 @@ func (m *Multicast) listen() {
 		if addr.IP.String() != from.IP.String() {
 			continue
 		}
-		if _, ok := m.Interfaces()[from.Zone]; ok {
+		var interfaces map[string]interfaceInfo
+		phony.Block(m, func() {
+			interfaces = m._interfaces
+		})
+		if _, ok := interfaces[from.Zone]; ok {
 			addr.Zone = ""
 			if err := m.core.CallPeer("tcp://"+addr.String(), from.Zone); err != nil {
 				m.log.Debugln("Call from multicast failed:", err)

src/multicast/multicast_darwin.go

@@ -31,16 +31,19 @@ import (
 
 var awdlGoroutineStarted bool
 
-func (m *Multicast) multicastStarted() {
+func (m *Multicast) _multicastStarted() {
+	if !m.isOpen {
+		return
+	}
 	C.StopAWDLBrowsing()
-	for intf := range m.Interfaces() {
+	for intf := range m._interfaces {
 		if intf == "awdl0" {
 			C.StartAWDLBrowsing()
 			break
 		}
 	}
-	m.platformhandler = time.AfterFunc(time.Minute, func() {
-		m.Act(m, m.multicastStarted)
+	time.AfterFunc(time.Minute, func() {
+		m.Act(nil, m._multicastStarted)
 	})
 }
 

src/multicast/multicast_other.go

@@ -4,7 +4,7 @@ package multicast
 
 import "syscall"
 
-func (m *Multicast) multicastStarted() {
+func (m *Multicast) _multicastStarted() {
 
 }
 

src/multicast/multicast_unix.go

@@ -5,7 +5,7 @@ package multicast
 import "syscall"
 import "golang.org/x/sys/unix"
 
-func (m *Multicast) multicastStarted() {
+func (m *Multicast) _multicastStarted() {
 
 }
 

src/multicast/multicast_windows.go

@@ -5,7 +5,7 @@ package multicast
 import "syscall"
 import "golang.org/x/sys/windows"
 
-func (m *Multicast) multicastStarted() {
+func (m *Multicast) _multicastStarted() {
 
 }
 

src/tuntap/admin.go

@@ -19,9 +19,8 @@ func (t *TunAdapter) SetupAdminHandlers(a *admin.AdminSocket) {
 		}()
 
 		return admin.Info{
-			t.iface.Name(): admin.Info{
-				"tap_mode": t.iface.IsTAP(),
-				"mtu":      t.mtu,
+			t.Name(): admin.Info{
+				"mtu": t.mtu,
 			},
 		}, nil
 	})
@@ -69,16 +68,14 @@ func (t *TunAdapter) SetupAdminHandlers(a *admin.AdminSocket) {
 	a.AddHandler("addLocalSubnet", []string{"subnet"}, func(in admin.Info) (admin.Info, error) {
 		if err := t.ckr.addLocalSubnet(in["subnet"].(string)); err == nil {
 			return admin.Info{"added": []string{in["subnet"].(string)}}, nil
-		} else {
-			return admin.Info{"not_added": []string{in["subnet"].(string)}}, errors.New("Failed to add source subnet")
 		}
+		return admin.Info{"not_added": []string{in["subnet"].(string)}}, errors.New("Failed to add source subnet")
 	})
 	a.AddHandler("addRemoteSubnet", []string{"subnet", "box_pub_key"}, func(in admin.Info) (admin.Info, error) {
 		if err := t.ckr.addRemoteSubnet(in["subnet"].(string), in["box_pub_key"].(string)); err == nil {
 			return admin.Info{"added": []string{fmt.Sprintf("%s via %s", in["subnet"].(string), in["box_pub_key"].(string))}}, nil
-		} else {
-			return admin.Info{"not_added": []string{fmt.Sprintf("%s via %s", in["subnet"].(string), in["box_pub_key"].(string))}}, errors.New("Failed to add route")
 		}
+		return admin.Info{"not_added": []string{fmt.Sprintf("%s via %s", in["subnet"].(string), in["box_pub_key"].(string))}}, errors.New("Failed to add route")
 	})
 	a.AddHandler("getSourceSubnets", []string{}, func(in admin.Info) (admin.Info, error) {
 		var subnets []string
@@ -105,15 +102,13 @@ func (t *TunAdapter) SetupAdminHandlers(a *admin.AdminSocket) {
 	a.AddHandler("removeLocalSubnet", []string{"subnet"}, func(in admin.Info) (admin.Info, error) {
 		if err := t.ckr.removeLocalSubnet(in["subnet"].(string)); err == nil {
 			return admin.Info{"removed": []string{in["subnet"].(string)}}, nil
-		} else {
-			return admin.Info{"not_removed": []string{in["subnet"].(string)}}, errors.New("Failed to remove source subnet")
 		}
+		return admin.Info{"not_removed": []string{in["subnet"].(string)}}, errors.New("Failed to remove source subnet")
 	})
 	a.AddHandler("removeRemoteSubnet", []string{"subnet", "box_pub_key"}, func(in admin.Info) (admin.Info, error) {
 		if err := t.ckr.removeRemoteSubnet(in["subnet"].(string), in["box_pub_key"].(string)); err == nil {
 			return admin.Info{"removed": []string{fmt.Sprintf("%s via %s", in["subnet"].(string), in["box_pub_key"].(string))}}, nil
-		} else {
-			return admin.Info{"not_removed": []string{fmt.Sprintf("%s via %s", in["subnet"].(string), in["box_pub_key"].(string))}}, errors.New("Failed to remove route")
 		}
+		return admin.Info{"not_removed": []string{fmt.Sprintf("%s via %s", in["subnet"].(string), in["box_pub_key"].(string))}}, errors.New("Failed to remove route")
 	})
 }

src/tuntap/conn.go

@@ -44,13 +44,11 @@ func (s *tunConn) _read(bs []byte) (err error) {
 	select {
 	case <-s.stop:
 		err = errors.New("session was already closed")
-		util.PutBytes(bs)
 		return
 	default:
 	}
 	if len(bs) == 0 {
 		err = errors.New("read packet with 0 size")
-		util.PutBytes(bs)
 		return
 	}
 	ipv4 := len(bs) > 20 && bs[0]&0xf0 == 0x40
@@ -63,7 +61,7 @@ func (s *tunConn) _read(bs []byte) (err error) {
 	default:
 		isCGA = false
 	}
-	// Check destiantion addresses
+	// Check destination addresses
 	switch {
 	case ipv6 && bs[24] == 0x02 && bytes.Equal(s.tun.addr[:16], bs[24:40]): // destination
 	case ipv6 && bs[24] == 0x03 && bytes.Equal(s.tun.subnet[:8], bs[24:32]): // destination
@@ -92,8 +90,7 @@ func (s *tunConn) _read(bs []byte) (err error) {
 			// The destination address isn't in our CKR allowed range
 			skip = true
 		} else if key, err := s.tun.ckr.getPublicKeyForAddress(srcAddr, addrlen); err == nil {
-			srcNodeID := crypto.GetNodeID(&key)
-			if *s.conn.RemoteAddr().(*crypto.NodeID) == *srcNodeID {
+			if *s.conn.RemoteAddr().(*crypto.BoxPubKey) == key {
 				// This is the one allowed CKR case, where source and destination addresses are both good
 			} else {
 				// The CKR key associated with this address doesn't match the sender's NodeID
@@ -108,7 +105,6 @@ func (s *tunConn) _read(bs []byte) (err error) {
 	}
 	if skip {
 		err = errors.New("address not allowed")
-		util.PutBytes(bs)
 		return
 	}
 	s.tun.writer.writeFrom(s, bs)
@@ -126,7 +122,6 @@ func (s *tunConn) _write(bs []byte) (err error) {
 	select {
 	case <-s.stop:
 		err = errors.New("session was already closed")
-		util.PutBytes(bs)
 		return
 	default:
 	}
@@ -169,8 +164,7 @@ func (s *tunConn) _write(bs []byte) (err error) {
 			// The source address isn't in our CKR allowed range
 			skip = true
 		} else if key, err := s.tun.ckr.getPublicKeyForAddress(dstAddr, addrlen); err == nil {
-			dstNodeID := crypto.GetNodeID(&key)
-			if *s.conn.RemoteAddr().(*crypto.NodeID) == *dstNodeID {
+			if *s.conn.RemoteAddr().(*crypto.BoxPubKey) == key {
 				// This is the one allowed CKR case, where source and destination addresses are both good
 			} else {
 				// The CKR key associated with this address doesn't match the sender's NodeID
@@ -185,7 +179,6 @@ func (s *tunConn) _write(bs []byte) (err error) {
 	}
 	if skip {
 		err = errors.New("address not allowed")
-		util.PutBytes(bs)
 		return
 	}
 	msg := yggdrasil.FlowKeyMessage{

src/tuntap/icmpv6.go

@@ -11,32 +11,16 @@ package tuntap
 
 import (
 	"encoding/binary"
-	"errors"
 	"net"
-	"sync"
-	"time"
 
 	"golang.org/x/net/icmp"
 	"golang.org/x/net/ipv6"
-
-	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 )
 
 const len_ETHER = 14
 
 type ICMPv6 struct {
-	tun           *TunAdapter
-	mylladdr      net.IP
-	mymac         net.HardwareAddr
-	peermacs      map[address.Address]neighbor
-	peermacsmutex sync.RWMutex
-}
-
-type neighbor struct {
-	mac               net.HardwareAddr
-	learned           bool
-	lastadvertisement time.Time
-	lastsolicitation  time.Time
+	tun *TunAdapter
 }
 
 // Marshal returns the binary encoding of h.
@@ -61,182 +45,6 @@ func ipv6Header_Marshal(h *ipv6.Header) ([]byte, error) {
 // addresses.
 func (i *ICMPv6) Init(t *TunAdapter) {
 	i.tun = t
-	i.peermacsmutex.Lock()
-	i.peermacs = make(map[address.Address]neighbor)
-	i.peermacsmutex.Unlock()
-
-	// Our MAC address and link-local address
-	i.mymac = net.HardwareAddr{
-		0x02, 0x00, 0x00, 0x00, 0x00, 0x02}
-	i.mylladdr = net.IP{
-		0xFE, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0xFE}
-	copy(i.mymac[:], i.tun.addr[:])
-	copy(i.mylladdr[9:], i.tun.addr[1:])
-}
-
-// Parses an incoming ICMPv6 packet. The packet provided may be either an
-// ethernet frame containing an IP packet, or the IP packet alone. This is
-// determined by whether the TUN/TAP adapter is running in TUN (layer 3) or
-// TAP (layer 2) mode. Returns an error condition which is nil if the ICMPv6
-// module handled the packet or contains the error if not.
-func (i *ICMPv6) ParsePacket(datain []byte) error {
-	var response []byte
-	var err error
-
-	// Parse the frame/packet
-	if i.tun.IsTAP() {
-		response, err = i.UnmarshalPacketL2(datain)
-	} else {
-		response, err = i.UnmarshalPacket(datain, nil)
-	}
-
-	if err != nil {
-		return err
-	}
-
-	// Write the packet to TUN/TAP
-	i.tun.iface.Write(response)
-	return nil
-}
-
-// Unwraps the ethernet headers of an incoming ICMPv6 packet and hands off
-// the IP packet to the ParsePacket function for further processing.
-// A response buffer is also created for the response message, also complete
-// with ethernet headers.
-func (i *ICMPv6) UnmarshalPacketL2(datain []byte) ([]byte, error) {
-	// Ignore non-IPv6 frames
-	if binary.BigEndian.Uint16(datain[12:14]) != uint16(0x86DD) {
-		return nil, errors.New("Ignoring non-IPv6 frame")
-	}
-
-	// Hand over to ParsePacket to interpret the IPv6 packet
-	mac := datain[6:12]
-	ipv6packet, err := i.UnmarshalPacket(datain[len_ETHER:], &mac)
-	if err != nil {
-		return nil, err
-	}
-
-	// Create the response buffer
-	dataout := make([]byte, len_ETHER+ipv6.HeaderLen+32)
-
-	// Populate the response ethernet headers
-	copy(dataout[:6], datain[6:12])
-	copy(dataout[6:12], i.mymac[:])
-	binary.BigEndian.PutUint16(dataout[12:14], uint16(0x86DD))
-
-	// Copy the returned packet to our response ethernet frame
-	copy(dataout[len_ETHER:], ipv6packet)
-	return dataout, nil
-}
-
-// Unwraps the IP headers of an incoming IPv6 packet and performs various
-// sanity checks on the packet - i.e. is the packet an ICMPv6 packet, does the
-// ICMPv6 message match a known expected type. The relevant handler function
-// is then called and a response packet may be returned.
-func (i *ICMPv6) UnmarshalPacket(datain []byte, datamac *[]byte) ([]byte, error) {
-	// Parse the IPv6 packet headers
-	ipv6Header, err := ipv6.ParseHeader(datain[:ipv6.HeaderLen])
-	if err != nil {
-		return nil, err
-	}
-
-	// Check if the packet is IPv6
-	if ipv6Header.Version != ipv6.Version {
-		return nil, errors.New("Ignoring non-IPv6 packet")
-	}
-
-	// Check if the packet is ICMPv6
-	if ipv6Header.NextHeader != 58 {
-		return nil, errors.New("Ignoring non-ICMPv6 packet")
-	}
-
-	// Parse the ICMPv6 message contents
-	icmpv6Header, err := icmp.ParseMessage(58, datain[ipv6.HeaderLen:])
-	if err != nil {
-		return nil, err
-	}
-
-	// Check for a supported message type
-	switch icmpv6Header.Type {
-	case ipv6.ICMPTypeNeighborSolicitation:
-		if !i.tun.IsTAP() {
-			return nil, errors.New("Ignoring Neighbor Solicitation in TUN mode")
-		}
-		response, err := i.HandleNDP(datain[ipv6.HeaderLen:])
-		if err == nil {
-			// Create our ICMPv6 response
-			responsePacket, err := CreateICMPv6(
-				ipv6Header.Src, i.mylladdr,
-				ipv6.ICMPTypeNeighborAdvertisement, 0,
-				&icmp.DefaultMessageBody{Data: response})
-			if err != nil {
-				return nil, err
-			}
-			// Send it back
-			return responsePacket, nil
-		} else {
-			return nil, err
-		}
-	case ipv6.ICMPTypeNeighborAdvertisement:
-		if !i.tun.IsTAP() {
-			return nil, errors.New("Ignoring Neighbor Advertisement in TUN mode")
-		}
-		if datamac != nil {
-			var addr address.Address
-			var target address.Address
-			mac := net.HardwareAddr{0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
-			copy(addr[:], ipv6Header.Src[:])
-			copy(target[:], datain[48:64])
-			copy(mac[:], (*datamac)[:])
-			i.peermacsmutex.Lock()
-			neighbor := i.peermacs[target]
-			neighbor.mac = mac
-			neighbor.learned = true
-			neighbor.lastadvertisement = time.Now()
-			i.peermacs[target] = neighbor
-			i.peermacsmutex.Unlock()
-			i.tun.log.Debugln("Learned peer MAC", mac.String(), "for", net.IP(target[:]).String())
-			/*
-				i.tun.log.Debugln("Peer MAC table:")
-				i.peermacsmutex.RLock()
-				for t, n := range i.peermacs {
-					if n.learned {
-						i.tun.log.Debugln("- Target", net.IP(t[:]).String(), "has MAC", n.mac.String())
-					} else {
-						i.tun.log.Debugln("- Target", net.IP(t[:]).String(), "is not learned yet")
-					}
-				}
-				i.peermacsmutex.RUnlock()
-			*/
-		}
-		return nil, errors.New("No response needed")
-	}
-
-	return nil, errors.New("ICMPv6 type not matched")
-}
-
-// Creates an ICMPv6 packet based on the given icmp.MessageBody and other
-// parameters, complete with ethernet and IP headers, which can be written
-// directly to a TAP adapter.
-func (i *ICMPv6) CreateICMPv6L2(dstmac net.HardwareAddr, dst net.IP, src net.IP, mtype ipv6.ICMPType, mcode int, mbody icmp.MessageBody) ([]byte, error) {
-	// Pass through to CreateICMPv6
-	ipv6packet, err := CreateICMPv6(dst, src, mtype, mcode, mbody)
-	if err != nil {
-		return nil, err
-	}
-
-	// Create the response buffer
-	dataout := make([]byte, len_ETHER+len(ipv6packet))
-
-	// Populate the response ethernet headers
-	copy(dataout[:6], dstmac[:6])
-	copy(dataout[6:12], i.mymac[:])
-	binary.BigEndian.PutUint16(dataout[12:14], uint16(0x86DD))
-
-	// Copy the returned packet to our response ethernet frame
-	copy(dataout[len_ETHER:], ipv6packet)
-	return dataout, nil
 }
 
 // Creates an ICMPv6 packet based on the given icmp.MessageBody and other
@@ -281,106 +89,3 @@ func CreateICMPv6(dst net.IP, src net.IP, mtype ipv6.ICMPType, mcode int, mbody
 	// Send it back
 	return responsePacket, nil
 }
-
-func (i *ICMPv6) Solicit(addr address.Address) {
-	retries := 5
-	for retries > 0 {
-		retries--
-		i.peermacsmutex.RLock()
-		if n, ok := i.peermacs[addr]; ok && n.learned {
-			i.tun.log.Debugln("MAC learned for", net.IP(addr[:]).String())
-			i.peermacsmutex.RUnlock()
-			return
-		}
-		i.peermacsmutex.RUnlock()
-		i.tun.log.Debugln("Sending neighbor solicitation for", net.IP(addr[:]).String())
-		i.peermacsmutex.Lock()
-		if n, ok := i.peermacs[addr]; !ok {
-			i.peermacs[addr] = neighbor{
-				lastsolicitation: time.Now(),
-			}
-		} else {
-			n.lastsolicitation = time.Now()
-		}
-		i.peermacsmutex.Unlock()
-		request, err := i.createNDPL2(addr)
-		if err != nil {
-			panic(err)
-		}
-		if _, err := i.tun.iface.Write(request); err != nil {
-			panic(err)
-		}
-		i.tun.log.Debugln("Sent neighbor solicitation for", net.IP(addr[:]).String())
-		time.Sleep(time.Second)
-	}
-}
-
-func (i *ICMPv6) getNeighbor(addr address.Address) (neighbor, bool) {
-	i.peermacsmutex.RLock()
-	defer i.peermacsmutex.RUnlock()
-
-	n, ok := i.peermacs[addr]
-	return n, ok
-}
-
-func (i *ICMPv6) createNDPL2(dst address.Address) ([]byte, error) {
-	// Create the ND payload
-	var payload [28]byte
-	copy(payload[:4], []byte{0x00, 0x00, 0x00, 0x00}) // Flags
-	copy(payload[4:20], dst[:])                       // Destination
-	copy(payload[20:22], []byte{0x01, 0x01})          // Type & length
-	copy(payload[22:28], i.mymac[:6])                 // Link layer address
-
-	// Create the ICMPv6 solicited-node address
-	var dstaddr address.Address
-	copy(dstaddr[:13], []byte{
-		0xFF, 0x02, 0x00, 0x00,
-		0x00, 0x00, 0x00, 0x00,
-		0x00, 0x00, 0x00, 0x01, 0xFF})
-	copy(dstaddr[13:], dst[13:16])
-
-	// Create the multicast MAC
-	dstmac := net.HardwareAddr{0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
-	copy(dstmac[:2], []byte{0x33, 0x33})
-	copy(dstmac[2:6], dstaddr[12:16])
-
-	// Create the ND request
-	requestPacket, err := i.CreateICMPv6L2(
-		dstmac, dstaddr[:], i.mylladdr,
-		ipv6.ICMPTypeNeighborSolicitation, 0,
-		&icmp.DefaultMessageBody{Data: payload[:]})
-	if err != nil {
-		return nil, err
-	}
-
-	return requestPacket, nil
-}
-
-// Generates a response to an NDP discovery packet. This is effectively called
-// when the host operating system generates an NDP request for any address in
-// the fd00::/8 range, so that the operating system knows to route that traffic
-// to the Yggdrasil TAP adapter.
-func (i *ICMPv6) HandleNDP(in []byte) ([]byte, error) {
-	// Ignore NDP requests for anything outside of fd00::/8
-	var source address.Address
-	copy(source[:], in[8:])
-	var snet address.Subnet
-	copy(snet[:], in[8:])
-	switch {
-	case source.IsValid():
-	case snet.IsValid():
-	default:
-		return nil, errors.New("Not an NDP for 0200::/7")
-	}
-
-	// Create our NDP message body response
-	body := make([]byte, 28)
-	binary.BigEndian.PutUint32(body[:4], uint32(0x40000000)) // Flags
-	copy(body[4:20], in[8:24])                               // Target address
-	body[20] = uint8(2)                                      // Type: Target link-layer address
-	body[21] = uint8(1)                                      // Length: 1x address (8 bytes)
-	copy(body[22:28], i.mymac[:6])
-
-	// Send it back
-	return body, nil
-}

src/tuntap/iface.go

@@ -1,22 +1,22 @@
 package tuntap
 
 import (
-	"bytes"
-	"net"
-	"time"
-
-	"github.com/songgao/packets/ethernet"
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
-	"github.com/yggdrasil-network/yggdrasil-go/src/util"
 	"github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil"
 
+	"golang.org/x/net/icmp"
+	"golang.org/x/net/ipv6"
+
 	"github.com/Arceliar/phony"
 )
 
+const TUN_OFFSET_BYTES = 4
+
 type tunWriter struct {
 	phony.Inbox
 	tun *TunAdapter
+	buf [TUN_OFFSET_BYTES + 65536]byte
 }
 
 func (w *tunWriter) writeFrom(from phony.Actor, b []byte) {
@@ -25,7 +25,8 @@ func (w *tunWriter) writeFrom(from phony.Actor, b []byte) {
 	})
 }
 
-// write is pretty loose with the memory safety rules, e.g. it assumes it can read w.tun.iface.IsTap() safely
+// write is pretty loose with the memory safety rules, e.g. it assumes it can
+// read w.tun.iface.IsTap() safely
 func (w *tunWriter) _write(b []byte) {
 	var written int
 	var err error
@@ -33,89 +34,41 @@ func (w *tunWriter) _write(b []byte) {
 	if n == 0 {
 		return
 	}
-	if w.tun.iface.IsTAP() {
-		sendndp := func(dstAddr address.Address) {
-			neigh, known := w.tun.icmpv6.getNeighbor(dstAddr)
-			known = known && (time.Since(neigh.lastsolicitation).Seconds() < 30)
-			if !known {
-				w.tun.icmpv6.Solicit(dstAddr)
-			}
-		}
-		peermac := net.HardwareAddr{0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
-		var dstAddr address.Address
-		var peerknown bool
-		if b[0]&0xf0 == 0x40 {
-			dstAddr = w.tun.addr
-		} else if b[0]&0xf0 == 0x60 {
-			if !bytes.Equal(w.tun.addr[:16], dstAddr[:16]) && !bytes.Equal(w.tun.subnet[:8], dstAddr[:8]) {
-				dstAddr = w.tun.addr
-			}
-		}
-		if neighbor, ok := w.tun.icmpv6.getNeighbor(dstAddr); ok && neighbor.learned {
-			// If we've learned the MAC of a 300::/7 address, for example, or a CKR
-			// address, use the MAC address of that
-			peermac = neighbor.mac
-			peerknown = true
-		} else if neighbor, ok := w.tun.icmpv6.getNeighbor(w.tun.addr); ok && neighbor.learned {
-			// Otherwise send directly to the MAC address of the host if that's
-			// known instead
-			peermac = neighbor.mac
-			peerknown = true
-		} else {
-			// Nothing has been discovered, try to discover the destination
-			sendndp(w.tun.addr)
-		}
-		if peerknown {
-			var proto ethernet.Ethertype
-			switch {
-			case b[0]&0xf0 == 0x60:
-				proto = ethernet.IPv6
-			case b[0]&0xf0 == 0x40:
-				proto = ethernet.IPv4
-			}
-			var frame ethernet.Frame
-			frame.Prepare(
-				peermac[:6],            // Destination MAC address
-				w.tun.icmpv6.mymac[:6], // Source MAC address
-				ethernet.NotTagged,     // VLAN tagging
-				proto,                  // Ethertype
-				len(b))                 // Payload length
-			copy(frame[tun_ETHER_HEADER_LENGTH:], b[:n])
-			n += tun_ETHER_HEADER_LENGTH
-			written, err = w.tun.iface.Write(frame[:n])
-		} else {
-			w.tun.log.Errorln("TUN/TAP iface write error: no peer MAC known for", net.IP(dstAddr[:]).String(), "- dropping packet")
-		}
-	} else {
-		written, err = w.tun.iface.Write(b[:n])
-		util.PutBytes(b)
-	}
+	temp := append(w.buf[:TUN_OFFSET_BYTES], b...)
+	written, err = w.tun.iface.Write(temp, TUN_OFFSET_BYTES)
 	if err != nil {
 		w.tun.Act(w, func() {
 			if !w.tun.isOpen {
-				w.tun.log.Errorln("TUN/TAP iface write error:", err)
+				w.tun.log.Errorln("TUN iface write error:", err)
 			}
 		})
 	}
-	if written != n {
-		w.tun.log.Errorln("TUN/TAP iface write mismatch:", written, "bytes written vs", n, "bytes given")
+	if written != n+TUN_OFFSET_BYTES {
+		// FIXME some platforms return the wrong number of bytes written, causing error spam
+		//w.tun.log.Errorln("TUN iface write mismatch:", written, "bytes written vs", n+TUN_OFFSET_BYTES, "bytes given")
 	}
 }
 
 type tunReader struct {
 	phony.Inbox
 	tun *TunAdapter
+	buf [TUN_OFFSET_BYTES + 65536]byte
 }
 
 func (r *tunReader) _read() {
 	// Get a slice to store the packet in
-	recvd := util.ResizeBytes(util.GetBytes(), 65535+tun_ETHER_HEADER_LENGTH)
-	// Wait for a packet to be delivered to us through the TUN/TAP adapter
-	n, err := r.tun.iface.Read(recvd)
-	if n <= 0 {
-		util.PutBytes(recvd)
+	// Wait for a packet to be delivered to us through the TUN adapter
+	n, err := r.tun.iface.Read(r.buf[:], TUN_OFFSET_BYTES)
+	if n <= TUN_OFFSET_BYTES || err != nil {
+		r.tun.log.Errorln("Error reading TUN:", err)
+		ferr := r.tun.iface.Flush()
+		if ferr != nil {
+			r.tun.log.Errorln("Unable to flush packets:", ferr)
+		}
 	} else {
-		r.tun.handlePacketFrom(r, recvd[:n], err)
+		bs := make([]byte, n, n+crypto.BoxOverhead) // extra capacity for later...
+		copy(bs, r.buf[TUN_OFFSET_BYTES:n+TUN_OFFSET_BYTES])
+		r.tun.handlePacketFrom(r, bs, err)
 	}
 	if err == nil {
 		// Now read again
@@ -132,43 +85,17 @@ func (tun *TunAdapter) handlePacketFrom(from phony.Actor, packet []byte, err err
 // does the work of reading a packet and sending it to the correct tunConn
 func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
 	if err != nil {
-		tun.log.Errorln("TUN/TAP iface read error:", err)
+		tun.log.Errorln("TUN iface read error:", err)
 		return
 	}
-	// If it's a TAP adapter, update the buffer slice so that we no longer
-	// include the ethernet headers
-	offset := 0
-	if tun.iface.IsTAP() {
-		// Set our offset to beyond the ethernet headers
-		offset = tun_ETHER_HEADER_LENGTH
-		// Check first of all that we can go beyond the ethernet headers
-		if len(recvd) <= offset {
-			return
-		}
-	}
 	// Offset the buffer from now on so that we can ignore ethernet frames if
 	// they are present
-	bs := recvd[offset:]
+	bs := recvd[:]
 	// Check if the packet is long enough to detect if it's an ICMP packet or not
 	if len(bs) < 7 {
-		tun.log.Traceln("TUN/TAP iface read undersized unknown packet, length:", len(bs))
+		tun.log.Traceln("TUN iface read undersized unknown packet, length:", len(bs))
 		return
 	}
-	// If we detect an ICMP packet then hand it to the ICMPv6 module
-	if bs[6] == 58 {
-		// Found an ICMPv6 packet - we need to make sure to give ICMPv6 the full
-		// Ethernet frame rather than just the IPv6 packet as this is needed for
-		// NDP to work correctly
-		if err := tun.icmpv6.ParsePacket(recvd); err == nil {
-			// We acted on the packet in the ICMPv6 module so don't forward or do
-			// anything else with it
-			return
-		}
-	}
-	if offset != 0 {
-		// Shift forward to avoid leaking bytes off the front of the slice when we eventually store it
-		bs = append(recvd[:0], bs...)
-	}
 	// From the IP header, work out what our source and destination addresses
 	// and node IDs are. We will need these in order to work out where to send
 	// the packet
@@ -181,7 +108,7 @@ func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
 	if bs[0]&0xf0 == 0x60 {
 		// Check if we have a fully-sized IPv6 header
 		if len(bs) < 40 {
-			tun.log.Traceln("TUN/TAP iface read undersized ipv6 packet, length:", len(bs))
+			tun.log.Traceln("TUN iface read undersized ipv6 packet, length:", len(bs))
 			return
 		}
 		// Check the packet size
@@ -195,7 +122,7 @@ func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
 	} else if bs[0]&0xf0 == 0x40 {
 		// Check if we have a fully-sized IPv4 header
 		if len(bs) < 20 {
-			tun.log.Traceln("TUN/TAP iface read undersized ipv4 packet, length:", len(bs))
+			tun.log.Traceln("TUN iface read undersized ipv4 packet, length:", len(bs))
 			return
 		}
 		// Check the packet size
@@ -223,6 +150,16 @@ func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
 	}
 	if addrlen != 16 || (!dstAddr.IsValid() && !dstSnet.IsValid()) {
 		// Couldn't find this node's ygg IP
+		dlen := len(bs)
+		if dlen > 900 {
+			dlen = 900
+		}
+		ptb := &icmp.DstUnreach{
+			Data: bs[:dlen],
+		}
+		if packet, err := CreateICMPv6(bs[8:24], bs[24:40], ipv6.ICMPTypeDestinationUnreachable, 0, ptb); err == nil {
+			tun.writer.writeFrom(nil, packet)
+		}
 		return
 	}
 	// Do we have an active connection for this node address?
@@ -250,7 +187,6 @@ func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
 		_, known := tun.dials[dstString]
 		tun.dials[dstString] = append(tun.dials[dstString], bs)
 		for len(tun.dials[dstString]) > 32 {
-			util.PutBytes(tun.dials[dstString][0])
 			tun.dials[dstString] = tun.dials[dstString][1:]
 		}
 		if !known {
@@ -267,14 +203,13 @@ func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
 					if tc, err = tun._wrap(conn.(*yggdrasil.Conn)); err != nil {
 						// Something went wrong when storing the connection, typically that
 						// something already exists for this address or subnet
-						tun.log.Debugln("TUN/TAP iface wrap:", err)
+						tun.log.Debugln("TUN iface wrap:", err)
 						return
 					}
 					for _, packet := range packets {
 						tc.writeFrom(nil, packet)
 					}
 				})
-				return
 			}()
 		}
 	}

src/tuntap/tun.go

@@ -18,22 +18,24 @@ import (
 
 	"github.com/Arceliar/phony"
 	"github.com/gologme/log"
-	"github.com/yggdrasil-network/water"
+	"golang.zx2c4.com/wireguard/tun"
 
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/config"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/defaults"
+	"github.com/yggdrasil-network/yggdrasil-go/src/types"
 	"github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil"
 )
 
-const tun_IPv6_HEADER_LENGTH = 40
-const tun_ETHER_HEADER_LENGTH = 14
+type MTU = types.MTU
 
-// TunAdapter represents a running TUN/TAP interface and extends the
-// yggdrasil.Adapter type. In order to use the TUN/TAP adapter with Yggdrasil,
-// you should pass this object to the yggdrasil.SetRouterAdapter() function
-// before calling yggdrasil.Start().
+const tun_IPv6_HEADER_LENGTH = 40
+
+// TunAdapter represents a running TUN interface and extends the
+// yggdrasil.Adapter type. In order to use the TUN adapter with Yggdrasil, you
+// should pass this object to the yggdrasil.SetRouterAdapter() function before
+// calling yggdrasil.Start().
 type TunAdapter struct {
 	core        *yggdrasil.Core
 	writer      tunWriter
@@ -47,8 +49,8 @@ type TunAdapter struct {
 	subnet      address.Subnet
 	ckr         cryptokey
 	icmpv6      ICMPv6
-	mtu         int
-	iface       *water.Interface
+	mtu         MTU
+	iface       tun.Device
 	phony.Inbox // Currently only used for _handlePacket from the reader, TODO: all the stuff that currently needs a mutex below
 	//mutex        sync.RWMutex // Protects the below
 	addrToConn   map[address.Address]*tunConn
@@ -64,12 +66,12 @@ type TunOptions struct {
 
 // Gets the maximum supported MTU for the platform based on the defaults in
 // defaults.GetDefaults().
-func getSupportedMTU(mtu int, istapmode bool) int {
+func getSupportedMTU(mtu MTU) MTU {
 	if mtu < 1280 {
 		return 1280
 	}
-	if mtu > MaximumMTU(istapmode) {
-		return MaximumMTU(istapmode)
+	if mtu > MaximumMTU() {
+		return MaximumMTU()
 	}
 	return mtu
 }
@@ -77,55 +79,38 @@ func getSupportedMTU(mtu int, istapmode bool) int {
 // Name returns the name of the adapter, e.g. "tun0". On Windows, this may
 // return a canonical adapter name instead.
 func (tun *TunAdapter) Name() string {
-	return tun.iface.Name()
+	if name, err := tun.iface.Name(); err == nil {
+		return name
+	}
+	return ""
 }
 
 // MTU gets the adapter's MTU. This can range between 1280 and 65535, although
 // the maximum value is determined by your platform. The returned value will
 // never exceed that of MaximumMTU().
-func (tun *TunAdapter) MTU() int {
-	return getSupportedMTU(tun.mtu, tun.IsTAP())
+func (tun *TunAdapter) MTU() MTU {
+	return getSupportedMTU(tun.mtu)
 }
 
-// IsTAP returns true if the adapter is a TAP adapter (Layer 2) or false if it
-// is a TUN adapter (Layer 3).
-func (tun *TunAdapter) IsTAP() bool {
-	return tun.iface.IsTAP()
-}
-
-// DefaultName gets the default TUN/TAP interface name for your platform.
+// DefaultName gets the default TUN interface name for your platform.
 func DefaultName() string {
 	return defaults.GetDefaults().DefaultIfName
 }
 
-// DefaultMTU gets the default TUN/TAP interface MTU for your platform. This can
+// DefaultMTU gets the default TUN interface MTU for your platform. This can
 // be as high as MaximumMTU(), depending on platform, but is never lower than 1280.
-func DefaultMTU() int {
-	ehbytes := 0
-	if DefaultIsTAP() {
-		ehbytes = tun_ETHER_HEADER_LENGTH
-	}
-	return defaults.GetDefaults().DefaultIfMTU - ehbytes
+func DefaultMTU() MTU {
+	return defaults.GetDefaults().DefaultIfMTU
 }
 
-// DefaultIsTAP returns true if the default adapter mode for the current
-// platform is TAP (Layer 2) and returns false for TUN (Layer 3).
-func DefaultIsTAP() bool {
-	return defaults.GetDefaults().DefaultIfTAPMode
-}
-
-// MaximumMTU returns the maximum supported TUN/TAP interface MTU for your
+// MaximumMTU returns the maximum supported TUN interface MTU for your
 // platform. This can be as high as 65535, depending on platform, but is never
 // lower than 1280.
-func MaximumMTU(iftapmode bool) int {
-	ehbytes := 0
-	if iftapmode {
-		ehbytes = tun_ETHER_HEADER_LENGTH
-	}
-	return defaults.GetDefaults().MaximumIfMTU - ehbytes
+func MaximumMTU() MTU {
+	return defaults.GetDefaults().MaximumIfMTU
 }
 
-// Init initialises the TUN/TAP module. You must have acquired a Listener from
+// Init initialises the TUN module. You must have acquired a Listener from
 // the Yggdrasil core before this point and it must not be in use elsewhere.
 func (tun *TunAdapter) Init(core *yggdrasil.Core, config *config.NodeState, log *log.Logger, options interface{}) error {
 	tunoptions, ok := options.(TunOptions)
@@ -145,7 +130,7 @@ func (tun *TunAdapter) Init(core *yggdrasil.Core, config *config.NodeState, log
 	return nil
 }
 
-// Start the setup process for the TUN/TAP adapter. If successful, starts the
+// Start the setup process for the TUN adapter. If successful, starts the
 // reader actor to handle packets on that interface.
 func (tun *TunAdapter) Start() error {
 	var err error
@@ -157,11 +142,11 @@ func (tun *TunAdapter) Start() error {
 
 func (tun *TunAdapter) _start() error {
 	if tun.isOpen {
-		return errors.New("TUN/TAP module is already started")
+		return errors.New("TUN module is already started")
 	}
 	current := tun.config.GetCurrent()
 	if tun.config == nil || tun.listener == nil || tun.dialer == nil {
-		return errors.New("no configuration available to TUN/TAP")
+		return errors.New("no configuration available to TUN")
 	}
 	var boxPub crypto.BoxPubKey
 	boxPubHex, err := hex.DecodeString(current.EncryptionPublicKey)
@@ -174,23 +159,19 @@ func (tun *TunAdapter) _start() error {
 	tun.subnet = *address.SubnetForNodeID(nodeID)
 	addr := fmt.Sprintf("%s/%d", net.IP(tun.addr[:]).String(), 8*len(address.GetPrefix())-1)
 	if current.IfName == "none" || current.IfName == "dummy" {
-		tun.log.Debugln("Not starting TUN/TAP as ifname is none or dummy")
+		tun.log.Debugln("Not starting TUN as ifname is none or dummy")
 		return nil
 	}
-	if err := tun.setup(current.IfName, current.IfTAPMode, addr, current.IfMTU); err != nil {
+	if err := tun.setup(current.IfName, addr, current.IfMTU); err != nil {
 		return err
 	}
 	if tun.MTU() != current.IfMTU {
-		tun.log.Warnf("Warning: Interface MTU %d automatically adjusted to %d (supported range is 1280-%d)", current.IfMTU, tun.MTU(), MaximumMTU(tun.IsTAP()))
+		tun.log.Warnf("Warning: Interface MTU %d automatically adjusted to %d (supported range is 1280-%d)", current.IfMTU, tun.MTU(), MaximumMTU())
 	}
-	tun.core.SetMaximumSessionMTU(uint16(tun.MTU()))
+	tun.core.SetMaximumSessionMTU(tun.MTU())
 	tun.isOpen = true
 	go tun.handler()
 	tun.reader.Act(nil, tun.reader._read) // Start the reader
-	tun.icmpv6.Init(tun)
-	if tun.IsTAP() {
-		go tun.icmpv6.Solicit(tun.addr)
-	}
 	tun.ckr.init(tun)
 	return nil
 }
@@ -204,7 +185,7 @@ func (tun *TunAdapter) IsStarted() bool {
 	return isOpen
 }
 
-// Start the setup process for the TUN/TAP adapter. If successful, starts the
+// Start the setup process for the TUN adapter. If successful, starts the
 // read/write goroutines to handle packets on that interface.
 func (tun *TunAdapter) Stop() error {
 	var err error
@@ -216,7 +197,7 @@ func (tun *TunAdapter) Stop() error {
 
 func (tun *TunAdapter) _stop() error {
 	tun.isOpen = false
-	// by TUN/TAP, e.g. readers/writers, sessions
+	// by TUN, e.g. readers/writers, sessions
 	if tun.iface != nil {
 		// Just in case we failed to start up the iface for some reason, this can apparently happen on Windows
 		tun.iface.Close()
@@ -224,16 +205,16 @@ func (tun *TunAdapter) _stop() error {
 	return nil
 }
 
-// UpdateConfig updates the TUN/TAP module with the provided config.NodeConfig
+// UpdateConfig updates the TUN module with the provided config.NodeConfig
 // and then signals the various module goroutines to reconfigure themselves if
 // needed.
 func (tun *TunAdapter) UpdateConfig(config *config.NodeConfig) {
-	tun.log.Debugln("Reloading TUN/TAP configuration...")
+	tun.log.Debugln("Reloading TUN configuration...")
 
 	// Replace the active configuration with the supplied one
 	tun.config.Replace(*config)
 
-	// If the MTU has changed in the TUN/TAP module then this is where we would
+	// If the MTU has changed in the TUN module then this is where we would
 	// tell the router so that updated session pings can be sent. However, we
 	// don't currently update the MTU of the adapter once it has been created so
 	// this doesn't actually happen in the real world yet.
@@ -248,14 +229,14 @@ func (tun *TunAdapter) handler() error {
 		// Accept the incoming connection
 		conn, err := tun.listener.Accept()
 		if err != nil {
-			tun.log.Errorln("TUN/TAP connection accept error:", err)
+			tun.log.Errorln("TUN connection accept error:", err)
 			return err
 		}
 		phony.Block(tun, func() {
 			if _, err := tun._wrap(conn.(*yggdrasil.Conn)); err != nil {
 				// Something went wrong when storing the connection, typically that
 				// something already exists for this address or subnet
-				tun.log.Debugln("TUN/TAP handler wrap:", err)
+				tun.log.Debugln("TUN handler wrap:", err)
 			}
 		})
 	}
@@ -270,7 +251,8 @@ func (tun *TunAdapter) _wrap(conn *yggdrasil.Conn) (c *tunConn, err error) {
 	}
 	c = &s
 	// Get the remote address and subnet of the other side
-	remoteNodeID := conn.RemoteAddr().(*crypto.NodeID)
+	remotePubKey := conn.RemoteAddr().(*crypto.BoxPubKey)
+	remoteNodeID := crypto.GetNodeID(remotePubKey)
 	s.addr = *address.AddrForNodeID(remoteNodeID)
 	s.snet = *address.SubnetForNodeID(remoteNodeID)
 	// Work out if this is already a destination we already know about

src/tuntap/tun_bsd.go

@@ -1,4 +1,4 @@
-// +build openbsd freebsd netbsd
+// +build openbsd freebsd
 
 package tuntap
 
@@ -12,7 +12,7 @@ import (
 
 	"golang.org/x/sys/unix"
 
-	"github.com/yggdrasil-network/water"
+	wgtun "golang.zx2c4.com/wireguard/tun"
 )
 
 const SIOCSIFADDR_IN6 = (0x80000000) | ((288 & 0x1fff) << 16) | uint32(byte('i'))<<8 | 12
@@ -72,34 +72,18 @@ type in6_ifreq_lifetime struct {
 	ifru_addrlifetime in6_addrlifetime
 }
 
-// Sets the IPv6 address of the utun adapter. On all BSD platforms (FreeBSD,
-// OpenBSD, NetBSD) an attempt is made to set the adapter properties by using
-// a system socket and making syscalls to the kernel. This is not refined though
-// and often doesn't work (if at all), therefore if a call fails, it resorts
-// to calling "ifconfig" instead.
-func (tun *TunAdapter) setup(ifname string, iftapmode bool, addr string, mtu int) error {
-	var config water.Config
-	if ifname[:4] == "auto" {
-		ifname = "/dev/tap0"
-	}
-	if len(ifname) < 9 {
-		panic("TUN/TAP name must be in format /dev/tunX or /dev/tapX")
-	}
-	switch {
-	case iftapmode || ifname[:8] == "/dev/tap":
-		config = water.Config{DeviceType: water.TAP}
-	case !iftapmode || ifname[:8] == "/dev/tun":
-		panic("TUN mode is not currently supported on this platform, please use TAP instead")
-	default:
-		panic("TUN/TAP name must be in format /dev/tunX or /dev/tapX")
-	}
-	config.Name = ifname
-	iface, err := water.New(config)
+// Configures the TUN adapter with the correct IPv6 address and MTU.
+func (tun *TunAdapter) setup(ifname string, addr string, mtu MTU) error {
+	iface, err := wgtun.CreateTUN(ifname, int(mtu))
 	if err != nil {
 		panic(err)
 	}
 	tun.iface = iface
-	tun.mtu = getSupportedMTU(mtu, iftapmode)
+	if mtu, err := iface.MTU(); err == nil {
+		tun.mtu = getSupportedMTU(MTU(mtu))
+	} else {
+		tun.mtu = 0
+	}
 	return tun.setupAddress(addr)
 }
 
@@ -114,13 +98,13 @@ func (tun *TunAdapter) setupAddress(addr string) error {
 	}
 
 	// Friendly output
-	tun.log.Infof("Interface name: %s", tun.iface.Name())
+	tun.log.Infof("Interface name: %s", tun.Name())
 	tun.log.Infof("Interface IPv6: %s", addr)
 	tun.log.Infof("Interface MTU: %d", tun.mtu)
 
 	// Create the MTU request
 	var ir in6_ifreq_mtu
-	copy(ir.ifr_name[:], tun.iface.Name())
+	copy(ir.ifr_name[:], tun.Name())
 	ir.ifru_mtu = int(tun.mtu)
 
 	// Set the MTU
@@ -129,7 +113,7 @@ func (tun *TunAdapter) setupAddress(addr string) error {
 		tun.log.Errorf("Error in SIOCSIFMTU: %v", errno)
 
 		// Fall back to ifconfig to set the MTU
-		cmd := exec.Command("ifconfig", tun.iface.Name(), "mtu", string(tun.mtu))
+		cmd := exec.Command("ifconfig", tun.Name(), "mtu", string(tun.mtu))
 		tun.log.Warnf("Using ifconfig as fallback: %v", strings.Join(cmd.Args, " "))
 		output, err := cmd.CombinedOutput()
 		if err != nil {
@@ -141,7 +125,7 @@ func (tun *TunAdapter) setupAddress(addr string) error {
 	// Create the address request
 	// FIXME: I don't work!
 	var ar in6_ifreq_addr
-	copy(ar.ifr_name[:], tun.iface.Name())
+	copy(ar.ifr_name[:], tun.Name())
 	ar.ifru_addr.sin6_len = uint8(unsafe.Sizeof(ar.ifru_addr))
 	ar.ifru_addr.sin6_family = unix.AF_INET6
 	parts := strings.Split(strings.Split(addr, "/")[0], ":")
@@ -158,7 +142,7 @@ func (tun *TunAdapter) setupAddress(addr string) error {
 		tun.log.Errorf("Error in SIOCSIFADDR_IN6: %v", errno)
 
 		// Fall back to ifconfig to set the address
-		cmd := exec.Command("ifconfig", tun.iface.Name(), "inet6", addr)
+		cmd := exec.Command("ifconfig", tun.Name(), "inet6", addr)
 		tun.log.Warnf("Using ifconfig as fallback: %v", strings.Join(cmd.Args, " "))
 		output, err := cmd.CombinedOutput()
 		if err != nil {

src/tuntap/tun_darwin.go

@@ -12,22 +12,24 @@ import (
 
 	"golang.org/x/sys/unix"
 
-	water "github.com/yggdrasil-network/water"
+	wgtun "golang.zx2c4.com/wireguard/tun"
 )
 
 // Configures the "utun" adapter with the correct IPv6 address and MTU.
-func (tun *TunAdapter) setup(ifname string, iftapmode bool, addr string, mtu int) error {
-	if iftapmode {
-		tun.log.Warnln("Warning: TAP mode is not supported on this platform, defaulting to TUN")
-		iftapmode = false
+func (tun *TunAdapter) setup(ifname string, addr string, mtu MTU) error {
+	if ifname == "auto" {
+		ifname = "utun"
 	}
-	config := water.Config{DeviceType: water.TUN}
-	iface, err := water.New(config)
+	iface, err := wgtun.CreateTUN(ifname, int(mtu))
 	if err != nil {
 		panic(err)
 	}
 	tun.iface = iface
-	tun.mtu = getSupportedMTU(mtu, iftapmode)
+	if mtu, err := iface.MTU(); err == nil {
+		tun.mtu = getSupportedMTU(MTU(mtu))
+	} else {
+		tun.mtu = 0
+	}
 	return tun.setupAddress(addr)
 }
 
@@ -80,7 +82,7 @@ func (tun *TunAdapter) setupAddress(addr string) error {
 	}
 
 	var ar in6_aliasreq
-	copy(ar.ifra_name[:], tun.iface.Name())
+	copy(ar.ifra_name[:], tun.Name())
 
 	ar.ifra_prefixmask.sin6_len = uint8(unsafe.Sizeof(ar.ifra_prefixmask))
 	b := make([]byte, 16)
@@ -104,7 +106,7 @@ func (tun *TunAdapter) setupAddress(addr string) error {
 	ar.ifra_lifetime.ia6t_pltime = darwin_ND6_INFINITE_LIFETIME
 
 	var ir ifreq
-	copy(ir.ifr_name[:], tun.iface.Name())
+	copy(ir.ifr_name[:], tun.Name())
 	ir.ifru_mtu = uint32(tun.mtu)
 
 	tun.log.Infof("Interface name: %s", ar.ifra_name)

src/tuntap/tun_linux.go

@@ -6,31 +6,24 @@ package tuntap
 
 import (
 	"github.com/vishvananda/netlink"
-
-	water "github.com/yggdrasil-network/water"
+	wgtun "golang.zx2c4.com/wireguard/tun"
 )
 
-// Configures the TAP adapter with the correct IPv6 address and MTU.
-func (tun *TunAdapter) setup(ifname string, iftapmode bool, addr string, mtu int) error {
-	var config water.Config
-	if iftapmode {
-		config = water.Config{DeviceType: water.TAP}
-	} else {
-		config = water.Config{DeviceType: water.TUN}
+// Configures the TUN adapter with the correct IPv6 address and MTU.
+func (tun *TunAdapter) setup(ifname string, addr string, mtu MTU) error {
+	if ifname == "auto" {
+		ifname = "\000"
 	}
-	if ifname != "" && ifname != "auto" {
-		config.Name = ifname
-	}
-	iface, err := water.New(config)
+	iface, err := wgtun.CreateTUN(ifname, int(mtu))
 	if err != nil {
 		panic(err)
 	}
 	tun.iface = iface
-	tun.mtu = getSupportedMTU(mtu, iftapmode)
-	// Friendly output
-	tun.log.Infof("Interface name: %s", tun.iface.Name())
-	tun.log.Infof("Interface IPv6: %s", addr)
-	tun.log.Infof("Interface MTU: %d", tun.mtu)
+	if mtu, err := iface.MTU(); err == nil {
+		tun.mtu = getSupportedMTU(MTU(mtu))
+	} else {
+		tun.mtu = 0
+	}
 	return tun.setupAddress(addr)
 }
 
@@ -43,18 +36,22 @@ func (tun *TunAdapter) setupAddress(addr string) error {
 	if err != nil {
 		return err
 	}
-	nlintf, err := netlink.LinkByName(tun.iface.Name())
+	nlintf, err := netlink.LinkByName(tun.Name())
 	if err != nil {
 		return err
 	}
 	if err := netlink.AddrAdd(nlintf, nladdr); err != nil {
 		return err
 	}
-	if err := netlink.LinkSetMTU(nlintf, tun.mtu); err != nil {
+	if err := netlink.LinkSetMTU(nlintf, int(tun.mtu)); err != nil {
 		return err
 	}
 	if err := netlink.LinkSetUp(nlintf); err != nil {
 		return err
 	}
+	// Friendly output
+	tun.log.Infof("Interface name: %s", tun.Name())
+	tun.log.Infof("Interface IPv6: %s", addr)
+	tun.log.Infof("Interface MTU: %d", tun.mtu)
 	return nil
 }

src/tuntap/tun_other.go

@@ -1,33 +1,32 @@
-// +build !linux,!darwin,!windows,!openbsd,!freebsd,!netbsd,!mobile
+// +build !linux,!darwin,!windows,!openbsd,!freebsd,!mobile
 
 package tuntap
 
-import water "github.com/yggdrasil-network/water"
-
 // This is to catch unsupported platforms
 // If your platform supports tun devices, you could try configuring it manually
 
-// Creates the TUN/TAP adapter, if supported by the Water library. Note that
-// no guarantees are made at this point on an unsupported platform.
-func (tun *TunAdapter) setup(ifname string, iftapmode bool, addr string, mtu int) error {
-	var config water.Config
-	if iftapmode {
-		config = water.Config{DeviceType: water.TAP}
-	} else {
-		config = water.Config{DeviceType: water.TUN}
-	}
-	iface, err := water.New(config)
+import (
+	wgtun "golang.zx2c4.com/wireguard/tun"
+)
+
+// Configures the TUN adapter with the correct IPv6 address and MTU.
+func (tun *TunAdapter) setup(ifname string, addr string, mtu int) error {
+	iface, err := wgtun.CreateTUN(ifname, mtu)
 	if err != nil {
 		panic(err)
 	}
 	tun.iface = iface
-	tun.mtu = getSupportedMTU(mtu, iftapmode)
+	if mtu, err := iface.MTU(); err == nil {
+		tun.mtu = getSupportedMTU(mtu)
+	} else {
+		tun.mtu = 0
+	}
 	return tun.setupAddress(addr)
 }
 
 // We don't know how to set the IPv6 address on an unknown platform, therefore
 // write about it to stdout and don't try to do anything further.
 func (tun *TunAdapter) setupAddress(addr string) error {
-	tun.log.Warnln("Warning: Platform not supported, you must set the address of", tun.iface.Name(), "to", addr)
+	tun.log.Warnln("Warning: Platform not supported, you must set the address of", tun.Name(), "to", addr)
 	return nil
 }

src/tuntap/tun_windows.go

@@ -1,116 +1,150 @@
+// +build windows
+
 package tuntap
 
 import (
+	"bytes"
 	"errors"
-	"fmt"
-	"os/exec"
-	"strings"
-	"time"
+	"log"
+	"net"
 
-	water "github.com/yggdrasil-network/water"
+	"github.com/yggdrasil-network/yggdrasil-go/src/defaults"
+	"golang.org/x/sys/windows"
+
+	wgtun "golang.zx2c4.com/wireguard/tun"
+	"golang.zx2c4.com/wireguard/windows/elevate"
+	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
 )
 
 // This is to catch Windows platforms
 
-// Configures the TAP adapter with the correct IPv6 address and MTU. On Windows
-// we don't make use of a direct operating system API to do this - we instead
-// delegate the hard work to "netsh".
-func (tun *TunAdapter) setup(ifname string, iftapmode bool, addr string, mtu int) error {
-	if !iftapmode {
-		tun.log.Warnln("Warning: TUN mode is not supported on this platform, defaulting to TAP")
-		iftapmode = true
-	}
-	config := water.Config{DeviceType: water.TAP}
-	config.PlatformSpecificParams.ComponentID = "tap0901"
-	config.PlatformSpecificParams.Network = "169.254.0.1/32"
+// Configures the TUN adapter with the correct IPv6 address and MTU.
+func (tun *TunAdapter) setup(ifname string, addr string, mtu MTU) error {
 	if ifname == "auto" {
-		config.PlatformSpecificParams.InterfaceName = ""
-	} else {
-		config.PlatformSpecificParams.InterfaceName = ifname
+		ifname = defaults.GetDefaults().DefaultIfName
 	}
-	iface, err := water.New(config)
-	if err != nil {
-		return err
-	}
-	if iface.Name() == "" {
-		return errors.New("unable to find TAP adapter with component ID " + config.PlatformSpecificParams.ComponentID)
-	}
-	// Reset the adapter - this invalidates iface so we'll need to get a new one
-	cmd := exec.Command("netsh", "interface", "set", "interface", iface.Name(), "admin=DISABLED")
-	tun.log.Debugln("netsh command:", strings.Join(cmd.Args, " "))
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		tun.log.Errorln("Windows netsh failed:", err)
-		tun.log.Traceln(string(output))
-		return err
-	}
-	time.Sleep(time.Second) // FIXME artifical delay to give netsh time to take effect
-	// Bring the interface back up
-	cmd = exec.Command("netsh", "interface", "set", "interface", iface.Name(), "admin=ENABLED")
-	tun.log.Debugln("netsh command:", strings.Join(cmd.Args, " "))
-	output, err = cmd.CombinedOutput()
-	if err != nil {
-		tun.log.Errorln("Windows netsh failed:", err)
-		tun.log.Traceln(string(output))
-		return err
-	}
-	time.Sleep(time.Second) // FIXME artifical delay to give netsh time to take effect
-	// Get a new iface
-	iface, err = water.New(config)
-	if err != nil {
-		panic(err)
-	}
-	tun.iface = iface
-	tun.mtu = getSupportedMTU(mtu, iftapmode)
-	err = tun.setupMTU(tun.mtu)
-	if err != nil {
-		panic(err)
-	}
-	// Friendly output
-	tun.log.Infof("Interface name: %s", tun.iface.Name())
-	tun.log.Infof("Interface IPv6: %s", addr)
-	tun.log.Infof("Interface MTU: %d", tun.mtu)
-	return tun.setupAddress(addr)
+	return elevate.DoAsSystem(func() error {
+		var err error
+		var iface wgtun.Device
+		var guid windows.GUID
+		if guid, err = windows.GUIDFromString("{8f59971a-7872-4aa6-b2eb-061fc4e9d0a7}"); err != nil {
+			return err
+		}
+		if iface, err = wgtun.CreateTUNWithRequestedGUID(ifname, &guid, int(mtu)); err != nil {
+			return err
+		}
+		tun.iface = iface
+		if err = tun.setupAddress(addr); err != nil {
+			tun.log.Errorln("Failed to set up TUN address:", err)
+			return err
+		}
+		if err = tun.setupMTU(getSupportedMTU(mtu)); err != nil {
+			tun.log.Errorln("Failed to set up TUN MTU:", err)
+			return err
+		}
+		if mtu, err := iface.MTU(); err == nil {
+			tun.mtu = MTU(mtu)
+		}
+		return nil
+	})
 }
 
 // Sets the MTU of the TAP adapter.
-func (tun *TunAdapter) setupMTU(mtu int) error {
-	if tun.iface == nil || tun.iface.Name() == "" {
-		return errors.New("Can't configure MTU as TAP adapter is not present")
+func (tun *TunAdapter) setupMTU(mtu MTU) error {
+	if tun.iface == nil || tun.Name() == "" {
+		return errors.New("Can't configure MTU as TUN adapter is not present")
 	}
-	// Set MTU
-	cmd := exec.Command("netsh", "interface", "ipv6", "set", "subinterface",
-		fmt.Sprintf("interface=%s", tun.iface.Name()),
-		fmt.Sprintf("mtu=%d", mtu),
-		"store=active")
-	tun.log.Debugln("netsh command:", strings.Join(cmd.Args, " "))
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		tun.log.Errorln("Windows netsh failed:", err)
-		tun.log.Traceln(string(output))
-		return err
+	if intf, ok := tun.iface.(*wgtun.NativeTun); ok {
+		luid := winipcfg.LUID(intf.LUID())
+		ipfamily, err := luid.IPInterface(windows.AF_INET6)
+		if err != nil {
+			return err
+		}
+
+		ipfamily.NLMTU = uint32(mtu)
+		intf.ForceMTU(int(ipfamily.NLMTU))
+		ipfamily.UseAutomaticMetric = false
+		ipfamily.Metric = 0
+		ipfamily.DadTransmits = 0
+		ipfamily.RouterDiscoveryBehavior = winipcfg.RouterDiscoveryDisabled
+
+		if err := ipfamily.Set(); err != nil {
+			return err
+		}
 	}
-	time.Sleep(time.Second) // FIXME artifical delay to give netsh time to take effect
+
 	return nil
 }
 
 // Sets the IPv6 address of the TAP adapter.
 func (tun *TunAdapter) setupAddress(addr string) error {
-	if tun.iface == nil || tun.iface.Name() == "" {
-		return errors.New("Can't configure IPv6 address as TAP adapter is not present")
+	if tun.iface == nil || tun.Name() == "" {
+		return errors.New("Can't configure IPv6 address as TUN adapter is not present")
 	}
-	// Set address
-	cmd := exec.Command("netsh", "interface", "ipv6", "add", "address",
-		fmt.Sprintf("interface=%s", tun.iface.Name()),
-		fmt.Sprintf("addr=%s", addr),
-		"store=active")
-	tun.log.Debugln("netsh command:", strings.Join(cmd.Args, " "))
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		tun.log.Errorln("Windows netsh failed:", err)
-		tun.log.Traceln(string(output))
-		return err
+	if intf, ok := tun.iface.(*wgtun.NativeTun); ok {
+		if ipaddr, ipnet, err := net.ParseCIDR(addr); err == nil {
+			luid := winipcfg.LUID(intf.LUID())
+			addresses := append([]net.IPNet{}, net.IPNet{
+				IP:   ipaddr,
+				Mask: ipnet.Mask,
+			})
+
+			err := luid.SetIPAddressesForFamily(windows.AF_INET6, addresses)
+			if err == windows.ERROR_OBJECT_ALREADY_EXISTS {
+				cleanupAddressesOnDisconnectedInterfaces(windows.AF_INET6, addresses)
+				err = luid.SetIPAddressesForFamily(windows.AF_INET6, addresses)
+			}
+			if err != nil {
+				return err
+			}
+		} else {
+			return err
+		}
+	} else {
+		return errors.New("unable to get NativeTUN")
 	}
-	time.Sleep(time.Second) // FIXME artifical delay to give netsh time to take effect
 	return nil
 }
+
+/*
+ * cleanupAddressesOnDisconnectedInterfaces
+ * SPDX-License-Identifier: MIT
+ * Copyright (C) 2019 WireGuard LLC. All Rights Reserved.
+ */
+func cleanupAddressesOnDisconnectedInterfaces(family winipcfg.AddressFamily, addresses []net.IPNet) {
+	if len(addresses) == 0 {
+		return
+	}
+	includedInAddresses := func(a net.IPNet) bool {
+		// TODO: this makes the whole algorithm O(n^2). But we can't stick net.IPNet in a Go hashmap. Bummer!
+		for _, addr := range addresses {
+			ip := addr.IP
+			if ip4 := ip.To4(); ip4 != nil {
+				ip = ip4
+			}
+			mA, _ := addr.Mask.Size()
+			mB, _ := a.Mask.Size()
+			if bytes.Equal(ip, a.IP) && mA == mB {
+				return true
+			}
+		}
+		return false
+	}
+	interfaces, err := winipcfg.GetAdaptersAddresses(family, winipcfg.GAAFlagDefault)
+	if err != nil {
+		return
+	}
+	for _, iface := range interfaces {
+		if iface.OperStatus == winipcfg.IfOperStatusUp {
+			continue
+		}
+		for address := iface.FirstUnicastAddress; address != nil; address = address.Next {
+			ip := address.Address.IP()
+			ipnet := net.IPNet{IP: ip, Mask: net.CIDRMask(int(address.OnLinkPrefixLength), 8*len(ip))}
+			if includedInAddresses(ipnet) {
+				log.Printf("Cleaning up stale address %s from interface ‘%s’", ipnet.String(), iface.FriendlyName())
+				iface.LUID.DeleteIPAddress(ipnet)
+			}
+		}
+	}
+}

src/types/types.go

@@ -0,0 +1,3 @@
+package types
+
+type MTU uint16

src/util/bytes_mobile.go

@@ -1,21 +0,0 @@
-//+build mobile
-
-package util
-
-import "runtime/debug"
-
-func init() {
-	debug.SetGCPercent(25)
-}
-
-// GetBytes always returns a nil slice on mobile platforms.
-func GetBytes() []byte {
-	return nil
-}
-
-// PutBytes does literally nothing on mobile platforms.
-// This is done rather than keeping a free list of bytes on platforms with memory constraints.
-// It's needed to help keep memory usage low enough to fall under the limits set for e.g. iOS NEPacketTunnelProvider apps.
-func PutBytes(bs []byte) {
-	return
-}

src/util/bytes_other.go

@@ -1,18 +0,0 @@
-//+build !mobile
-
-package util
-
-import "sync"
-
-// This is used to buffer recently used slices of bytes, to prevent allocations in the hot loops.
-var byteStore = sync.Pool{New: func() interface{} { return []byte(nil) }}
-
-// GetBytes returns a 0-length (possibly nil) slice of bytes from a free list, so it may have a larger capacity.
-func GetBytes() []byte {
-	return byteStore.Get().([]byte)[:0]
-}
-
-// PutBytes stores a slice in a free list, where it can potentially be reused to prevent future allocations.
-func PutBytes(bs []byte) {
-	byteStore.Put(bs)
-}

src/util/cancellation.go

@@ -54,12 +54,11 @@ func (c *cancellation) Cancel(err error) error {
 	defer c.mutex.Unlock()
 	if c.done {
 		return c.err
-	} else {
-		c.err = err
-		c.done = true
-		close(c.cancel)
-		return nil
 	}
+	c.err = err
+	c.done = true
+	close(c.cancel)
+	return nil
 }
 
 // Error returns the error provided to Cancel, or nil if no error has been provided.

src/util/util.go

@@ -30,9 +30,8 @@ func UnlockThread() {
 func ResizeBytes(bs []byte, length int) []byte {
 	if cap(bs) >= length {
 		return bs[:length]
-	} else {
-		return make([]byte, length)
 	}
+	return make([]byte, length)
 }
 
 // TimerStop stops a timer and makes sure the channel is drained, returns true if the timer was stopped before firing.
@@ -103,11 +102,11 @@ func GetFlowKey(bs []byte) uint64 {
 	// Get the IP protocol version from the packet
 	switch bs[0] & 0xf0 {
 	case 0x40: // IPv4 packet
-		// Check the packet meets minimum UDP packet length
-		if len(bs) >= 24 {
-			// Is the protocol TCP, UDP or SCTP?
-			if bs[9] == 0x06 || bs[9] == 0x11 || bs[9] == 0x84 {
-				ihl := bs[0] & 0x0f * 4 // Header length
+		ihl := (bs[0] & 0x0f) * 4 // whole IPv4 header length (min 20)
+		// 8 is minimum UDP packet length
+		if ihl >= 20 && len(bs)-int(ihl) >= 8 {
+			switch bs[9] /* protocol */ {
+			case 0x06 /* TCP */, 0x11 /* UDP */, 0x84 /* SCTP */ :
 				flowkey = uint64(bs[9])<<32 /* proto */ |
 					uint64(bs[ihl+0])<<24 | uint64(bs[ihl+1])<<16 /* sport */ |
 					uint64(bs[ihl+2])<<8 | uint64(bs[ihl+3]) /* dport */
@@ -119,8 +118,8 @@ func GetFlowKey(bs []byte) uint64 {
 		// If the flowlabel isn't present, make protokey from proto | sport | dport
 		// if the packet meets minimum UDP packet length
 		if flowkey == 0 && len(bs) >= 48 {
-			// Is the protocol TCP, UDP or SCTP?
-			if bs[6] == 0x06 || bs[6] == 0x11 || bs[6] == 0x84 {
+			switch bs[9] /* protocol */ {
+			case 0x06 /* TCP */, 0x11 /* UDP */, 0x84 /* SCTP */ :
 				flowkey = uint64(bs[6])<<32 /* proto */ |
 					uint64(bs[40])<<24 | uint64(bs[41])<<16 /* sport */ |
 					uint64(bs[42])<<8 | uint64(bs[43]) /* dport */

src/yggdrasil/api.go

@@ -99,7 +99,7 @@ type Session struct {
 	Coords      []uint64         // The coordinates of the remote node
 	BytesSent   uint64           // Bytes sent to the session
 	BytesRecvd  uint64           // Bytes received from the session
-	MTU         uint16           // The maximum supported message size of the session
+	MTU         MTU              // The maximum supported message size of the session
 	Uptime      time.Duration    // How long this session has been active for
 	WasMTUFixed bool             // This field is no longer used
 }
@@ -110,7 +110,8 @@ type Session struct {
 // there is exactly one entry then this node is not connected to any other nodes
 // and is therefore isolated.
 func (c *Core) GetPeers() []Peer {
-	ports := c.peers.ports.Load().(map[switchPort]*peer)
+	var ports map[switchPort]*peer
+	phony.Block(&c.peers, func() { ports = c.peers.ports })
 	var peers []Peer
 	var ps []switchPort
 	for port := range ports {
@@ -122,10 +123,10 @@ func (c *Core) GetPeers() []Peer {
 		var info Peer
 		phony.Block(p, func() {
 			info = Peer{
-				Endpoint:   p.intf.name,
+				Endpoint:   p.intf.name(),
 				BytesSent:  p.bytesSent,
 				BytesRecvd: p.bytesRecvd,
-				Protocol:   p.intf.info.linkType,
+				Protocol:   p.intf.interfaceType(),
 				Port:       uint64(port),
 				Uptime:     time.Since(p.firstSeen),
 			}
@@ -143,10 +144,14 @@ func (c *Core) GetPeers() []Peer {
 // isolated or not connected to any peers.
 func (c *Core) GetSwitchPeers() []SwitchPeer {
 	var switchpeers []SwitchPeer
-	table := c.switchTable.table.Load().(lookupTable)
-	peers := c.peers.ports.Load().(map[switchPort]*peer)
+	var table *lookupTable
+	var ports map[switchPort]*peer
+	phony.Block(&c.peers, func() {
+		table = c.peers.table
+		ports = c.peers.ports
+	})
 	for _, elem := range table.elems {
-		peer, isIn := peers[elem.port]
+		peer, isIn := ports[elem.port]
 		if !isIn {
 			continue
 		}
@@ -158,8 +163,8 @@ func (c *Core) GetSwitchPeers() []SwitchPeer {
 				BytesSent:  peer.bytesSent,
 				BytesRecvd: peer.bytesRecvd,
 				Port:       uint64(elem.port),
-				Protocol:   peer.intf.info.linkType,
-				Endpoint:   peer.intf.info.remote,
+				Protocol:   peer.intf.interfaceType(),
+				Endpoint:   peer.intf.remote(),
 			}
 			copy(info.PublicKey[:], peer.box[:])
 		})
@@ -194,34 +199,6 @@ func (c *Core) GetDHT() []DHTEntry {
 	return dhtentries
 }
 
-// GetSwitchQueues returns information about the switch queues that are
-// currently in effect. These values can change within an instant.
-func (c *Core) GetSwitchQueues() SwitchQueues {
-	var switchqueues SwitchQueues
-	switchTable := &c.switchTable
-	getSwitchQueues := func() {
-		switchqueues = SwitchQueues{
-			Count:        uint64(len(switchTable.queues.bufs)),
-			Size:         switchTable.queues.size,
-			HighestCount: uint64(switchTable.queues.maxbufs),
-			HighestSize:  switchTable.queues.maxsize,
-			MaximumSize:  switchTable.queues.totalMaxSize,
-		}
-		for k, v := range switchTable.queues.bufs {
-			nexthop := switchTable.bestPortForCoords([]byte(k))
-			queue := SwitchQueue{
-				ID:      k,
-				Size:    v.size,
-				Packets: uint64(len(v.packets)),
-				Port:    uint64(nexthop),
-			}
-			switchqueues.Queues = append(switchqueues.Queues, queue)
-		}
-	}
-	phony.Block(&c.switchTable, getSwitchQueues)
-	return switchqueues
-}
-
 // GetSessions returns a list of open sessions from this node to other nodes.
 func (c *Core) GetSessions() []Session {
 	var sessions []Session
@@ -280,14 +257,14 @@ func (c *Core) ConnDialer() (*Dialer, error) {
 // "Listen" configuration item, e.g.
 // 		tcp://a.b.c.d:e
 func (c *Core) ListenTCP(uri string) (*TcpListener, error) {
-	return c.link.tcp.listen(uri, nil)
+	return c.links.tcp.listen(uri, nil)
 }
 
 // ListenTLS starts a new TLS listener. The input URI should match that of the
 // "Listen" configuration item, e.g.
 // 		tls://a.b.c.d:e
 func (c *Core) ListenTLS(uri string) (*TcpListener, error) {
-	return c.link.tcp.listen(uri, c.link.tcp.tls.forListener)
+	return c.links.tcp.listen(uri, c.links.tcp.tls.forListener)
 }
 
 // NodeID gets the node ID. This is derived from your router encryption keys.
@@ -324,8 +301,11 @@ func (c *Core) EncryptionPublicKey() string {
 // connected to any other nodes (effectively making you the root of a
 // single-node network).
 func (c *Core) Coords() []uint64 {
-	table := c.switchTable.table.Load().(lookupTable)
-	return wire_coordsBytestoUint64s(table.self.getCoords())
+	var coords []byte
+	phony.Block(&c.router, func() {
+		coords = c.router.table.self.getCoords()
+	})
+	return wire_coordsBytestoUint64s(coords)
 }
 
 // Address gets the IPv6 address of the Yggdrasil node. This is always a /128
@@ -364,8 +344,8 @@ func (c *Core) SetNodeInfo(nodeinfo interface{}, nodeinfoprivacy bool) {
 }
 
 // GetMaximumSessionMTU returns the maximum allowed session MTU size.
-func (c *Core) GetMaximumSessionMTU() uint16 {
-	var mtu uint16
+func (c *Core) GetMaximumSessionMTU() MTU {
+	var mtu MTU
 	phony.Block(&c.router, func() {
 		mtu = c.router.sessions.myMaximumMTU
 	})
@@ -375,7 +355,7 @@ func (c *Core) GetMaximumSessionMTU() uint16 {
 // SetMaximumSessionMTU sets the maximum allowed session MTU size. The default
 // value is 65535 bytes. Session pings will be sent to update all open sessions
 // if the MTU has changed.
-func (c *Core) SetMaximumSessionMTU(mtu uint16) {
+func (c *Core) SetMaximumSessionMTU(mtu MTU) {
 	phony.Block(&c.router, func() {
 		if c.router.sessions.myMaximumMTU != mtu {
 			c.router.sessions.myMaximumMTU = mtu
@@ -390,17 +370,15 @@ func (c *Core) SetMaximumSessionMTU(mtu uint16) {
 // necessary when, e.g. crawling the network.
 func (c *Core) GetNodeInfo(key crypto.BoxPubKey, coords []uint64, nocache bool) (NodeInfoPayload, error) {
 	response := make(chan *NodeInfoPayload, 1)
-	sendNodeInfoRequest := func() {
-		c.router.nodeinfo.addCallback(key, func(nodeinfo *NodeInfoPayload) {
-			defer func() { recover() }()
-			select {
-			case response <- nodeinfo:
-			default:
-			}
-		})
-		c.router.nodeinfo.sendNodeInfo(key, wire_coordsUint64stoBytes(coords), false)
-	}
-	phony.Block(&c.router, sendNodeInfoRequest)
+	c.router.nodeinfo.addCallback(key, func(nodeinfo *NodeInfoPayload) {
+		defer func() { recover() }()
+		select {
+		case response <- nodeinfo:
+		default:
+		}
+	})
+	c.router.nodeinfo.sendNodeInfo(key, wire_coordsUint64stoBytes(coords), false)
+	phony.Block(&c.router.nodeinfo, func() {}) // Wait for sendNodeInfo before starting timer
 	timer := time.AfterFunc(6*time.Second, func() { close(response) })
 	defer timer.Stop()
 	for res := range response {
@@ -470,12 +448,33 @@ func (c *Core) AddPeer(addr string, sintf string) error {
 	return nil
 }
 
-// RemovePeer is not implemented yet.
 func (c *Core) RemovePeer(addr string, sintf string) error {
-	// TODO: Implement a reverse of AddPeer, where we look up the port number
-	// based on the addr and sintf, disconnect it and then remove it from the
-	// peers list so we don't reconnect to it later
-	return errors.New("not implemented")
+	if sintf == "" {
+		for i, peer := range c.config.Current.Peers {
+			if peer == addr {
+				c.config.Current.Peers = append(c.config.Current.Peers[:i], c.config.Current.Peers[i+1:]...)
+				break
+			}
+		}
+	} else if _, ok := c.config.Current.InterfacePeers[sintf]; ok {
+		for i, peer := range c.config.Current.InterfacePeers[sintf] {
+			if peer == addr {
+				c.config.Current.InterfacePeers[sintf] = append(c.config.Current.InterfacePeers[sintf][:i], c.config.Current.InterfacePeers[sintf][i+1:]...)
+				break
+			}
+		}
+	}
+
+	c.peers.Act(nil, func() {
+		ports := c.peers.ports
+		for _, peer := range ports {
+			if addr == peer.intf.name() {
+				c.peers._removePeer(peer)
+			}
+		}
+	})
+
+	return nil
 }
 
 // CallPeer calls a peer once. This should be specified in the peer URI format,
@@ -485,13 +484,17 @@ func (c *Core) RemovePeer(addr string, sintf string) error {
 // This does not add the peer to the peer list, so if the connection drops, the
 // peer will not be called again automatically.
 func (c *Core) CallPeer(addr string, sintf string) error {
-	return c.link.call(addr, sintf)
+	return c.links.call(addr, sintf)
 }
 
 // DisconnectPeer disconnects a peer once. This should be specified as a port
 // number.
 func (c *Core) DisconnectPeer(port uint64) error {
-	c.peers.removePeer(switchPort(port))
+	c.peers.Act(nil, func() {
+		if p, isIn := c.peers.ports[switchPort(port)]; isIn {
+			p.Act(&c.peers, p._removeSelf)
+		}
+	})
 	return nil
 }
 

src/yggdrasil/conn.go

@@ -7,11 +7,14 @@ import (
 	"time"
 
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
+	"github.com/yggdrasil-network/yggdrasil-go/src/types"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
 
 	"github.com/Arceliar/phony"
 )
 
+type MTU = types.MTU
+
 // ConnError implements the net.Error interface
 type ConnError struct {
 	error
@@ -65,7 +68,7 @@ type Conn struct {
 	nodeID        *crypto.NodeID
 	nodeMask      *crypto.NodeID
 	session       *sessionInfo
-	mtu           uint16
+	mtu           MTU
 	readCallback  func([]byte)
 	readBuffer    chan []byte
 }
@@ -93,7 +96,7 @@ func (c *Conn) String() string {
 	return s
 }
 
-func (c *Conn) setMTU(from phony.Actor, mtu uint16) {
+func (c *Conn) setMTU(from phony.Actor, mtu MTU) {
 	c.Act(from, func() { c.mtu = mtu })
 }
 
@@ -128,7 +131,7 @@ func (c *Conn) search() error {
 				}
 			}
 			sinfo := c.core.router.searches.newIterSearch(c.nodeID, c.nodeMask, searchCompleted)
-			sinfo.continueSearch()
+			sinfo.startSearch()
 		} else {
 			err = errors.New("search already exists")
 			close(done)
@@ -142,7 +145,8 @@ func (c *Conn) search() error {
 }
 
 // Used in session keep-alive traffic
-func (c *Conn) doSearch() {
+func (c *Conn) _doSearch() {
+	s := fmt.Sprintf("conn=%p", c)
 	routerWork := func() {
 		// Check to see if there is a search already matching the destination
 		sinfo, isIn := c.core.router.searches.searches[*c.nodeID]
@@ -150,9 +154,9 @@ func (c *Conn) doSearch() {
 			// Nothing was found, so create a new search
 			searchCompleted := func(sinfo *sessionInfo, e error) {}
 			sinfo = c.core.router.searches.newIterSearch(c.nodeID, c.nodeMask, searchCompleted)
-			c.core.log.Debugf("%s DHT search started: %p", c.String(), sinfo)
+			c.core.log.Debugf("%s DHT search started: %p", s, sinfo)
 			// Start the search
-			sinfo.continueSearch()
+			sinfo.startSearch()
 		}
 	}
 	c.core.router.Act(c.session, routerWork)
@@ -163,10 +167,9 @@ func (c *Conn) _getDeadlineCancellation(t *time.Time) (util.Cancellation, bool)
 		// A deadline is set, so return a Cancellation that uses it
 		c := util.CancellationWithDeadline(c.session.cancel, *t)
 		return c, true
-	} else {
-		// No deadline was set, so just return the existinc cancellation and a dummy value
-		return c.session.cancel, false
 	}
+	// No deadline was set, so just return the existing cancellation and a dummy value
+	return c.session.cancel, false
 }
 
 // SetReadCallback allows you to specify a function that will be called whenever
@@ -221,9 +224,8 @@ func (c *Conn) readNoCopy() ([]byte, error) {
 	case <-cancel.Finished():
 		if cancel.Error() == util.CancellationTimeoutError {
 			return nil, ConnError{errors.New("read timeout"), true, false, false, 0}
-		} else {
-			return nil, ConnError{errors.New("session closed"), false, false, true, 0}
 		}
+		return nil, ConnError{errors.New("session closed"), false, false, true, 0}
 	case bs := <-c.readBuffer:
 		return bs, nil
 	}
@@ -249,7 +251,6 @@ func (c *Conn) Read(b []byte) (int, error) {
 	}
 	// Copy results to the output slice and clean up
 	copy(b, bs)
-	util.PutBytes(bs)
 	// Return the number of bytes copied to the slice, along with any error
 	return n, err
 }
@@ -266,7 +267,7 @@ func (c *Conn) _write(msg FlowKeyMessage) error {
 		case time.Since(c.session.time) > 6*time.Second:
 			if c.session.time.Before(c.session.pingTime) && time.Since(c.session.pingTime) > 6*time.Second {
 				// TODO double check that the above condition is correct
-				c.doSearch()
+				c._doSearch()
 			} else {
 				c.session.ping(c.session) // TODO send from self if this becomes an actor
 			}
@@ -279,7 +280,7 @@ func (c *Conn) _write(msg FlowKeyMessage) error {
 }
 
 // WriteFrom should be called by a phony.Actor, and tells the Conn to send a
-// message. This is used internaly by Write. If the callback is called with a
+// message. This is used internally by Write. If the callback is called with a
 // non-nil value, then it is safe to reuse the argument FlowKeyMessage.
 func (c *Conn) WriteFrom(from phony.Actor, msg FlowKeyMessage, callback func(error)) {
 	c.Act(from, func() {
@@ -320,10 +321,11 @@ func (c *Conn) writeNoCopy(msg FlowKeyMessage) error {
 // returned.
 func (c *Conn) Write(b []byte) (int, error) {
 	written := len(b)
-	msg := FlowKeyMessage{Message: append(util.GetBytes(), b...)}
+	bs := make([]byte, 0, len(b)+crypto.BoxOverhead)
+	bs = append(bs, b...)
+	msg := FlowKeyMessage{Message: bs}
 	err := c.writeNoCopy(msg)
 	if err != nil {
-		util.PutBytes(msg.Message)
 		written = 0
 	}
 	return written, err
@@ -347,16 +349,19 @@ func (c *Conn) Close() (err error) {
 	return
 }
 
-// LocalAddr returns the complete node ID of the local side of the connection.
-// This is always going to return your own node's node ID.
+// LocalAddr returns the complete public key of the local side of the
+// connection. This is always going to return your own node's public key.
 func (c *Conn) LocalAddr() net.Addr {
-	return crypto.GetNodeID(&c.core.boxPub)
+	return &c.core.boxPub
 }
 
-// RemoteAddr returns the complete node ID of the remote side of the connection.
+// RemoteAddr returns the complete public key of the remote side of the
+// connection.
 func (c *Conn) RemoteAddr() net.Addr {
-	// RemoteAddr is set during the dial or accept, and isn't changed, so it's safe to access directly
-	return c.nodeID
+	if c.session != nil {
+		return &c.session.theirPermPub
+	}
+	return nil
 }
 
 // SetDeadline is equivalent to calling both SetReadDeadline and

src/yggdrasil/core.go

@@ -29,7 +29,7 @@ type Core struct {
 	switchTable  switchTable
 	peers        peers
 	router       router
-	link         link
+	links        links
 	log          *log.Logger
 	addPeerTimer *time.Timer
 }
@@ -160,9 +160,12 @@ func (c *Core) _start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState
 	}
 
 	c.log.Infoln("Starting up...")
-	c._init()
+	if err := c._init(); err != nil {
+		c.log.Errorln("Failed to initialize core")
+		return nil, err
+	}
 
-	if err := c.link.init(c); err != nil {
+	if err := c.links.init(c); err != nil {
 		c.log.Errorln("Failed to start link interfaces")
 		return nil, err
 	}
@@ -194,9 +197,11 @@ func (c *Core) _stop() {
 	if c.addPeerTimer != nil {
 		c.addPeerTimer.Stop()
 	}
-	c.link.stop()
+	c.links.stop()
+	/* FIXME this deadlocks, need a waitgroup or something to coordinate shutdown
 	for _, peer := range c.GetPeers() {
 		c.DisconnectPeer(peer.Port)
 	}
+	*/
 	c.log.Infoln("Stopped")
 }

src/yggdrasil/core_test.go

@@ -22,7 +22,7 @@ func GenerateConfig() *config.NodeConfig {
 	return cfg
 }
 
-// GetLoggerWithPrefix creates a new logger instance wih prefix.
+// GetLoggerWithPrefix creates a new logger instance with prefix.
 // If verbose is set to true, three log levels are enabled: "info", "warn", "error".
 func GetLoggerWithPrefix(prefix string, verbose bool) *log.Logger {
 	l := log.New(os.Stderr, prefix, log.Flags())

src/yggdrasil/dht.go

@@ -20,7 +20,7 @@ const (
 )
 
 // dhtInfo represents everything we know about a node in the DHT.
-// This includes its key, a cache of it's NodeID, coords, and timing/ping related info for deciding who/when to ping nodes for maintenance.
+// This includes its key, a cache of its NodeID, coords, and timing/ping related info for deciding who/when to ping nodes for maintenance.
 type dhtInfo struct {
 	nodeID_hidden *crypto.NodeID
 	key           crypto.BoxPubKey
@@ -89,6 +89,11 @@ func (t *dht) reconfigure() {
 // Resets the DHT in response to coord changes.
 // This empties all info from the DHT and drops outstanding requests.
 func (t *dht) reset() {
+	for _, info := range t.table {
+		if t.isImportant(info) {
+			t.ping(info, nil)
+		}
+	}
 	t.reqs = make(map[dhtReqKey]time.Time)
 	t.table = make(map[crypto.NodeID]*dhtInfo)
 	t.imp = nil
@@ -144,12 +149,8 @@ func (t *dht) insert(info *dhtInfo) {
 
 // Insert a peer into the table if it hasn't been pinged lately, to keep peers from dropping
 func (t *dht) insertPeer(info *dhtInfo) {
-	oldInfo, isIn := t.table[*info.getNodeID()]
-	if !isIn || time.Since(oldInfo.recv) > dht_max_delay+30*time.Second {
-		// TODO? also check coords?
-		newInfo := *info // Insert a copy
-		t.insert(&newInfo)
-	}
+	t.insert(info)    // FIXME this resets timers / ping counts / etc, so it seems kind of dangerous
+	t.ping(info, nil) // This is a quick fix to the above, ping them immediately...
 }
 
 // Return true if first/second/third are (partially) ordered correctly.
@@ -186,11 +187,9 @@ func dht_ordered(first, second, third *crypto.NodeID) bool {
 // Update info about the node that sent the request.
 func (t *dht) handleReq(req *dhtReq) {
 	// Send them what they asked for
-	loc := t.router.core.switchTable.getLocator()
-	coords := loc.getCoords()
 	res := dhtRes{
 		Key:    t.router.core.boxPub,
-		Coords: coords,
+		Coords: t.router.table.self.getCoords(),
 		Dest:   req.Dest,
 		Infos:  t.lookup(&req.Dest, false),
 	}
@@ -302,11 +301,9 @@ func (t *dht) ping(info *dhtInfo, target *crypto.NodeID) {
 	if target == nil {
 		target = &t.nodeID
 	}
-	loc := t.router.core.switchTable.getLocator()
-	coords := loc.getCoords()
 	req := dhtReq{
 		Key:    t.router.core.boxPub,
-		Coords: coords,
+		Coords: t.router.table.self.getCoords(),
 		Dest:   *target,
 	}
 	t.sendReq(&req, info)
@@ -380,13 +377,15 @@ func (t *dht) getImportant() []*dhtInfo {
 		})
 		// Keep the ones that are no further than the closest seen so far
 		minDist := ^uint64(0)
-		loc := t.router.core.switchTable.getLocator()
+		loc := t.router.table.self
 		important := infos[:0]
 		for _, info := range infos {
 			dist := uint64(loc.dist(info.coords))
 			if dist < minDist {
 				minDist = dist
 				important = append(important, info)
+			} else if len(important) < 2 {
+				important = append(important, info)
 			}
 		}
 		var temp []*dhtInfo
@@ -397,6 +396,8 @@ func (t *dht) getImportant() []*dhtInfo {
 			if dist < minDist {
 				minDist = dist
 				temp = append(temp, info)
+			} else if len(temp) < 2 {
+				temp = append(temp, info)
 			}
 		}
 		for idx := len(temp) - 1; idx >= 0; idx-- {
@@ -414,7 +415,7 @@ func (t *dht) isImportant(ninfo *dhtInfo) bool {
 	}
 	important := t.getImportant()
 	// Check if ninfo is of equal or greater importance to what we already know
-	loc := t.router.core.switchTable.getLocator()
+	loc := t.router.table.self
 	ndist := uint64(loc.dist(ninfo.coords))
 	minDist := ^uint64(0)
 	for _, info := range important {

src/yggdrasil/dialer.go

@@ -17,19 +17,32 @@ type Dialer struct {
 	core *Core
 }
 
-// Dial opens a session to the given node. The first paramter should be "nodeid"
-// and the second parameter should contain a hexadecimal representation of the
-// target node ID. It uses DialContext internally.
+// Dial opens a session to the given node. The first parameter should be
+// "curve25519" or "nodeid" and the second parameter should contain a
+// hexadecimal representation of the target. It uses DialContext internally.
 func (d *Dialer) Dial(network, address string) (net.Conn, error) {
 	return d.DialContext(nil, network, address)
 }
 
-// DialContext is used internally by Dial, and should only be used with a context that includes a timeout. It uses DialByNodeIDandMask internally.
+// DialContext is used internally by Dial, and should only be used with a
+// context that includes a timeout. It uses DialByNodeIDandMask internally when
+// the network is "nodeid", or DialByPublicKey when the network is "curve25519".
 func (d *Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
 	var nodeID crypto.NodeID
 	var nodeMask crypto.NodeID
 	// Process
 	switch network {
+	case "curve25519":
+		dest, err := hex.DecodeString(address)
+		if err != nil {
+			return nil, err
+		}
+		if len(dest) != crypto.BoxPubKeyLen {
+			return nil, errors.New("invalid key length supplied")
+		}
+		var pubKey crypto.BoxPubKey
+		copy(pubKey[:], dest)
+		return d.DialByPublicKey(ctx, &pubKey)
 	case "nodeid":
 		// A node ID was provided - we don't need to do anything special with it
 		if tokens := strings.Split(address, "/"); len(tokens) == 2 {
@@ -62,15 +75,19 @@ func (d *Dialer) DialContext(ctx context.Context, network, address string) (net.
 	}
 }
 
-// DialByNodeIDandMask opens a session to the given node based on raw
-// NodeID parameters. If ctx is nil or has no timeout, then a default timeout of 6 seconds will apply, beginning *after* the search finishes.
+// DialByNodeIDandMask opens a session to the given node based on raw NodeID
+// parameters. If ctx is nil or has no timeout, then a default timeout of 6
+// seconds will apply, beginning *after* the search finishes.
 func (d *Dialer) DialByNodeIDandMask(ctx context.Context, nodeID, nodeMask *crypto.NodeID) (net.Conn, error) {
+	startDial := time.Now()
 	conn := newConn(d.core, nodeID, nodeMask, nil)
 	if err := conn.search(); err != nil {
 		// TODO: make searches take a context, so they can be cancelled early
 		conn.Close()
 		return nil, err
 	}
+	endSearch := time.Now()
+	d.core.log.Debugln("Dial searched for:", nodeID, "in time:", endSearch.Sub(startDial))
 	conn.session.setConn(nil, conn)
 	var cancel context.CancelFunc
 	if ctx == nil {
@@ -80,9 +97,24 @@ func (d *Dialer) DialByNodeIDandMask(ctx context.Context, nodeID, nodeMask *cryp
 	defer cancel()
 	select {
 	case <-conn.session.init:
+		endInit := time.Now()
+		d.core.log.Debugln("Dial initialized session for:", nodeID, "in time:", endInit.Sub(endSearch))
+		d.core.log.Debugln("Finished dial for:", nodeID, "in time:", endInit.Sub(startDial))
 		return conn, nil
 	case <-ctx.Done():
 		conn.Close()
 		return nil, errors.New("session handshake timeout")
 	}
 }
+
+// DialByPublicKey opens a session to the given node based on the public key. If
+// ctx is nil or has no timeout, then a default timeout of 6 seconds will apply,
+// beginning *after* the search finishes.
+func (d *Dialer) DialByPublicKey(ctx context.Context, pubKey *crypto.BoxPubKey) (net.Conn, error) {
+	nodeID := crypto.GetNodeID(pubKey)
+	var nodeMask crypto.NodeID
+	for i := range nodeMask {
+		nodeMask[i] = 0xFF
+	}
+	return d.DialByNodeIDandMask(ctx, nodeID, &nodeMask)
+}

src/yggdrasil/doc.go

@@ -113,9 +113,9 @@ a Dialer:
     // ...
   }
 
-You can then dial using the 16-byte node ID in hexadecimal format, for example:
+You can then dial using the node's public key in hexadecimal format, for example:
 
-  conn, err := dialer.Dial("nodeid", "24a58cfce691ec016b0f698f7be1bee983cea263781017e99ad3ef62b4ef710a45d6c1a072c5ce46131bd574b78818c9957042cafeeed13966f349e94eb771bf")
+  conn, err := dialer.Dial("curve25519", "55071be281f50d0abbda63aadc59755624280c44b2f1f47684317aa4e0325604")
   if err != nil {
     // ...
   }

src/yggdrasil/link.go

@@ -16,16 +16,17 @@ import (
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
+	"golang.org/x/net/proxy"
 
 	"github.com/Arceliar/phony"
 )
 
-type link struct {
-	core       *Core
-	mutex      sync.RWMutex // protects interfaces below
-	interfaces map[linkInfo]*linkInterface
-	tcp        tcp // TCP interface support
-	stopped    chan struct{}
+type links struct {
+	core    *Core
+	mutex   sync.RWMutex // protects links below
+	links   map[linkInfo]*link
+	tcp     tcp // TCP interface support
+	stopped chan struct{}
 	// TODO timeout (to remove from switch), read from config.ReadTimeout
 }
 
@@ -37,7 +38,7 @@ type linkInfo struct {
 	remote   string           // Remote name or address
 }
 
-type linkInterfaceMsgIO interface {
+type linkMsgIO interface {
 	readMsg() ([]byte, error)
 	writeMsgs([][]byte) (int, error)
 	close() error
@@ -46,30 +47,37 @@ type linkInterfaceMsgIO interface {
 	_recvMetaBytes() ([]byte, error)
 }
 
-type linkInterface struct {
-	name           string
-	link           *link
+type link struct {
+	lname          string
+	links          *links
 	peer           *peer
-	msgIO          linkInterfaceMsgIO
+	options        linkOptions
+	msgIO          linkMsgIO
 	info           linkInfo
 	incoming       bool
 	force          bool
 	closed         chan struct{}
-	reader         linkReader  // Reads packets, notifies this linkInterface, passes packets to switch
-	writer         linkWriter  // Writes packets, notifies this linkInterface
+	reader         linkReader  // Reads packets, notifies this link, passes packets to switch
+	writer         linkWriter  // Writes packets, notifies this link
 	phony.Inbox                // Protects the below
 	sendTimer      *time.Timer // Fires to signal that sending is blocked
 	keepAliveTimer *time.Timer // Fires to send keep-alive traffic
 	stallTimer     *time.Timer // Fires to signal that no incoming traffic (including keep-alive) has been seen
 	closeTimer     *time.Timer // Fires when the link has been idle so long we need to close it
-	inSwitch       bool        // True if the switch is tracking this link
-	stalled        bool        // True if we haven't been receiving any response traffic
+	readUnblocked  bool        // True if we've sent a read message unblocking this peer in the switch
+	writeUnblocked bool        // True if we've sent a write message unblocking this peer in the swithc
+	shutdown       bool        // True if we're shutting down, avoids sending some messages that could race with new peers being crated in the same port
 }
 
-func (l *link) init(c *Core) error {
+type linkOptions struct {
+	pinnedCurve25519Keys map[crypto.BoxPubKey]struct{}
+	pinnedEd25519Keys    map[crypto.SigPubKey]struct{}
+}
+
+func (l *links) init(c *Core) error {
 	l.core = c
 	l.mutex.Lock()
-	l.interfaces = make(map[linkInfo]*linkInterface)
+	l.links = make(map[linkInfo]*link)
 	l.mutex.Unlock()
 	l.stopped = make(chan struct{})
 
@@ -81,30 +89,58 @@ func (l *link) init(c *Core) error {
 	return nil
 }
 
-func (l *link) reconfigure() {
+func (l *links) reconfigure() {
 	l.tcp.reconfigure()
 }
 
-func (l *link) call(uri string, sintf string) error {
+func (l *links) call(uri string, sintf string) error {
 	u, err := url.Parse(uri)
 	if err != nil {
 		return fmt.Errorf("peer %s is not correctly formatted (%s)", uri, err)
 	}
 	pathtokens := strings.Split(strings.Trim(u.Path, "/"), "/")
+	tcpOpts := tcpOptions{}
+	if pubkeys, ok := u.Query()["curve25519"]; ok && len(pubkeys) > 0 {
+		tcpOpts.pinnedCurve25519Keys = make(map[crypto.BoxPubKey]struct{})
+		for _, pubkey := range pubkeys {
+			if boxPub, err := hex.DecodeString(pubkey); err == nil {
+				var boxPubKey crypto.BoxPubKey
+				copy(boxPubKey[:], boxPub)
+				tcpOpts.pinnedCurve25519Keys[boxPubKey] = struct{}{}
+			}
+		}
+	}
+	if pubkeys, ok := u.Query()["ed25519"]; ok && len(pubkeys) > 0 {
+		tcpOpts.pinnedEd25519Keys = make(map[crypto.SigPubKey]struct{})
+		for _, pubkey := range pubkeys {
+			if sigPub, err := hex.DecodeString(pubkey); err == nil {
+				var sigPubKey crypto.SigPubKey
+				copy(sigPubKey[:], sigPub)
+				tcpOpts.pinnedEd25519Keys[sigPubKey] = struct{}{}
+			}
+		}
+	}
 	switch u.Scheme {
 	case "tcp":
-		l.tcp.call(u.Host, nil, sintf, nil)
+		l.tcp.call(u.Host, tcpOpts, sintf)
 	case "socks":
-		l.tcp.call(pathtokens[0], u.Host, sintf, nil)
+		tcpOpts.socksProxyAddr = u.Host
+		if u.User != nil {
+			tcpOpts.socksProxyAuth = &proxy.Auth{}
+			tcpOpts.socksProxyAuth.User = u.User.Username()
+			tcpOpts.socksProxyAuth.Password, _ = u.User.Password()
+		}
+		l.tcp.call(pathtokens[0], tcpOpts, sintf)
 	case "tls":
-		l.tcp.call(u.Host, nil, sintf, l.tcp.tls.forDialer)
+		tcpOpts.upgrade = l.tcp.tls.forDialer
+		l.tcp.call(u.Host, tcpOpts, sintf)
 	default:
 		return errors.New("unknown call scheme: " + u.Scheme)
 	}
 	return nil
 }
 
-func (l *link) listen(uri string) error {
+func (l *links) listen(uri string) error {
 	u, err := url.Parse(uri)
 	if err != nil {
 		return fmt.Errorf("listener %s is not correctly formatted (%s)", uri, err)
@@ -121,12 +157,13 @@ func (l *link) listen(uri string) error {
 	}
 }
 
-func (l *link) create(msgIO linkInterfaceMsgIO, name, linkType, local, remote string, incoming, force bool) (*linkInterface, error) {
-	// Technically anything unique would work for names, but lets pick something human readable, just for debugging
-	intf := linkInterface{
-		name:  name,
-		link:  l,
-		msgIO: msgIO,
+func (l *links) create(msgIO linkMsgIO, name, linkType, local, remote string, incoming, force bool, options linkOptions) (*link, error) {
+	// Technically anything unique would work for names, but let's pick something human readable, just for debugging
+	intf := link{
+		lname:   name,
+		links:   l,
+		options: options,
+		msgIO:   msgIO,
 		info: linkInfo{
 			linkType: linkType,
 			local:    local,
@@ -136,12 +173,13 @@ func (l *link) create(msgIO linkInterfaceMsgIO, name, linkType, local, remote st
 		force:    force,
 	}
 	intf.writer.intf = &intf
+	intf.writer.worker = make(chan [][]byte, 1)
 	intf.reader.intf = &intf
 	intf.reader.err = make(chan error)
 	return &intf, nil
 }
 
-func (l *link) stop() error {
+func (l *links) stop() error {
 	close(l.stopped)
 	if err := l.tcp.stop(); err != nil {
 		return err
@@ -149,94 +187,114 @@ func (l *link) stop() error {
 	return nil
 }
 
-func (intf *linkInterface) handler() error {
+func (intf *link) handler() (chan struct{}, error) {
 	// TODO split some of this into shorter functions, so it's easier to read, and for the FIXME duplicate peer issue mentioned later
+	go func() {
+		for bss := range intf.writer.worker {
+			intf.msgIO.writeMsgs(bss)
+		}
+	}()
+	defer intf.writer.Act(nil, func() {
+		intf.writer.closed = true
+		close(intf.writer.worker)
+	})
 	myLinkPub, myLinkPriv := crypto.NewBoxKeys()
 	meta := version_getBaseMetadata()
-	meta.box = intf.link.core.boxPub
-	meta.sig = intf.link.core.sigPub
+	meta.box = intf.links.core.boxPub
+	meta.sig = intf.links.core.sigPub
 	meta.link = *myLinkPub
 	metaBytes := meta.encode()
 	// TODO timeouts on send/recv (goroutine for send/recv, channel select w/ timer)
 	var err error
 	if !util.FuncTimeout(func() { err = intf.msgIO._sendMetaBytes(metaBytes) }, 30*time.Second) {
-		return errors.New("timeout on metadata send")
+		return nil, errors.New("timeout on metadata send")
 	}
 	if err != nil {
-		return err
+		return nil, err
 	}
 	if !util.FuncTimeout(func() { metaBytes, err = intf.msgIO._recvMetaBytes() }, 30*time.Second) {
-		return errors.New("timeout on metadata recv")
+		return nil, errors.New("timeout on metadata recv")
 	}
 	if err != nil {
-		return err
+		return nil, err
 	}
 	meta = version_metadata{}
 	if !meta.decode(metaBytes) || !meta.check() {
-		return errors.New("failed to decode metadata")
+		return nil, errors.New("failed to decode metadata")
 	}
 	base := version_getBaseMetadata()
 	if meta.ver > base.ver || meta.ver == base.ver && meta.minorVer > base.minorVer {
-		intf.link.core.log.Errorln("Failed to connect to node: " + intf.name + " version: " + fmt.Sprintf("%d.%d", meta.ver, meta.minorVer))
-		return errors.New("failed to connect: wrong version")
+		intf.links.core.log.Errorln("Failed to connect to node: " + intf.lname + " version: " + fmt.Sprintf("%d.%d", meta.ver, meta.minorVer))
+		return nil, errors.New("failed to connect: wrong version")
+	}
+	// Check if the remote side matches the keys we expected. This is a bit of a weak
+	// check - in future versions we really should check a signature or something like that.
+	if pinned := intf.options.pinnedCurve25519Keys; pinned != nil {
+		if _, allowed := pinned[meta.box]; !allowed {
+			intf.links.core.log.Errorf("Failed to connect to node: %q sent curve25519 key that does not match pinned keys", intf.name)
+			return nil, fmt.Errorf("failed to connect: host sent curve25519 key that does not match pinned keys")
+		}
+	}
+	if pinned := intf.options.pinnedEd25519Keys; pinned != nil {
+		if _, allowed := pinned[meta.sig]; !allowed {
+			intf.links.core.log.Errorf("Failed to connect to node: %q sent ed25519 key that does not match pinned keys", intf.name)
+			return nil, fmt.Errorf("failed to connect: host sent ed25519 key that does not match pinned keys")
+		}
 	}
 	// Check if we're authorized to connect to this key / IP
-	if intf.incoming && !intf.force && !intf.link.core.peers.isAllowedEncryptionPublicKey(&meta.box) {
-		intf.link.core.log.Warnf("%s connection from %s forbidden: AllowedEncryptionPublicKeys does not contain key %s",
+	if intf.incoming && !intf.force && !intf.links.core.peers.isAllowedEncryptionPublicKey(&meta.box) {
+		intf.links.core.log.Warnf("%s connection from %s forbidden: AllowedEncryptionPublicKeys does not contain key %s",
 			strings.ToUpper(intf.info.linkType), intf.info.remote, hex.EncodeToString(meta.box[:]))
 		intf.msgIO.close()
-		return nil
+		return nil, nil
 	}
 	// Check if we already have a link to this node
 	intf.info.box = meta.box
 	intf.info.sig = meta.sig
-	intf.link.mutex.Lock()
-	if oldIntf, isIn := intf.link.interfaces[intf.info]; isIn {
-		intf.link.mutex.Unlock()
+	intf.links.mutex.Lock()
+	if oldIntf, isIn := intf.links.links[intf.info]; isIn {
+		intf.links.mutex.Unlock()
 		// FIXME we should really return an error and let the caller block instead
 		// That lets them do things like close connections on its own, avoid printing a connection message in the first place, etc.
-		intf.link.core.log.Debugln("DEBUG: found existing interface for", intf.name)
+		intf.links.core.log.Debugln("DEBUG: found existing interface for", intf.name)
 		intf.msgIO.close()
-		if !intf.incoming {
-			// Block outgoing connection attempts until the existing connection closes
-			<-oldIntf.closed
-		}
-		return nil
+		return oldIntf.closed, nil
 	} else {
 		intf.closed = make(chan struct{})
-		intf.link.interfaces[intf.info] = intf
+		intf.links.links[intf.info] = intf
 		defer func() {
-			intf.link.mutex.Lock()
-			delete(intf.link.interfaces, intf.info)
-			intf.link.mutex.Unlock()
+			intf.links.mutex.Lock()
+			delete(intf.links.links, intf.info)
+			intf.links.mutex.Unlock()
 			close(intf.closed)
 		}()
-		intf.link.core.log.Debugln("DEBUG: registered interface for", intf.name)
+		intf.links.core.log.Debugln("DEBUG: registered interface for", intf.name)
 	}
-	intf.link.mutex.Unlock()
+	intf.links.mutex.Unlock()
 	// Create peer
 	shared := crypto.GetSharedKey(myLinkPriv, &meta.link)
-	intf.peer = intf.link.core.peers.newPeer(&meta.box, &meta.sig, shared, intf, func() { intf.msgIO.close() })
+	phony.Block(&intf.links.core.peers, func() {
+		// FIXME don't use phony.Block, it's bad practice, even if it's safe here
+		intf.peer = intf.links.core.peers._newPeer(&meta.box, &meta.sig, shared, intf)
+	})
 	if intf.peer == nil {
-		return errors.New("failed to create peer")
+		return nil, errors.New("failed to create peer")
 	}
 	defer func() {
 		// More cleanup can go here
-		intf.link.core.peers.removePeer(intf.peer.port)
+		intf.Act(nil, func() {
+			intf.shutdown = true
+			intf.peer.Act(intf, intf.peer._removeSelf)
+		})
 	}()
-	intf.peer.out = func(msgs [][]byte) {
-		intf.writer.sendFrom(intf.peer, msgs, false)
-	}
-	intf.peer.linkOut = func(bs []byte) {
-		intf.writer.sendFrom(intf.peer, [][]byte{bs}, true)
-	}
 	themAddr := address.AddrForNodeID(crypto.GetNodeID(&intf.info.box))
 	themAddrString := net.IP(themAddr[:]).String()
 	themString := fmt.Sprintf("%s@%s", themAddrString, intf.info.remote)
-	intf.link.core.log.Infof("Connected %s: %s, source %s",
+	intf.links.core.log.Infof("Connected %s: %s, source %s",
 		strings.ToUpper(intf.info.linkType), themString, intf.info.local)
 	// Start things
 	go intf.peer.start()
+	intf.Act(nil, intf._notifyIdle)
 	intf.reader.Act(nil, intf.reader._read)
 	// Wait for the reader to finish
 	// TODO find a way to do this without keeping live goroutines around
@@ -244,7 +302,7 @@ func (intf *linkInterface) handler() error {
 	defer close(done)
 	go func() {
 		select {
-		case <-intf.link.stopped:
+		case <-intf.links.stopped:
 			intf.msgIO.close()
 		case <-done:
 		}
@@ -252,17 +310,71 @@ func (intf *linkInterface) handler() error {
 	err = <-intf.reader.err
 	// TODO don't report an error if it's just a 'use of closed network connection'
 	if err != nil {
-		intf.link.core.log.Infof("Disconnected %s: %s, source %s; error: %s",
+		intf.links.core.log.Infof("Disconnected %s: %s, source %s; error: %s",
 			strings.ToUpper(intf.info.linkType), themString, intf.info.local, err)
 	} else {
-		intf.link.core.log.Infof("Disconnected %s: %s, source %s",
+		intf.links.core.log.Infof("Disconnected %s: %s, source %s",
 			strings.ToUpper(intf.info.linkType), themString, intf.info.local)
 	}
-	return err
+	return nil, err
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 
+// link needs to match the linkInterface type needed by the peers
+
+type linkInterface interface {
+	out([][]byte)
+	linkOut([]byte)
+	close()
+	// These next ones are only used by the API
+	name() string
+	local() string
+	remote() string
+	interfaceType() string
+}
+
+func (intf *link) out(bss [][]byte) {
+	intf.Act(nil, func() {
+		// nil to prevent it from blocking if the link is somehow frozen
+		// this is safe because another packet won't be sent until the link notifies
+		//  the peer that it's ready for one
+		intf.writer.sendFrom(nil, bss)
+	})
+}
+
+func (intf *link) linkOut(bs []byte) {
+	intf.Act(nil, func() {
+		// nil to prevent it from blocking if the link is somehow frozen
+		// FIXME this is hypothetically not safe, the peer shouldn't be sending
+		//  additional packets until this one finishes, otherwise this could leak
+		//  memory if writing happens slower than link packets are generated...
+		//  that seems unlikely, so it's a lesser evil than deadlocking for now
+		intf.writer.sendFrom(nil, [][]byte{bs})
+	})
+}
+
+func (intf *link) close() {
+	intf.Act(nil, func() { intf.msgIO.close() })
+}
+
+func (intf *link) name() string {
+	return intf.lname
+}
+
+func (intf *link) local() string {
+	return intf.info.local
+}
+
+func (intf *link) remote() string {
+	return intf.info.remote
+}
+
+func (intf *link) interfaceType() string {
+	return intf.info.linkType
+}
+
+////////////////////////////////////////////////////////////////////////////////
 const (
 	sendTime      = 1 * time.Second    // How long to wait before deciding a send is blocked
 	keepAliveTime = 2 * time.Second    // How long to wait before sending a keep-alive response if we have no real traffic to send
@@ -271,111 +383,108 @@ const (
 )
 
 // notify the intf that we're currently sending
-func (intf *linkInterface) notifySending(size int, isLinkTraffic bool) {
+func (intf *link) notifySending(size int) {
 	intf.Act(&intf.writer, func() {
-		if !isLinkTraffic {
-			intf.inSwitch = false
-		}
 		intf.sendTimer = time.AfterFunc(sendTime, intf.notifyBlockedSend)
-		intf._cancelStallTimer()
+		if intf.keepAliveTimer != nil {
+			intf.keepAliveTimer.Stop()
+			intf.keepAliveTimer = nil
+		}
+		intf.peer.notifyBlocked(intf)
 	})
 }
 
-// called by an AfterFunc if we seem to be blocked in a send syscall for a long time
-func (intf *linkInterface) _notifySyscall() {
-	intf.link.core.switchTable.Act(intf, func() {
-		intf.link.core.switchTable._sendingIn(intf.peer.port)
-	})
-}
-
-// we just sent something, so cancel any pending timer to send keep-alive traffic
-func (intf *linkInterface) _cancelStallTimer() {
-	if intf.stallTimer != nil {
-		intf.stallTimer.Stop()
-		intf.stallTimer = nil
-	}
-}
-
 // This gets called from a time.AfterFunc, and notifies the switch that we appear
 // to have gotten blocked on a write, so the switch should start routing traffic
 // through other links, if alternatives exist
-func (intf *linkInterface) notifyBlockedSend() {
+func (intf *link) notifyBlockedSend() {
 	intf.Act(nil, func() {
 		if intf.sendTimer != nil {
 			//As far as we know, we're still trying to send, and the timer fired.
-			intf.link.core.switchTable.blockPeer(intf.peer.port)
+			intf.sendTimer.Stop()
+			intf.sendTimer = nil
+			if !intf.shutdown && intf.writeUnblocked {
+				intf.writeUnblocked = false
+				intf.links.core.switchTable.blockPeer(intf, intf.peer.port, true)
+			}
 		}
 	})
 }
 
 // notify the intf that we've finished sending, returning the peer to the switch
-func (intf *linkInterface) notifySent(size int, isLinkTraffic bool) {
+func (intf *link) notifySent(size int) {
 	intf.Act(&intf.writer, func() {
-		intf.sendTimer.Stop()
-		intf.sendTimer = nil
-		if !isLinkTraffic {
-			intf._notifySwitch()
+		if intf.sendTimer != nil {
+			intf.sendTimer.Stop()
+			intf.sendTimer = nil
 		}
+		if intf.keepAliveTimer != nil {
+			// TODO? unset this when we start sending, not when we finish...
+			intf.keepAliveTimer.Stop()
+			intf.keepAliveTimer = nil
+		}
+		intf._notifyIdle()
 		if size > 0 && intf.stallTimer == nil {
 			intf.stallTimer = time.AfterFunc(stallTime, intf.notifyStalled)
 		}
+		if !intf.shutdown && !intf.writeUnblocked {
+			intf.writeUnblocked = true
+			intf.links.core.switchTable.unblockPeer(intf, intf.peer.port, true)
+		}
 	})
 }
 
-// Notify the switch that we're ready for more traffic, assuming we're not in a stalled state
-func (intf *linkInterface) _notifySwitch() {
-	if !intf.inSwitch && !intf.stalled {
-		intf.inSwitch = true
-		intf.link.core.switchTable.Act(intf, func() {
-			intf.link.core.switchTable._idleIn(intf.peer.port)
-		})
-	}
+// Notify the peer that we're ready for more traffic
+func (intf *link) _notifyIdle() {
+	intf.peer.Act(intf, intf.peer._handleIdle)
 }
 
 // Set the peer as stalled, to prevent them from returning to the switch until a read succeeds
-func (intf *linkInterface) notifyStalled() {
+func (intf *link) notifyStalled() {
 	intf.Act(nil, func() { // Sent from a time.AfterFunc
 		if intf.stallTimer != nil {
 			intf.stallTimer.Stop()
 			intf.stallTimer = nil
-			intf.stalled = true
-			intf.link.core.switchTable.blockPeer(intf.peer.port)
+			if !intf.shutdown && intf.readUnblocked {
+				intf.readUnblocked = false
+				intf.links.core.switchTable.blockPeer(intf, intf.peer.port, false)
+			}
 		}
 	})
 }
 
 // reset the close timer
-func (intf *linkInterface) notifyReading() {
+func (intf *link) notifyReading() {
 	intf.Act(&intf.reader, func() {
-		if intf.closeTimer != nil {
-			intf.closeTimer.Stop()
-		}
 		intf.closeTimer = time.AfterFunc(closeTime, func() { intf.msgIO.close() })
 	})
 }
 
 // wake up the link if it was stalled, and (if size > 0) prepare to send keep-alive traffic
-func (intf *linkInterface) notifyRead(size int) {
+func (intf *link) notifyRead(size int) {
 	intf.Act(&intf.reader, func() {
+		intf.closeTimer.Stop()
 		if intf.stallTimer != nil {
 			intf.stallTimer.Stop()
 			intf.stallTimer = nil
 		}
-		intf.stalled = false
-		intf._notifySwitch()
-		if size > 0 && intf.stallTimer == nil {
-			intf.stallTimer = time.AfterFunc(keepAliveTime, intf.notifyDoKeepAlive)
+		if size > 0 && intf.keepAliveTimer == nil {
+			intf.keepAliveTimer = time.AfterFunc(keepAliveTime, intf.notifyDoKeepAlive)
+		}
+		if !intf.shutdown && !intf.readUnblocked {
+			intf.readUnblocked = true
+			intf.links.core.switchTable.unblockPeer(intf, intf.peer.port, false)
 		}
 	})
 }
 
 // We need to send keep-alive traffic now
-func (intf *linkInterface) notifyDoKeepAlive() {
+func (intf *link) notifyDoKeepAlive() {
 	intf.Act(nil, func() { // Sent from a time.AfterFunc
-		if intf.stallTimer != nil {
-			intf.stallTimer.Stop()
-			intf.stallTimer = nil
-			intf.writer.sendFrom(nil, [][]byte{nil}, true) // Empty keep-alive traffic
+		if intf.keepAliveTimer != nil {
+			intf.keepAliveTimer.Stop()
+			intf.keepAliveTimer = nil
+			intf.writer.sendFrom(nil, [][]byte{nil}) // Empty keep-alive traffic
 		}
 	})
 }
@@ -384,34 +493,23 @@ func (intf *linkInterface) notifyDoKeepAlive() {
 
 type linkWriter struct {
 	phony.Inbox
-	intf *linkInterface
+	intf   *link
+	worker chan [][]byte
+	closed bool
 }
 
-func (w *linkWriter) sendFrom(from phony.Actor, bss [][]byte, isLinkTraffic bool) {
+func (w *linkWriter) sendFrom(from phony.Actor, bss [][]byte) {
 	w.Act(from, func() {
+		if w.closed {
+			return
+		}
 		var size int
 		for _, bs := range bss {
 			size += len(bs)
 		}
-		w.intf.notifySending(size, isLinkTraffic)
-		// start a timer that will fire if we get stuck in writeMsgs for an oddly long time
-		var once sync.Once
-		timer := time.AfterFunc(time.Millisecond, func() {
-			// 1 ms is kind of arbitrary
-			// the rationale is that this should be very long compared to a syscall
-			// but it's still short compared to end-to-end latency or human perception
-			once.Do(func() {
-				w.intf.Act(nil, w.intf._notifySyscall)
-			})
-		})
-		w.intf.msgIO.writeMsgs(bss)
-		// Make sure we either stop the timer from doing anything or wait until it's done
-		once.Do(func() { timer.Stop() })
-		w.intf.notifySent(size, isLinkTraffic)
-		// Cleanup
-		for _, bs := range bss {
-			util.PutBytes(bs)
-		}
+		w.intf.notifySending(size)
+		w.worker <- bss
+		w.intf.notifySent(size)
 	})
 }
 
@@ -419,7 +517,7 @@ func (w *linkWriter) sendFrom(from phony.Actor, bss [][]byte, isLinkTraffic bool
 
 type linkReader struct {
 	phony.Inbox
-	intf *linkInterface
+	intf *link
 	err  chan error
 }
 

src/yggdrasil/listener.go

@@ -39,7 +39,7 @@ func (l *Listener) Close() (err error) {
 	return nil
 }
 
-// Addr is not implemented for this type yet
+// Addr returns the address of the listener
 func (l *Listener) Addr() net.Addr {
-	return nil
+	return &l.core.boxPub
 }

src/yggdrasil/nodeinfo.go

@@ -5,21 +5,20 @@ import (
 	"errors"
 	"runtime"
 	"strings"
-	"sync"
 	"time"
 
+	"github.com/Arceliar/phony"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/version"
 )
 
 type nodeinfo struct {
-	core            *Core
-	myNodeInfo      NodeInfoPayload
-	myNodeInfoMutex sync.RWMutex
-	callbacks       map[crypto.BoxPubKey]nodeinfoCallback
-	callbacksMutex  sync.Mutex
-	cache           map[crypto.BoxPubKey]nodeinfoCached
-	cacheMutex      sync.RWMutex
+	phony.Inbox
+	core       *Core
+	myNodeInfo NodeInfoPayload
+	callbacks  map[crypto.BoxPubKey]nodeinfoCallback
+	cache      map[crypto.BoxPubKey]nodeinfoCached
+	table      *lookupTable
 }
 
 type nodeinfoCached struct {
@@ -43,35 +42,43 @@ type nodeinfoReqRes struct {
 // Initialises the nodeinfo cache/callback maps, and starts a goroutine to keep
 // the cache/callback maps clean of stale entries
 func (m *nodeinfo) init(core *Core) {
+	m.Act(nil, func() {
+		m._init(core)
+	})
+}
+
+func (m *nodeinfo) _init(core *Core) {
 	m.core = core
 	m.callbacks = make(map[crypto.BoxPubKey]nodeinfoCallback)
 	m.cache = make(map[crypto.BoxPubKey]nodeinfoCached)
 
-	var f func()
-	f = func() {
-		m.callbacksMutex.Lock()
-		for boxPubKey, callback := range m.callbacks {
-			if time.Since(callback.created) > time.Minute {
-				delete(m.callbacks, boxPubKey)
-			}
+	m._cleanup()
+}
+
+func (m *nodeinfo) _cleanup() {
+	for boxPubKey, callback := range m.callbacks {
+		if time.Since(callback.created) > time.Minute {
+			delete(m.callbacks, boxPubKey)
 		}
-		m.callbacksMutex.Unlock()
-		m.cacheMutex.Lock()
-		for boxPubKey, cache := range m.cache {
-			if time.Since(cache.created) > time.Hour {
-				delete(m.cache, boxPubKey)
-			}
-		}
-		m.cacheMutex.Unlock()
-		time.AfterFunc(time.Second*30, f)
 	}
-	go f()
+	for boxPubKey, cache := range m.cache {
+		if time.Since(cache.created) > time.Hour {
+			delete(m.cache, boxPubKey)
+		}
+	}
+	time.AfterFunc(time.Second*30, func() {
+		m.Act(nil, m._cleanup)
+	})
 }
 
 // Add a callback for a nodeinfo lookup
 func (m *nodeinfo) addCallback(sender crypto.BoxPubKey, call func(nodeinfo *NodeInfoPayload)) {
-	m.callbacksMutex.Lock()
-	defer m.callbacksMutex.Unlock()
+	m.Act(nil, func() {
+		m._addCallback(sender, call)
+	})
+}
+
+func (m *nodeinfo) _addCallback(sender crypto.BoxPubKey, call func(nodeinfo *NodeInfoPayload)) {
 	m.callbacks[sender] = nodeinfoCallback{
 		created: time.Now(),
 		call:    call,
@@ -79,9 +86,7 @@ func (m *nodeinfo) addCallback(sender crypto.BoxPubKey, call func(nodeinfo *Node
 }
 
 // Handles the callback, if there is one
-func (m *nodeinfo) callback(sender crypto.BoxPubKey, nodeinfo NodeInfoPayload) {
-	m.callbacksMutex.Lock()
-	defer m.callbacksMutex.Unlock()
+func (m *nodeinfo) _callback(sender crypto.BoxPubKey, nodeinfo NodeInfoPayload) {
 	if callback, ok := m.callbacks[sender]; ok {
 		callback.call(&nodeinfo)
 		delete(m.callbacks, sender)
@@ -89,16 +94,26 @@ func (m *nodeinfo) callback(sender crypto.BoxPubKey, nodeinfo NodeInfoPayload) {
 }
 
 // Get the current node's nodeinfo
-func (m *nodeinfo) getNodeInfo() NodeInfoPayload {
-	m.myNodeInfoMutex.RLock()
-	defer m.myNodeInfoMutex.RUnlock()
+func (m *nodeinfo) getNodeInfo() (p NodeInfoPayload) {
+	phony.Block(m, func() {
+		p = m._getNodeInfo()
+	})
+	return
+}
+
+func (m *nodeinfo) _getNodeInfo() NodeInfoPayload {
 	return m.myNodeInfo
 }
 
 // Set the current node's nodeinfo
-func (m *nodeinfo) setNodeInfo(given interface{}, privacy bool) error {
-	m.myNodeInfoMutex.Lock()
-	defer m.myNodeInfoMutex.Unlock()
+func (m *nodeinfo) setNodeInfo(given interface{}, privacy bool) (err error) {
+	phony.Block(m, func() {
+		err = m._setNodeInfo(given, privacy)
+	})
+	return
+}
+
+func (m *nodeinfo) _setNodeInfo(given interface{}, privacy bool) error {
 	defaults := map[string]interface{}{
 		"buildname":     version.BuildName(),
 		"buildversion":  version.BuildVersion(),
@@ -122,21 +137,19 @@ func (m *nodeinfo) setNodeInfo(given interface{}, privacy bool) error {
 			newnodeinfo[key] = value
 		}
 	}
-	if newjson, err := json.Marshal(newnodeinfo); err == nil {
+	newjson, err := json.Marshal(newnodeinfo)
+	if err == nil {
 		if len(newjson) > 16384 {
 			return errors.New("NodeInfo exceeds max length of 16384 bytes")
 		}
 		m.myNodeInfo = newjson
 		return nil
-	} else {
-		return err
 	}
+	return err
 }
 
 // Add nodeinfo into the cache for a node
-func (m *nodeinfo) addCachedNodeInfo(key crypto.BoxPubKey, payload NodeInfoPayload) {
-	m.cacheMutex.Lock()
-	defer m.cacheMutex.Unlock()
+func (m *nodeinfo) _addCachedNodeInfo(key crypto.BoxPubKey, payload NodeInfoPayload) {
 	m.cache[key] = nodeinfoCached{
 		created: time.Now(),
 		payload: payload,
@@ -144,9 +157,7 @@ func (m *nodeinfo) addCachedNodeInfo(key crypto.BoxPubKey, payload NodeInfoPaylo
 }
 
 // Get a nodeinfo entry from the cache
-func (m *nodeinfo) getCachedNodeInfo(key crypto.BoxPubKey) (NodeInfoPayload, error) {
-	m.cacheMutex.RLock()
-	defer m.cacheMutex.RUnlock()
+func (m *nodeinfo) _getCachedNodeInfo(key crypto.BoxPubKey) (NodeInfoPayload, error) {
 	if nodeinfo, ok := m.cache[key]; ok {
 		return nodeinfo.payload, nil
 	}
@@ -154,22 +165,34 @@ func (m *nodeinfo) getCachedNodeInfo(key crypto.BoxPubKey) (NodeInfoPayload, err
 }
 
 // Handles a nodeinfo request/response - called from the router
-func (m *nodeinfo) handleNodeInfo(nodeinfo *nodeinfoReqRes) {
+func (m *nodeinfo) handleNodeInfo(from phony.Actor, nodeinfo *nodeinfoReqRes) {
+	m.Act(from, func() {
+		m._handleNodeInfo(nodeinfo)
+	})
+}
+
+func (m *nodeinfo) _handleNodeInfo(nodeinfo *nodeinfoReqRes) {
 	if nodeinfo.IsResponse {
-		m.callback(nodeinfo.SendPermPub, nodeinfo.NodeInfo)
-		m.addCachedNodeInfo(nodeinfo.SendPermPub, nodeinfo.NodeInfo)
+		m._callback(nodeinfo.SendPermPub, nodeinfo.NodeInfo)
+		m._addCachedNodeInfo(nodeinfo.SendPermPub, nodeinfo.NodeInfo)
 	} else {
-		m.sendNodeInfo(nodeinfo.SendPermPub, nodeinfo.SendCoords, true)
+		m._sendNodeInfo(nodeinfo.SendPermPub, nodeinfo.SendCoords, true)
 	}
 }
 
 // Send nodeinfo request or response - called from the router
 func (m *nodeinfo) sendNodeInfo(key crypto.BoxPubKey, coords []byte, isResponse bool) {
-	table := m.core.switchTable.table.Load().(lookupTable)
+	m.Act(nil, func() {
+		m._sendNodeInfo(key, coords, isResponse)
+	})
+}
+
+func (m *nodeinfo) _sendNodeInfo(key crypto.BoxPubKey, coords []byte, isResponse bool) {
+	loc := m.table.self
 	nodeinfo := nodeinfoReqRes{
-		SendCoords: table.self.getCoords(),
+		SendCoords: loc.getCoords(),
 		IsResponse: isResponse,
-		NodeInfo:   m.getNodeInfo(),
+		NodeInfo:   m._getNodeInfo(),
 	}
 	bs := nodeinfo.encode()
 	shared := m.core.router.sessions.getSharedKey(&m.core.boxPriv, &key)

src/yggdrasil/packetqueue.go

@@ -0,0 +1,118 @@
+package yggdrasil
+
+import (
+	"container/heap"
+	"time"
+)
+
+// TODO separate queues per e.g. traffic flow
+//  For now, we put everything in queue
+
+type pqStreamID string
+
+type pqPacketInfo struct {
+	packet []byte
+	time   time.Time
+}
+
+type pqStream struct {
+	id    pqStreamID
+	infos []pqPacketInfo
+	size  uint64
+}
+
+type packetQueue struct {
+	streams []pqStream
+	size    uint64
+}
+
+// drop will remove a packet from the queue, returning it to the pool
+//  returns true if a packet was removed, false otherwise
+func (q *packetQueue) drop() bool {
+	if q.size == 0 {
+		return false
+	}
+	var longestIdx int
+	for idx := range q.streams {
+		if q.streams[idx].size > q.streams[longestIdx].size {
+			longestIdx = idx
+		}
+	}
+	stream := q.streams[longestIdx]
+	info := stream.infos[0]
+	if len(stream.infos) > 1 {
+		stream.infos = stream.infos[1:]
+		stream.size -= uint64(len(info.packet))
+		q.streams[longestIdx] = stream
+		q.size -= uint64(len(info.packet))
+		heap.Fix(q, longestIdx)
+	} else {
+		heap.Remove(q, longestIdx)
+	}
+	pool_putBytes(info.packet)
+	return true
+}
+
+func (q *packetQueue) push(packet []byte) {
+	id := pqStreamID(peer_getPacketCoords(packet)) // just coords for now
+	info := pqPacketInfo{packet: packet, time: time.Now()}
+	for idx := range q.streams {
+		if q.streams[idx].id == id {
+			q.streams[idx].infos = append(q.streams[idx].infos, info)
+			q.streams[idx].size += uint64(len(packet))
+			q.size += uint64(len(packet))
+			return
+		}
+	}
+	stream := pqStream{id: id, size: uint64(len(packet))}
+	stream.infos = append(stream.infos, info)
+	heap.Push(q, stream)
+}
+
+func (q *packetQueue) pop() ([]byte, bool) {
+	if q.size > 0 {
+		stream := q.streams[0]
+		info := stream.infos[0]
+		if len(stream.infos) > 1 {
+			stream.infos = stream.infos[1:]
+			stream.size -= uint64(len(info.packet))
+			q.streams[0] = stream
+			q.size -= uint64(len(info.packet))
+			heap.Fix(q, 0)
+		} else {
+			heap.Remove(q, 0)
+		}
+		return info.packet, true
+	}
+	return nil, false
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+// Interface methods for packetQueue to satisfy heap.Interface
+
+func (q *packetQueue) Len() int {
+	return len(q.streams)
+}
+
+func (q *packetQueue) Less(i, j int) bool {
+	return q.streams[i].infos[0].time.Before(q.streams[j].infos[0].time)
+}
+
+func (q *packetQueue) Swap(i, j int) {
+	q.streams[i], q.streams[j] = q.streams[j], q.streams[i]
+}
+
+func (q *packetQueue) Push(x interface{}) {
+	stream := x.(pqStream)
+	q.streams = append(q.streams, stream)
+	q.size += stream.size
+}
+
+func (q *packetQueue) Pop() interface{} {
+	idx := len(q.streams) - 1
+	stream := q.streams[idx]
+	q.streams = q.streams[:idx]
+	q.size -= stream.size
+	return stream
+}

src/yggdrasil/peer.go

@@ -6,12 +6,9 @@ package yggdrasil
 
 import (
 	"encoding/hex"
-	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
-	"github.com/yggdrasil-network/yggdrasil-go/src/util"
 
 	"github.com/Arceliar/phony"
 )
@@ -19,19 +16,19 @@ import (
 // The peers struct represents peers with an active connection.
 // Incoming packets are passed to the corresponding peer, which handles them somehow.
 // In most cases, this involves passing the packet to the handler for outgoing traffic to another peer.
-// In other cases, it's link protocol traffic used to build the spanning tree, in which case this checks signatures and passes the message along to the switch.
+// In other cases, its link protocol traffic is used to build the spanning tree, in which case this checks signatures and passes the message along to the switch.
 type peers struct {
+	phony.Inbox
 	core  *Core
-	mutex sync.Mutex   // Synchronize writes to atomic
-	ports atomic.Value //map[switchPort]*peer, use CoW semantics
+	ports map[switchPort]*peer // use CoW semantics, share updated version with each peer
+	table *lookupTable         // Sent from switch, share updated version with each peer
 }
 
 // Initializes the peers struct.
 func (ps *peers) init(c *Core) {
-	ps.mutex.Lock()
-	defer ps.mutex.Unlock()
-	ps.putPorts(make(map[switchPort]*peer))
 	ps.core = c
+	ps.ports = make(map[switchPort]*peer)
+	ps.table = new(lookupTable)
 }
 
 func (ps *peers) reconfigure() {
@@ -80,54 +77,62 @@ func (ps *peers) getAllowedEncryptionPublicKeys() []string {
 	return ps.core.config.Current.AllowedEncryptionPublicKeys
 }
 
-// Atomically gets a map[switchPort]*peer of known peers.
-func (ps *peers) getPorts() map[switchPort]*peer {
-	return ps.ports.Load().(map[switchPort]*peer)
-}
-
-// Stores a map[switchPort]*peer (note that you should take a mutex before store operations to avoid conflicts with other nodes attempting to read/change/store at the same time).
-func (ps *peers) putPorts(ports map[switchPort]*peer) {
-	ps.ports.Store(ports)
-}
-
-// Information known about a peer, including thier box/sig keys, precomputed shared keys (static and ephemeral) and a handler for their outgoing traffic
+// Information known about a peer, including their box/sig keys, precomputed shared keys (static and ephemeral) and a handler for their outgoing traffic
 type peer struct {
 	phony.Inbox
 	core       *Core
-	intf       *linkInterface
+	intf       linkInterface
 	port       switchPort
 	box        crypto.BoxPubKey
 	sig        crypto.SigPubKey
 	shared     crypto.BoxSharedKey
 	linkShared crypto.BoxSharedKey
 	endpoint   string
-	firstSeen  time.Time       // To track uptime for getPeers
-	linkOut    func([]byte)    // used for protocol traffic (bypasses the switch)
-	dinfo      *dhtInfo        // used to keep the DHT working
-	out        func([][]byte)  // Set up by whatever created the peers struct, used to send packets to other nodes
-	done       (chan struct{}) // closed to exit the linkLoop
-	close      func()          // Called when a peer is removed, to close the underlying connection, or via admin api
+	firstSeen  time.Time // To track uptime for getPeers
+	dinfo      *dhtInfo  // used to keep the DHT working
 	// The below aren't actually useful internally, they're just gathered for getPeers statistics
 	bytesSent  uint64
 	bytesRecvd uint64
+	ports      map[switchPort]*peer
+	table      *lookupTable
+	queue      packetQueue
+	max        uint64
+	seq        uint64 // this and idle are used to detect when to drop packets from queue
+	idle       bool
+	drop       bool // set to true if we're dropping packets from the queue
+}
+
+func (ps *peers) updateTables(from phony.Actor, table *lookupTable) {
+	ps.Act(from, func() {
+		ps.table = table
+		ps._updatePeers()
+	})
+}
+
+func (ps *peers) _updatePeers() {
+	ports := ps.ports
+	table := ps.table
+	for _, peer := range ps.ports {
+		p := peer // peer is mutated during iteration
+		p.Act(ps, func() {
+			p.ports = ports
+			p.table = table
+		})
+	}
 }
 
 // Creates a new peer with the specified box, sig, and linkShared keys, using the lowest unoccupied port number.
-func (ps *peers) newPeer(box *crypto.BoxPubKey, sig *crypto.SigPubKey, linkShared *crypto.BoxSharedKey, intf *linkInterface, closer func()) *peer {
+func (ps *peers) _newPeer(box *crypto.BoxPubKey, sig *crypto.SigPubKey, linkShared *crypto.BoxSharedKey, intf linkInterface) *peer {
 	now := time.Now()
 	p := peer{box: *box,
+		core:       ps.core,
+		intf:       intf,
 		sig:        *sig,
 		shared:     *crypto.GetSharedKey(&ps.core.boxPriv, box),
 		linkShared: *linkShared,
 		firstSeen:  now,
-		done:       make(chan struct{}),
-		close:      closer,
-		core:       ps.core,
-		intf:       intf,
 	}
-	ps.mutex.Lock()
-	defer ps.mutex.Unlock()
-	oldPorts := ps.getPorts()
+	oldPorts := ps.ports
 	newPorts := make(map[switchPort]*peer)
 	for k, v := range oldPorts {
 		newPorts[k] = v
@@ -139,63 +144,62 @@ func (ps *peers) newPeer(box *crypto.BoxPubKey, sig *crypto.SigPubKey, linkShare
 			break
 		}
 	}
-	ps.putPorts(newPorts)
+	ps.ports = newPorts
+	ps._updatePeers()
 	return &p
 }
 
-// Removes a peer for a given port, if one exists.
-func (ps *peers) removePeer(port switchPort) {
-	if port == 0 {
-		return
-	} // Can't remove self peer
-	phony.Block(&ps.core.router, func() {
-		ps.core.switchTable.forgetPeer(port)
+func (p *peer) _removeSelf() {
+	p.core.peers.Act(p, func() {
+		p.core.peers._removePeer(p)
 	})
-	ps.mutex.Lock()
-	oldPorts := ps.getPorts()
-	p, isIn := oldPorts[port]
+}
+
+// Removes a peer for a given port, if one exists.
+func (ps *peers) _removePeer(p *peer) {
+	if q := ps.ports[p.port]; p.port == 0 || q != p {
+		return
+	} // Can't remove self peer or nonexistant peer
+	ps.core.switchTable.forgetPeer(ps, p.port)
+	oldPorts := ps.ports
 	newPorts := make(map[switchPort]*peer)
 	for k, v := range oldPorts {
 		newPorts[k] = v
 	}
-	delete(newPorts, port)
-	ps.putPorts(newPorts)
-	ps.mutex.Unlock()
-	if isIn {
-		if p.close != nil {
-			p.close()
-		}
-		close(p.done)
-	}
+	delete(newPorts, p.port)
+	p.intf.close()
+	ps.ports = newPorts
+	ps._updatePeers()
 }
 
 // If called, sends a notification to each peer that they should send a new switch message.
 // Mainly called by the switch after an update.
 func (ps *peers) sendSwitchMsgs(from phony.Actor) {
-	ports := ps.getPorts()
-	for _, p := range ports {
-		if p.port == 0 {
-			continue
+	ps.Act(from, func() {
+		for _, peer := range ps.ports {
+			p := peer
+			if p.port == 0 {
+				continue
+			}
+			p.Act(ps, p._sendSwitchMsg)
 		}
-		p.Act(from, p._sendSwitchMsg)
-	}
+	})
+}
+
+func (ps *peers) updateDHT(from phony.Actor) {
+	ps.Act(from, func() {
+		for _, peer := range ps.ports {
+			p := peer
+			if p.port == 0 {
+				continue
+			}
+			p.Act(ps, p._updateDHT)
+		}
+	})
 }
 
 // This must be launched in a separate goroutine by whatever sets up the peer struct.
-// It handles link protocol traffic.
 func (p *peer) start() {
-	var updateDHT func()
-	updateDHT = func() {
-		phony.Block(p, func() {
-			select {
-			case <-p.done:
-			default:
-				p._updateDHT()
-				time.AfterFunc(time.Second, updateDHT)
-			}
-		})
-	}
-	updateDHT()
 	// Just for good measure, immediately send a switch message to this peer when we start
 	p.Act(nil, p._sendSwitchMsg)
 }
@@ -229,37 +233,81 @@ func (p *peer) _handlePacket(packet []byte) {
 	case wire_LinkProtocolTraffic:
 		p._handleLinkTraffic(packet)
 	default:
-		util.PutBytes(packet)
 	}
 }
 
+// Get the coords of a packet without decoding
+func peer_getPacketCoords(packet []byte) []byte {
+	_, pTypeLen := wire_decode_uint64(packet)
+	coords, _ := wire_decode_coords(packet[pTypeLen:])
+	return coords
+}
+
 // Called to handle traffic or protocolTraffic packets.
 // In either case, this reads from the coords of the packet header, does a switch lookup, and forwards to the next node.
 func (p *peer) _handleTraffic(packet []byte) {
-	table := p.core.switchTable.getTable()
-	if _, isIn := table.elems[p.port]; !isIn && p.port != 0 {
+	if _, isIn := p.table.elems[p.port]; !isIn && p.port != 0 {
 		// Drop traffic if the peer isn't in the switch
 		return
 	}
-	p.core.switchTable.packetInFrom(p, packet)
+	coords := peer_getPacketCoords(packet)
+	next := p.table.lookup(coords)
+	if nPeer, isIn := p.ports[next]; isIn {
+		nPeer.sendPacketFrom(p, packet)
+	}
+	//p.core.switchTable.packetInFrom(p, packet)
 }
 
-func (p *peer) sendPacketsFrom(from phony.Actor, packets [][]byte) {
+func (p *peer) sendPacketFrom(from phony.Actor, packet []byte) {
 	p.Act(from, func() {
-		p._sendPackets(packets)
+		p._sendPacket(packet)
 	})
 }
 
-// This just calls p.out(packet) for now.
-func (p *peer) _sendPackets(packets [][]byte) {
-	// Is there ever a case where something more complicated is needed?
-	// What if p.out blocks?
-	var size int
-	for _, packet := range packets {
-		size += len(packet)
+func (p *peer) _sendPacket(packet []byte) {
+	p.queue.push(packet)
+	if p.idle {
+		p.idle = false
+		p._handleIdle()
+	} else if p.drop {
+		for p.queue.size > p.max {
+			p.queue.drop()
+		}
 	}
-	p.bytesSent += uint64(size)
-	p.out(packets)
+}
+
+func (p *peer) _handleIdle() {
+	var packets [][]byte
+	var size uint64
+	for {
+		if packet, success := p.queue.pop(); success {
+			packets = append(packets, packet)
+			size += uint64(len(packet))
+		} else {
+			break
+		}
+	}
+	p.seq++
+	if len(packets) > 0 {
+		p.bytesSent += uint64(size)
+		p.intf.out(packets)
+		p.max = p.queue.size
+	} else {
+		p.idle = true
+	}
+	p.drop = false
+}
+
+func (p *peer) notifyBlocked(from phony.Actor) {
+	p.Act(from, func() {
+		seq := p.seq
+		p.Act(nil, func() {
+			if seq == p.seq {
+				p.drop = true
+				p.max = 2*p.queue.size + streamMsgSize
+			}
+		})
+	})
 }
 
 // This wraps the packet in the inner (ephemeral) and outer (permanent) crypto layers.
@@ -277,7 +325,7 @@ func (p *peer) _sendLinkPacket(packet []byte) {
 		Payload: bs,
 	}
 	packet = linkPacket.encode()
-	p.linkOut(packet)
+	p.intf.linkOut(packet)
 }
 
 // Decrypts the outer (permanent) and inner (ephemeral) crypto layers on link traffic.
@@ -307,13 +355,12 @@ func (p *peer) _handleLinkTraffic(bs []byte) {
 	case wire_SwitchMsg:
 		p._handleSwitchMsg(payload)
 	default:
-		util.PutBytes(bs)
 	}
 }
 
 // Gets a switchMsg from the switch, adds signed next-hop info for this peer, and sends it to them.
 func (p *peer) _sendSwitchMsg() {
-	msg := p.core.switchTable.getMsg()
+	msg := p.table.getMsg()
 	if msg == nil {
 		return
 	}
@@ -335,7 +382,8 @@ func (p *peer) _handleSwitchMsg(packet []byte) {
 		return
 	}
 	if len(msg.Hops) < 1 {
-		p.core.peers.removePeer(p.port)
+		p._removeSelf()
+		return
 	}
 	var loc switchLocator
 	prevKey := msg.Root
@@ -346,23 +394,31 @@ func (p *peer) _handleSwitchMsg(packet []byte) {
 		loc.coords = append(loc.coords, hop.Port)
 		bs := getBytesForSig(&hop.Next, &sigMsg)
 		if !crypto.Verify(&prevKey, bs, &hop.Sig) {
-			p.core.peers.removePeer(p.port)
+			p._removeSelf()
+			return
 		}
 		prevKey = hop.Next
 	}
-	p.core.switchTable.handleMsg(&msg, p.port)
-	if !p.core.switchTable.checkRoot(&msg) {
-		// Bad switch message
-		p.dinfo = nil
-		return
-	}
-	// Pass a mesage to the dht informing it that this peer (still) exists
-	loc.coords = loc.coords[:len(loc.coords)-1]
-	p.dinfo = &dhtInfo{
-		key:    p.box,
-		coords: loc.getCoords(),
-	}
-	p._updateDHT()
+	p.core.switchTable.Act(p, func() {
+		if !p.core.switchTable._checkRoot(&msg) {
+			// Bad switch message
+			p.Act(&p.core.switchTable, func() {
+				p.dinfo = nil
+			})
+		} else {
+			// handle the message
+			p.core.switchTable._handleMsg(&msg, p.port, false)
+			p.Act(&p.core.switchTable, func() {
+				// Pass a message to the dht informing it that this peer (still) exists
+				loc.coords = loc.coords[:len(loc.coords)-1]
+				p.dinfo = &dhtInfo{
+					key:    p.box,
+					coords: loc.getCoords(),
+				}
+				p._updateDHT()
+			})
+		}
+	})
 }
 
 // This generates the bytes that we sign or check the signature of for a switchMsg.

src/yggdrasil/pool.go

@@ -0,0 +1,20 @@
+package yggdrasil
+
+import "sync"
+
+// Used internally to reduce allocations in the hot loop
+//  I.e. packets being switched or between the crypto and the switch
+// For safety reasons, these must not escape this package
+var pool = sync.Pool{New: func() interface{} { return []byte(nil) }}
+
+func pool_getBytes(size int) []byte {
+	bs := pool.Get().([]byte)
+	if cap(bs) < size {
+		bs = make([]byte, size)
+	}
+	return bs[:size]
+}
+
+func pool_putBytes(bs []byte) {
+	pool.Put(bs)
+}

src/yggdrasil/router.go

@@ -29,7 +29,6 @@ import (
 
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
-	"github.com/yggdrasil-network/yggdrasil-go/src/util"
 
 	"github.com/Arceliar/phony"
 )
@@ -46,6 +45,9 @@ type router struct {
 	nodeinfo nodeinfo
 	searches searches
 	sessions sessions
+	intf     routerInterface
+	peer     *peer
+	table    *lookupTable // has a copy of our locator
 }
 
 // Initializes the router struct, which includes setting up channels to/from the adapter.
@@ -53,17 +55,15 @@ func (r *router) init(core *Core) {
 	r.core = core
 	r.addr = *address.AddrForNodeID(&r.dht.nodeID)
 	r.subnet = *address.SubnetForNodeID(&r.dht.nodeID)
-	self := linkInterface{
-		name: "(self)",
-		info: linkInfo{
-			local:    "(self)",
-			remote:   "(self)",
-			linkType: "self",
-		},
+	r.intf.router = r
+	phony.Block(&r.core.peers, func() {
+		// FIXME don't block here!
+		r.peer = r.core.peers._newPeer(&r.core.boxPub, &r.core.sigPub, &crypto.BoxSharedKey{}, &r.intf)
+	})
+	r.peer.Act(r, r.peer._handleIdle)
+	r.out = func(bs []byte) {
+		r.peer.handlePacketFrom(r, bs)
 	}
-	p := r.core.peers.newPeer(&r.core.boxPub, &r.core.sigPub, &crypto.BoxSharedKey{}, &self, nil)
-	p.out = func(packets [][]byte) { r.handlePackets(p, packets) }
-	r.out = func(bs []byte) { p.handlePacketFrom(r, bs) }
 	r.nodeinfo.init(r.core)
 	r.core.config.Mutex.RLock()
 	r.nodeinfo.setNodeInfo(r.core.config.Current.NodeInfo, r.core.config.Current.NodeInfoPrivacy)
@@ -73,11 +73,27 @@ func (r *router) init(core *Core) {
 	r.sessions.init(r)
 }
 
+func (r *router) updateTable(from phony.Actor, table *lookupTable) {
+	r.Act(from, func() {
+		r.table = table
+		r.nodeinfo.Act(r, func() {
+			r.nodeinfo.table = table
+		})
+		for _, ses := range r.sessions.sinfos {
+			sinfo := ses
+			sinfo.Act(r, func() {
+				sinfo.table = table
+			})
+		}
+	})
+}
+
 // Reconfigures the router and any child modules. This should only ever be run
 // by the router actor.
 func (r *router) reconfigure() {
 	// Reconfigure the router
 	current := r.core.config.GetCurrent()
+	r.core.log.Println("Reloading NodeInfo...")
 	if err := r.nodeinfo.setNodeInfo(current.NodeInfo, current.NodeInfoPrivacy); err != nil {
 		r.core.log.Errorln("Error reloading NodeInfo:", err)
 	} else {
@@ -96,15 +112,6 @@ func (r *router) start() error {
 	return nil
 }
 
-// In practice, the switch will call this with 1 packet
-func (r *router) handlePackets(from phony.Actor, packets [][]byte) {
-	r.Act(from, func() {
-		for _, packet := range packets {
-			r._handlePacket(packet)
-		}
-	})
-}
-
 // Insert a peer info into the dht, TODO? make the dht a separate actor
 func (r *router) insertPeer(from phony.Actor, info *dhtInfo) {
 	r.Act(from, func() {
@@ -125,7 +132,7 @@ func (r *router) reset(from phony.Actor) {
 func (r *router) doMaintenance() {
 	phony.Block(r, func() {
 		// Any periodic maintenance stuff goes here
-		r.core.switchTable.doMaintenance()
+		r.core.switchTable.doMaintenance(r)
 		r.dht.doMaintenance()
 		r.sessions.cleanup()
 	})
@@ -150,14 +157,12 @@ func (r *router) _handlePacket(packet []byte) {
 // Handles incoming traffic, i.e. encapuslated ordinary IPv6 packets.
 // Passes them to the crypto session worker to be decrypted and sent to the adapter.
 func (r *router) _handleTraffic(packet []byte) {
-	defer util.PutBytes(packet)
 	p := wire_trafficPacket{}
 	if !p.decode(packet) {
 		return
 	}
 	sinfo, isIn := r.sessions.getSessionForHandle(&p.Handle)
 	if !isIn {
-		util.PutBytes(p.Payload)
 		return
 	}
 	sinfo.recv(r, &p)
@@ -203,7 +208,6 @@ func (r *router) _handleProto(packet []byte) {
 	case wire_DHTLookupResponse:
 		r._handleDHTRes(bs, &p.FromKey)
 	default:
-		util.PutBytes(packet)
 	}
 }
 
@@ -249,5 +253,37 @@ func (r *router) _handleNodeInfo(bs []byte, fromKey *crypto.BoxPubKey) {
 		return
 	}
 	req.SendPermPub = *fromKey
-	r.nodeinfo.handleNodeInfo(&req)
+	r.nodeinfo.handleNodeInfo(r, &req)
 }
+
+////////////////////////////////////////////////////////////////////////////////
+
+// routerInterface is a helper that implements linkInterface
+type routerInterface struct {
+	router *router
+}
+
+func (intf *routerInterface) out(bss [][]byte) {
+	// Note that this is run in the peer's goroutine
+	intf.router.Act(intf.router.peer, func() {
+		for _, bs := range bss {
+			intf.router._handlePacket(bs)
+		}
+	})
+	// This should now immediately make the peer idle again
+	// So the self-peer shouldn't end up buffering anything
+	// We let backpressure act as a throttle instead
+	intf.router.peer._handleIdle()
+}
+
+func (intf *routerInterface) linkOut(_ []byte) {}
+
+func (intf *routerInterface) close() {}
+
+func (intf *routerInterface) name() string { return "(self)" }
+
+func (intf *routerInterface) local() string { return "(self)" }
+
+func (intf *routerInterface) remote() string { return "(self)" }
+
+func (intf *routerInterface) interfaceType() string { return "self" }

src/yggdrasil/search.go

@@ -4,15 +4,13 @@ package yggdrasil
 
 // The basic idea is as follows:
 //  We may know a NodeID (with a mask) and want to connect
-//  We begin a search by initializing a list of all nodes in our DHT, sorted by closest to the destination
-//  We then iteratively ping nodes from the search, marking each pinged node as visited
-//  We add any unvisited nodes from ping responses to the search, truncating to some maximum search size
-//  This stops when we either run out of nodes to ping (we hit a dead end where we can't make progress without going back), or we reach the destination
-//  A new search packet is sent immediately after receiving a response
-//  A new search packet is sent periodically, once per second, in case a packet was dropped (this slowly causes the search to become parallel if the search doesn't timeout but also doesn't finish within 1 second for whatever reason)
-
-// TODO?
-//  Some kind of max search steps, in case the node is offline, so we don't crawl through too much of the network looking for a destination that isn't there?
+//  We begin a search by sending a dht lookup to ourself
+//  Each time a node responds, we sort the results and filter to only include useful nodes
+//  We then periodically send a packet to the first node from the list (after re-filtering)
+//  This happens in parallel for each node that replies
+//  Meanwhile, we keep a list of the (up to) 16 closest nodes to the destination that we've visited
+//  We only consider an unvisited node useful if either the list isn't full or the unvisited node is closer to the destination than the furthest node on the list
+//  That gives the search some chance to recover if it hits a dead end where a node doesn't know everyone it should
 
 import (
 	"errors"
@@ -22,13 +20,10 @@ import (
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 )
 
-// This defines the maximum number of dhtInfo that we keep track of for nodes to query in an ongoing search.
-const search_MAX_SEARCH_SIZE = 16
-
-// This defines the time after which we send a new search packet.
-// Search packets are sent automatically immediately after a response is received.
-// So this allows for timeouts and for long searches to become increasingly parallel.
-const search_RETRY_TIME = time.Second
+// This defines the time after which we time out a search (so it can restart).
+const search_RETRY_TIME = 3 * time.Second
+const search_STEP_TIME = time.Second
+const search_MAX_RESULTS = dht_lookup_size
 
 // Information about an ongoing search.
 // Includes the target NodeID, the bitmask to match it to an IP, and the list of nodes to visit / already visited.
@@ -37,10 +32,11 @@ type searchInfo struct {
 	dest     crypto.NodeID
 	mask     crypto.NodeID
 	time     time.Time
-	toVisit  []*dhtInfo
-	visited  map[crypto.NodeID]bool
+	visited  []*crypto.NodeID // Closest addresses visited so far
 	callback func(*sessionInfo, error)
 	// TODO context.Context for timeout and cancellation
+	send uint64 // log number of requests sent
+	recv uint64 // log number of responses received
 }
 
 // This stores a map of active searches.
@@ -78,100 +74,127 @@ func (s *searches) createSearch(dest *crypto.NodeID, mask *crypto.NodeID, callba
 // If there is, it adds the response info to the search and triggers a new search step.
 // If there's no ongoing search, or we if the dhtRes finished the search (it was from the target node), then don't do anything more.
 func (sinfo *searchInfo) handleDHTRes(res *dhtRes) {
-	if res == nil || sinfo.checkDHTRes(res) {
-		// Either we don't recognize this search, or we just finished it
-		return
+	if nfo := sinfo.searches.searches[sinfo.dest]; nfo != sinfo {
+		return // already done
+	}
+	if res != nil {
+		sinfo.recv++
+		if sinfo.checkDHTRes(res) {
+			return // Search finished successfully
+		}
+		// Use results to start an additional search thread
+		infos := append([]*dhtInfo(nil), res.Infos...)
+		infos = sinfo.getAllowedInfos(infos)
+		if len(infos) > 0 {
+			sinfo.continueSearch(infos)
+		}
 	}
-	// Add to the search and continue
-	sinfo.addToSearch(res)
-	sinfo.doSearchStep()
 }
 
-// Adds the information from a dhtRes to an ongoing search.
-// Info about a node that has already been visited is not re-added to the search.
-// Duplicate information about nodes toVisit is deduplicated (the newest information is kept).
-// The toVisit list is sorted in ascending order of keyspace distance from the destination.
-func (sinfo *searchInfo) addToSearch(res *dhtRes) {
-	// Add responses to toVisit if closer to dest than the res node
-	from := dhtInfo{key: res.Key, coords: res.Coords}
-	sinfo.visited[*from.getNodeID()] = true
-	for _, info := range res.Infos {
-		if *info.getNodeID() == sinfo.searches.router.dht.nodeID || sinfo.visited[*info.getNodeID()] {
+// If there has been no response in too long, then this cleans up the search.
+// Otherwise, it pops the closest node to the destination (in keyspace) off of the toVisit list and sends a dht ping.
+func (sinfo *searchInfo) doSearchStep(infos []*dhtInfo) {
+	if len(infos) > 0 {
+		// Send to the next search target
+		next := infos[0]
+		rq := dhtReqKey{next.key, sinfo.dest}
+		sinfo.searches.router.dht.addCallback(&rq, sinfo.handleDHTRes)
+		sinfo.searches.router.dht.ping(next, &sinfo.dest)
+		sinfo.send++
+	}
+}
+
+// Get a list of search targets that are close enough to the destination to try
+// Requires an initial list as input
+func (sinfo *searchInfo) getAllowedInfos(infos []*dhtInfo) []*dhtInfo {
+	var temp []*dhtInfo
+	for _, info := range infos {
+		if false && len(sinfo.visited) < search_MAX_RESULTS {
+			// We're not full on results yet, so don't block anything yet
+		} else if !dht_ordered(&sinfo.dest, info.getNodeID(), sinfo.visited[len(sinfo.visited)-1]) {
+			// Too far away
 			continue
 		}
-		if dht_ordered(&sinfo.dest, info.getNodeID(), from.getNodeID()) {
-			// Response is closer to the destination
-			sinfo.toVisit = append(sinfo.toVisit, info)
+		var known bool
+		for _, nfo := range sinfo.visited {
+			if *nfo == *info.getNodeID() {
+				known = true
+				break
+			}
+		}
+		if !known {
+			temp = append(temp, info)
 		}
 	}
-	// Deduplicate
-	vMap := make(map[crypto.NodeID]*dhtInfo)
-	for _, info := range sinfo.toVisit {
-		vMap[*info.getNodeID()] = info
-	}
-	sinfo.toVisit = sinfo.toVisit[:0]
-	for _, info := range vMap {
-		sinfo.toVisit = append(sinfo.toVisit, info)
-	}
-	// Sort
-	sort.SliceStable(sinfo.toVisit, func(i, j int) bool {
+	infos = append(infos[:0], temp...) // restrict to only the allowed infos
+	sort.SliceStable(infos, func(i, j int) bool {
 		// Should return true if i is closer to the destination than j
-		return dht_ordered(&res.Dest, sinfo.toVisit[i].getNodeID(), sinfo.toVisit[j].getNodeID())
-	})
-	// Truncate to some maximum size
-	if len(sinfo.toVisit) > search_MAX_SEARCH_SIZE {
-		sinfo.toVisit = sinfo.toVisit[:search_MAX_SEARCH_SIZE]
+		return dht_ordered(&sinfo.dest, infos[i].getNodeID(), infos[j].getNodeID())
+	}) // Sort infos to start with the closest
+	if len(infos) > search_MAX_RESULTS {
+		infos = infos[:search_MAX_RESULTS] // Limit max number of infos
 	}
+	return infos
 }
 
-// If there are no nodes left toVisit, then this cleans up the search.
-// Otherwise, it pops the closest node to the destination (in keyspace) off of the toVisit list and sends a dht ping.
-func (sinfo *searchInfo) doSearchStep() {
-	if len(sinfo.toVisit) == 0 {
-		if time.Since(sinfo.time) > search_RETRY_TIME {
-			// Dead end and no response in too long, do cleanup
-			delete(sinfo.searches.searches, sinfo.dest)
-			sinfo.callback(nil, errors.New("search reached dead end"))
-		}
-		return
-	}
-	// Send to the next search target
-	var next *dhtInfo
-	next, sinfo.toVisit = sinfo.toVisit[0], sinfo.toVisit[1:]
-	rq := dhtReqKey{next.key, sinfo.dest}
-	sinfo.searches.router.dht.addCallback(&rq, sinfo.handleDHTRes)
-	sinfo.searches.router.dht.ping(next, &sinfo.dest)
-	sinfo.time = time.Now()
-}
-
-// If we've recenty sent a ping for this search, do nothing.
-// Otherwise, doSearchStep and schedule another continueSearch to happen after search_RETRY_TIME.
-func (sinfo *searchInfo) continueSearch() {
-	sinfo.doSearchStep()
-	// In case the search dies, try to spawn another thread later
-	// Note that this will spawn multiple parallel searches as time passes
-	// Any that die aren't restarted, but a new one will start later
-	time.AfterFunc(search_RETRY_TIME, func() {
+// Run doSearchStep and schedule another continueSearch to happen after search_RETRY_TIME.
+// Must not be called with an empty list of infos
+func (sinfo *searchInfo) continueSearch(infos []*dhtInfo) {
+	sinfo.doSearchStep(infos)
+	infos = infos[1:] // Remove the node we just tried
+	// In case there's no response, try the next node in infos later
+	time.AfterFunc(search_STEP_TIME, func() {
 		sinfo.searches.router.Act(nil, func() {
 			// FIXME this keeps the search alive forever if not for the searches map, fix that
 			newSearchInfo := sinfo.searches.searches[sinfo.dest]
 			if newSearchInfo != sinfo {
 				return
 			}
-			sinfo.continueSearch()
+			// Get good infos here instead of at the top, to make sure we can always start things off with a continueSearch call to ourself
+			infos = sinfo.getAllowedInfos(infos)
+			if len(infos) > 0 {
+				sinfo.continueSearch(infos)
+			}
 		})
 	})
 }
 
+// Initially start a search
+func (sinfo *searchInfo) startSearch() {
+	var infos []*dhtInfo
+	infos = append(infos, &dhtInfo{
+		key:    sinfo.searches.router.core.boxPub,
+		coords: sinfo.searches.router.table.self.getCoords(),
+	})
+	// Start the search by asking ourself, useful if we're the destination
+	sinfo.continueSearch(infos)
+	// Start a timer to clean up the search if everything times out
+	var cleanupFunc func()
+	cleanupFunc = func() {
+		sinfo.searches.router.Act(nil, func() {
+			// FIXME this keeps the search alive forever if not for the searches map, fix that
+			newSearchInfo := sinfo.searches.searches[sinfo.dest]
+			if newSearchInfo != sinfo {
+				return
+			}
+			elapsed := time.Since(sinfo.time)
+			if elapsed > search_RETRY_TIME {
+				// cleanup
+				delete(sinfo.searches.searches, sinfo.dest)
+				sinfo.searches.router.core.log.Debugln("search timeout:", &sinfo.dest, sinfo.send, sinfo.recv)
+				sinfo.callback(nil, errors.New("search reached dead end"))
+				return
+			}
+			time.AfterFunc(search_RETRY_TIME-elapsed, cleanupFunc)
+		})
+	}
+	time.AfterFunc(search_RETRY_TIME, cleanupFunc)
+}
+
 // Calls create search, and initializes the iterative search parts of the struct before returning it.
 func (s *searches) newIterSearch(dest *crypto.NodeID, mask *crypto.NodeID, callback func(*sessionInfo, error)) *searchInfo {
 	sinfo := s.createSearch(dest, mask, callback)
-	sinfo.visited = make(map[crypto.NodeID]bool)
-	loc := s.router.core.switchTable.getLocator()
-	sinfo.toVisit = append(sinfo.toVisit, &dhtInfo{
-		key:    s.router.core.boxPub,
-		coords: loc.getCoords(),
-	}) // Start the search by asking ourself, useful if we're the destination
+	sinfo.visited = append(sinfo.visited, &s.router.dht.nodeID)
 	return sinfo
 }
 
@@ -179,7 +202,30 @@ func (s *searches) newIterSearch(dest *crypto.NodeID, mask *crypto.NodeID, callb
 // If the response is from the target, get/create a session, trigger a session ping, and return true.
 // Otherwise return false.
 func (sinfo *searchInfo) checkDHTRes(res *dhtRes) bool {
-	them := crypto.GetNodeID(&res.Key)
+	from := dhtInfo{key: res.Key, coords: res.Coords}
+	them := from.getNodeID()
+	var known bool
+	for _, v := range sinfo.visited {
+		if *v == *them {
+			known = true
+			break
+		}
+	}
+	if !known {
+		if len(sinfo.visited) < search_MAX_RESULTS || dht_ordered(&sinfo.dest, them, sinfo.visited[len(sinfo.visited)-1]) {
+			// Closer to the destination than the threshold, so update visited
+			sinfo.searches.router.core.log.Debugln("Updating search:", &sinfo.dest, them, sinfo.send, sinfo.recv)
+			sinfo.visited = append(sinfo.visited, them)
+			sort.SliceStable(sinfo.visited, func(i, j int) bool {
+				// Should return true if i is closer to the destination than j
+				return dht_ordered(&sinfo.dest, sinfo.visited[i], sinfo.visited[j])
+			}) // Sort infos to start with the closest
+			if len(sinfo.visited) > search_MAX_RESULTS {
+				sinfo.visited = sinfo.visited[:search_MAX_RESULTS]
+			}
+			sinfo.time = time.Now()
+		}
+	}
 	var destMasked crypto.NodeID
 	var themMasked crypto.NodeID
 	for idx := 0; idx < crypto.NodeIDLen; idx++ {
@@ -201,7 +247,10 @@ func (sinfo *searchInfo) checkDHTRes(res *dhtRes) bool {
 			sinfo.callback(sess, nil)
 		}
 		// Cleanup
-		delete(sinfo.searches.searches, res.Dest)
+		if _, isIn := sinfo.searches.searches[sinfo.dest]; isIn {
+			sinfo.searches.router.core.log.Debugln("Finished search:", &sinfo.dest, sinfo.send, sinfo.recv)
+			delete(sinfo.searches.searches, res.Dest)
+		}
 	}
 	// They match, so create a session and send a sessionRequest
 	var err error

src/yggdrasil/session.go

@@ -16,9 +16,6 @@ import (
 	"github.com/Arceliar/phony"
 )
 
-// Duration that we keep track of old nonces per session, to allow some out-of-order packet delivery
-const nonceWindow = time.Second
-
 // All the information we know about an active session.
 // This includes coords, permanent and ephemeral keys, handles and nonces, various sorts of timing information for timeout and maintenance, and some metadata for the admin API.
 type sessionInfo struct {
@@ -36,14 +33,13 @@ type sessionInfo struct {
 	myHandle      crypto.Handle       //
 	theirNonce    crypto.BoxNonce     //
 	myNonce       crypto.BoxNonce     //
-	theirMTU      uint16              //
-	myMTU         uint16              //
+	theirMTU      MTU                 //
+	myMTU         MTU                 //
 	wasMTUFixed   bool                // Was the MTU fixed by a receive error?
-	timeOpened    time.Time           // Time the sessino was opened
+	timeOpened    time.Time           // Time the session was opened
 	time          time.Time           // Time we last received a packet
 	mtuTime       time.Time           // time myMTU was last changed
 	pingTime      time.Time           // time the first ping was sent since the last received packet
-	pingSend      time.Time           // time the last ping was sent
 	coords        []byte              // coords of destination
 	reset         bool                // reset if coords change
 	tstamp        int64               // ATOMIC - tstamp from their last session ping, replay attack mitigation
@@ -53,9 +49,10 @@ type sessionInfo struct {
 	cancel        util.Cancellation   // Used to terminate workers
 	conn          *Conn               // The associated Conn object
 	callbacks     []chan func()       // Finished work from crypto workers
+	table         *lookupTable        // table.self is a locator where we get our coords
 }
 
-// Represents a session ping/pong packet, andincludes information like public keys, a session handle, coords, a timestamp to prevent replays, and the tun/tap MTU.
+// Represents a session ping/pong packet, and includes information like public keys, a session handle, coords, a timestamp to prevent replays, and the tun/tap MTU.
 type sessionPing struct {
 	SendPermPub crypto.BoxPubKey // Sender's permanent key
 	Handle      crypto.Handle    // Random number to ID session
@@ -63,7 +60,7 @@ type sessionPing struct {
 	Coords      []byte           //
 	Tstamp      int64            // unix time, but the only real requirement is that it increases
 	IsPong      bool             //
-	MTU         uint16           //
+	MTU         MTU              //
 }
 
 // Updates session info in response to a ping, after checking that the ping is OK.
@@ -117,7 +114,7 @@ type sessions struct {
 	lastCleanup      time.Time
 	isAllowedHandler func(pubkey *crypto.BoxPubKey, initiator bool) bool // Returns true or false if session setup is allowed
 	isAllowedMutex   sync.RWMutex                                        // Protects the above
-	myMaximumMTU     uint16                                              // Maximum allowed session MTU
+	myMaximumMTU     MTU                                                 // Maximum allowed session MTU
 	permShared       map[crypto.BoxPubKey]*crypto.BoxSharedKey           // Maps known permanent keys to their shared key, used by DHT a lot
 	sinfos           map[crypto.Handle]*sessionInfo                      // Maps handle onto session info
 	byTheirPerm      map[crypto.BoxPubKey]*crypto.Handle                 // Maps theirPermPub onto handle
@@ -175,7 +172,7 @@ func (ss *sessions) getByTheirPerm(key *crypto.BoxPubKey) (*sessionInfo, bool) {
 }
 
 // Creates a new session and lazily cleans up old existing sessions. This
-// includse initializing session info to sane defaults (e.g. lowest supported
+// includes initializing session info to sane defaults (e.g. lowest supported
 // MTU).
 func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 	// TODO: this check definitely needs to be moved
@@ -197,7 +194,6 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 	sinfo.time = now
 	sinfo.mtuTime = now
 	sinfo.pingTime = now
-	sinfo.pingSend = now
 	sinfo.init = make(chan struct{})
 	sinfo.cancel = util.NewCancellation()
 	higher := false
@@ -219,6 +215,7 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 	sinfo.myHandle = *crypto.NewHandle()
 	sinfo.theirAddr = *address.AddrForNodeID(crypto.GetNodeID(&sinfo.theirPermPub))
 	sinfo.theirSubnet = *address.SubnetForNodeID(crypto.GetNodeID(&sinfo.theirPermPub))
+	sinfo.table = ss.router.table
 	ss.sinfos[sinfo.myHandle] = &sinfo
 	ss.byTheirPerm[sinfo.theirPermPub] = &sinfo.myHandle
 	return &sinfo
@@ -268,8 +265,7 @@ func (ss *sessions) removeSession(sinfo *sessionInfo) {
 
 // Returns a session ping appropriate for the given session info.
 func (sinfo *sessionInfo) _getPing() sessionPing {
-	loc := sinfo.sessions.router.core.switchTable.getLocator()
-	coords := loc.getCoords()
+	coords := sinfo.table.self.getCoords()
 	ping := sessionPing{
 		SendPermPub: sinfo.sessions.router.core.boxPub,
 		Handle:      sinfo.myHandle,
@@ -385,7 +381,7 @@ func (ss *sessions) handlePing(ping *sessionPing) {
 // Get the MTU of the session.
 // Will be equal to the smaller of this node's MTU or the remote node's MTU.
 // If sending over links with a maximum message size (this was a thing with the old UDP code), it could be further lowered, to a minimum of 1280.
-func (sinfo *sessionInfo) _getMTU() uint16 {
+func (sinfo *sessionInfo) _getMTU() MTU {
 	if sinfo.theirMTU == 0 || sinfo.myMTU == 0 {
 		return 0
 	}
@@ -395,14 +391,9 @@ func (sinfo *sessionInfo) _getMTU() uint16 {
 	return sinfo.myMTU
 }
 
-// Checks if a packet's nonce is recent enough to fall within the window of allowed packets, and not already received.
+// Checks if a packet's nonce is newer than any previously received
 func (sinfo *sessionInfo) _nonceIsOK(theirNonce *crypto.BoxNonce) bool {
-	// The bitmask is to allow for some non-duplicate out-of-order packets
-	if theirNonce.Minus(&sinfo.theirNonce) > 0 {
-		// This is newer than the newest nonce we've seen
-		return true
-	}
-	return time.Since(sinfo.time) < nonceWindow
+	return theirNonce.Minus(&sinfo.theirNonce) > 0
 }
 
 // Updates the nonce mask by (possibly) shifting the bitmask and setting the bit corresponding to this nonce to 1, and then updating the most recent nonce
@@ -415,7 +406,7 @@ func (sinfo *sessionInfo) _updateNonce(theirNonce *crypto.BoxNonce) {
 }
 
 // Resets all sessions to an uninitialized state.
-// Called after coord changes, so attemtps to use a session will trigger a new ping and notify the remote end of the coord change.
+// Called after coord changes, so attempts to use a session will trigger a new ping and notify the remote end of the coord change.
 // Only call this from the router actor.
 func (ss *sessions) reset() {
 	for _, _sinfo := range ss.sinfos {
@@ -457,12 +448,9 @@ func (sinfo *sessionInfo) _recvPacket(p *wire_trafficPacket) {
 	select {
 	case <-sinfo.init:
 	default:
-		// TODO find a better way to drop things until initialized
-		util.PutBytes(p.Payload)
 		return
 	}
 	if !sinfo._nonceIsOK(&p.Nonce) {
-		util.PutBytes(p.Payload)
 		return
 	}
 	k := sinfo.sharedSesKey
@@ -472,11 +460,9 @@ func (sinfo *sessionInfo) _recvPacket(p *wire_trafficPacket) {
 	poolFunc := func() {
 		bs, isOK = crypto.BoxOpen(&k, p.Payload, &p.Nonce)
 		callback := func() {
-			util.PutBytes(p.Payload)
 			if !isOK || k != sinfo.sharedSesKey || !sinfo._nonceIsOK(&p.Nonce) {
 				// Either we failed to decrypt, or the session was updated, or we
 				// received this packet in the mean time
-				util.PutBytes(bs)
 				return
 			}
 			sinfo._updateNonce(&p.Nonce)
@@ -494,8 +480,6 @@ func (sinfo *sessionInfo) _send(msg FlowKeyMessage) {
 	select {
 	case <-sinfo.init:
 	default:
-		// TODO find a better way to drop things until initialized
-		util.PutBytes(msg.Message)
 		return
 	}
 	sinfo.bytesSent += uint64(len(msg.Message))
@@ -514,14 +498,8 @@ func (sinfo *sessionInfo) _send(msg FlowKeyMessage) {
 	ch := make(chan func(), 1)
 	poolFunc := func() {
 		p.Payload, _ = crypto.BoxSeal(&k, msg.Message, &p.Nonce)
+		packet := p.encode()
 		callback := func() {
-			// Encoding may block on a util.GetBytes(), so kept out of the worker pool
-			packet := p.encode()
-			// Cleanup
-			util.PutBytes(msg.Message)
-			util.PutBytes(p.Payload)
-			// Send the packet
-			// TODO replace this with a send to the peer struct if that becomes an actor
 			sinfo.sessions.router.Act(sinfo, func() {
 				sinfo.sessions.router.out(packet)
 			})

src/yggdrasil/simlink.go

@@ -0,0 +1,91 @@
+package yggdrasil
+
+import (
+	"errors"
+	"github.com/Arceliar/phony"
+)
+
+type Simlink struct {
+	phony.Inbox
+	rch     chan []byte
+	dest    *Simlink
+	link    *link
+	started bool
+}
+
+func (s *Simlink) readMsg() ([]byte, error) {
+	bs, ok := <-s.rch
+	if !ok {
+		return nil, errors.New("read from closed Simlink")
+	}
+	return bs, nil
+}
+
+func (s *Simlink) _recvMetaBytes() ([]byte, error) {
+	return s.readMsg()
+}
+
+func (s *Simlink) _sendMetaBytes(bs []byte) error {
+	_, err := s.writeMsgs([][]byte{bs})
+	return err
+}
+
+func (s *Simlink) close() error {
+	defer func() { recover() }()
+	close(s.rch)
+	return nil
+}
+
+func (s *Simlink) writeMsgs(msgs [][]byte) (int, error) {
+	if s.dest == nil {
+		return 0, errors.New("write to unpaired Simlink")
+	}
+	var size int
+	for _, msg := range msgs {
+		size += len(msg)
+		bs := append([]byte(nil), msg...)
+		phony.Block(s, func() {
+			s.dest.Act(s, func() {
+				defer func() { recover() }()
+				s.dest.rch <- bs
+			})
+		})
+	}
+	return size, nil
+}
+
+func (c *Core) NewSimlink() *Simlink {
+	s := &Simlink{rch: make(chan []byte, 1)}
+	n := "Simlink"
+	var err error
+	s.link, err = c.links.create(s, n, n, n, n, false, true, linkOptions{})
+	if err != nil {
+		panic(err)
+	}
+	return s
+}
+
+func (s *Simlink) SetDestination(dest *Simlink) error {
+	var err error
+	phony.Block(s, func() {
+		if s.dest != nil {
+			err = errors.New("destination already set")
+		} else {
+			s.dest = dest
+		}
+	})
+	return err
+}
+
+func (s *Simlink) Start() error {
+	var err error
+	phony.Block(s, func() {
+		if s.started {
+			err = errors.New("already started")
+		} else {
+			s.started = true
+			go s.link.handler()
+		}
+	})
+	return err
+}

src/yggdrasil/stream.go

@@ -6,12 +6,10 @@ import (
 	"fmt"
 	"io"
 	"net"
-
-	"github.com/yggdrasil-network/yggdrasil-go/src/util"
 )
 
 // Test that this matches the interface we expect
-var _ = linkInterfaceMsgIO(&stream{})
+var _ = linkMsgIO(&stream{})
 
 type stream struct {
 	rwc          io.ReadWriteCloser
@@ -46,6 +44,9 @@ func (s *stream) writeMsgs(bss [][]byte) (int, error) {
 	}
 	s.outputBuffer = buf[:0] // So we can reuse the same underlying array later
 	_, err := buf.WriteTo(s.rwc)
+	for _, bs := range bss {
+		pool_putBytes(bs)
+	}
 	// TODO only include number of bytes from bs *successfully* written?
 	return written, err
 }
@@ -112,7 +113,7 @@ func (s *stream) readMsgFromBuffer() ([]byte, error) {
 	if msgLen > streamMsgSize {
 		return nil, errors.New("oversized message")
 	}
-	msg := util.ResizeBytes(util.GetBytes(), int(msgLen))
+	msg := pool_getBytes(int(msgLen))
 	_, err = io.ReadFull(s.inputBuffer, msg)
 	return msg, err
 }

src/yggdrasil/switch.go

@@ -12,13 +12,9 @@ package yggdrasil
 //  A little annoying to do with constant changes from backpressure
 
 import (
-	"math/rand"
-	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
-	"github.com/yggdrasil-network/yggdrasil-go/src/util"
 
 	"github.com/Arceliar/phony"
 )
@@ -97,6 +93,20 @@ func (l *switchLocator) dist(dest []byte) int {
 	return dist
 }
 
+func (l *switchLocator) ldist(sl *switchLocator) int {
+	lca := -1
+	for idx := 0; idx < len(l.coords); idx++ {
+		if idx >= len(sl.coords) {
+			break
+		}
+		if l.coords[idx] != sl.coords[idx] {
+			break
+		}
+		lca = idx
+	}
+	return len(l.coords) + len(sl.coords) - 2*(lca+1)
+}
+
 // Gets coords in wire encoded format, with *no* length prefix.
 func (l *switchLocator) getCoords() []byte {
 	bs := make([]byte, 0, len(l.coords))
@@ -107,7 +117,7 @@ func (l *switchLocator) getCoords() []byte {
 	return bs
 }
 
-// Returns true if the this locator represents an ancestor of the locator given as an argument.
+// Returns true if this locator represents an ancestor of the locator given as an argument.
 // Ancestor means that it's the parent node, or the parent of parent, and so on...
 func (x *switchLocator) isAncestorOf(y *switchLocator) bool {
 	if x.root != y.root {
@@ -126,14 +136,19 @@ func (x *switchLocator) isAncestorOf(y *switchLocator) bool {
 
 // Information about a peer, used by the switch to build the tree and eventually make routing decisions.
 type peerInfo struct {
-	key     crypto.SigPubKey      // ID of this peer
-	locator switchLocator         // Should be able to respond with signatures upon request
-	degree  uint64                // Self-reported degree
-	time    time.Time             // Time this node was last seen
-	faster  map[switchPort]uint64 // Counter of how often a node is faster than the current parent, penalized extra if slower
-	port    switchPort            // Interface number of this peer
-	msg     switchMsg             // The wire switchMsg used
-	blocked bool                  // True if the link is blocked, used to avoid parenting a blocked link
+	key        crypto.SigPubKey      // ID of this peer
+	locator    switchLocator         // Should be able to respond with signatures upon request
+	degree     uint64                // Self-reported degree
+	time       time.Time             // Time this node was last seen
+	faster     map[switchPort]uint64 // Counter of how often a node is faster than the current parent, penalized extra if slower
+	port       switchPort            // Interface number of this peer
+	msg        switchMsg             // The wire switchMsg used
+	readBlock  bool                  // True if the link notified us of a read that blocked too long
+	writeBlock bool                  // True of the link notified us of a write that blocked too long
+}
+
+func (pinfo *peerInfo) blocked() bool {
+	return pinfo.readBlock || pinfo.writeBlock
 }
 
 // This is just a uint64 with a named type for clarity reasons.
@@ -144,12 +159,15 @@ type tableElem struct {
 	port    switchPort
 	locator switchLocator
 	time    time.Time
+	next    map[switchPort]*tableElem
 }
 
 // This is the subset of the information about all peers needed to make routing decisions, and it stored separately in an atomically accessed table, which gets hammered in the "hot loop" of the routing logic (see: peer.handleTraffic in peers.go).
 type lookupTable struct {
-	self  switchLocator
-	elems map[switchPort]tableElem
+	self   switchLocator
+	elems  map[switchPort]tableElem // all switch peers, just for sanity checks + API/debugging
+	_start tableElem                // used for lookups
+	_msg   switchMsg
 }
 
 // This is switch information which is mutable and needs to be modified by other goroutines, but is not accessed atomically.
@@ -158,7 +176,6 @@ type switchData struct {
 	// All data that's mutable and used by exported Table methods
 	// To be read/written with atomic.Value Store/Load calls
 	locator switchLocator
-	seq     uint64 // Sequence number, reported to peers, so they know about changes
 	peers   map[switchPort]peerInfo
 	msg     *switchMsg
 }
@@ -167,17 +184,11 @@ type switchData struct {
 type switchTable struct {
 	core        *Core
 	key         crypto.SigPubKey           // Our own key
+	phony.Inbox                            // Owns the below
 	time        time.Time                  // Time when locator.tstamp was last updated
 	drop        map[crypto.SigPubKey]int64 // Tstamp associated with a dropped root
-	mutex       sync.RWMutex               // Lock for reads/writes of switchData
 	parent      switchPort                 // Port of whatever peer is our parent, or self if we're root
 	data        switchData                 //
-	updater     atomic.Value               // *sync.Once
-	table       atomic.Value               // lookupTable
-	phony.Inbox                            // Owns the below
-	queues      switch_buffers             // Queues - not atomic so ONLY use through the actor
-	idle        map[switchPort]struct{}    // idle peers - not atomic so ONLY use through the actor
-	sending     map[switchPort]struct{}    // peers known to be blocked in a send (somehow)
 }
 
 // Minimum allowed total size of switch queues.
@@ -191,47 +202,27 @@ func (t *switchTable) init(core *Core) {
 	locator := switchLocator{root: t.key, tstamp: now.Unix()}
 	peers := make(map[switchPort]peerInfo)
 	t.data = switchData{locator: locator, peers: peers}
-	t.updater.Store(&sync.Once{})
-	t.table.Store(lookupTable{})
 	t.drop = make(map[crypto.SigPubKey]int64)
-	phony.Block(t, func() {
-		core.config.Mutex.RLock()
-		if core.config.Current.SwitchOptions.MaxTotalQueueSize > SwitchQueueTotalMinSize {
-			t.queues.totalMaxSize = core.config.Current.SwitchOptions.MaxTotalQueueSize
-		} else {
-			t.queues.totalMaxSize = SwitchQueueTotalMinSize
-		}
-		core.config.Mutex.RUnlock()
-		t.queues.bufs = make(map[string]switch_buffer)
-		t.idle = make(map[switchPort]struct{})
-		t.sending = make(map[switchPort]struct{})
-	})
+	phony.Block(t, t._updateTable)
 }
 
 func (t *switchTable) reconfigure() {
 	// This is where reconfiguration would go, if we had anything useful to do.
-	t.core.link.reconfigure()
+	t.core.links.reconfigure()
 	t.core.peers.reconfigure()
 }
 
-// Safely gets a copy of this node's locator.
-func (t *switchTable) getLocator() switchLocator {
-	t.mutex.RLock()
-	defer t.mutex.RUnlock()
-	return t.data.locator.clone()
-}
-
 // Regular maintenance to possibly timeout/reset the root and similar.
-func (t *switchTable) doMaintenance() {
-	// Periodic maintenance work to keep things internally consistent
-	t.mutex.Lock()         // Write lock
-	defer t.mutex.Unlock() // Release lock when we're done
-	t.cleanRoot()
-	t.cleanDropped()
+func (t *switchTable) doMaintenance(from phony.Actor) {
+	t.Act(from, func() {
+		// Periodic maintenance work to keep things internally consistent
+		t._cleanRoot()
+		t._cleanDropped()
+	})
 }
 
 // Updates the root periodically if it is ourself, or promotes ourself to root if we're better than the current root or if the current root has timed out.
-func (t *switchTable) cleanRoot() {
+func (t *switchTable) _cleanRoot() {
 	// TODO rethink how this is done?...
 	// Get rid of the root if it looks like its timed out
 	now := time.Now()
@@ -255,59 +246,79 @@ func (t *switchTable) cleanRoot() {
 		t.parent = switchPort(0)
 		t.time = now
 		if t.data.locator.root != t.key {
-			t.data.seq++
-			t.updater.Store(&sync.Once{})
-			t.core.router.reset(nil)
+			defer t.core.router.reset(nil)
 		}
 		t.data.locator = switchLocator{root: t.key, tstamp: now.Unix()}
+		t._updateTable() // updates base copy of switch msg in lookupTable
 		t.core.peers.sendSwitchMsgs(t)
 	}
 }
 
 // Blocks and, if possible, unparents a peer
-func (t *switchTable) blockPeer(port switchPort) {
-	t.mutex.Lock()
-	defer t.mutex.Unlock()
-	peer, isIn := t.data.peers[port]
-	if !isIn {
-		return
-	}
-	peer.blocked = true
-	t.data.peers[port] = peer
-	if port != t.parent {
-		return
-	}
-	t.parent = 0
-	for _, info := range t.data.peers {
-		if info.port == port {
-			continue
+func (t *switchTable) blockPeer(from phony.Actor, port switchPort, isWrite bool) {
+	t.Act(from, func() {
+		peer, isIn := t.data.peers[port]
+		switch {
+		case isIn && !isWrite && !peer.readBlock:
+			peer.readBlock = true
+		case isIn && isWrite && !peer.writeBlock:
+			peer.writeBlock = true
+		default:
+			return
 		}
-		t.unlockedHandleMsg(&info.msg, info.port, true)
-	}
-	t.unlockedHandleMsg(&peer.msg, peer.port, true)
+		t.data.peers[port] = peer
+		defer t._updateTable()
+		if port != t.parent {
+			return
+		}
+		t.parent = 0
+		for _, info := range t.data.peers {
+			if info.port == port {
+				continue
+			}
+			t._handleMsg(&info.msg, info.port, true)
+		}
+		t._handleMsg(&peer.msg, peer.port, true)
+	})
+}
+
+func (t *switchTable) unblockPeer(from phony.Actor, port switchPort, isWrite bool) {
+	t.Act(from, func() {
+		peer, isIn := t.data.peers[port]
+		switch {
+		case isIn && !isWrite && peer.readBlock:
+			peer.readBlock = false
+		case isIn && isWrite && peer.writeBlock:
+			peer.writeBlock = false
+		default:
+			return
+		}
+		t.data.peers[port] = peer
+		t._updateTable()
+	})
 }
 
 // Removes a peer.
 // Must be called by the router actor with a lambda that calls this.
 // If the removed peer was this node's parent, it immediately tries to find a new parent.
-func (t *switchTable) forgetPeer(port switchPort) {
-	t.mutex.Lock()
-	defer t.mutex.Unlock()
-	delete(t.data.peers, port)
-	t.updater.Store(&sync.Once{})
-	if port != t.parent {
-		return
-	}
-	t.parent = 0
-	for _, info := range t.data.peers {
-		t.unlockedHandleMsg(&info.msg, info.port, true)
-	}
+func (t *switchTable) forgetPeer(from phony.Actor, port switchPort) {
+	t.Act(from, func() {
+		delete(t.data.peers, port)
+		defer t._updateTable()
+		if port != t.parent {
+			return
+		}
+		t.parent = 0
+		for _, info := range t.data.peers {
+			t._handleMsg(&info.msg, info.port, true)
+		}
+	})
 }
 
 // Dropped is a list of roots that are better than the current root, but stopped sending new timestamps.
 // If we switch to a new root, and that root is better than an old root that previously timed out, then we can clean up the old dropped root infos.
 // This function is called periodically to do that cleanup.
-func (t *switchTable) cleanDropped() {
+func (t *switchTable) _cleanDropped() {
 	// TODO? only call this after root changes, not periodically
 	for root := range t.drop {
 		if !firstIsBetter(&root, &t.data.locator.root) {
@@ -333,9 +344,7 @@ type switchMsgHop struct {
 }
 
 // This returns a *switchMsg to a copy of this node's current switchMsg, which can safely have additional information appended to Hops and sent to a peer.
-func (t *switchTable) getMsg() *switchMsg {
-	t.mutex.RLock()
-	defer t.mutex.RUnlock()
+func (t *switchTable) _getMsg() *switchMsg {
 	if t.parent == 0 {
 		return &switchMsg{Root: t.key, TStamp: t.data.locator.tstamp}
 	} else if parent, isIn := t.data.peers[t.parent]; isIn {
@@ -347,14 +356,18 @@ func (t *switchTable) getMsg() *switchMsg {
 	}
 }
 
+func (t *lookupTable) getMsg() *switchMsg {
+	msg := t._msg
+	msg.Hops = append([]switchMsgHop(nil), t._msg.Hops...)
+	return &msg
+}
+
 // This function checks that the root information in a switchMsg is OK.
 // In particular, that the root is better, or else the same as the current root but with a good timestamp, and that this root+timestamp haven't been dropped due to timeout.
-func (t *switchTable) checkRoot(msg *switchMsg) bool {
+func (t *switchTable) _checkRoot(msg *switchMsg) bool {
 	// returns false if it's a dropped root, not a better root, or has an older timestamp
 	// returns true otherwise
 	// used elsewhere to keep inserting peers into the dht only if root info is OK
-	t.mutex.RLock()
-	defer t.mutex.RUnlock()
 	dropTstamp, isIn := t.drop[msg.Root]
 	switch {
 	case isIn && dropTstamp >= msg.TStamp:
@@ -370,20 +383,13 @@ func (t *switchTable) checkRoot(msg *switchMsg) bool {
 	}
 }
 
-// This is a mutexed wrapper to unlockedHandleMsg, and is called by the peer structs in peers.go to pass a switchMsg for that peer into the switch.
-func (t *switchTable) handleMsg(msg *switchMsg, fromPort switchPort) {
-	t.mutex.Lock()
-	defer t.mutex.Unlock()
-	t.unlockedHandleMsg(msg, fromPort, false)
-}
-
 // This updates the switch with information about a peer.
 // Then the tricky part, it decides if it should update our own locator as a result.
 // That happens if this node is already our parent, or is advertising a better root, or is advertising a better path to the same root, etc...
 // There are a lot of very delicate order sensitive checks here, so its' best to just read the code if you need to understand what it's doing.
-// It's very important to not change the order of the statements in the case function unless you're absolutely sure that it's safe, including safe if used along side nodes that used the previous order.
+// It's very important to not change the order of the statements in the case function unless you're absolutely sure that it's safe, including safe if used alongside nodes that used the previous order.
 // Set the third arg to true if you're reprocessing an old message, e.g. to find a new parent after one disconnects, to avoid updating some timing related things.
-func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort, reprocessing bool) {
+func (t *switchTable) _handleMsg(msg *switchMsg, fromPort switchPort, reprocessing bool) {
 	// TODO directly use a switchMsg instead of switchMessage + sigs
 	now := time.Now()
 	// Set up the sender peerInfo
@@ -397,6 +403,9 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort, rep
 		sender.key = prevKey
 		prevKey = hop.Next
 	}
+	if sender.key == t.key {
+		return // Don't peer with ourself via different interfaces
+	}
 	sender.msg = *msg
 	sender.port = fromPort
 	sender.time = now
@@ -426,7 +435,8 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort, rep
 	if reprocessing {
 		sender.faster = oldSender.faster
 		sender.time = oldSender.time
-		sender.blocked = oldSender.blocked
+		sender.readBlock = oldSender.readBlock
+		sender.writeBlock = oldSender.writeBlock
 	} else {
 		sender.faster = make(map[switchPort]uint64, len(oldSender.faster))
 		for port, peer := range t.data.peers {
@@ -449,6 +459,9 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort, rep
 			}
 		}
 	}
+	if sender.blocked() != oldSender.blocked() {
+		doUpdate = true
+	}
 	// Update sender
 	t.data.peers[fromPort] = sender
 	// Decide if we should also update our root info to make the sender our parent
@@ -486,10 +499,10 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort, rep
 	case sender.faster[t.parent] >= switch_faster_threshold:
 		// The is reliably faster than the current parent.
 		updateRoot = true
-	case !sender.blocked && oldParent.blocked:
+	case !sender.blocked() && oldParent.blocked():
 		// Replace a blocked parent
 		updateRoot = true
-	case reprocessing && sender.blocked && !oldParent.blocked:
+	case reprocessing && sender.blocked() && !oldParent.blocked():
 		// Don't replace an unblocked parent when reprocessing
 	case reprocessing && sender.faster[t.parent] > oldParent.faster[sender.port]:
 		// The sender seems to be reliably faster than the current parent, so switch to them instead.
@@ -506,75 +519,109 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort, rep
 			if peer.port == sender.port {
 				continue
 			}
-			t.unlockedHandleMsg(&peer.msg, peer.port, true)
+			t._handleMsg(&peer.msg, peer.port, true)
 		}
 		// Process the sender last, to avoid keeping them as a parent if at all possible.
-		t.unlockedHandleMsg(&sender.msg, sender.port, true)
+		t._handleMsg(&sender.msg, sender.port, true)
 	case now.Sub(t.time) < switch_throttle:
 		// We've already gotten an update from this root recently, so ignore this one to avoid flooding.
 	case sender.locator.tstamp > t.data.locator.tstamp:
 		// The timestamp was updated, so we need to update locally and send to our peers.
 		updateRoot = true
 	}
+	// Note that we depend on the LIFO order of the stack of defers here...
 	if updateRoot {
+		doUpdate = true
 		if !equiv(&sender.locator, &t.data.locator) {
-			doUpdate = true
-			t.data.seq++
-			t.core.router.reset(nil)
+			defer t.core.router.reset(t)
 		}
 		if t.data.locator.tstamp != sender.locator.tstamp {
 			t.time = now
 		}
 		t.data.locator = sender.locator
 		t.parent = sender.port
-		t.core.peers.sendSwitchMsgs(t)
+		defer t.core.peers.sendSwitchMsgs(t)
 	}
-	if true || doUpdate {
-		t.updater.Store(&sync.Once{})
+	if doUpdate {
+		t._updateTable()
 	}
-	return
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 
-// The rest of these are related to the switch worker
+// The rest of these are related to the switch lookup table
 
-// This is called via a sync.Once to update the atomically readable subset of switch information that gets used for routing decisions.
-func (t *switchTable) updateTable() {
-	// WARNING this should only be called from within t.data.updater.Do()
-	//  It relies on the sync.Once for synchronization with messages and lookups
-	// TODO use a pre-computed faster lookup table
-	//  Instead of checking distance for every destination every time
-	//  Array of structs, indexed by first coord that differs from self
-	//  Each struct has stores the best port to forward to, and a next coord map
-	//  Move to struct, then iterate over coord maps until you dead end
-	//  The last port before the dead end should be the closest
-	t.mutex.RLock()
-	defer t.mutex.RUnlock()
+func (t *switchTable) _updateTable() {
 	newTable := lookupTable{
 		self:  t.data.locator.clone(),
 		elems: make(map[switchPort]tableElem, len(t.data.peers)),
+		_msg:  *t._getMsg(),
 	}
+	newTable._init()
 	for _, pinfo := range t.data.peers {
-		//if !pinfo.forward { continue }
-		if pinfo.locator.root != newTable.self.root {
+		if pinfo.blocked() || pinfo.locator.root != newTable.self.root {
 			continue
 		}
 		loc := pinfo.locator.clone()
 		loc.coords = loc.coords[:len(loc.coords)-1] // Remove the them->self link
-		newTable.elems[pinfo.port] = tableElem{
+		elem := tableElem{
 			locator: loc,
 			port:    pinfo.port,
 			time:    pinfo.time,
 		}
+		newTable._insert(&elem)
+		newTable.elems[pinfo.port] = elem
 	}
-	t.table.Store(newTable)
+	t.core.peers.updateTables(t, &newTable)
+	t.core.router.updateTable(t, &newTable)
 }
 
-// Returns a copy of the atomically-updated table used for switch lookups
-func (t *switchTable) getTable() lookupTable {
-	t.updater.Load().(*sync.Once).Do(t.updateTable)
-	return t.table.Load().(lookupTable)
+func (t *lookupTable) _init() {
+	// WARNING: this relies on the convention that the self port is 0
+	self := tableElem{locator: t.self} // create self elem
+	t._start = self                    // initialize _start to self
+	t._insert(&self)                   // insert self into table
+}
+
+func (t *lookupTable) _insert(elem *tableElem) {
+	// This is a helper that should only be run during _updateTable
+	here := &t._start
+	for idx := 0; idx <= len(elem.locator.coords); idx++ {
+		refLoc := here.locator
+		refLoc.coords = refLoc.coords[:idx] // Note that this is length idx (starts at length 0)
+		oldDist := refLoc.ldist(&here.locator)
+		newDist := refLoc.ldist(&elem.locator)
+		var update bool
+		switch {
+		case newDist < oldDist: // new elem is closer to this point in the tree
+			update = true
+		case newDist > oldDist: // new elem is too far
+		case elem.locator.tstamp > refLoc.tstamp: // new elem has a closer timestamp
+			update = true
+		case elem.locator.tstamp < refLoc.tstamp: // new elem's timestamp is too old
+		case elem.time.Before(here.time): // same dist+timestamp, but new elem delivered it faster
+			update = true
+		}
+		if update {
+			here.port = elem.port
+			here.locator = elem.locator
+			here.time = elem.time
+			// Problem: here is a value, so this doesn't actually update anything...
+		}
+		if idx < len(elem.locator.coords) {
+			if here.next == nil {
+				here.next = make(map[switchPort]*tableElem)
+			}
+			var next *tableElem
+			var ok bool
+			if next, ok = here.next[elem.locator.coords[idx]]; !ok {
+				nextVal := *elem
+				next = &nextVal
+				here.next[next.locator.coords[idx]] = next
+			}
+			here = next
+		}
+	}
 }
 
 // Starts the switch worker
@@ -584,306 +631,17 @@ func (t *switchTable) start() error {
 	return nil
 }
 
-type closerInfo struct {
-	elem tableElem
-	dist int
-}
-
-// Return a map of ports onto distance, keeping only ports closer to the destination than this node
-// If the map is empty (or nil), then no peer is closer
-func (t *switchTable) getCloser(dest []byte) []closerInfo {
-	table := t.getTable()
-	myDist := table.self.dist(dest)
-	if myDist == 0 {
-		// Skip the iteration step if it's impossible to be closer
-		return nil
-	}
-	t.queues.closer = t.queues.closer[:0]
-	for _, info := range table.elems {
-		dist := info.locator.dist(dest)
-		if dist < myDist {
-			t.queues.closer = append(t.queues.closer, closerInfo{info, dist})
-		}
-	}
-	return t.queues.closer
-}
-
-// Returns true if the peer is closer to the destination than ourself
-func (t *switchTable) portIsCloser(dest []byte, port switchPort) bool {
-	table := t.getTable()
-	if info, isIn := table.elems[port]; isIn {
-		theirDist := info.locator.dist(dest)
-		myDist := table.self.dist(dest)
-		return theirDist < myDist
-	} else {
-		return false
-	}
-}
-
-// Get the coords of a packet without decoding
-func switch_getPacketCoords(packet []byte) []byte {
-	_, pTypeLen := wire_decode_uint64(packet)
-	coords, _ := wire_decode_coords(packet[pTypeLen:])
-	return coords
-}
-
-// Returns a unique string for each stream of traffic
-// Equal to coords
-// The sender may append arbitrary info to the end of coords (as long as it's begins with a 0x00) to designate separate traffic streams
-// Currently, it's the IPv6 next header type and the first 2 uint16 of the next header
-// This is equivalent to the TCP/UDP protocol numbers and the source / dest ports
-// TODO figure out if something else would make more sense (other transport protocols?)
-func switch_getPacketStreamID(packet []byte) string {
-	return string(switch_getPacketCoords(packet))
-}
-
-// Returns the flowlabel from a given set of coords
-func switch_getFlowLabelFromCoords(in []byte) []byte {
-	for i, v := range in {
-		if v == 0 {
-			return in[i+1:]
-		}
-	}
-	return []byte{}
-}
-
-// Find the best port for a given set of coords
-func (t *switchTable) bestPortForCoords(coords []byte) switchPort {
-	table := t.getTable()
-	var best switchPort
-	bestDist := table.self.dist(coords)
-	for to, elem := range table.elems {
-		dist := elem.locator.dist(coords)
-		if !(dist < bestDist) {
-			continue
-		}
-		best = to
-		bestDist = dist
-	}
-	return best
-}
-
-// Handle an incoming packet
-// Either send it to ourself, or to the first idle peer that's free
-// Returns true if the packet has been handled somehow, false if it should be queued
-func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]struct{}, sending map[switchPort]struct{}) bool {
-	coords := switch_getPacketCoords(packet)
-	closer := t.getCloser(coords)
-	if len(closer) == 0 {
-		// TODO? call the router directly, and remove the whole concept of a self peer?
-		self := t.core.peers.getPorts()[0]
-		self.sendPacketsFrom(t, [][]byte{packet})
-		return true
-	}
-	var best *closerInfo
-	ports := t.core.peers.getPorts()
-	for _, cinfo := range closer {
-		to := ports[cinfo.elem.port]
-		//_, isIdle := idle[cinfo.elem.port]
-		_, isSending := sending[cinfo.elem.port]
-		var update bool
-		switch {
-		case to == nil:
-			// no port was found, ignore it
-		case isSending:
-			// the port is busy, ignore it
-		case best == nil:
-			// this is the first idle port we've found, so select it until we find a
-			// better candidate port to use instead
-			update = true
-		case cinfo.dist < best.dist:
-			// the port takes a shorter path/is more direct than our current
-			// candidate, so select that instead
-			update = true
-		case cinfo.dist > best.dist:
-			// the port takes a longer path/is less direct than our current candidate,
-			// ignore it
-		case cinfo.elem.locator.tstamp > best.elem.locator.tstamp:
-			// has a newer tstamp from the root, so presumably a better path
-			update = true
-		case cinfo.elem.locator.tstamp < best.elem.locator.tstamp:
-			// has a n older tstamp, so presumably a worse path
-		case cinfo.elem.time.Before(best.elem.time):
-			// same tstamp, but got it earlier, so presumably a better path
-			//t.core.log.Println("DEBUG new best:", best.elem.time, cinfo.elem.time)
-			update = true
-		default:
-			// the search for a port has finished
-		}
-		if update {
-			b := cinfo // because cinfo gets mutated by the iteration
-			best = &b
-		}
-	}
-	if best != nil {
-		if _, isIdle := idle[best.elem.port]; isIdle {
-			delete(idle, best.elem.port)
-			ports[best.elem.port].sendPacketsFrom(t, [][]byte{packet})
-			return true
-		}
-	}
-	// Didn't find anyone idle to send it to
-	return false
-}
-
-// Info about a buffered packet
-type switch_packetInfo struct {
-	bytes []byte
-	time  time.Time // Timestamp of when the packet arrived
-}
-
-// Used to keep track of buffered packets
-type switch_buffer struct {
-	packets []switch_packetInfo // Currently buffered packets, which may be dropped if it grows too large
-	size    uint64              // Total queue size in bytes
-}
-
-type switch_buffers struct {
-	totalMaxSize uint64
-	bufs         map[string]switch_buffer // Buffers indexed by StreamID
-	size         uint64                   // Total size of all buffers, in bytes
-	maxbufs      int
-	maxsize      uint64
-	closer       []closerInfo // Scratch space
-}
-
-func (b *switch_buffers) _cleanup(t *switchTable) {
-	for streamID, buf := range b.bufs {
-		// Remove queues for which we have no next hop
-		packet := buf.packets[0]
-		coords := switch_getPacketCoords(packet.bytes)
-		if len(t.getCloser(coords)) == 0 {
-			for _, packet := range buf.packets {
-				util.PutBytes(packet.bytes)
-			}
-			b.size -= buf.size
-			delete(b.bufs, streamID)
-		}
-	}
-
-	for b.size > b.totalMaxSize {
-		// Drop a random queue
-		target := rand.Uint64() % b.size
-		var size uint64 // running total
-		for streamID, buf := range b.bufs {
-			size += buf.size
-			if size < target {
-				continue
-			}
-			var packet switch_packetInfo
-			packet, buf.packets = buf.packets[0], buf.packets[1:]
-			buf.size -= uint64(len(packet.bytes))
-			b.size -= uint64(len(packet.bytes))
-			util.PutBytes(packet.bytes)
-			if len(buf.packets) == 0 {
-				delete(b.bufs, streamID)
-			} else {
-				// Need to update the map, since buf was retrieved by value
-				b.bufs[streamID] = buf
-			}
-			break
-		}
-	}
-}
-
-// Handles incoming idle notifications
-// Loops over packets and sends the newest one that's OK for this peer to send
-// Returns true if the peer is no longer idle, false if it should be added to the idle list
-func (t *switchTable) _handleIdle(port switchPort) bool {
-	// TODO? only send packets for which this is the best next hop that isn't currently blocked sending
-	to := t.core.peers.getPorts()[port]
-	if to == nil {
-		return true
-	}
-	var packets [][]byte
-	var psize int
-	t.queues._cleanup(t)
-	now := time.Now()
-	for psize < 65535 {
-		var best string
-		var bestPriority float64
-		for streamID, buf := range t.queues.bufs {
-			// Filter over the streams that this node is closer to
-			// Keep the one with the smallest queue
-			packet := buf.packets[0]
-			coords := switch_getPacketCoords(packet.bytes)
-			priority := float64(now.Sub(packet.time)) / float64(buf.size)
-			if priority > bestPriority && t.portIsCloser(coords, port) {
-				best = streamID
-				bestPriority = priority
-			}
-		}
-		if bestPriority != 0 {
-			buf := t.queues.bufs[best]
-			var packet switch_packetInfo
-			// TODO decide if this should be LIFO or FIFO
-			packet, buf.packets = buf.packets[0], buf.packets[1:]
-			buf.size -= uint64(len(packet.bytes))
-			t.queues.size -= uint64(len(packet.bytes))
-			if len(buf.packets) == 0 {
-				delete(t.queues.bufs, best)
-			} else {
-				// Need to update the map, since buf was retrieved by value
-				t.queues.bufs[best] = buf
-			}
-			packets = append(packets, packet.bytes)
-			psize += len(packet.bytes)
+func (t *lookupTable) lookup(coords []byte) switchPort {
+	var offset int
+	here := &t._start
+	for offset < len(coords) {
+		port, l := wire_decode_uint64(coords[offset:])
+		offset += l
+		if next, ok := here.next[switchPort(port)]; ok {
+			here = next
 		} else {
-			// Finished finding packets
 			break
 		}
 	}
-	if len(packets) > 0 {
-		to.sendPacketsFrom(t, packets)
-		return true
-	}
-	return false
-}
-
-func (t *switchTable) packetInFrom(from phony.Actor, bytes []byte) {
-	t.Act(from, func() {
-		t._packetIn(bytes)
-	})
-}
-
-func (t *switchTable) _packetIn(bytes []byte) {
-	// Try to send it somewhere (or drop it if it's corrupt or at a dead end)
-	if !t._handleIn(bytes, t.idle, t.sending) {
-		// There's nobody free to take it right now, so queue it for later
-		packet := switch_packetInfo{bytes, time.Now()}
-		streamID := switch_getPacketStreamID(packet.bytes)
-		buf, bufExists := t.queues.bufs[streamID]
-		buf.packets = append(buf.packets, packet)
-		buf.size += uint64(len(packet.bytes))
-		t.queues.size += uint64(len(packet.bytes))
-		// Keep a track of the max total queue size
-		if t.queues.size > t.queues.maxsize {
-			t.queues.maxsize = t.queues.size
-		}
-		t.queues.bufs[streamID] = buf
-		if !bufExists {
-			// Keep a track of the max total queue count. Only recalculate this
-			// when the queue is new because otherwise repeating len(dict) might
-			// cause unnecessary processing overhead
-			if len(t.queues.bufs) > t.queues.maxbufs {
-				t.queues.maxbufs = len(t.queues.bufs)
-			}
-		}
-		t.queues._cleanup(t)
-	}
-}
-
-func (t *switchTable) _idleIn(port switchPort) {
-	// Try to find something to send to this peer
-	delete(t.sending, port)
-	if !t._handleIdle(port) {
-		// Didn't find anything ready to send yet, so stay idle
-		t.idle[port] = struct{}{}
-	}
-}
-
-func (t *switchTable) _sendingIn(port switchPort) {
-	if _, isIn := t.idle[port]; !isIn {
-		t.sending[port] = struct{}{}
-	}
+	return here.port
 }

src/yggdrasil/tcp.go

@@ -25,6 +25,7 @@ import (
 
 	"golang.org/x/net/proxy"
 
+	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
 )
 
@@ -33,7 +34,7 @@ const tcp_ping_interval = (default_timeout * 2 / 3)
 
 // The TCP listener and information about active TCP connections, to avoid duplication.
 type tcp struct {
-	link      *link
+	links     *links
 	waitgroup sync.WaitGroup
 	mutex     sync.Mutex // Protecting the below
 	listeners map[string]*TcpListener
@@ -57,6 +58,14 @@ type TcpUpgrade struct {
 	name    string
 }
 
+type tcpOptions struct {
+	linkOptions
+	upgrade        *TcpUpgrade
+	socksProxyAddr string
+	socksProxyAuth *proxy.Auth
+	socksPeerAddr  string
+}
+
 func (l *TcpListener) Stop() {
 	defer func() { recover() }()
 	close(l.stop)
@@ -86,8 +95,8 @@ func (t *tcp) getAddr() *net.TCPAddr {
 }
 
 // Initializes the struct.
-func (t *tcp) init(l *link) error {
-	t.link = l
+func (t *tcp) init(l *links) error {
+	t.links = l
 	t.tls.init(t)
 	t.mutex.Lock()
 	t.calls = make(map[string]struct{})
@@ -95,9 +104,9 @@ func (t *tcp) init(l *link) error {
 	t.listeners = make(map[string]*TcpListener)
 	t.mutex.Unlock()
 
-	t.link.core.config.Mutex.RLock()
-	defer t.link.core.config.Mutex.RUnlock()
-	for _, listenaddr := range t.link.core.config.Current.Listen {
+	t.links.core.config.Mutex.RLock()
+	defer t.links.core.config.Mutex.RUnlock()
+	for _, listenaddr := range t.links.core.config.Current.Listen {
 		switch listenaddr[:6] {
 		case "tcp://":
 			if _, err := t.listen(listenaddr[6:], nil); err != nil {
@@ -108,7 +117,7 @@ func (t *tcp) init(l *link) error {
 				return err
 			}
 		default:
-			t.link.core.log.Errorln("Failed to add listener: listener", listenaddr, "is not correctly formatted, ignoring")
+			t.links.core.log.Errorln("Failed to add listener: listener", listenaddr, "is not correctly formatted, ignoring")
 		}
 	}
 
@@ -126,35 +135,35 @@ func (t *tcp) stop() error {
 }
 
 func (t *tcp) reconfigure() {
-	t.link.core.config.Mutex.RLock()
-	added := util.Difference(t.link.core.config.Current.Listen, t.link.core.config.Previous.Listen)
-	deleted := util.Difference(t.link.core.config.Previous.Listen, t.link.core.config.Current.Listen)
-	t.link.core.config.Mutex.RUnlock()
+	t.links.core.config.Mutex.RLock()
+	added := util.Difference(t.links.core.config.Current.Listen, t.links.core.config.Previous.Listen)
+	deleted := util.Difference(t.links.core.config.Previous.Listen, t.links.core.config.Current.Listen)
+	t.links.core.config.Mutex.RUnlock()
 	if len(added) > 0 || len(deleted) > 0 {
 		for _, a := range added {
 			switch a[:6] {
 			case "tcp://":
 				if _, err := t.listen(a[6:], nil); err != nil {
-					t.link.core.log.Errorln("Error adding TCP", a[6:], "listener:", err)
+					t.links.core.log.Errorln("Error adding TCP", a[6:], "listener:", err)
 				}
 			case "tls://":
 				if _, err := t.listen(a[6:], t.tls.forListener); err != nil {
-					t.link.core.log.Errorln("Error adding TLS", a[6:], "listener:", err)
+					t.links.core.log.Errorln("Error adding TLS", a[6:], "listener:", err)
 				}
 			default:
-				t.link.core.log.Errorln("Failed to add listener: listener", a, "is not correctly formatted, ignoring")
+				t.links.core.log.Errorln("Failed to add listener: listener", a, "is not correctly formatted, ignoring")
 			}
 		}
 		for _, d := range deleted {
 			if d[:6] != "tcp://" && d[:6] != "tls://" {
-				t.link.core.log.Errorln("Failed to delete listener: listener", d, "is not correctly formatted, ignoring")
+				t.links.core.log.Errorln("Failed to delete listener: listener", d, "is not correctly formatted, ignoring")
 				continue
 			}
 			t.mutex.Lock()
 			if listener, ok := t.listeners[d[6:]]; ok {
 				t.mutex.Unlock()
 				listener.Stop()
-				t.link.core.log.Infoln("Stopped TCP listener:", d[6:])
+				t.links.core.log.Infoln("Stopped TCP listener:", d[6:])
 			} else {
 				t.mutex.Unlock()
 			}
@@ -196,19 +205,18 @@ func (t *tcp) listener(l *TcpListener, listenaddr string) {
 		t.mutex.Unlock()
 		l.Listener.Close()
 		return
-	} else {
-		t.listeners[listenaddr] = l
-		t.mutex.Unlock()
 	}
+	t.listeners[listenaddr] = l
+	t.mutex.Unlock()
 	// And here we go!
 	defer func() {
-		t.link.core.log.Infoln("Stopping TCP listener on:", l.Listener.Addr().String())
+		t.links.core.log.Infoln("Stopping TCP listener on:", l.Listener.Addr().String())
 		l.Listener.Close()
 		t.mutex.Lock()
 		delete(t.listeners, listenaddr)
 		t.mutex.Unlock()
 	}()
-	t.link.core.log.Infoln("Listening for TCP on:", l.Listener.Addr().String())
+	t.links.core.log.Infoln("Listening for TCP on:", l.Listener.Addr().String())
 	go func() {
 		<-l.stop
 		l.Listener.Close()
@@ -217,11 +225,20 @@ func (t *tcp) listener(l *TcpListener, listenaddr string) {
 	for {
 		sock, err := l.Listener.Accept()
 		if err != nil {
-			t.link.core.log.Errorln("Failed to accept connection:", err)
-			return
+			t.links.core.log.Errorln("Failed to accept connection:", err)
+			select {
+			case <-l.stop:
+				return
+			default:
+			}
+			time.Sleep(time.Second) // So we don't busy loop
+			continue
 		}
 		t.waitgroup.Add(1)
-		go t.handler(sock, true, nil, l.upgrade)
+		options := tcpOptions{
+			upgrade: l.upgrade,
+		}
+		go t.handler(sock, true, options)
 	}
 }
 
@@ -239,12 +256,12 @@ func (t *tcp) startCalling(saddr string) bool {
 // If the dial is successful, it launches the handler.
 // When finished, it removes the outgoing call, so reconnection attempts can be made later.
 // This all happens in a separate goroutine that it spawns.
-func (t *tcp) call(saddr string, options interface{}, sintf string, upgrade *TcpUpgrade) {
+func (t *tcp) call(saddr string, options tcpOptions, sintf string) {
 	go func() {
 		callname := saddr
 		callproto := "TCP"
-		if upgrade != nil {
-			callproto = strings.ToUpper(upgrade.name)
+		if options.upgrade != nil {
+			callproto = strings.ToUpper(options.upgrade.name)
 		}
 		if sintf != "" {
 			callname = fmt.Sprintf("%s/%s/%s", callproto, saddr, sintf)
@@ -263,17 +280,16 @@ func (t *tcp) call(saddr string, options interface{}, sintf string, upgrade *Tcp
 		}()
 		var conn net.Conn
 		var err error
-		socksaddr, issocks := options.(string)
-		if issocks {
+		if options.socksProxyAddr != "" {
 			if sintf != "" {
 				return
 			}
-			dialerdst, er := net.ResolveTCPAddr("tcp", socksaddr)
+			dialerdst, er := net.ResolveTCPAddr("tcp", options.socksProxyAddr)
 			if er != nil {
 				return
 			}
 			var dialer proxy.Dialer
-			dialer, err = proxy.SOCKS5("tcp", dialerdst.String(), nil, proxy.Direct)
+			dialer, err = proxy.SOCKS5("tcp", dialerdst.String(), options.socksProxyAuth, proxy.Direct)
 			if err != nil {
 				return
 			}
@@ -282,7 +298,10 @@ func (t *tcp) call(saddr string, options interface{}, sintf string, upgrade *Tcp
 				return
 			}
 			t.waitgroup.Add(1)
-			t.handler(conn, false, saddr, nil)
+			options.socksPeerAddr = conn.RemoteAddr().String()
+			if ch := t.handler(conn, false, options); ch != nil {
+				<-ch
+			}
 		} else {
 			dst, err := net.ResolveTCPAddr("tcp", saddr)
 			if err != nil {
@@ -344,40 +363,41 @@ func (t *tcp) call(saddr string, options interface{}, sintf string, upgrade *Tcp
 			}
 			conn, err = dialer.Dial("tcp", dst.String())
 			if err != nil {
-				t.link.core.log.Debugf("Failed to dial %s: %s", callproto, err)
+				t.links.core.log.Debugf("Failed to dial %s: %s", callproto, err)
 				return
 			}
 			t.waitgroup.Add(1)
-			t.handler(conn, false, nil, upgrade)
+			if ch := t.handler(conn, false, options); ch != nil {
+				<-ch
+			}
 		}
 	}()
 }
 
-func (t *tcp) handler(sock net.Conn, incoming bool, options interface{}, upgrade *TcpUpgrade) {
+func (t *tcp) handler(sock net.Conn, incoming bool, options tcpOptions) chan struct{} {
 	defer t.waitgroup.Done() // Happens after sock.close
 	defer sock.Close()
 	t.setExtraOptions(sock)
 	var upgraded bool
-	if upgrade != nil {
+	if options.upgrade != nil {
 		var err error
-		if sock, err = upgrade.upgrade(sock); err != nil {
-			t.link.core.log.Errorln("TCP handler upgrade failed:", err)
-			return
-		} else {
-			upgraded = true
+		if sock, err = options.upgrade.upgrade(sock); err != nil {
+			t.links.core.log.Errorln("TCP handler upgrade failed:", err)
+			return nil
 		}
+		upgraded = true
 	}
 	stream := stream{}
 	stream.init(sock)
 	var name, proto, local, remote string
-	if socksaddr, issocks := options.(string); issocks {
-		name = "socks://" + sock.RemoteAddr().String() + "/" + socksaddr
+	if options.socksProxyAddr != "" {
+		name = "socks://" + sock.RemoteAddr().String() + "/" + options.socksPeerAddr
 		proto = "socks"
 		local, _, _ = net.SplitHostPort(sock.LocalAddr().String())
-		remote, _, _ = net.SplitHostPort(socksaddr)
+		remote, _, _ = net.SplitHostPort(options.socksPeerAddr)
 	} else {
 		if upgraded {
-			proto = upgrade.name
+			proto = options.upgrade.name
 			name = proto + "://" + sock.RemoteAddr().String()
 		} else {
 			proto = "tcp"
@@ -386,13 +406,30 @@ func (t *tcp) handler(sock net.Conn, incoming bool, options interface{}, upgrade
 		local, _, _ = net.SplitHostPort(sock.LocalAddr().String())
 		remote, _, _ = net.SplitHostPort(sock.RemoteAddr().String())
 	}
+	localIP := net.ParseIP(local)
+	if localIP = localIP.To16(); localIP != nil {
+		var laddr address.Address
+		var lsubnet address.Subnet
+		copy(laddr[:], localIP)
+		copy(lsubnet[:], localIP)
+		if laddr.IsValid() || lsubnet.IsValid() {
+			// The local address is with the network address/prefix range
+			// This would route ygg over ygg, which we don't want
+			// FIXME ideally this check should happen outside of the core library
+			//  Maybe dial/listen at the application level
+			//  Then pass a net.Conn to the core library (after these kinds of checks are done)
+			t.links.core.log.Debugln("Dropping ygg-tunneled connection", local, remote)
+			return nil
+		}
+	}
 	force := net.ParseIP(strings.Split(remote, "%")[0]).IsLinkLocalUnicast()
-	link, err := t.link.core.link.create(&stream, name, proto, local, remote, incoming, force)
+	link, err := t.links.create(&stream, name, proto, local, remote, incoming, force, options.linkOptions)
 	if err != nil {
-		t.link.core.log.Println(err)
+		t.links.core.log.Println(err)
 		panic(err)
 	}
-	t.link.core.log.Debugln("DEBUG: starting handler for", name)
-	err = link.handler()
-	t.link.core.log.Debugln("DEBUG: stopped handler for", name, err)
+	t.links.core.log.Debugln("DEBUG: starting handler for", name)
+	ch, err := link.handler()
+	t.links.core.log.Debugln("DEBUG: stopped handler for", name, err)
+	return ch
 }

src/yggdrasil/tcp_linux.go

@@ -20,10 +20,10 @@ func (t *tcp) tcpContext(network, address string, c syscall.RawConn) error {
 
 	// Log any errors
 	if bbr != nil {
-		t.link.core.log.Debugln("Failed to set tcp_congestion_control to bbr for socket, SetsockoptString error:", bbr)
+		t.links.core.log.Debugln("Failed to set tcp_congestion_control to bbr for socket, SetsockoptString error:", bbr)
 	}
 	if control != nil {
-		t.link.core.log.Debugln("Failed to set tcp_congestion_control to bbr for socket, Control error:", control)
+		t.links.core.log.Debugln("Failed to set tcp_congestion_control to bbr for socket, Control error:", control)
 	}
 
 	// Return nil because errors here are not considered fatal for the connection, it just means congestion control is suboptimal
@@ -38,7 +38,7 @@ func (t *tcp) getControl(sintf string) func(string, string, syscall.RawConn) err
 		}
 		c.Control(btd)
 		if err != nil {
-			t.link.core.log.Debugln("Failed to set SO_BINDTODEVICE:", sintf)
+			t.links.core.log.Debugln("Failed to set SO_BINDTODEVICE:", sintf)
 		}
 		return t.tcpContext(network, address, c)
 	}

src/yggdrasil/tls.go

@@ -34,7 +34,7 @@ func (t *tcptls) init(tcp *tcp) {
 	}
 
 	edpriv := make(ed25519.PrivateKey, ed25519.PrivateKeySize)
-	copy(edpriv[:], tcp.link.core.sigPriv[:])
+	copy(edpriv[:], tcp.links.core.sigPriv[:])
 
 	certBuf := &bytes.Buffer{}
 
@@ -42,7 +42,7 @@ func (t *tcptls) init(tcp *tcp) {
 	pubtemp := x509.Certificate{
 		SerialNumber: big.NewInt(1),
 		Subject: pkix.Name{
-			CommonName: hex.EncodeToString(tcp.link.core.sigPub[:]),
+			CommonName: hex.EncodeToString(tcp.links.core.sigPub[:]),
 		},
 		NotBefore:             time.Now(),
 		NotAfter:              time.Now().Add(time.Hour * 24 * 365),

src/yggdrasil/version.go

@@ -1,14 +1,14 @@
 package yggdrasil
 
 // This file contains the version metadata struct
-// Used in the inital connection setup and key exchange
+// Used in the initial connection setup and key exchange
 // Some of this could arguably go in wire.go instead
 
 import "github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 
 // This is the version-specific metadata exchanged at the start of a connection.
-// It must always beign with the 4 bytes "meta" and a wire formatted uint64 major version number.
-// The current version also includes a minor version number, and the box/sig/link keys that need to be exchanged to open an connection.
+// It must always begin with the 4 bytes "meta" and a wire formatted uint64 major version number.
+// The current version also includes a minor version number, and the box/sig/link keys that need to be exchanged to open a connection.
 type version_metadata struct {
 	meta [4]byte
 	ver  uint64 // 1 byte in this version
@@ -28,11 +28,11 @@ func version_getBaseMetadata() version_metadata {
 	}
 }
 
-// Gest the length of the metadata for this version, used to know how many bytes to read from the start of a connection.
+// Gets the length of the metadata for this version, used to know how many bytes to read from the start of a connection.
 func version_getMetaLength() (mlen int) {
 	mlen += 4                   // meta
-	mlen += 1                   // ver, as long as it's < 127, which it is in this version
-	mlen += 1                   // minorVer, as long as it's < 127, which it is in this version
+	mlen++                      // ver, as long as it's < 127, which it is in this version
+	mlen++                      // minorVer, as long as it's < 127, which it is in this version
 	mlen += crypto.BoxPubKeyLen // box
 	mlen += crypto.SigPubKeyLen // sig
 	mlen += crypto.BoxPubKeyLen // link

src/yggdrasil/wire.go

@@ -9,7 +9,6 @@ package yggdrasil
 
 import (
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
-	"github.com/yggdrasil-network/yggdrasil-go/src/util"
 )
 
 const (
@@ -230,8 +229,9 @@ type wire_trafficPacket struct {
 }
 
 // Encodes a wire_trafficPacket into its wire format.
+// The returned slice was taken from the pool.
 func (p *wire_trafficPacket) encode() []byte {
-	bs := util.GetBytes()
+	bs := pool_getBytes(0)
 	bs = wire_put_uint64(wire_Traffic, bs)
 	bs = wire_put_coords(p.Coords, bs)
 	bs = append(bs, p.Handle[:]...)
@@ -241,7 +241,9 @@ func (p *wire_trafficPacket) encode() []byte {
 }
 
 // Decodes an encoded wire_trafficPacket into the struct, returning true if successful.
+// Either way, the argument slice is added to the pool.
 func (p *wire_trafficPacket) decode(bs []byte) bool {
+	defer pool_putBytes(bs)
 	var pType uint64
 	switch {
 	case !wire_chop_uint64(&pType, &bs):
@@ -255,7 +257,7 @@ func (p *wire_trafficPacket) decode(bs []byte) bool {
 	case !wire_chop_slice(p.Nonce[:], &bs):
 		return false
 	}
-	p.Payload = append(util.GetBytes(), bs...)
+	p.Payload = append(p.Payload, bs...)
 	return true
 }
 
@@ -380,7 +382,7 @@ func (p *sessionPing) decode(bs []byte) bool {
 	if pType == wire_SessionPong {
 		p.IsPong = true
 	}
-	p.MTU = uint16(mtu)
+	p.MTU = MTU(mtu)
 	return true
 }